@1aboveio/skills 1.0.9 → 1.0.11

package/skills/risk-compliance/tld-detection/scripts/detect_patterns.py

@@ -0,0 +1,346 @@
+#!/usr/bin/env python3
+"""Detect independent patterns in a prepared dataset.
+
+Pricing rates adapter: {"base":"USD","rates":{"YYYY-MM-DD":{"EUR":"0.9"}}}.
+Rates are units of each currency per one base unit, from validated exact-date v2
+quotes. Include no carried or pegged quotes. Same-currency conversion needs no quote.
+"""
+from __future__ import annotations
+
+import argparse
+import hashlib
+import json
+import math
+import random
+import sys
+from collections import Counter, defaultdict
+from datetime import date, datetime, timezone
+from decimal import Decimal, InvalidOperation
+from fractions import Fraction
+from pathlib import Path
+from types import SimpleNamespace
+
+import detect_ladders as ladders
+import detect_streams as streams
+from tld_common import load_dataset, view_rows, scope_of, validate_groups, write_json
+
+
+class ExactRates:
+    def __init__(self, payload):
+        if not isinstance(payload, dict) or set(payload) != {"base", "rates"} or not isinstance(payload["rates"], dict):
+            raise ValueError("pricing rates must contain only base and rates")
+        self.base = currency(payload["base"])
+        self.rates = {}
+        for day, quotes in payload["rates"].items():
+            if date.fromisoformat(day).isoformat() != day or not isinstance(quotes, dict):
+                raise ValueError("invalid pricing rate date or quotes")
+            self.rates[day] = {}
+            for code, value in quotes.items():
+                code = currency(code)
+                if not isinstance(value, str):
+                    raise ValueError("pricing rates must be decimal strings")
+                rate = Decimal(value)
+                if not rate.is_finite() or rate <= 0 or (code == self.base and rate != 1):
+                    raise ValueError("pricing rates must be positive and base must equal one")
+                bounded_float(rate, f"rate {day} {code}")
+                self.rates[day][code] = value
+
+    def rate(self, day, cur):
+        value = self.quote(day, cur)
+        return bounded_float(Decimal(value), f"rate {day} {cur}") if value is not None else None
+
+    def quote(self, day, cur):
+        return "1" if cur == self.base else self.rates.get(day, {}).get(cur)
+
+    def market_move(self, cur, day_a, day_b, source=None):
+        source = source or self.base
+        a, b, sa, sb = (self.rate(day_a, cur), self.rate(day_b, cur), self.rate(day_a, source), self.rate(day_b, source))
+        return abs((b / sb) / (a / sa) - 1) if all((a, b, sa, sb)) else None
+
+
+class SourceRates:
+    """Rebase exact quotes on the charge currency; identity needs no quote."""
+    def __init__(self, rates, source):
+        self.rates, self.base = rates, source
+
+    def rate(self, day, cur):
+        if cur == self.base:
+            return 1.0
+        source = self.rates.rate(day, self.base)
+        target = self.rates.rate(day, cur)
+        if source is None or target is None:
+            return None
+        return bounded_float(Decimal(str(target)) / Decimal(str(source)), f"cross-rate {day} {self.base}/{cur}")
+
+    def market_move(self, cur, day_a, day_b, source=None):
+        return self.rates.market_move(cur, day_a, day_b, source or self.base)
+
+
+def bounded_float(value, label):
+    # The legacy engine rounds peaks to cents and uses binary floats. Refuse
+    # values outside a conservative inference range rather than misclassify them.
+    if not Decimal('1e-100') <= value <= Decimal('1e100'):
+        raise ValueError(f"unsupported pricing inference range for {label}: {value}")
+    result = float(value)
+    if not math.isfinite(result) or result == 0:
+        raise ValueError(f"unsupported pricing inference range for {label}: {value}")
+    return result
+
+
+def retail_exact(amount, constant):
+    value = Fraction(Decimal(amount)) / Fraction(Decimal(str(constant)))
+    nearest = round(value)
+    return any(base > 0 and abs(value - base) <= Fraction(6, 1000)
+               for base in (Fraction(nearest), Fraction(nearest) - Fraction(1, 100),
+                            Fraction(nearest) - Fraction(5, 100), Fraction(nearest) - Fraction(2, 100)))
+
+
+def currency(value):
+    if not isinstance(value, str) or len(value) != 3 or not value.isascii() or not value.isalpha() or value != value.upper():
+        raise ValueError("currency must be a three-letter uppercase code")
+    return value
+
+
+def decimal_arg(value):
+    try:
+        n = Decimal(value)
+        if not n.is_finite() or n < 0:
+            raise ValueError()
+        return n
+    except (InvalidOperation, ValueError):
+        raise argparse.ArgumentTypeError("expected a finite non-negative decimal")
+
+
+def cap_arg(value):
+    try:
+        code, amount = value.split("=", 1)
+        return currency(code), decimal_arg(amount)
+    except (ValueError, argparse.ArgumentTypeError) as exc:
+        raise argparse.ArgumentTypeError("expected CCY=non-negative-decimal") from exc
+
+
+def jsonable(value):
+    if isinstance(value, Decimal):
+        return str(value)
+    if isinstance(value, set):
+        return [jsonable(v) for v in sorted(value)]
+    if isinstance(value, (tuple, list)):
+        return [jsonable(v) for v in value]
+    if isinstance(value, dict):
+        return {str(k): jsonable(v) for k, v in value.items()}
+    return value
+
+
+def scope_key(row):
+    scope = scope_of(row)
+    return tuple(scope[k] for k in ("mid", "descriptor", "view", "currency"))
+
+
+def main(argv=None):
+    p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
+    p.add_argument("dataset", type=Path)
+    p.add_argument("output", type=Path)
+    p.add_argument("--pattern", action="append", choices=("concentration", "escalating", "pricing"), required=True)
+    p.add_argument("--concentration-min-count", type=int)
+    p.add_argument("--concentration-min-share", type=decimal_arg)
+    p.add_argument("--cap", action="append", type=cap_arg, default=[])
+    p.add_argument("--sensitivity-cap", action="append", type=cap_arg, default=[])
+    p.add_argument("--min-levels", type=int, default=3)
+    p.add_argument("--min-cluster-count", type=int, default=100)
+    p.add_argument("--multi-anchor-min-count", type=int, default=24)
+    p.add_argument("--pricing-rates", type=Path)
+    p.add_argument("--pricing-currency", action="append", default=[])
+    p.add_argument("--overwrite", action="store_true")
+    a = p.parse_args(argv)
+    patterns = sorted(set(a.pattern))
+    if "concentration" in patterns and (a.concentration_min_count is None or a.concentration_min_count < 1 or a.concentration_min_share is None or a.concentration_min_share > 1):
+        p.error("concentration requires --concentration-min-count >= 1 and --concentration-min-share between 0 and 1")
+    if a.min_levels < 3 or a.min_cluster_count < 1 or a.multi_anchor_min_count < 1:
+        p.error("invalid escalation thresholds")
+    if len(dict(a.cap)) != len(a.cap):
+        p.error("duplicate currency cap")
+    if "escalating" in patterns and not a.cap:
+        p.error("escalating requires per-currency --cap")
+    try:
+        candidates = sorted({currency(x) for x in a.pricing_currency})
+        rates = ExactRates(json.loads(a.pricing_rates.read_text(encoding="utf-8"))) if a.pricing_rates else ExactRates({"base": "USD", "rates": {}})
+        data = load_dataset(a.dataset)
+        original = {r["transaction_id"]: r for r in data["transactions"]}
+        rows = sorted(view_rows(data), key=lambda r: (scope_key(r), r["transaction_id"]))
+        partitions = defaultdict(list)
+        for row in rows:
+            partitions[scope_key(row)].append(row)
+        groups, diagnostics = [], []
+        config = ladders.LadderConfig(dict(a.cap), a.min_levels, a.min_cluster_count, a.multi_anchor_min_count)
+        # These are the discovery helper's defaults, not calibrated thresholds.
+        opt = SimpleNamespace(peak_tol=.012, peak_min_share=.02, top_k=10, jitter_lo=.02, jitter_hi=.05,
+            scheme_tol=.015, wallet_min_excess=.25, wallet_min_coverage=.45,
+            wallet_min_coverage_with_scheme=.15, scheme_min_share=.5, drift_bucket=.004,
+            retail_min_rows=60, retail_k_min=1., retail_k_max=1.15, retail_k_step=.0025,
+            retail_min_excess=.2, retail_min_share=.3, retail_strong_distinct=8,
+            retail_weak_share=.2, retail_min_distinct=5)
+        def add(pattern, scope, members, details):
+            ids = sorted({r["transaction_id"] for r in members})
+            if not ids:
+                return
+            identity = json.dumps([pattern, scope, ids, details], sort_keys=True, ensure_ascii=False, separators=(",", ":"))
+            groups.append({"id": "G-" + hashlib.sha256(identity.encode()).hexdigest()[:24], "pattern": pattern,
+                           "scope": scope, "member_ids": ids, "details": details})
+        for key, part in sorted(partitions.items()):
+            scope = scope_of(part[0]); counts = Counter(Decimal(r["amount"]) for r in part)
+            if "concentration" in patterns:
+                for amount, count in sorted(counts.items()):
+                    selected = count >= a.concentration_min_count and Decimal(count) / len(part) >= a.concentration_min_share
+                    info = {"amount": str(amount), "count": count, "denominator": len(part), "selected": selected}
+                    diagnostics.append({"pattern": "concentration", "scope": scope, **info})
+                    if selected:
+                        add("concentration", scope, [r for r in part if Decimal(r["amount"]) == amount], info)
+            if "escalating" in patterns:
+                if key[3] not in dict(a.cap):
+                    raise ValueError(f"missing absolute residual cap for {key[3]}")
+                pairs = {(key[3], amount): count for amount, count in counts.items()}
+                found, meta = ladders.detect_ladders(pairs, len(part), config)
+                sensitivity = ladders.sensitivity_analysis(pairs, len(part), config, key[3], [cap for cur, cap in a.sensitivity_cap if cur == key[3]])
+                diagnostics.append({"pattern": "escalating", "scope": scope, "metadata": jsonable(meta), "sensitivity": jsonable(sensitivity)})
+                for ladder in found:
+                    amounts = {pair[1] for pair in ladder["pairs"]}
+                    add("escalating", scope, [r for r in part if Decimal(r["amount"]) in amounts], jsonable(ladder))
+        if "pricing" in patterns:
+            chunks = defaultdict(list)
+            for tx in data["transactions"]:
+                chunks[(tx["mid"], tx["billing_descriptor"], tx["currency"], tx.get("card_bin_country") or "")].append(tx)
+            for (mid, descriptor, source, country), chunk in sorted(chunks.items()):
+                positive = [r for r in chunk if Decimal(r["amount"]) > 0]
+                prepared = [{"transaction_id": r["transaction_id"],
+                             "amount": bounded_float(Decimal(r["amount"]), f"amount {r['transaction_id']}"),
+                             "currency": source,
+                             "day": datetime.fromisoformat(r["transaction_time"]).astimezone(timezone.utc).date().isoformat()} for r in positive]
+                rng = random.Random(7)
+                scope_parts = [(scope_of(part[0]), part) for key, part in sorted(partitions.items()) if key[:2] == (mid, descriptor)]
+                diagnostic = {"pattern": "pricing", "source": source, "country": country,
+                              "count": len(chunk), "zero_excluded": len(chunk) - len(positive)}
+                discoveries = []
+                if not prepared:
+                    diagnostic["reason"] = "no positive pricing support"
+                elif len(positive) < 80:
+                    diagnostic["reason"] = "small group"
+                else:
+                    retail = streams.retail_scan(prepared, opt, rng,
+                        exact_match=lambda r, k: retail_exact(original[r["transaction_id"]]["amount"], k))
+                    source_rates = SourceRates(rates, source)
+                    for r in prepared:
+                        for target in sorted(set(candidates + [source])):
+                            cross = source_rates.rate(r["day"], target)
+                            if cross is not None and (not math.isfinite(r["amount"] * cross) or r["amount"] * cross < 0.005):
+                                raise ValueError(f"unsupported pricing inference range for converted amount {r['transaction_id']} {target}")
+                    walls = streams.infer_wallet(prepared, source_rates, sorted(set(candidates + [source])), opt, rng)
+                    diagnostic.update({"candidates": walls, "retail": retail})
+                    by_id = {r["transaction_id"]: r for r in positive}
+                    if streams.retail_passes(retail, opt):
+                        evidence = [{"transaction_id": r["transaction_id"], "date": r["day"],
+                                     "charge_amount": original[r["transaction_id"]]["amount"]} for r in prepared
+                                    if retail_exact(original[r["transaction_id"]]["amount"], retail["constant"])]
+                        discoveries.append(({"form": "retail", "charge_currency": source, "country": country, "fit": retail}, evidence))
+                    final_candidates = {}
+                    diagnostic["final_candidates"] = final_candidates
+                    for target, info in sorted(walls.items()):
+                        if info["scheme"] not in streams.SCHEMES:
+                            final_candidates[target] = {"accepted": False, "reason": "no scheme candidate"}
+                            continue
+                        # Float peaks propose centers only; membership and gates use
+                        # the exact source amounts and quotes against those centers.
+                        menu = streams.SCHEMES[info["scheme"]]
+                        scale = info["scheme_scale"]
+                        centers = [Decimal(str(x[0])) for x in info["scheme_peaks"]
+                                   if any(abs(Fraction(Decimal(str(x[0]))) - price * scale)
+                                          <= Fraction(Decimal(str(opt.scheme_tol))) * price * scale
+                                          for price in menu)]
+                        all_centers = [Decimal(str(x[0])) for x in info["peaks"]]
+                        evidence = []
+                        all_matched = 0
+                        peak_counts = Counter()
+                        scheme_counts = Counter()
+                        control_matched = 0
+                        control_rng = random.Random(7)
+                        tolerance = Fraction(Decimal(str(opt.peak_tol)))
+                        for r in prepared:
+                            day = r["day"]
+                            src = rates.quote(day, source); dst = rates.quote(day, target)
+                            if source == target:
+                                src = dst = "1"
+                            if src is None or dst is None:
+                                continue
+                            value = Fraction(Decimal(by_id[r["transaction_id"]]["amount"])) * Fraction(Decimal(dst)) / Fraction(Decimal(src))
+                            def matches_center(v, c):
+                                return abs(v - Fraction(c)) <= tolerance * Fraction(c)
+                            peak_matches = [c for c in all_centers if matches_center(value, c)]
+                            if peak_matches:
+                                all_matched += 1
+                                peak_counts[min(peak_matches, key=lambda c: abs(value - Fraction(c)))] += 1
+                            factor = 1 + control_rng.choice((-1, 1)) * control_rng.uniform(opt.jitter_lo, opt.jitter_hi)
+                            control_value = value * Fraction(Decimal(str(factor)))
+                            control_matched += any(matches_center(control_value, c) for c in centers)
+                            matches = [c for c in centers if matches_center(value, c)]
+                            if not matches:
+                                continue
+                            center = min(matches, key=lambda c: abs(value - Fraction(c)))
+                            scheme_counts[center] += 1
+                            evidence.append({"transaction_id": r["transaction_id"], "date": day,
+                                             "charge_amount": original[r["transaction_id"]]["amount"],
+                                             "matched_price": str(center), "exact_rate": str(Decimal(dst) / Decimal(src)),
+                                             "rate_base": rates.base, "source_quote": src, "target_quote": dst})
+                        total = len(prepared)
+                        coverage = len(evidence) / total
+                        control = control_matched / total
+                        scheme_share = len(evidence) / all_matched if all_matched else 0.0
+                        final = {**info, "coverage": coverage,
+                                 "usable_coverage": len(evidence) / info["usable_rows"] if info["usable_rows"] else None,
+                                 "scheme_coverage": coverage, "scheme_share": scheme_share,
+                                 "control": control, "excess": coverage - control,
+                                 "peaks": [(float(c), peak_counts[c]) for c in all_centers],
+                                 "scheme_peaks": [(float(c), scheme_counts[c]) for c in centers]}
+                        exact_coverage = Fraction(len(evidence), total)
+                        exact_excess = Fraction(len(evidence) - control_matched, total)
+                        exact_share = Fraction(len(evidence), all_matched) if all_matched else Fraction(0)
+                        minimum_share = Fraction(Decimal(str(opt.scheme_min_share)))
+                        accepted = (exact_share >= minimum_share
+                                    and exact_excess >= Fraction(Decimal(str(opt.wallet_min_excess))) * Fraction(2, 5)
+                                    and ((exact_coverage >= Fraction(Decimal(str(opt.wallet_min_coverage)))
+                                          and exact_excess >= Fraction(Decimal(str(opt.wallet_min_excess))))
+                                         or exact_coverage >= Fraction(Decimal(str(opt.wallet_min_coverage_with_scheme)))))
+                        final_candidates[target] = {"accepted": accepted, "fit": final, "frozen_count": len(evidence)}
+                        if accepted:
+                            discoveries.append(({"form": "direct" if source == target else "fx-converted", "charge_currency": source,
+                                                 "pricing_currency": target, "country": country, "fit": final}, evidence))
+                for scope, part in scope_parts:
+                    diagnostics.append({**diagnostic, "scope": scope})
+                    eligible = {r["transaction_id"]: r for r in part}
+                    for details, evidence in discoveries:
+                        projected = [e for e in evidence if e["transaction_id"] in eligible]
+                        excluded = [e["transaction_id"] for e in evidence if e["transaction_id"] not in eligible]
+                        diagnostics.append({"pattern": "pricing", "scope": scope, "source": source, "country": country,
+                                            "form": details["form"], "pricing_currency": details.get("pricing_currency"),
+                                            "frozen_count": len(evidence), "projected_count": len(projected),
+                                            "projected_excluded_ids": excluded})
+                        add("pricing", scope, [eligible[e["transaction_id"]] for e in projected], {**details, "evidence": projected})
+        groups.sort(key=lambda g: (g["pattern"], tuple(g["scope"][k] for k in ("mid", "descriptor", "view", "currency")), g["id"]))
+        payload = {"schema_version": 1, "dataset_sha256": data["dataset_sha256"], "period": data["period"], "groups": groups,
+                   "parameters": {"patterns": patterns, "concentration_min_count": a.concentration_min_count,
+                                  "concentration_min_share": str(a.concentration_min_share) if a.concentration_min_share is not None else None,
+                                  "caps": jsonable(dict(a.cap)), "sensitivity_caps": jsonable(a.sensitivity_cap),
+                                  "min_levels": a.min_levels, "min_cluster_count": a.min_cluster_count,
+                                  "multi_anchor_min_count": a.multi_anchor_min_count, "pricing": vars(opt),
+                                  "pricing_min_group": 80, "pricing_seed": 7, "pricing_currencies": candidates,
+                                  "pricing_inference_range": "positive amounts, quotes and cross-rates from 1e-100 through 1e100 inclusive; converted candidates must be at least 0.005; heuristic candidates use float",
+                                  "pricing_rates": {"kind": "exact-date adapter", "sha256": hashlib.sha256(a.pricing_rates.read_bytes()).hexdigest()} if a.pricing_rates else None},
+                   "census": data.get("census", {}), "diagnostics": diagnostics}
+        validate_groups(data, payload)
+        write_json(a.output, payload, overwrite=a.overwrite)
+    except (ValueError, OSError, KeyError, InvalidOperation) as exc:
+        print(f"detect_patterns: {exc}", file=sys.stderr)
+        return 1
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

package/skills/risk-compliance/tld-detection/scripts/detect_streams.py

@@ -144,6 +144,8 @@ def day_drift(rows: list[dict], converted: list[float], centers: list[tuple[floa
     total = 0.0
     weight = 0
     for center, cnt in centers[:5]:
+        if center <= 0:
+            continue
         per_day: dict[str, list[float]] = defaultdict(list)
         for r, v in zip(rows, converted):
             if abs(v - center) <= tol * center:
@@ -210,32 +212,47 @@ def parse_rows(path: Path, cols: dict) -> list[dict]:
     return rows
 
 
+def candidate_rows(rows: list[dict], rates: Rates, cur: str) -> tuple[list[dict], list[float], dict]:
+    usable_rows, vals = [], []
+    missing = Counter()
+    for r in rows:
+        rt = rates.rate(r["day"], cur)
+        src = rates.rate(r["day"], r["currency"])
+        if not rt or not src:
+            reason = "missing_candidate_and_source_rate" if not rt and not src else (
+                "missing_candidate_rate" if not rt else "missing_source_rate")
+            missing[reason] += 1
+            continue
+        usable_rows.append(r)
+        vals.append(r["amount"] / src * rt)
+    return usable_rows, vals, {"total_rows": len(rows), "usable_rows": len(vals),
+                               "missing_rows": len(rows) - len(vals), "missing_reasons": dict(missing)}
+
+
 def infer_wallet(rows: list[dict], rates: Rates, candidates: list[str], opt, rng) -> dict:
     """Per group: coverage under each candidate currency, control against the same centres, day drift."""
     out = {}
     for cur in candidates:
-        vals, ok = [], True
-        for r in rows:
-            rt = rates.rate(r["day"], cur)
-            src = rates.rate(r["day"], r["currency"])
-            if rt is None or src is None:
-                ok = False
-                break
-            vals.append(r["amount"] / src * rt)
-        if not ok or not vals:
+        usable_rows, vals, diagnostics = candidate_rows(rows, rates, cur)
+        if not vals:
+            out[cur] = {**diagnostics, "coverage": None, "usable_coverage": None, "control": None, "excess": None,
+                        "day_drift": None, "peaks": [], "scheme": "none", "scheme_scale": 1,
+                        "scheme_share": 0.0, "scheme_coverage": None, "scheme_peaks": []}
             continue
         cov, peaks = coverage(vals, opt.peak_tol, opt.peak_min_share, opt.top_k)
         ctrl = coverage_against(jitter(vals, rng, opt.jitter_lo, opt.jitter_hi), [c for c, _ in peaks], opt.peak_tol)
-        drift = day_drift(rows, vals, peaks, opt.peak_tol)
+        drift = day_drift(usable_rows, vals, peaks, opt.peak_tol)
         name, scale, sshare, on = match_scheme_peaks(peaks, opt.scheme_tol)
-        out[cur] = {"coverage": round(cov, 4), "control": round(ctrl, 4), "excess": round(cov - ctrl, 4), "day_drift": None if drift is None else round(drift, 4),
+        fraction = len(vals) / len(rows)
+        out[cur] = {**diagnostics, "coverage": round(cov * fraction, 4), "usable_coverage": round(cov, 4),
+                    "control": round(ctrl * fraction, 4), "excess": round((cov - ctrl) * fraction, 4), "day_drift": None if drift is None else round(drift, 4),
                     "peaks": [(round(c, 2), n) for c, n in peaks], "scheme": name, "scheme_scale": scale, "scheme_share": round(sshare, 4),
-                    "scheme_coverage": round(sum(n for _, n in on) / len(vals), 4), "scheme_peaks": [(round(c, 2), n) for c, n in on]}
+                    "scheme_coverage": round(sum(n for _, n in on) / len(rows), 4), "scheme_peaks": [(round(c, 2), n) for c, n in on]}
     return out
 
 
 def wallet_acceptable(info: dict, opt) -> bool:
-    if info["excess"] < opt.wallet_min_excess * 0.4:
+    if info["excess"] is None or info["excess"] < opt.wallet_min_excess * 0.4:
         return False
     if info["coverage"] >= opt.wallet_min_coverage and info["excess"] >= opt.wallet_min_excess:
         return True
@@ -288,7 +305,7 @@ def rate_consistency(rows: list[dict], rates: Rates, wallet: str, peaks: list, o
     return result
 
 
-def retail_scan(rows: list[dict], opt, rng) -> dict:
+def retail_scan(rows: list[dict], opt, rng, exact_match=None) -> dict:
     """Find one constant k such that amount / k is retail-shaped for a large share of rows, with many distinct bases."""
     amounts = [r["amount"] for r in rows]
     empty = {"constant": None, "share": 0.0, "control": 0.0, "excess": 0.0, "distinct_bases": 0, "rows": len(amounts)}
@@ -297,11 +314,12 @@ def retail_scan(rows: list[dict], opt, rng) -> dict:
     best = (1.0, 0.0)
     k = opt.retail_k_min
     while k <= opt.retail_k_max + 1e-9:
-        share = sum(retail_shaped(a / k) for a in amounts) / len(amounts)
+        share = sum(exact_match(r, round(k, 4)) if exact_match else retail_shaped(r["amount"] / k) for r in rows) / len(amounts)
         if share > best[1]:
             best = (round(k, 4), share)
         k += opt.retail_k_step
-    bases = Counter(round(a / best[0], 2) for a in amounts if retail_shaped(a / best[0]))
+    bases = Counter(round(r["amount"] / best[0], 2) for r in rows
+                    if (exact_match(r, best[0]) if exact_match else retail_shaped(r["amount"] / best[0])))
     distinct = sum(1 for _, c in bases.items() if c >= 3)
     ctrl = sum(retail_shaped(a / best[0]) for a in jitter(amounts, rng, 0.004, 0.02)) / len(amounts)
     return {"constant": best[0], "share": round(best[1], 4), "control": round(ctrl, 4), "excess": round(best[1] - ctrl, 4), "distinct_bases": distinct, "rows": len(amounts)}
@@ -383,6 +401,7 @@ def extract_substreams(grows: list[dict], legal: str | None, rates: Rates, opt,
     """Peel streams off a country group: menu (wallet + scheme), retail list, then remainder as off-menu."""
     subs = []
     residual = list(grows)
+    cands = [c for c in dict.fromkeys([legal, *opt.candidate_currencies]) if c]
     for _round in range(opt.max_substreams):
         if len(residual) < opt.min_group:
             break
@@ -393,7 +412,6 @@ def extract_substreams(grows: list[dict], legal: str | None, rates: Rates, opt,
             residual = [r for r in residual if not retail_shaped(r["amount"] / k)]
             subs.append({"stream": f"retail x{k}", "rows": inside, "retail": retail})
             continue
-        cands = [c for c in dict.fromkeys([legal, *opt.candidate_currencies]) if c]
         walls = infer_wallet(residual, rates, cands, opt, rng)
         wallet = choose_wallet(walls, legal, opt)
         if wallet:
@@ -406,7 +424,8 @@ def extract_substreams(grows: list[dict], legal: str | None, rates: Rates, opt,
             label = f"{wallet} menu-{scheme if sshare >= opt.scheme_min_share else 'none'}{' frozen' if frozen else ' live'}"
             subs.append({"stream": label, "rows": inside, "wallet": wallet, "coverage": info["coverage"], "day_drift": info["day_drift"], "scheme": scheme,
                          "scheme_scale": scale, "scheme_share": round(sshare, 3), "peaks": use_peaks, "all_peaks": info["peaks"], "frozen": frozen, "rate_consistency": cons,
-                         "alternatives": {c: i["coverage"] for c, i in walls.items() if c != wallet}})
+                         "alternatives": {c: i["coverage"] for c, i in walls.items() if c != wallet},
+                         "candidate_diagnostics": walls})
             residual = outside
             continue
         if retail_passes(retail, opt):
@@ -417,7 +436,20 @@ def extract_substreams(grows: list[dict], legal: str | None, rates: Rates, opt,
             continue
         break
     if residual:
-        subs.append({"stream": "off-menu", "rows": residual})
+        unpriced, priced = [], []
+        for r in residual:
+            target = priced if any(rates.rate(r["day"], cur) and rates.rate(r["day"], r["currency"])
+                                   for cur in cands) else unpriced
+            target.append(r)
+        for label, stream_rows in (("missing FX coverage", unpriced), ("off-menu", priced)):
+            if stream_rows:
+                diagnostics = {}
+                for cur in cands:
+                    _, vals, info = candidate_rows(stream_rows, rates, cur)
+                    if not vals:
+                        info.update({"coverage": None, "usable_coverage": None})
+                    diagnostics[cur] = info
+                subs.append({"stream": label, "rows": stream_rows, "candidate_diagnostics": diagnostics})
     return subs
 
 

package/skills/risk-compliance/tld-detection/scripts/profile_groups.py

@@ -0,0 +1,109 @@
+#!/usr/bin/env python3
+"""Build reproducible, same-scope group and remainder profiles."""
+import argparse
+import bisect
+from datetime import date, timedelta
+from decimal import Decimal, InvalidOperation
+import json
+
+from tld_common import load_dataset, validate_groups, view_rows, scope_of, write_json
+
+
+def tier_scheme(rows, edges):
+    if edges is None:
+        values = sorted({Decimal(row['amount']) for row in rows})
+        return {'kind': 'exact', 'tiers': [{'label': str(x), 'amount': str(x)} for x in values] + [{'label': 'other'}]}
+    if not isinstance(edges, (list, tuple)) or not edges:
+        raise ValueError('tier edges must be a nonempty sorted list of decimal strings')
+    try:
+        if any(not isinstance(x, str) for x in edges):
+            raise ValueError('tier edges must be decimal strings')
+        values = [Decimal(x) for x in edges]
+        if any(not x.is_finite() or x < 0 for x in values) or any(a >= b for a, b in zip(values, values[1:])):
+            raise ValueError('tier edges must be finite, nonnegative and strictly increasing')
+    except InvalidOperation as exc:
+        raise ValueError('invalid tier edge') from exc
+    bounds = [None] + [str(x) for x in values] + [None]
+    tiers = [{'label': f"[{bounds[i] if bounds[i] is not None else '-∞'}, {bounds[i+1] if bounds[i+1] is not None else '∞'})", 'lower': bounds[i], 'upper': bounds[i+1]} for i in range(len(bounds)-1)]
+    return {'kind': 'edges', 'edges': [str(x) for x in values], 'tiers': tiers}
+
+
+def tier_index(amount, scheme):
+    value = Decimal(amount)
+    if scheme['kind'] == 'edges':
+        return bisect.bisect_right([Decimal(x) for x in scheme['edges']], value)
+    for i, tier in enumerate(scheme['tiers'][:-1]):
+        if value == Decimal(tier['amount']):
+            return i
+    return len(scheme['tiers']) - 1
+
+
+def metrics(rows, days, scheme):
+    def record(items):
+        n = len(items)
+        result = {'count': n}
+        for field, name in [('authorized', 'success'), ('suspected_fraud', 'fraud')]:
+            yes = sum(r.get(field) is True for r in items)
+            unknown = sum(r.get(field) is None for r in items)
+            result[name] = {'numerator': yes, 'denominator': n, 'unknown': unknown,
+                            'rate': yes / n if n and not unknown else None,
+                            'coverage': (n - unknown) / n if n else None}
+        return result
+    def hours(items):
+        counts = [0] * 24
+        for r in items:
+            counts[int(r['hour'])] += 1
+        return counts
+    daily = []
+    for day in days:
+        items = [r for r in rows if str(r['date']) == day]
+        entry = {'date': day, **record(items), 'share': len(items) / len(rows) if rows else None,
+                 'hours': hours(items), 'tier_counts': [0] * len(scheme['tiers'])}
+        for row in items:
+            entry['tier_counts'][tier_index(row['amount'], scheme)] += 1
+        daily.append(entry)
+    total = record(rows)
+    total['tier_counts'] = [sum(day['tier_counts'][i] for day in daily) for i in range(len(scheme['tiers']))]
+    hour_counts = hours(rows)
+    if sum(total['tier_counts']) != len(rows) or sum(hour_counts) != len(rows) or any(sum(day['hours']) != day['count'] or sum(day['tier_counts']) != day['count'] for day in daily):
+        raise ValueError('profile counts do not reconcile')
+    return {'total': total, 'daily': daily, 'hours': hour_counts}
+
+
+def profile_dataset(data, manifest, tier_edges=None):
+    groups = validate_groups(data, manifest)
+    if tier_edges is not None:
+        tier_scheme([], tier_edges)
+    rows = list(view_rows(data))
+    period = data['period']
+    start, end = date.fromisoformat(period['start']), date.fromisoformat(period['end'])
+    days = [(start + timedelta(days=i)).isoformat() for i in range((end-start).days+1)]
+    results = []
+    exposed = set()
+    for group in sorted(groups, key=lambda g: g['id']):
+        members = set(group['member_ids'])
+        scoped = [row for row in rows if scope_of(row) == group['scope']]
+        selected = [row for row in scoped if row['transaction_id'] in members]
+        remainder = [row for row in scoped if row['transaction_id'] not in members]
+        exposed.update(members)
+        scheme = tier_scheme(selected, tier_edges)
+        results.append({'id': group['id'], 'pattern': group['pattern'], 'scope': group['scope'],
+                        'member_ids': sorted(members), 'out_group_member_ids': sorted(r['transaction_id'] for r in remainder),
+                        'tiers': scheme, 'group': metrics(selected, days, scheme), 'out_group': metrics(remainder, days, scheme)})
+    return {'schema_version': 1, 'dataset_sha256': data['dataset_sha256'], 'period': period,
+            'census': data['census'], 'combined_exposure': {'count': len(exposed), 'member_ids': sorted(exposed)}, 'groups': results}
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument('dataset'); parser.add_argument('manifest'); parser.add_argument('output')
+    parser.add_argument('--tier-edges', nargs='+', help='strictly increasing decimal boundaries')
+    parser.add_argument('--overwrite', action='store_true')
+    args = parser.parse_args()
+    with open(args.manifest, encoding='utf-8') as f:
+        manifest = json.load(f)
+    write_json(args.output, profile_dataset(load_dataset(args.dataset), manifest, args.tier_edges), overwrite=args.overwrite)
+
+
+if __name__ == '__main__':
+    main()