@1aboveio/skills 0.0.0-stage → 1.0.0

package/skills/risk-compliance/tld-detection/scripts/detect_streams.py

@@ -0,0 +1,650 @@
+#!/usr/bin/env python3
+"""Infer payment-pattern hypotheses inside a descriptor: pricing currency, preset menus, rate consistency, retail lists.
+
+Discovery layer that runs before exact currency-amount-pair clustering. It reads a cleaned
+authorization CSV plus a daily FX cache and, per descriptor and BIN-country group, reports:
+
+* inferred wallet currency (the currency under which charges collapse onto the fewest peaks)
+* the preset menu (peak set) and which known scheme it matches (10-scheme, 12-scheme)
+* rate consistency: source-specific cross-rate tests for frozen-rate evidence
+* a retail-list constant (list prices ending .99/.95/.98/.00 times one factor)
+* off-menu groups (no peak structure)
+* behaviour metrics: repeat cards, retry after decline, burst regularity, hour concentration,
+  address verification, 3-D Secure, prepaid share, issuer-program concentration
+
+Every threshold is a CLI option and is echoed in the output. Stdlib only.
+"""
+from __future__ import annotations
+
+import argparse
+import csv
+import json
+import math
+import random
+import statistics
+import sys
+import urllib.request
+from collections import Counter, defaultdict
+from datetime import date, datetime, timedelta
+from pathlib import Path
+
+PEGS = {"USD": 1.0, "SAR": 3.75, "AED": 3.6725, "QAR": 3.64, "KWD": 0.3066, "BHD": 0.376, "OMR": 0.3845, "HKD": 7.8, "JOD": 0.709}
+
+COUNTRY_CURRENCY = {
+    "united states of america": "USD", "united states": "USD", "us": "USD", "usa": "USD",
+    "united kingdom": "GBP", "gb": "GBP", "uk": "GBP", "australia": "AUD", "new zealand": "NZD", "canada": "CAD",
+    "switzerland": "CHF", "poland": "PLN", "czech republic": "CZK", "czechia": "CZK", "hungary": "HUF", "norway": "NOK",
+    "sweden": "SEK", "denmark": "DKK", "romania": "RON", "bulgaria": "BGN", "iceland": "ISK", "brazil": "BRL",
+    "mexico": "MXN", "japan": "JPY", "india": "INR", "south africa": "ZAR", "turkey": "TRY", "israel": "ILS",
+    "singapore": "SGD", "hong kong": "HKD", "korea, republic of": "KRW", "south korea": "KRW", "thailand": "THB",
+    "malaysia": "MYR", "philippines": "PHP", "indonesia": "IDR", "saudi arabia": "SAR", "united arab emirates": "AED",
+    "qatar": "QAR", "kuwait": "KWD", "bahrain": "BHD", "oman": "OMR", "jordan": "JOD",
+}
+EURO_COUNTRIES = {
+    "germany", "italy", "france", "spain", "portugal", "greece", "slovakia", "finland", "ireland", "austria",
+    "netherlands", "belgium", "lithuania", "croatia", "slovenia", "estonia", "latvia", "cyprus", "luxembourg", "malta",
+    "moldova, republic of", "moldova", "montenegro", "kosovo",
+}
+for _c in EURO_COUNTRIES:
+    COUNTRY_CURRENCY[_c] = "EUR"
+
+SCHEMES = {
+    "10": [5, 10, 15, 20, 25, 30, 40, 50, 60, 75, 80, 100, 150, 200, 250, 300, 400, 500, 750, 1000, 1500, 2000],
+    "12": [12, 24, 36, 48, 72, 96, 108, 144, 192, 288, 384, 480],
+}
+SCHEME_SCALES = [1, 5, 10, 25, 100, 400, 1000]
+
+
+# ----------------------------------------------------------------------------- rates
+
+def fetch_rates(start: str, end: str, base: str, symbols: list[str]) -> dict:
+    url = f"https://api.frankfurter.dev/v1/{start}..{end}?base={base}&symbols={','.join(sorted(set(symbols)))}"
+    with urllib.request.urlopen(url, timeout=60) as resp:  # noqa: S310 - explicit user action
+        payload = json.load(resp)
+    return {"base": base, "source": url, "rates": payload["rates"]}
+
+
+class Rates:
+    """Daily rates quoted as units of currency per one unit of base; prior-day carry up to ``carry`` days."""
+
+    def __init__(self, cache: dict, carry: int = 7):
+        self.base = cache.get("base", "USD")
+        self.rates = cache["rates"]
+        self.days = sorted(self.rates)
+        self.carry = carry
+        self.carried: Counter = Counter()
+
+    def rate(self, day: str, cur: str) -> float | None:
+        if cur == self.base:
+            return 1.0
+        if cur in PEGS and self.base == "USD":
+            return PEGS[cur]
+        if not self.days:
+            return None
+        d = date.fromisoformat(day)
+        for lag in range(self.carry + 1):
+            dd = (d - timedelta(days=lag)).isoformat()
+            if dd in self.rates and cur in self.rates[dd]:
+                if lag:
+                    self.carried[cur] += 1
+                return self.rates[dd][cur]
+        return None
+
+    def market_move(self, cur: str, day_a: str, day_b: str, source: str | None = None) -> float | None:
+        source = self.base if source is None else source
+        if cur == source:
+            return 0.0
+        a, b = self.rate(day_a, cur), self.rate(day_b, cur)
+        src_a, src_b = self.rate(day_a, source), self.rate(day_b, source)
+        if not a or not b or not src_a or not src_b:
+            return None
+        return abs((b / src_b) / (a / src_a) - 1)
+
+
+# ----------------------------------------------------------------------------- helpers
+
+def find_peaks(values: list[float], tol: float, min_share: float) -> list[tuple[float, int]]:
+    """Group values within ±tol (relative) of the modal value, largest first; return (center, count)."""
+    if not values:
+        return []
+    counts = Counter(round(v, 2) for v in values)
+    n = len(values)
+    used: set[float] = set()
+    peaks = []
+    for v, _ in counts.most_common():
+        if v in used:
+            continue
+        grp = [w for w in counts if w not in used and abs(w - v) <= tol * max(v, 1e-9)]
+        cnt = sum(counts[w] for w in grp)
+        if cnt / n < min_share:
+            continue
+        used.update(grp)
+        center = sum(w * counts[w] for w in grp) / cnt
+        peaks.append((center, cnt))
+    return sorted(peaks, key=lambda p: -p[1])
+
+
+def coverage(values: list[float], tol: float, min_share: float, top_k: int) -> tuple[float, list[tuple[float, int]]]:
+    peaks = find_peaks(values, tol, min_share)[:top_k]
+    return (sum(c for _, c in peaks) / len(values) if values else 0.0), peaks
+
+
+def coverage_against(values: list[float], centers: list[float], tol: float) -> float:
+    """Share of values within tol of any given centre (used for the jittered control)."""
+    if not values or not centers:
+        return 0.0
+    return sum(1 for v in values if any(abs(v - c) <= tol * c for c in centers)) / len(values)
+
+
+def day_drift(rows: list[dict], converted: list[float], centers: list[tuple[float, int]], tol: float, min_per_day: int = 3) -> float | None:
+    """Weighted mean, over the top peaks, of (max - min) / centre of the daily median converted value.
+
+    Under the true wallet currency a preset converts to the same local value every day; under a wrong
+    currency the value follows the cross rate day by day."""
+    total = 0.0
+    weight = 0
+    for center, cnt in centers[:5]:
+        per_day: dict[str, list[float]] = defaultdict(list)
+        for r, v in zip(rows, converted):
+            if abs(v - center) <= tol * center:
+                per_day[r["day"]].append(v)
+        meds = [statistics.median(v) for v in per_day.values() if len(v) >= min_per_day]
+        if len(meds) < 2:
+            continue
+        total += (max(meds) - min(meds)) / center * cnt
+        weight += cnt
+    return total / weight if weight else None
+
+
+def jitter(values: list[float], rng: random.Random, lo: float, hi: float) -> list[float]:
+    return [v * (1 + rng.choice((-1, 1)) * rng.uniform(lo, hi)) for v in values]
+
+
+def retail_shaped(x: float, tol: float = 0.006) -> bool:
+    r = round(x)
+    return any(g > 0 and abs(x - g) <= tol for g in (r, r - 0.01, r - 0.05, r - 0.02))
+
+
+def match_scheme(peaks: list[tuple[float, int]], tol: float) -> tuple[str, int, float]:
+    """Return (scheme name, scale, share of peak attempts on the scheme)."""
+    name, scale, share, _ = match_scheme_peaks(peaks, tol)
+    return name, scale, share
+
+
+def match_scheme_peaks(peaks: list[tuple[float, int]], tol: float) -> tuple[str, int, float, list[tuple[float, int]]]:
+    total = sum(c for _, c in peaks) or 1
+    best = ("none", 1, 0.0, [])
+    for name, menu in SCHEMES.items():
+        for scale in SCHEME_SCALES:
+            on = [(center, cnt) for center, cnt in peaks if any(abs(center - m * scale) <= tol * m * scale for m in menu)]
+            share = sum(c for _, c in on) / total
+            if share > best[2]:
+                best = (name, scale, share, on)
+    return best
+
+
+# ----------------------------------------------------------------------------- core
+
+def parse_rows(path: Path, cols: dict) -> list[dict]:
+    rows = []
+    with path.open(newline="", encoding="utf-8-sig") as fh:
+        for r in csv.DictReader(fh):
+            try:
+                amt = float(r[cols["amount"]])
+            except (KeyError, ValueError):
+                continue
+            ts = r.get(cols["date"], "")[:19]
+            rows.append({
+                "amount": amt,
+                "currency": r.get(cols["currency"], "USD").strip().upper() or "USD",
+                "day": ts[:10],
+                "ts": ts,
+                "descriptor": (r.get(cols["descriptor"], "") or "UNKNOWN_DESCRIPTOR").strip(),
+                "country": (r.get(cols["country"], "") or "UNKNOWN").strip(),
+                "response": (r.get(cols["response"], "") if cols.get("response") else "").strip(),
+                "account": (r.get(cols["account"], "") if cols.get("account") else "").strip(),
+                "card_type": (r.get(cols["card_type"], "") if cols.get("card_type") else "").strip(),
+                "avs": (r.get(cols["avs"], "") if cols.get("avs") else "").strip(),
+                "auth": (r.get(cols["auth"], "") if cols.get("auth") else "").strip(),
+            })
+    return rows
+
+
+def infer_wallet(rows: list[dict], rates: Rates, candidates: list[str], opt, rng) -> dict:
+    """Per group: coverage under each candidate currency, control against the same centres, day drift."""
+    out = {}
+    for cur in candidates:
+        vals, ok = [], True
+        for r in rows:
+            rt = rates.rate(r["day"], cur)
+            src = rates.rate(r["day"], r["currency"])
+            if rt is None or src is None:
+                ok = False
+                break
+            vals.append(r["amount"] / src * rt)
+        if not ok or not vals:
+            continue
+        cov, peaks = coverage(vals, opt.peak_tol, opt.peak_min_share, opt.top_k)
+        ctrl = coverage_against(jitter(vals, rng, opt.jitter_lo, opt.jitter_hi), [c for c, _ in peaks], opt.peak_tol)
+        drift = day_drift(rows, vals, peaks, opt.peak_tol)
+        name, scale, sshare, on = match_scheme_peaks(peaks, opt.scheme_tol)
+        out[cur] = {"coverage": round(cov, 4), "control": round(ctrl, 4), "excess": round(cov - ctrl, 4), "day_drift": None if drift is None else round(drift, 4),
+                    "peaks": [(round(c, 2), n) for c, n in peaks], "scheme": name, "scheme_scale": scale, "scheme_share": round(sshare, 4),
+                    "scheme_coverage": round(sum(n for _, n in on) / len(vals), 4), "scheme_peaks": [(round(c, 2), n) for c, n in on]}
+    return out
+
+
+def wallet_acceptable(info: dict, opt) -> bool:
+    if info["excess"] < opt.wallet_min_excess * 0.4:
+        return False
+    if info["coverage"] >= opt.wallet_min_coverage and info["excess"] >= opt.wallet_min_excess:
+        return True
+    return info["scheme_coverage"] >= opt.wallet_min_coverage_with_scheme and info["scheme_share"] >= opt.scheme_min_share
+
+
+def choose_wallet(walls: dict, legal: str | None, opt) -> str | None:
+    """Prefer a currency whose peaks land on a known menu, then the lowest day drift, then legal tender, EUR, USD."""
+    ok = [(cur, info) for cur, info in walls.items() if wallet_acceptable(info, opt)]
+    if not ok:
+        return None
+    prio = {legal: 0, "EUR": 1, "USD": 2}
+
+    def key(kv):
+        cur, info = kv
+        drift = info["day_drift"] if info["day_drift"] is not None else 1.0
+        return (0 if info["scheme_share"] >= opt.scheme_min_share else 1, round(drift / opt.drift_bucket), prio.get(cur, 3), -info["coverage"])
+    return sorted(ok, key=key)[0][0]
+
+
+def rate_consistency(rows: list[dict], rates: Rates, wallet: str, peaks: list, opt) -> dict:
+    """Per peak and source currency: modal processed amount by day versus the source-to-wallet cross-rate."""
+    result = {"frozen_peaks": 0, "live_peaks": 0, "frozen_attempts": 0, "tested_attempts": 0, "detail": []}
+    for center, _ in peaks:
+        by_source: dict[str, dict[str, Counter]] = defaultdict(lambda: defaultdict(Counter))
+        for r in rows:
+            src = rates.rate(r["day"], r["currency"])
+            rt = rates.rate(r["day"], wallet)
+            if not src or not rt:
+                continue
+            local = r["amount"] / src * rt
+            if abs(local - center) <= opt.peak_tol * center:
+                by_source[r["currency"]][r["day"]][round(r["amount"], 2)] += 1
+        for source, by_day in by_source.items():
+            days = sorted(d for d in by_day if sum(by_day[d].values()) >= opt.frozen_min_per_day)
+            if len(days) < opt.frozen_days:
+                continue
+            modal = [by_day[d].most_common(1)[0][0] for d in days]
+            attempts = sum(sum(by_day[d].values()) for d in days)
+            result["tested_attempts"] += attempts
+            move = rates.market_move(wallet, days[0], days[-1], source) or 0.0
+            distinct = len(set(modal))
+            frozen = distinct == 1 and move >= opt.frozen_market_move and source != wallet
+            if frozen:
+                result["frozen_peaks"] += 1
+                result["frozen_attempts"] += attempts
+            else:
+                result["live_peaks"] += 1
+            result["detail"].append({"peak": round(center, 2), "source_currency": source, "days": len(days), "modal_amounts": sorted(set(modal)), "market_move": round(move, 4), "frozen": frozen, "attempts": attempts})
+    return result
+
+
+def retail_scan(rows: list[dict], opt, rng) -> dict:
+    """Find one constant k such that amount / k is retail-shaped for a large share of rows, with many distinct bases."""
+    amounts = [r["amount"] for r in rows]
+    empty = {"constant": None, "share": 0.0, "control": 0.0, "excess": 0.0, "distinct_bases": 0, "rows": len(amounts)}
+    if len(amounts) < opt.retail_min_rows:
+        return empty
+    best = (1.0, 0.0)
+    k = opt.retail_k_min
+    while k <= opt.retail_k_max + 1e-9:
+        share = sum(retail_shaped(a / k) for a in amounts) / len(amounts)
+        if share > best[1]:
+            best = (round(k, 4), share)
+        k += opt.retail_k_step
+    bases = Counter(round(a / best[0], 2) for a in amounts if retail_shaped(a / best[0]))
+    distinct = sum(1 for _, c in bases.items() if c >= 3)
+    ctrl = sum(retail_shaped(a / best[0]) for a in jitter(amounts, rng, 0.004, 0.02)) / len(amounts)
+    return {"constant": best[0], "share": round(best[1], 4), "control": round(ctrl, 4), "excess": round(best[1] - ctrl, 4), "distinct_bases": distinct, "rows": len(amounts)}
+
+
+def retail_passes(res: dict, opt, strong: bool = False) -> bool:
+    if not res["constant"] or res["excess"] < opt.retail_min_excess:
+        return False
+    if strong:
+        return res["share"] >= opt.retail_min_share and res["distinct_bases"] >= opt.retail_strong_distinct
+    return res["share"] >= opt.retail_weak_share and res["distinct_bases"] >= opt.retail_min_distinct
+
+
+def behaviour(rows: list[dict], opt) -> dict:
+    n = len(rows)
+    acc = Counter(r["account"] for r in rows if r["account"])
+    repeat = sum(1 for v in acc.values() if v >= 2) / len(acc) if acc else None
+    # retry after decline within window
+    by_acc: dict[str, list] = defaultdict(list)
+    for r in rows:
+        if r["account"] and r["ts"]:
+            by_acc[r["account"]].append(r)
+    declines = retries = 0
+    approvals = {v for v in opt.approval_values}
+    for lst in by_acc.values():
+        lst.sort(key=lambda r: r["ts"])
+        for i, r in enumerate(lst):
+            if opt.response_column and r["response"] not in approvals:
+                declines += 1
+                if i + 1 < len(lst):
+                    try:
+                        gap = (datetime.fromisoformat(lst[i + 1]["ts"]) - datetime.fromisoformat(r["ts"])).total_seconds()
+                    except ValueError:
+                        continue
+                    if gap <= opt.retry_window_seconds:
+                        retries += 1
+    hours = Counter(int(r["ts"][11:13]) for r in rows if len(r["ts"]) >= 13)
+    top3 = sum(v for _, v in hours.most_common(3)) / n if hours else None
+    # burst regularity: coefficient of variation of gaps inside the busiest hour-day
+    gaps = []
+    if hours:
+        buckets: dict[str, list] = defaultdict(list)
+        for r in rows:
+            if len(r["ts"]) >= 13:
+                buckets[r["ts"][:13]].append(r["ts"])
+        busiest = max(buckets.values(), key=len)
+        ts = sorted(datetime.fromisoformat(t) for t in busiest if len(t) >= 19)
+        gaps = [(ts[i + 1] - ts[i]).total_seconds() for i in range(len(ts) - 1)]
+    cv = (statistics.pstdev(gaps) / statistics.mean(gaps)) if len(gaps) >= 5 and statistics.mean(gaps) > 0 else None
+    avs = sum(1 for r in rows if r["avs"] and "not checked" not in r["avs"].lower()) / n if opt.avs_column else None
+    tds = sum(1 for r in rows if r["auth"] and r["auth"].lower() not in ("none", "", "n/a")) / n if opt.auth_column else None
+    prepaid = sum(1 for r in rows if "prepaid" in r["card_type"].lower()) / n if opt.card_type_column else None
+    bins = Counter(r["account"][:6] for r in rows if len(r["account"]) >= 6)
+    top5 = sum(v for _, v in bins.most_common(5)) / n if bins else None
+    approved = sum(1 for r in rows if r["response"] in approvals) / n if opt.response_column else None
+    fraud = sum(1 for r in rows if r["response"] in set(opt.suspected_fraud_values)) / n if opt.response_column else None
+    return {
+        "attempts": n, "cards": len(acc) or None, "repeat_card_share": repeat,
+        "retry_after_decline": (retries / declines) if declines else None, "declines": declines,
+        "top3_hour_share_utc": top3, "burst_gap_cv": cv, "avs_present_share": avs, "three_ds_share": tds,
+        "prepaid_share": prepaid, "top5_bin_share": top5, "approval_rate": approved, "suspected_fraud_share": fraud,
+        "first_day": min(r["day"] for r in rows), "last_day": max(r["day"] for r in rows),
+    }
+
+
+def on_peak_rows(rows: list[dict], rates: Rates, wallet: str, centers: list[float], tol: float) -> tuple[list[dict], list[dict]]:
+    inside, outside = [], []
+    for r in rows:
+        src = rates.rate(r["day"], r["currency"])
+        rt = rates.rate(r["day"], wallet)
+        if src and rt and any(abs(r["amount"] / src * rt - c) <= tol * c for c in centers):
+            inside.append(r)
+        else:
+            outside.append(r)
+    return inside, outside
+
+
+def extract_substreams(grows: list[dict], legal: str | None, rates: Rates, opt, rng) -> list[dict]:
+    """Peel streams off a country group: menu (wallet + scheme), retail list, then remainder as off-menu."""
+    subs = []
+    residual = list(grows)
+    for _round in range(opt.max_substreams):
+        if len(residual) < opt.min_group:
+            break
+        retail = retail_scan(residual, opt, rng)
+        if retail_passes(retail, opt, strong=True):
+            k = retail["constant"]
+            inside = [r for r in residual if retail_shaped(r["amount"] / k)]
+            residual = [r for r in residual if not retail_shaped(r["amount"] / k)]
+            subs.append({"stream": f"retail x{k}", "rows": inside, "retail": retail})
+            continue
+        cands = [c for c in dict.fromkeys([legal, *opt.candidate_currencies]) if c]
+        walls = infer_wallet(residual, rates, cands, opt, rng)
+        wallet = choose_wallet(walls, legal, opt)
+        if wallet:
+            info = walls[wallet]
+            scheme, scale, sshare = info["scheme"], info["scheme_scale"], info["scheme_share"]
+            use_peaks = info["scheme_peaks"] if sshare >= opt.scheme_min_share else info["peaks"]
+            inside, outside = on_peak_rows(residual, rates, wallet, [c for c, _ in use_peaks], opt.peak_tol)
+            cons = rate_consistency(inside, rates, wallet, use_peaks, opt)
+            frozen = bool(cons["tested_attempts"] and cons["frozen_attempts"] / cons["tested_attempts"] >= opt.frozen_share)
+            label = f"{wallet} menu-{scheme if sshare >= opt.scheme_min_share else 'none'}{' frozen' if frozen else ' live'}"
+            subs.append({"stream": label, "rows": inside, "wallet": wallet, "coverage": info["coverage"], "day_drift": info["day_drift"], "scheme": scheme,
+                         "scheme_scale": scale, "scheme_share": round(sshare, 3), "peaks": use_peaks, "all_peaks": info["peaks"], "frozen": frozen, "rate_consistency": cons,
+                         "alternatives": {c: i["coverage"] for c, i in walls.items() if c != wallet}})
+            residual = outside
+            continue
+        if retail_passes(retail, opt):
+            k = retail["constant"]
+            inside = [r for r in residual if retail_shaped(r["amount"] / k)]
+            residual = [r for r in residual if not retail_shaped(r["amount"] / k)]
+            subs.append({"stream": f"retail x{k}", "rows": inside, "retail": retail})
+            continue
+        break
+    if residual:
+        subs.append({"stream": "off-menu", "rows": residual})
+    return subs
+
+
+# ----------------------------------------------------------------------------- presentation
+
+def fixed_amount_characteristics(rows: list[dict]) -> list[str]:
+    charge_days: dict[tuple[str, float], set[str]] = defaultdict(set)
+    for row in rows:
+        charge_days[(row["currency"], row["amount"])].add(row["day"])
+    return ["fixed amount"] if any(len(days) >= 2 for days in charge_days.values()) else []
+
+
+def payment_pattern_presentation(sdef: dict, rows: list[dict]) -> dict:
+    """Describe detected rows without splitting or renaming canonical streams."""
+    label = sdef["stream"]
+    wallet = sdef.get("wallet")
+    established_menu = bool(wallet and " menu-" in label and " menu-none" not in label)
+    repeated = bool(wallet and " menu-none" in label)
+    if label.startswith("retail x"):
+        basis = "retail-list price factor"
+    elif repeated:
+        basis = "unidentified repeated-amount pattern"
+    elif established_menu:
+        basis = "established menu"
+    else:
+        basis = label
+    frozen_by_source: Counter = Counter()
+    for detail in sdef.get("rate_consistency", {}).get("detail", []):
+        if detail["frozen"] and detail["source_currency"] != wallet:
+            frozen_by_source[detail["source_currency"]] += detail["attempts"]
+    parts = []
+    for currency, attempts in sorted(Counter(r["currency"] for r in rows).items()):
+        if established_menu:
+            pattern = "Direct top-up pattern" if currency == wallet else "FX-converted top-up pattern"
+        else:
+            pattern = "E-commerce pattern" if label.startswith("retail x") else "Other pattern"
+        parts.append({"pattern": pattern, "charge_currency": currency,
+                      "inferred_top_up_currency": wallet if established_menu else None,
+                      "attempts": attempts, "basis": basis,
+                      "frozen_fx_attempts": frozen_by_source[currency] if established_menu and currency != wallet else 0})
+    return summarize_payment_patterns([{"by_charge_currency": parts,
+                                        "pricing_characteristics": fixed_amount_characteristics(rows)}])
+
+
+def summarize_payment_patterns(presentations: list[dict]) -> dict:
+    """Aggregate display counts along the existing raw-label grouping only."""
+    combined = {}
+    pricing = set()
+    for presentation in presentations:
+        pricing.update(presentation["pricing_characteristics"])
+        for part in presentation["by_charge_currency"]:
+            key = (part["pattern"], part["charge_currency"], part["inferred_top_up_currency"], part["basis"])
+            if key not in combined:
+                combined[key] = {**part, "attempts": 0, "frozen_fx_attempts": 0}
+            combined[key]["attempts"] += part["attempts"]
+            combined[key]["frozen_fx_attempts"] += part["frozen_fx_attempts"]
+    parts = list(combined.values())
+    patterns: Counter = Counter()
+    currencies: Counter = Counter()
+    inferred = set()
+    for part in parts:
+        patterns[part["pattern"]] += part["attempts"]
+        currencies[part["charge_currency"]] += part["attempts"]
+        if part["inferred_top_up_currency"]:
+            inferred.add(part["inferred_top_up_currency"])
+    return {"pattern_counts": dict(sorted(patterns.items())),
+            "charge_currencies": dict(sorted(currencies.items())),
+            "inferred_top_up_currencies": sorted(inferred),
+            "by_charge_currency": parts, "pricing_characteristics": sorted(pricing)}
+
+
+def presentation_cells(stream: dict) -> str:
+    presentation = stream.get("presentation")
+    if not presentation:
+        return "n/a | n/a | n/a | n/a"
+    patterns = []
+    for pattern, attempts in presentation["pattern_counts"].items():
+        sources = []
+        for part in presentation["by_charge_currency"]:
+            if part["pattern"] != pattern:
+                continue
+            source = f"{part['charge_currency']}: {part['attempts']}"
+            if part["frozen_fx_attempts"]:
+                source += f" ({part['frozen_fx_attempts']} frozen FX tested attempts)"
+            sources.append(source)
+        text = f"{pattern}: {attempts} [{', '.join(sources)}]"
+        patterns.append(text)
+    if any(p["basis"] == "unidentified repeated-amount pattern" for p in presentation["by_charge_currency"]):
+        patterns.append("unidentified repeated-amount pattern")
+    inferred = ", ".join(presentation["inferred_top_up_currencies"]) or "n/a"
+    charges = ", ".join(f"{cur}: {n}" for cur, n in presentation["charge_currencies"].items())
+    pricing = ", ".join(presentation["pricing_characteristics"]) or "n/a"
+    return f"{'; '.join(patterns)} | {inferred} | {charges} | {pricing}"
+
+
+def analyse(rows: list[dict], rates: Rates, opt) -> dict:
+    rng = random.Random(opt.seed)
+    report = {"parameters": {k: v for k, v in vars(opt).items() if k not in ("csv_path", "output", "markdown", "fetch_rates", "rates")}, "descriptors": []}
+    by_desc: dict[str, list] = defaultdict(list)
+    for r in rows:
+        by_desc[r["descriptor"]].append(r)
+    for desc, drows in sorted(by_desc.items(), key=lambda x: -len(x[1])):
+        entry = {"descriptor": desc, "attempts": len(drows), "groups": [], "streams": []}
+        by_ctry: dict[str, list] = defaultdict(list)
+        for r in drows:
+            by_ctry[r["country"]].append(r)
+        stream_rows: dict[str, list] = defaultdict(list)
+        stream_presentations: dict[str, list] = defaultdict(list)
+        small = 0
+        for ctry, grows in sorted(by_ctry.items(), key=lambda x: -len(x[1])):
+            if len(grows) < opt.min_group:
+                small += len(grows)
+                stream_rows["small groups"].extend(grows)
+                stream_presentations["small groups"].append(payment_pattern_presentation({"stream": "small groups"}, grows))
+                continue
+            legal = COUNTRY_CURRENCY.get(ctry.lower())
+            subs = extract_substreams(grows, legal, rates, opt, rng)
+            group = {"country": ctry, "attempts": len(grows), "legal_tender": legal, "substreams": []}
+            for sdef in subs:
+                srows = sdef.pop("rows")
+                sdef["attempts"] = len(srows)
+                sdef["share_of_group"] = round(len(srows) / len(grows), 3)
+                sdef["behaviour"] = behaviour(srows, opt)
+                sdef["presentation"] = payment_pattern_presentation(sdef, srows)
+                group["substreams"].append(sdef)
+                stream_rows[sdef["stream"]].extend(srows)
+                stream_presentations[sdef["stream"]].append(sdef["presentation"])
+            entry["groups"].append(group)
+        for label, srows in sorted(stream_rows.items(), key=lambda kv: -len(kv[1])):
+            presentation = summarize_payment_patterns(stream_presentations[label])
+            presentation["pricing_characteristics"] = fixed_amount_characteristics(srows)
+            entry["streams"].append({"stream": label, "attempts": len(srows), "share_of_descriptor": round(len(srows) / len(drows), 3),
+                                     "countries": [c for c, _ in Counter(r["country"] for r in srows).most_common(6)], "behaviour": behaviour(srows, opt),
+                                     "presentation": presentation})
+        report["descriptors"].append(entry)
+    report["rates_carried_days"] = dict(rates.carried)
+    return report
+
+
+def to_markdown(report: dict) -> str:
+    fmt = lambda v: "n/a" if v is None else f"{v:.0%}"  # noqa: E731
+    out = ["# Payment patterns", "",
+           "Payment patterns describe amount evidence, not confirmed wallets, deposits, players, casinos or shops. "
+           "Inferred top-up currencies are distinct from actual charge currencies. Fixed amount is a supplementary pricing characteristic, "
+           "not proof of fixed FX or generated traffic. Raw live labels do not establish floating FX; "
+           "frozen FX counts refer only to source-specific cross-rate tests.", ""]
+    for d in report["descriptors"]:
+        out += [f"## {d['descriptor']} ({d['attempts']} attempts)", "", "| Raw stream | Payment pattern | Inferred top-up currencies | Actual charge currencies (attempts) | Pricing characteristic | Attempts | Share | Countries | Approval | Suspected fraud | Retry after decline | Repeat cards | Top-3-hour share | AVS present |", "|---|---|---|---|---|---|---|---|---|---|---|---|---|---|"]
+        for s in d["streams"]:
+            b = s["behaviour"]
+            out.append(f"| {s['stream']} | {presentation_cells(s)} | {s['attempts']} | {s['share_of_descriptor']:.0%} | {', '.join(s['countries'])} | {fmt(b['approval_rate'])} | {fmt(b['suspected_fraud_share'])} | {fmt(b['retry_after_decline'])} | {fmt(b['repeat_card_share'])} | {fmt(b['top3_hour_share_utc'])} | {fmt(b['avs_present_share'])} |")
+        out += ["", "| Country | n | Raw sub-stream | Payment pattern | Inferred top-up currencies | Actual charge currencies (attempts) | Pricing characteristic | Rows | Coverage | Day drift | Scheme share | Peaks |", "|---|---|---|---|---|---|---|---|---|---|---|---|"]
+        for g in d["groups"]:
+            for sdef in g["substreams"]:
+                peaks = " ".join(f"{c:g}x{n}" for c, n in sdef.get("peaks", [])[:6])
+                cov = sdef.get("coverage"); dr = sdef.get("day_drift"); ss = sdef.get("scheme_share")
+                out.append(f"| {g['country']} | {g['attempts']} | {sdef['stream']} | {presentation_cells(sdef)} | {sdef['attempts']} | {'' if cov is None else f'{cov:.0%}'} | {'' if dr is None else f'{dr:.2%}'} | {'' if ss is None else f'{ss:.0%}'} | {peaks} |")
+        out.append("")
+    return "\n".join(out)
+
+
+def main(argv=None) -> None:
+    p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
+    p.add_argument("csv_path", type=Path)
+    p.add_argument("--rates", type=Path, required=True, help="daily FX cache JSON: {base, rates:{YYYY-MM-DD:{CCY:rate}}}")
+    p.add_argument("--fetch-rates", nargs=2, metavar=("START", "END"), help="download Frankfurter v1 rates into --rates before analysis")
+    p.add_argument("--amount-column", required=True)
+    p.add_argument("--currency-column", required=True)
+    p.add_argument("--date-column", required=True, help="timestamp or date column (ISO)")
+    p.add_argument("--descriptor-column", required=True)
+    p.add_argument("--bin-country-column", required=True)
+    p.add_argument("--response-column")
+    p.add_argument("--approval-values", nargs="*", default=["A001"])
+    p.add_argument("--suspected-fraud-values", nargs="*", default=["D102"])
+    p.add_argument("--account-column")
+    p.add_argument("--card-type-column")
+    p.add_argument("--avs-column")
+    p.add_argument("--auth-column")
+    p.add_argument("--candidate-currencies", nargs="*", default=["EUR", "USD", "GBP", "CHF"])
+    p.add_argument("--min-group", type=int, default=80)
+    p.add_argument("--peak-tol", type=float, default=0.012)
+    p.add_argument("--peak-min-share", type=float, default=0.02)
+    p.add_argument("--top-k", type=int, default=10)
+    p.add_argument("--jitter-lo", type=float, default=0.02)
+    p.add_argument("--jitter-hi", type=float, default=0.05)
+    p.add_argument("--wallet-min-coverage", type=float, default=0.45, help="share of a group on its top-k peaks needed to call it menu-driven")
+    p.add_argument("--wallet-min-excess", type=float, default=0.25)
+    p.add_argument("--wallet-min-coverage-with-scheme", type=float, default=0.15, help="lower coverage bar when the peaks sit on a known menu")
+    p.add_argument("--drift-bucket", type=float, default=0.004, help="day-drift values within one bucket are ties broken by currency priority")
+    p.add_argument("--scheme-min-share", type=float, default=0.5)
+    p.add_argument("--scheme-tol", type=float, default=0.015, help="relative tolerance when matching peaks to a menu scheme (cashier markups run 0-3%%)")
+    p.add_argument("--max-substreams", type=int, default=3)
+    p.add_argument("--frozen-days", type=int, default=4)
+    p.add_argument("--frozen-min-per-day", type=int, default=3)
+    p.add_argument("--frozen-market-move", type=float, default=0.001)
+    p.add_argument("--frozen-share", type=float, default=0.5)
+    p.add_argument("--retail-k-min", type=float, default=1.0)
+    p.add_argument("--retail-k-max", type=float, default=1.15)
+    p.add_argument("--retail-k-step", type=float, default=0.0025)
+    p.add_argument("--retail-min-excess", type=float, default=0.2)
+    p.add_argument("--retail-min-share", type=float, default=0.3)
+    p.add_argument("--retail-min-distinct", type=int, default=5, help="distinct retail base prices (>=3 rows each) needed")
+    p.add_argument("--retail-strong-distinct", type=int, default=8, help="distinct bases for retail to be tried before wallet inference")
+    p.add_argument("--retail-weak-share", type=float, default=0.2)
+    p.add_argument("--retail-min-rows", type=int, default=60)
+    p.add_argument("--retry-window-seconds", type=int, default=1800)
+    p.add_argument("--seed", type=int, default=7)
+    p.add_argument("--output", type=Path)
+    p.add_argument("--markdown", type=Path)
+    opt = p.parse_args(argv)
+    cols = {"amount": opt.amount_column, "currency": opt.currency_column, "date": opt.date_column, "descriptor": opt.descriptor_column, "country": opt.bin_country_column,
+            "response": opt.response_column, "account": opt.account_column, "card_type": opt.card_type_column, "avs": opt.avs_column, "auth": opt.auth_column}
+    rows = parse_rows(opt.csv_path, cols)
+    if opt.fetch_rates:
+        symbols = sorted({c for c in COUNTRY_CURRENCY.values() if c not in PEGS} | set(opt.candidate_currencies))
+        cache = fetch_rates(opt.fetch_rates[0], opt.fetch_rates[1], "USD", [s for s in symbols if s != "USD"])
+        opt.rates.write_text(json.dumps(cache, indent=1))
+    rates = Rates(json.loads(opt.rates.read_text()))
+    report = analyse(rows, rates, opt)
+    text = json.dumps(report, indent=1, default=str)
+    if opt.output:
+        opt.output.write_text(text)
+    else:
+        sys.stdout.write(text + "\n")
+    if opt.markdown:
+        opt.markdown.write_text(to_markdown(report))
+
+
+if __name__ == "__main__":
+    main()