@1aboveio/skills 0.19.2 → 0.20.1

package/skills/payment/tld-detection/scripts/detect_ladders.py

@@ -0,0 +1,390 @@
+#!/usr/bin/env python3
+"""Detect exact currency-amount-pair ladders with absolute rung residuals."""
+from __future__ import annotations
+
+import argparse
+import csv
+import json
+import re
+from collections import Counter, defaultdict
+from dataclasses import dataclass
+from decimal import Decimal
+from pathlib import Path
+from typing import Iterable, Mapping, Sequence
+
+
+Pair = tuple[str, Decimal]
+Partition = str
+CLUSTER_MULTIPLIER = Decimal("2")
+
+
+@dataclass(frozen=True)
+class LadderConfig:
+    absolute_caps: Mapping[str, Decimal]
+    min_levels: int = 3
+    min_cluster_attempts: int = 100
+    multi_anchor_min_attempts: int = 24
+
+
+def _rank(keys: Iterable[Pair], counts: Mapping[Pair, int]) -> list[Pair]:
+    return sorted(keys, key=lambda key: (-counts[key], key[1], key[0]))
+
+
+def _residual(lower: Pair, upper: Pair, multiplier: Decimal) -> Decimal:
+    return abs(upper[1] - lower[1] * multiplier)
+
+
+def _expand_levels(
+    seed: Pair,
+    multiplier: Decimal,
+    available: set[Pair],
+    counts: Mapping[Pair, int],
+    cap: Decimal,
+    multi_anchor_min_attempts: int,
+) -> dict[int, set[Pair]]:
+    """Expand from a seed; every retained key is an explicit exact anchor."""
+    levels: dict[int, set[Pair]] = {0: {seed}}
+    claimed = {seed}
+    for direction in (1, -1):
+        previous = {seed}
+        for step in range(1, 21):
+            if direction == 1:
+                candidates = {
+                    candidate
+                    for candidate in available - claimed
+                    if candidate[0] == seed[0]
+                    and any(_residual(anchor, candidate, multiplier) <= cap for anchor in previous)
+                }
+            else:
+                candidates = {
+                    candidate
+                    for candidate in available - claimed
+                    if candidate[0] == seed[0]
+                    and any(_residual(candidate, anchor, multiplier) <= cap for anchor in previous)
+                }
+            if not candidates:
+                break
+
+            def candidate_residual(candidate: Pair) -> Decimal:
+                if direction == 1:
+                    return min(_residual(anchor, candidate, multiplier) for anchor in previous)
+                return min(_residual(candidate, anchor, multiplier) for anchor in previous)
+
+            primary = min(candidates, key=lambda candidate: (-counts[candidate], candidate_residual(candidate), candidate[1]))
+            next_level = {primary} | {
+                candidate for candidate in candidates if counts[candidate] >= multi_anchor_min_attempts
+            }
+            levels[direction * step] = next_level
+            claimed.update(next_level)
+            previous = next_level
+    return levels
+
+
+def _fit_residual(levels: Mapping[int, set[Pair]], multiplier: Decimal) -> Decimal:
+    total = Decimal("0")
+    for level, anchors in levels.items():
+        if level > 0:
+            total += sum(min(_residual(lower, anchor, multiplier) for lower in levels[level - 1]) for anchor in anchors)
+        elif level < 0:
+            total += sum(min(_residual(anchor, upper, multiplier) for upper in levels[level + 1]) for anchor in anchors)
+    return total
+
+
+def detect_ladders(
+    counts: Mapping[Pair, int], universe_count: int, config: LadderConfig
+) -> tuple[list[dict[str, object]], dict[str, object]]:
+    """Return disjoint ladders and deterministic threshold metadata."""
+    if universe_count <= 0:
+        raise ValueError("universe_count must be positive")
+    normalized = Counter({(currency.upper(), Decimal(amount)): count for (currency, amount), count in counts.items()})
+    high_threshold = max(30, 0.005 * universe_count)
+    rung_threshold = max(5, 0.001 * universe_count)
+    ranked = _rank(normalized, normalized)
+    global_top = set(ranked[:25])
+    by_currency: dict[str, list[Pair]] = defaultdict(list)
+    for key in normalized:
+        by_currency[key[0]].append(key)
+    high = {key for key, count in normalized.items() if count >= high_threshold} | global_top
+    for currency, keys in by_currency.items():
+        currency_top = _rank(keys, normalized)[:10]
+        high.update(key for key in currency_top if normalized[key] >= 30)
+
+    rung_pool = {key for key, count in normalized.items() if count >= rung_threshold}
+    rung_currencies = {currency for currency, _ in rung_pool}
+    missing_caps = sorted(rung_currencies - set(config.absolute_caps))
+    if missing_caps:
+        raise ValueError(f"missing absolute residual cap for rung-pool currencies: {', '.join(missing_caps)}")
+    unused = set(rung_pool)
+    found: list[dict[str, object]] = []
+    below_threshold: dict[frozenset[Pair], dict[str, object]] = {}
+    for seed in _rank(rung_pool, normalized):
+        if seed not in unused:
+            continue
+        candidates: list[tuple[tuple[object, ...], dict[int, set[Pair]], Decimal, Decimal]] = []
+        for multiplier in (CLUSTER_MULTIPLIER,):
+            cap = config.absolute_caps[seed[0]]
+            levels = _expand_levels(
+                seed,
+                multiplier,
+                unused,
+                normalized,
+                cap,
+                config.multi_anchor_min_attempts,
+            )
+            level_count = len(levels)
+            if level_count < max(3, config.min_levels):
+                continue
+            family = set().union(*levels.values())
+            fit_residual = _fit_residual(levels, multiplier)
+            score = (
+                level_count,
+                sum(normalized[key] for key in family),
+                -fit_residual,
+            )
+            candidates.append((score, levels, multiplier, fit_residual))
+        if not candidates:
+            continue
+        _, levels, multiplier, fit_residual = max(candidates, key=lambda item: item[0])
+        family = set().union(*levels.values())
+        candidate = {
+            "seed": seed,
+            "multiplier": multiplier,
+            "levels": levels,
+            "pairs": family,
+            "rule": "adjacent-absolute-residual",
+            "absolute_cap": config.absolute_caps[seed[0]],
+            "fit_residual": fit_residual,
+            "multi_anchor_levels": {level: anchors for level, anchors in levels.items() if len(anchors) > 1},
+            "alternatives": {},
+        }
+        family_attempts = sum(normalized[key] for key in family)
+        if family_attempts < config.min_cluster_attempts:
+            below_threshold.setdefault(frozenset(family), {**candidate, "attempts": family_attempts})
+            continue
+        found.append(candidate)
+        unused -= family
+
+    found.sort(key=lambda cluster: -sum(normalized[key] for key in cluster["pairs"]))
+    for index, cluster in enumerate(found, 1):
+        cluster["id"] = f"C{index}"
+    metadata = {
+        "method": "adjacent-absolute-residual",
+        "absolute_caps": dict(config.absolute_caps),
+        "high": high_threshold,
+        "rung": rung_threshold,
+        "high_pairs": _rank(high, normalized),
+        "rung_pool_size": len(rung_pool),
+        "pair_count": len(normalized),
+        "below_threshold": sorted(below_threshold.values(), key=lambda item: (-item["attempts"], item["seed"])),
+    }
+    return found, metadata
+
+
+def detect_partitioned_ladders(
+    entries: Iterable[tuple[Partition, Pair]],
+    config: LadderConfig,
+) -> tuple[Counter[tuple[Partition, Pair]], list[dict[str, object]], dict[str, object]]:
+    """Detect ladders independently inside frozen descriptor partitions."""
+    counts: Counter[tuple[Partition, Pair]] = Counter(entries)
+    partition_counts: Counter[Partition] = Counter()
+    pair_counts: dict[Partition, Counter[Pair]] = defaultdict(Counter)
+    for (partition, pair), count in counts.items():
+        partition_counts[partition] += count
+        pair_counts[partition][pair] += count
+
+    found: list[dict[str, object]] = []
+    partition_metadata: dict[Partition, dict[str, object]] = {}
+    high_entries: list[dict[str, object]] = []
+    for partition in sorted(partition_counts):
+        ladders, metadata = detect_ladders(pair_counts[partition], partition_counts[partition], config)
+        partition_metadata[partition] = metadata
+        for ladder in ladders:
+            ladder["partition"] = partition
+            ladder["partition_attempts"] = partition_counts[partition]
+            found.append(ladder)
+        for pair in metadata["high_pairs"]:
+            high_entries.append(
+                {
+                    "partition": partition,
+                    "pair": pair,
+                    "attempts": pair_counts[partition][pair],
+                    "partition_attempts": partition_counts[partition],
+                }
+            )
+
+    found.sort(key=lambda ladder: -sum(counts[(ladder["partition"], pair)] for pair in ladder["pairs"]))
+    for index, ladder in enumerate(found, 1):
+        ladder["id"] = f"C{index}"
+    high_entries.sort(key=lambda item: (-item["attempts"], item["pair"][1], item["partition"]))
+    return counts, found, {
+        "method": "partitioned-adjacent-absolute-residual",
+        "absolute_caps": dict(config.absolute_caps),
+        "min_cluster_attempts": config.min_cluster_attempts,
+        "partition_counts": partition_counts,
+        "partitions": partition_metadata,
+        "high_entries": high_entries,
+    }
+
+
+def sensitivity_analysis(
+    counts: Mapping[Pair, int],
+    universe_count: int,
+    config: LadderConfig,
+    currency: str,
+    caps: Sequence[Decimal],
+) -> list[dict[str, object]]:
+    """Run one-currency cap sensitivity while holding all other settings fixed."""
+    output: list[dict[str, object]] = []
+    currency = currency.upper()
+    for cap in caps:
+        varied_caps = dict(config.absolute_caps)
+        varied_caps[currency] = cap
+        ladders, _ = detect_ladders(
+            counts,
+            universe_count,
+            LadderConfig(
+                varied_caps,
+                min_levels=config.min_levels,
+                min_cluster_attempts=config.min_cluster_attempts,
+                multi_anchor_min_attempts=config.multi_anchor_min_attempts,
+            ),
+        )
+        output.append(
+            {
+                "currency": currency,
+                "cap": cap,
+                "ladder_count": len(ladders),
+                "covered_attempts": sum(sum(counts[key] for key in ladder["pairs"]) for ladder in ladders),
+                "ladders": ladders,
+            }
+        )
+    return output
+
+
+def partitioned_sensitivity_analysis(
+    entries: Iterable[tuple[Partition, Pair]],
+    config: LadderConfig,
+    currency: str,
+    caps: Sequence[Decimal],
+) -> list[dict[str, object]]:
+    """Run cap sensitivity over frozen partitions."""
+    entries = tuple(entries)
+    output: list[dict[str, object]] = []
+    currency = currency.upper()
+    for cap in caps:
+        varied_caps = dict(config.absolute_caps)
+        varied_caps[currency] = cap
+        counts, ladders, _ = detect_partitioned_ladders(
+            entries,
+            LadderConfig(
+                varied_caps,
+                min_levels=config.min_levels,
+                min_cluster_attempts=config.min_cluster_attempts,
+                multi_anchor_min_attempts=config.multi_anchor_min_attempts,
+            ),
+        )
+        output.append(
+            {
+                "currency": currency,
+                "cap": cap,
+                "ladder_count": len(ladders),
+                "covered_attempts": sum(sum(counts[(ladder["partition"], key)] for key in ladder["pairs"]) for ladder in ladders),
+                "ladders": ladders,
+            }
+        )
+    return output
+
+
+def normalize_descriptor(value: str) -> str:
+    return re.sub(r"\s+", " ", value.strip()).casefold() or "UNKNOWN_DESCRIPTOR"
+
+
+def normalize_bin_country(value: str) -> str:
+    normalized = re.sub(r"\s+", " ", value.strip()).upper()
+    aliases = {
+        "UNITED STATES": "US",
+        "UNITED STATES OF AMERICA": "US",
+        "USA": "US",
+        "UNITED KINGDOM": "GB",
+        "GREAT BRITAIN": "GB",
+    }
+    return aliases.get(normalized, normalized or "UNKNOWN_BIN_COUNTRY")
+
+
+def _parse_cap(value: str) -> tuple[str, Decimal]:
+    try:
+        currency, amount = value.split("=", 1)
+        cap = Decimal(amount)
+    except Exception as exc:
+        raise argparse.ArgumentTypeError("cap must be CURRENCY=AMOUNT") from exc
+    if not currency.strip() or cap < 0:
+        raise argparse.ArgumentTypeError("cap must use a currency and a non-negative amount")
+    return currency.strip().upper(), cap
+
+
+def _json_value(value: object) -> object:
+    if isinstance(value, Decimal):
+        return format(value, "f")
+    if isinstance(value, tuple) and len(value) == 2:
+        return {"currency": value[0], "amount": _json_value(value[1])}
+    if isinstance(value, set):
+        return [_json_value(item) for item in sorted(value)]
+    if isinstance(value, dict):
+        return {str(key): _json_value(item) for key, item in value.items()}
+    if isinstance(value, list):
+        return [_json_value(item) for item in value]
+    return value
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("csv_path", type=Path)
+    parser.add_argument("--currency-column", required=True)
+    parser.add_argument("--amount-column", required=True)
+    parser.add_argument("--descriptor-column", required=True)
+    parser.add_argument("--bin-country-column")
+    parser.add_argument("--cap", action="append", default=[], type=_parse_cap, metavar="CCY=AMOUNT")
+    parser.add_argument("--sensitivity-cap", action="append", default=[], type=_parse_cap, metavar="CCY=AMOUNT")
+    parser.add_argument("--multi-anchor-min-attempts", type=int, default=24)
+    parser.add_argument("--output", type=Path)
+    args = parser.parse_args()
+
+    entries: list[tuple[Partition, Pair]] = []
+    country_counts: Counter[tuple[Partition, Pair, str]] = Counter()
+    with args.csv_path.open(encoding="utf-8-sig", newline="") as handle:
+        for row in csv.DictReader(handle):
+            partition = normalize_descriptor(row[args.descriptor_column])
+            country = normalize_bin_country(row[args.bin_country_column]) if args.bin_country_column else "UNKNOWN_BIN_COUNTRY"
+            entries.append((partition, (row[args.currency_column].strip().upper(), Decimal(row[args.amount_column].strip()))))
+            country_counts[(partition, entries[-1][1], country)] += 1
+    config = LadderConfig(
+        dict(args.cap),
+        multi_anchor_min_attempts=args.multi_anchor_min_attempts,
+    )
+    _, ladders, metadata = detect_partitioned_ladders(entries, config)
+    for ladder in ladders:
+        components: Counter[str] = Counter()
+        for pair in ladder["pairs"]:
+            for (descriptor, candidate, country), count in country_counts.items():
+                if descriptor == ladder["partition"] and candidate == pair:
+                    components[country] += count
+        ladder["bin_country_components"] = dict(components.most_common())
+    sensitivity: dict[str, list[dict[str, object]]] = {}
+    sensitivity_args: dict[str, list[Decimal]] = defaultdict(list)
+    for currency, cap in args.sensitivity_cap:
+        sensitivity_args[currency].append(cap)
+    for currency, caps in sensitivity_args.items():
+        sensitivity[currency] = partitioned_sensitivity_analysis(entries, config, currency, caps)
+    payload = json.dumps(
+        _json_value({"metadata": metadata, "ladders": ladders, "sensitivity": sensitivity}),
+        ensure_ascii=False,
+        indent=2,
+    )
+    if args.output:
+        args.output.write_text(payload + "\n", encoding="utf-8")
+    else:
+        print(payload)
+
+
+if __name__ == "__main__":
+    main()

package/skills/payment/tld-detection/tests/test_detect_ladders.py

@@ -0,0 +1,132 @@
+from collections import Counter
+import csv
+from decimal import Decimal
+import importlib.util
+import json
+from pathlib import Path
+import subprocess
+import sys
+import tempfile
+import unittest
+
+
+sys.dont_write_bytecode = True
+SCRIPT = Path(__file__).parents[1] / "scripts" / "detect_ladders.py"
+SPEC = importlib.util.spec_from_file_location("detect_ladders", SCRIPT)
+MODULE = importlib.util.module_from_spec(SPEC)
+assert SPEC.loader is not None
+sys.modules[SPEC.name] = MODULE
+SPEC.loader.exec_module(MODULE)
+
+
+def pair(amount: str) -> tuple[str, Decimal]:
+    return "USD", Decimal(amount)
+
+
+class DetectLaddersTest(unittest.TestCase):
+    def detect(self, values: dict[str, int], cap: str = "0.15"):
+        counts = Counter({pair(amount): count for amount, count in values.items()})
+        return MODULE.detect_ladders(
+            counts,
+            sum(counts.values()),
+            MODULE.LadderConfig({"USD": Decimal(cap)}),
+        )[0]
+
+    def test_detects_tight_exact_ladder(self):
+        ladders = self.detect({"28.03": 131, "56.04": 117, "112.09": 65, "224.19": 27, "70.00": 100})
+        amounts = {amount for _, amount in ladders[0]["pairs"]}
+        self.assertEqual(amounts, {Decimal("28.03"), Decimal("56.04"), Decimal("112.09"), Decimal("224.19")})
+
+    def test_keeps_multiple_exact_anchors_at_one_level(self):
+        ladders = self.detect({"14.41": 34, "14.52": 25, "28.94": 43, "57.96": 29, "80.00": 100})
+        levels = next(ladder["levels"] for ladder in ladders if pair("28.94") in ladder["pairs"])
+        self.assertEqual(levels[-1], {pair("14.41"), pair("14.52")})
+
+    def test_rejects_upper_amount_outside_absolute_cap(self):
+        ladders = self.detect({"14.41": 34, "28.94": 43, "57.96": 29, "116.85": 29, "80.00": 100})
+        ladder = next(ladder for ladder in ladders if pair("14.41") in ladder["pairs"])
+        self.assertNotIn(pair("116.85"), ladder["pairs"])
+
+    def test_requires_explicit_currency_cap(self):
+        with self.assertRaisesRegex(ValueError, "missing absolute residual cap"):
+            MODULE.detect_ladders(Counter({pair("10"): 10}), 10, MODULE.LadderConfig({}))
+
+    def test_ignores_currency_below_rung_pool_threshold(self):
+        counts = Counter({pair("10"): 1000, ("EUR", Decimal("9.99")): 1})
+        ladders, _ = MODULE.detect_ladders(counts, 1001, MODULE.LadderConfig({"USD": Decimal("0.15")}))
+        self.assertEqual(ladders, [])
+
+    def test_sensitivity_reports_coverage_by_cap(self):
+        counts = Counter({pair("14.41"): 34, pair("28.94"): 43, pair("57.96"): 29})
+        rows = MODULE.sensitivity_analysis(
+            counts,
+            sum(counts.values()),
+            MODULE.LadderConfig({"USD": Decimal("0.15")}),
+            "USD",
+            [Decimal("0.10"), Decimal("0.15")],
+        )
+        self.assertEqual([row["ladder_count"] for row in rows], [0, 1])
+
+    def test_keeps_below_threshold_family_out_of_clusters(self):
+        counts = Counter({pair("10"): 32, pair("20"): 32, pair("40"): 32})
+        ladders, metadata = MODULE.detect_ladders(
+            counts,
+            96,
+            MODULE.LadderConfig({"USD": Decimal("0.15")}),
+        )
+        self.assertEqual(ladders, [])
+        self.assertEqual(metadata["below_threshold"][0]["attempts"], 96)
+
+    def test_same_pair_can_belong_to_different_partitions(self):
+        entries = []
+        amounts = (("10", 40), ("20", 40), ("40", 40))
+        for partition in ("alpha", "beta"):
+            for amount, count in amounts:
+                entries.extend([(partition, pair(amount))] * count)
+        _, ladders, _ = MODULE.detect_partitioned_ladders(
+            entries,
+            MODULE.LadderConfig({"USD": Decimal("0.15")}),
+        )
+        self.assertEqual(len(ladders), 2)
+        self.assertEqual({ladder["partition"] for ladder in ladders}, {"alpha", "beta"})
+
+    def test_requires_at_least_three_levels(self):
+        self.assertEqual(self.detect({"10": 60, "20": 60, "70": 100}), [])
+
+    def test_rejects_non_two_x_relationships(self):
+        self.assertEqual(self.detect({"10": 40, "30": 40, "90": 40}), [])
+
+    def test_cli_attributes_bin_country_after_descriptor_ladder(self):
+        with tempfile.TemporaryDirectory() as directory:
+            csv_path = Path(directory) / "clean.csv"
+            with csv_path.open("w", encoding="utf-8", newline="") as handle:
+                writer = csv.DictWriter(handle, fieldnames=["descriptor", "country", "currency", "amount"])
+                writer.writeheader()
+                for country in ("US", "GB"):
+                    for amount in ("10", "20", "40"):
+                        writer.writerows(
+                            {"descriptor": "ACME", "country": country, "currency": "USD", "amount": amount}
+                            for _ in range(20)
+                        )
+            result = subprocess.run(
+                [
+                    sys.executable,
+                    str(SCRIPT),
+                    str(csv_path),
+                    "--descriptor-column", "descriptor",
+                    "--bin-country-column", "country",
+                    "--currency-column", "currency",
+                    "--amount-column", "amount",
+                    "--cap", "USD=0.15",
+                ],
+                check=True,
+                capture_output=True,
+                text=True,
+            )
+        payload = json.loads(result.stdout)
+        self.assertEqual(len(payload["ladders"]), 1)
+        self.assertEqual(payload["ladders"][0]["bin_country_components"], {"US": 60, "GB": 60})
+
+
+if __name__ == "__main__":
+    unittest.main()