@1aboveio/skills 1.0.10 → 1.0.12

package/skills/risk-compliance/portfolio-payment-analysis/scripts/analyze.py

@@ -0,0 +1,636 @@
+#!/usr/bin/env python3
+"""Analyze Worldpay CSV exports without network access. Amounts use original transaction dates."""
+import argparse
+import calendar
+import csv
+import hashlib
+import json
+from collections import Counter, defaultdict
+from datetime import date, datetime, timedelta
+from decimal import Decimal, InvalidOperation
+from pathlib import Path
+
+import risk_history
+from csv_export import safe_writer
+
+LAYERS = {'Authorizations': ('auth', 'Timestamp', ('Transaction ID', 'Attempt')),
+          'Sales-and-refunds': ('settlement', 'Settlement date', ('Transaction ID',)),
+          'Chargebacks': ('dispute', 'Chargeback date', ('Dispute ID',))}
+
+
+def sha(path):
+    return hashlib.sha256(path.read_bytes()).hexdigest()
+
+
+def day(value):
+    if not value:
+        raise ValueError('Missing event date')
+    parsed = datetime.fromisoformat(value)
+    if parsed.tzinfo is not None:
+        raise ValueError('Zoned source timestamps require calendar alignment')
+    return parsed.date().isoformat()
+
+
+def shift(d, offset):
+    m = d.year * 12 + d.month - 1 + offset
+    y, m = divmod(m, 12)
+    return date(y, m + 1, min(d.day, calendar.monthrange(y, m + 1)[1]))
+
+
+def read_sources(root):
+    layers = defaultdict(list)
+    sources = []
+    for path in sorted(root.iterdir()):
+        if not path.is_file() or path.suffix.lower() != '.csv':
+            continue
+        match = next((v for k, v in LAYERS.items() if path.name.startswith(k + '_')), None)
+        if not match:
+            continue
+        layer, field, keys = match
+        with path.open(encoding='utf-8-sig', newline='') as f:
+            reader = csv.DictReader(f)
+            required = {field, *keys, 'Sub merchant ID', 'Billing descriptor'}
+            if layer == 'auth':
+                required.add('Response type')
+            elif layer == 'settlement':
+                required.update({'Settlement type', 'Settlement amount', 'Settlement amount currency', 'Timestamp'})
+            if not reader.fieldnames or not required.issubset(reader.fieldnames):
+                raise ValueError(f'{path.name}: missing required columns {sorted(required - set(reader.fieldnames or []))}')
+            rows = list(reader)
+        if any(None in row for row in rows):
+            raise ValueError(f'{path.name}: malformed CSV row')
+        if layer == 'auth' and any(not (row.get('Response type') or '').strip() for row in rows):
+            raise ValueError(f'{path.name}: empty Response type')
+        dates = sorted(day(r[field]) for r in rows)
+        sources.append({'file': path.name, 'layer': layer, 'sha256': sha(path), 'rows': len(rows),
+                        'min_date': dates[0] if dates else None, 'max_date': dates[-1] if dates else None})
+        layers[layer].extend(rows)
+    if not layers['auth']:
+        raise ValueError('Nonempty authorization CSV layer is required')
+    if any(not any(s['max_date'] for s in sources if s['layer'] == layer) for layer in {s['layer'] for s in sources}):
+        raise ValueError('Present layers must contain dated rows to establish a cutoff')
+    return layers, sources
+
+
+def dedup_provider(value):
+    normalized = ' '.join((value or '').split()).casefold()
+    return normalized if normalized in ('worldpay - wpg', 'worldpay - acquiring') else value
+
+
+def row_variant(row):
+    return tuple(sorted((k, dedup_provider(v) if k == 'Provider' else v) for k, v in row.items()))
+
+
+def dedup(rows, keys, snapshots=False):
+    groups = defaultdict(list)
+    for row in rows:
+        key = tuple(dedup_provider(row.get(k, '')) if k == 'Provider' else row.get(k, '') for k in keys)
+        # A missing key does not establish identity. Keep every such observation.
+        if not all((v or '').strip() for v in key):
+            key = ('__missing__', str(len(groups)), str(id(row)))
+        groups[key].append(row)
+    retained, duplicate, conflicts, missing = [], 0, 0, 0
+    for key, group in groups.items():
+        variants = {row_variant(row): row for row in group}
+        duplicate += len(group) - len(variants)
+        missing += key[0] == '__missing__'
+        if len(variants) > 1:
+            conflicts += 1
+        if snapshots or len(variants) == 1:
+            retained.extend(variants.values())
+        # Conflicting auth/settlement variants are counted here; analyze refuses the report.
+    return retained, {'raw_rows': len(rows), 'exact_duplicates_removed': duplicate,
+                      'conflicting_identity_groups': conflicts, 'missing_identity_rows': missing,
+                      'retained_rows': len(retained),
+                      'conflicting_rows_excluded': 0 if snapshots else sum(len({row_variant(r) for r in g}) for g in groups.values() if len({row_variant(r) for r in g}) > 1),
+                      'identity_fields': list(keys), 'conflict_policy': 'retain distinct snapshots' if snapshots else 'refuse report on conflicts'}
+
+
+def fx_load(path):
+    if not path:
+        return {}, None
+    data = json.loads(path.read_text(encoding='utf-8'))
+    if not isinstance(data, dict):
+        raise ValueError('FX rates must be an object: date -> currency -> USD rate')
+    rates = {}
+    for d, currencies in data.items():
+        date.fromisoformat(d)
+        for c, value in currencies.items():
+            n = Decimal(str(value))
+            if not n.is_finite() or n <= 0:
+                raise ValueError('FX rates must be finite positive USD-per-unit values')
+            rates[d, c.upper()] = n
+    return rates, {'file': str(path), 'sha256': sha(path), 'format': 'date -> currency -> USD per unit', 'provenance': 'caller supplied; not independently verified'}
+
+
+def provider_namespace(row):
+    provider = ' '.join((row.get('Provider') or '').split()).casefold()
+    if provider in ('worldpay - wpg', 'worldpay - acquiring'):
+        return 'worldpay'
+    return provider
+
+
+def counts(rows, field):
+    return dict(sorted(Counter(r.get(field, '') or 'Unknown' for r in rows).items()))
+
+
+def amounts(rows, rates):
+    total, native, missing = Decimal(0), defaultdict(Decimal), Counter()
+    for r in rows:
+        try:
+            amount = Decimal(r['Settlement amount'])
+            if not amount.is_finite():
+                raise ValueError('Non-finite settlement amount')
+        except (InvalidOperation, KeyError) as e:
+            raise ValueError('Invalid settlement amount') from e
+        currency = r.get('Settlement amount currency', '').upper()
+        d = day(r['Timestamp'])
+        native[currency or 'Unknown'] += amount
+        rate = Decimal(1) if currency == 'USD' else rates.get((d, currency))
+        if rate is None:
+            missing[d, currency or 'Unknown'] += 1
+        else:
+            total += amount * rate
+    return {'usd': str(total) if not missing else None,
+            'native': {k: str(v) for k, v in sorted(native.items())},
+            'missing_fx': [{'date': d, 'currency': c, 'rows': n} for (d, c), n in sorted(missing.items())]}
+
+
+def approved_amounts(auth, rates):
+    """Sum successful attempt amounts, never substitute settled sales or partial FX totals."""
+    if not auth:
+        return None
+    total, native, missing_fx = Decimal(0), defaultdict(Decimal), Counter()
+    missing_amount_rows = 0
+    for row in auth:
+        if row.get('Response type', '').lower() != 'approval':
+            continue
+        try:
+            value = Decimal(row.get('Transaction amount', ''))
+            if not value.is_finite() or value < 0:
+                raise InvalidOperation
+        except (InvalidOperation, TypeError):
+            missing_amount_rows += 1
+            continue
+        currency = row.get('Transaction amount currency', '').strip().upper()
+        rate_date = day(row['Timestamp'])
+        native[currency or 'Unknown'] += value
+        rate = Decimal(1) if currency == 'USD' else rates.get((rate_date, currency))
+        if rate is None:
+            missing_fx[rate_date, currency or 'Unknown'] += 1
+        else:
+            total += value * rate
+    return {'usd': str(total) if not missing_fx and not missing_amount_rows else None,
+            'native': {k: str(v) for k, v in sorted(native.items())},
+            'missing_fx': [{'date': d, 'currency': c, 'rows': n} for (d, c), n in sorted(missing_fx.items())],
+            'missing_amount_rows': missing_amount_rows}
+
+
+def merchant_breakdown(auth, weeks, rates):
+    """Group one merchant's observed attempts using full-period membership for every week."""
+    edges = (Decimal(0), Decimal(25), Decimal(50), Decimal(100), Decimal(200), Decimal(500))
+    bucket_labels = ['[0,25)', '[25,50)', '[50,100)', '[100,200)', '[200,500)', '[500,+inf)']
+    unknown = ('unknown',)
+    other = ('other',)
+
+    def value(row, field):
+        return (row.get(field) or '').strip()
+
+    def amount(row):
+        try:
+            n = Decimal(value(row, 'Transaction amount'))
+            if not n.is_finite() or n < 0:
+                return None
+            return n
+        except InvalidOperation:
+            return None
+
+    def usd(row):
+        n = amount(row)
+        currency = value(row, 'Transaction amount currency').upper()
+        rate = Decimal(1) if currency == 'USD' else rates.get((day(row['Timestamp']), currency))
+        return n * rate if n is not None and rate is not None else None
+
+    def category(row, dimension):
+        if dimension == 'bin_country':
+            v = value(row, 'Issuing bank country')
+            return ('value', v) if v else unknown
+        if dimension == 'card_product':
+            method, card = value(row, 'Payment method'), value(row, 'Card type')
+            return ('value', method, card) if method or card else unknown
+        if dimension == 'authentication':
+            v = value(row, 'Authentication method')
+            return ('value', v) if v else unknown
+        if dimension == 'decline_reason':
+            code, message = value(row, 'Pazien response code'), value(row, 'Response message')
+            return ('value', code, message) if code or message else unknown
+        converted = usd(row)
+        if converted is None:
+            return unknown
+        return ('bucket', str(max(i for i, edge in enumerate(edges) if converted >= edge)))
+
+    def label(key):
+        if key == unknown:
+            return 'Unknown'
+        if key == other:
+            return 'Other'
+        if key[0] == 'bucket':
+            return bucket_labels[int(key[1])]
+        return ' × '.join(part or 'Unknown' for part in key[1:])
+
+    def amount_sum(rows):
+        total, native, missing_fx = Decimal(0), defaultdict(Decimal), Counter()
+        missing_amount = 0
+        for row in rows:
+            n = amount(row)
+            if n is None:
+                missing_amount += 1
+                continue
+            currency = value(row, 'Transaction amount currency').upper()
+            d = day(row['Timestamp'])
+            native[currency or 'Unknown'] += n
+            rate = Decimal(1) if currency == 'USD' else rates.get((d, currency))
+            if rate is None:
+                missing_fx[d, currency or 'Unknown'] += 1
+            else:
+                total += n * rate
+        return {'usd': str(total) if not missing_fx and not missing_amount else None,
+                'native': {k: str(v) for k, v in sorted(native.items())},
+                'missing_fx': [{'date': d, 'currency': c, 'rows': n} for (d, c), n in sorted(missing_fx.items())],
+                'missing_amount_rows': missing_amount}
+
+    def metrics(rows, denominator, observed):
+        if not observed:
+            return dict.fromkeys(('count', 'approvals', 'auth_rate', 'share', 'attempt_amount', 'approved_amount'))
+        approved = [r for r in rows if r.get('Response type', '').lower() == 'approval']
+        return {'count': len(rows), 'approvals': len(approved),
+                'auth_rate': len(approved) / len(rows) if rows else None,
+                'share': len(rows) / denominator if denominator else None,
+                'attempt_amount': amount_sum(rows), 'approved_amount': amount_sum(approved)}
+
+    dimensions = []
+    for dimension, title in [('bin_country', '发卡行国家'), ('card_product', '卡产品：支付方式 × 卡类型'),
+                             ('amount_bucket', '授权尝试金额区间 USD'), ('authentication', '持卡人认证状态'),
+                             ('decline_reason', '未成功响应原因')]:
+        population = [r for r in auth if dimension != 'decline_reason' or r.get('Response type', '').lower() != 'approval']
+        categories = Counter(category(r, dimension) for r in population)
+        if dimension in ('bin_country', 'decline_reason'):
+            ranked = sorted((k for k in categories if k != unknown), key=lambda k: (-categories[k], k))
+            selected = set(ranked[:10])
+            keys = ranked[:10] + [other, unknown]
+        elif dimension == 'amount_bucket':
+            selected = None
+            keys = [('bucket', str(i)) for i in range(len(edges))] + [unknown]
+        else:
+            selected = None
+            keys = sorted(k for k in categories if k != unknown) + [unknown]
+
+        def group(row):
+            k = category(row, dimension)
+            return other if selected is not None and k != unknown and k not in selected else k
+
+        groups = []
+        for key in keys:
+            rows = [r for r in population if group(r) == key]
+            weekly = []
+            for week in weeks:
+                week_auth = [r for r in auth if week['start'] <= day(r['Timestamp']) <= week['end']]
+                week_population = [r for r in population if week['start'] <= day(r['Timestamp']) <= week['end']]
+                week_rows = [r for r in week_population if group(r) == key]
+                weekly.append({**{k: week[k] for k in ('week_start', 'week_end', 'start', 'end', 'days', 'partial_week')},
+                               'metrics': metrics(week_rows, len(week_population), bool(week_auth))})
+            groups.append({'key': json.dumps(key, ensure_ascii=False, separators=(',', ':')),
+                           'label': label(key), 'metrics': metrics(rows, len(population), bool(auth)), 'weeks': weekly})
+        dimensions.append({'key': dimension, 'label': title,
+                           'population': 'non_approvals' if dimension == 'decline_reason' else 'attempts',
+                           'groups': groups})
+    return dimensions
+
+
+def summary(auth, settlement, dispute, rates, present):
+    approvals = sum(r.get('Response type', '').lower() == 'approval' for r in auth)
+    settled = [r for r in settlement if r.get('Settlement status') == 'Settled']
+    sales = [r for r in settled if r.get('Settlement type') == 'Sale']
+    refunds = [r for r in settled if r.get('Settlement type') == 'Refund']
+    other = len(settled) - len(sales) - len(refunds)
+    def settlement_metrics(rows):
+        result = amounts(rows, rates)
+        missing_identity = sum(not (r.get('Transaction ID') or '').strip() for r in rows)
+        if missing_identity:
+            result['usd'] = None
+            result['native'] = None
+        result['missing_identity_rows'] = missing_identity
+        return (None if missing_identity else len(rows)), result
+    sale_count, sales_amount = settlement_metrics(sales)
+    refund_count, refunds_amount = settlement_metrics(refunds)
+    def breakdown(field):
+        groups = defaultdict(list)
+        for row in auth:
+            groups[row.get(field, '') or 'Unknown'].append(row)
+        return [{'category': key, 'attempts': len(rows),
+                 'approvals': sum(r.get('Response type', '').lower() == 'approval' for r in rows),
+                 'auth_rate': sum(r.get('Response type', '').lower() == 'approval' for r in rows) / len(rows)}
+                for key, rows in sorted(groups.items())]
+    declined = [r for r in auth if r.get('Response type', '').lower() != 'approval']
+    reasons = Counter((r.get('Pazien response code', '') or 'Unknown', r.get('Response message', '') or 'Unknown') for r in declined)
+    daily = defaultdict(list)
+    for r in auth:
+        daily[day(r['Timestamp'])].append(r)
+    result = {'layer_observed': {k: present[k] and bool(rows) for k, rows in
+                                 (('auth', auth), ('settlement', settlement), ('dispute', dispute))},
+            'attempts': len(auth), 'approvals': approvals, 'non_approvals': len(auth) - approvals,
+            'response_types': counts(auth, 'Response type'),
+            'approved_amount': approved_amounts(auth, rates),
+            'auth_rate': approvals / len(auth) if auth else None,
+            'settlement_statuses': counts(settlement, 'Settlement status'),
+            'sale_count': sale_count, 'refund_count': refund_count, 'other_settlement_count': other,
+            'sales': sales_amount, 'refunds': refunds_amount,
+            'decline_codes': counts([r for r in auth if r.get('Response type', '').lower() != 'approval'], 'Pazien response code'),
+            'countries': counts(auth, 'Issuing bank country'), 'authentication_methods': counts(auth, 'Authentication method'),
+            'dispute_snapshots': len(dispute), 'dispute_lifecycle': counts(dispute, 'Lifecycle state'),
+            'dispute_cycles': counts(dispute, 'Original cycle'), 'formal_dispute_rate': None,
+            'daily_auth': [{'date': d, 'attempts': len(rows), 'approvals': sum(r.get('Response type', '').lower() == 'approval' for r in rows),
+                            'auth_rate': sum(r.get('Response type', '').lower() == 'approval' for r in rows) / len(rows)} for d, rows in sorted(daily.items())],
+            'country_metrics': breakdown('Issuing bank country'), 'authentication_metrics': breakdown('Authentication method'),
+            'decline_reasons': [{'code': code, 'message': message, 'count': n} for (code, message), n in sorted(reasons.items(), key=lambda x: (-x[1], x[0]))]}
+    if not result['layer_observed']['settlement']:
+        for key in ('sale_count', 'refund_count', 'other_settlement_count', 'sales', 'refunds', 'settlement_statuses'):
+            result[key] = None
+    if not result['layer_observed']['dispute']:
+        for key in ('dispute_snapshots', 'dispute_lifecycle', 'dispute_cycles'):
+            result[key] = None
+    return result
+
+
+def analyze(root, start, end, fxpath):
+    raw, sources = read_sources(root)
+    rates, fxmeta = fx_load(fxpath)
+    present = {k: any(s['layer'] == k for s in sources) for k in ('auth', 'settlement', 'dispute')}
+    last_dates = {k: max((s['max_date'] for s in sources if s['layer'] == k and s['max_date']), default=None) for k in present}
+    requested_end = end
+    end = min(end, *(date.fromisoformat(d) for d in last_dates.values() if d))
+    if start > end:
+        raise ValueError('main-start is after the effective data ceiling')
+    layers, audit = {}, {}
+    for _, (layer, field, keys) in LAYERS.items():
+        layers[layer], audit[layer] = dedup(raw[layer], keys, layer == 'dispute')
+    for layer in ('auth', 'settlement'):
+        conflicts = audit[layer]['conflicting_identity_groups']
+        if conflicts:
+            raise ValueError(f'{layer}: {conflicts} conflicting identity group(s); report generation refused')
+    # Explicit IDs cannot safely share the output's bare merchant key across providers.
+    namespaces = defaultdict(set)
+    for rows in raw.values():
+        for row in rows:
+            merchant, namespace = row.get('Sub merchant ID', ''), provider_namespace(row)
+            if merchant:
+                namespaces[merchant].add(namespace)
+    if any(len(values) > 1 for values in namespaces.values()):
+        raise ValueError('Sub merchant ID reused across provider namespaces; report generation refused')
+    # A reference is evidence only within a nonempty provider namespace.
+    identities = defaultdict(set)
+    for layer in ('auth', 'settlement'):
+        for row in raw[layer]:
+            namespace = provider_namespace(row)
+            if namespace and row.get('Transaction ID') and row.get('Sub merchant ID'):
+                identities[namespace, row['Transaction ID']].add(row['Sub merchant ID'])
+    mapping = Counter()
+    for layer, rows in layers.items():
+        for row in rows:
+            merchant = row.get('Sub merchant ID', '')
+            method = 'source'
+            if not merchant:
+                ref = row.get('Original transaction ID' if layer == 'dispute' else 'Transaction ID', '')
+                namespace = provider_namespace(row)
+                candidates = identities.get((namespace, ref), set()) if namespace and ref else set()
+                if len(candidates) == 1:
+                    merchant, method = next(iter(candidates)), 'unique_transaction_reference'
+                else:
+                    method = 'unknown_ambiguous' if candidates else 'unknown_unmatched'
+            row['_merchant'] = merchant
+            mapping[layer, merchant, row.get('Billing descriptor', ''), method, provider_namespace(row)] += 1
+    fields = {'auth': 'Timestamp', 'settlement': 'Settlement date', 'dispute': 'Chargeback date'}
+    def window(a, b):
+        return {k: [r for r in v if a <= day(r[fields[k]]) <= b] for k, v in layers.items()}
+    def calc(w):
+        return summary(w['auth'], w['settlement'], w['dispute'], rates, present)
+    main = window(start.isoformat(), end.isoformat())
+    previous_start, previous_end = shift(start, -1), shift(end, -1)
+    previous = window(previous_start.isoformat(), previous_end.isoformat())
+    names = defaultdict(Counter)
+    for (_, merchant, descriptor, _, _), n in mapping.items():
+        if descriptor:
+            names[merchant][descriptor] += n
+    ids = sorted({r['_merchant'] for rows in main.values() for r in rows})
+    merchants = []
+    for mid in ids:
+        sub = {k: [r for r in v if r['_merchant'] == mid] for k, v in main.items()}
+        name = sorted(names[mid], key=lambda x: (-names[mid][x], x))[0] if names[mid] else 'Unknown'
+        merchants.append({'id': mid or None, 'name': name, **calc(sub)})
+    merchant_weeks = []
+    for merchant in merchants:
+        mid = merchant['id'] or ''
+        weeks = []
+        monday = start - timedelta(days=start.weekday())
+        while monday <= end:
+            sunday = monday + timedelta(days=6)
+            first, last = max(monday, start), min(sunday, end)
+            observed = window(first.isoformat(), last.isoformat())
+            metrics = calc({k: [r for r in rows if r['_merchant'] == mid] for k, rows in observed.items()})
+            days = (last - first).days + 1
+            comparison = {'eligible': False, 'reason': '', 'attempts_pct': None,
+                          'auth_rate_pp': None, 'sales_usd_pct': None}
+            if not weeks:
+                comparison['reason'] = 'No previous week; observed exports only, continuous coverage unverified'
+            elif days != 7 or weeks[-1]['days'] != 7:
+                comparison['reason'] = 'Partial week; observed exports only, continuous coverage unverified'
+            else:
+                prior = weeks[-1]['metrics']
+                if metrics['layer_observed']['auth'] and prior['layer_observed']['auth']:
+                    comparison['attempts_pct'] = (metrics['attempts'] - prior['attempts']) / prior['attempts']
+                    comparison['auth_rate_pp'] = 100 * (metrics['auth_rate'] - prior['auth_rate'])
+                if metrics['layer_observed']['settlement'] and prior['layer_observed']['settlement']:
+                    current_sales, prior_sales = metrics['sales']['usd'], prior['sales']['usd']
+                    if current_sales is not None and prior_sales is not None and Decimal(prior_sales) > 0:
+                        comparison['sales_usd_pct'] = float((Decimal(current_sales) - Decimal(prior_sales)) / Decimal(prior_sales))
+                comparison['eligible'] = any(comparison[k] is not None for k in ('attempts_pct', 'auth_rate_pp', 'sales_usd_pct'))
+                comparison['reason'] = ('Available metrics compare observed exports only; continuous coverage unverified'
+                                        if comparison['eligible'] else
+                                        'No comparable observed layers or positive baseline; observed exports only, continuous coverage unverified')
+            weeks.append({'week_start': monday.isoformat(), 'week_end': sunday.isoformat(),
+                          'start': first.isoformat(), 'end': last.isoformat(), 'days': days,
+                          'partial_week': days != 7, 'metrics': metrics, 'comparison': comparison})
+            monday += timedelta(days=7)
+        merchant_weeks.append({'id': merchant['id'], 'name': merchant['name'], 'weeks': weeks})
+    merchant_breakdowns = [{'id': merchant['id'], 'name': merchant['name'],
+                            'dimensions': merchant_breakdown(
+                                [r for r in main['auth'] if r['_merchant'] == (merchant['id'] or '')],
+                                merchant_weeks[i]['weeks'], rates)}
+                           for i, merchant in enumerate(merchants)]
+    current_start = max(start, end - timedelta(days=end.weekday()))
+    detail_periods = [
+        {'key': 'previous', 'label': '上周同期', 'start': (current_start - timedelta(days=7)).isoformat(),
+         'end': (end - timedelta(days=7)).isoformat()},
+        {'key': 'current', 'label': '本周', 'start': current_start.isoformat(), 'end': end.isoformat()},
+    ]
+    detail_merchants = []
+    for merchant in merchants:
+        mid = merchant['id'] or ''
+        period_rows = [{k: [r for r in rows if r['_merchant'] == mid and
+                            period['start'] <= day(r[fields[k]]) <= period['end']]
+                        for k, rows in main.items()} for period in detail_periods]
+        grid = []
+        for period in detail_periods:
+            first, last = date.fromisoformat(period['start']), date.fromisoformat(period['end'])
+            monday = first - timedelta(days=first.weekday())
+            grid.append({'week_start': monday.isoformat(), 'week_end': (monday + timedelta(days=6)).isoformat(),
+                         'start': period['start'], 'end': period['end'], 'days': (last - first).days + 1,
+                         'partial_week': (last - first).days != 6})
+        dimensions = merchant_breakdown([r for rows in period_rows for r in rows['auth']], grid, rates)
+        for dimension in dimensions:
+            for group in dimension['groups']:
+                group['periods'] = [{'key': period['key'], 'start': period['start'], 'end': period['end'],
+                                     'metrics': week['metrics']}
+                                    for period, week in zip(detail_periods, group.pop('weeks'))]
+        detail_merchants.append({'id': merchant['id'], 'name': merchant['name'],
+                                 'periods': [{'key': p['key'], 'start': p['start'], 'end': p['end'],
+                                              'metrics': calc(rows)} for p, rows in zip(detail_periods, period_rows)],
+                                 'dimensions': dimensions})
+    merchant_detail_comparison = {'periods': detail_periods, 'merchants': detail_merchants,
+                                  'scope': {'reason': 'same weekdays; observed exports; unavailable outside main window',
+                                            'coverage': 'Observed rows only; continuous coverage unverified'}}
+    portfolio = calc(main)
+    all_ids = sorted({r['_merchant'] for rows in layers.values() for r in rows})
+    comparison_merchants = []
+    for mid in all_ids:
+        current = [r for r in main['auth'] if r['_merchant'] == mid]
+        prior = [r for r in previous['auth'] if r['_merchant'] == mid]
+        c, p = len(current), len(prior)
+        ca = sum(r.get('Response type', '').lower() == 'approval' for r in current)
+        pa = sum(r.get('Response type', '').lower() == 'approval' for r in prior)
+        comparison_merchants.append({'id': mid or None, 'name': sorted(names[mid], key=lambda x: (-names[mid][x], x))[0] if names[mid] else 'Unknown',
+                                     'current_attempts': c, 'current_approvals': ca, 'current_auth_rate': ca / c if c else None,
+                                     'previous_attempts': p, 'previous_approvals': pa, 'previous_auth_rate': pa / p if p else None})
+    nc, np = len(main['auth']), len(previous['auth'])
+    components = {'common_rate_pp': 0.0, 'common_mix_pp': 0.0, 'current_only_pp': 0.0, 'prior_only_pp': 0.0}
+    for item in comparison_merchants:
+        parts = dict.fromkeys(components, 0.0)
+        c, p = item['current_attempts'], item['previous_attempts']
+        if nc and np:
+            if c and p:
+                wc, wp = c / nc, p / np
+                rc, rp = item['current_auth_rate'], item['previous_auth_rate']
+                parts['common_rate_pp'] = 100 * (rc - rp) * (wc + wp) / 2
+                parts['common_mix_pp'] = 100 * (wc - wp) * (rc + rp) / 2
+            elif c:
+                parts['current_only_pp'] = 100 * item['current_approvals'] / nc
+            elif p:
+                parts['prior_only_pp'] = -100 * item['previous_approvals'] / np
+            for k in components:
+                components[k] += parts[k]
+        item['decomposition_pp'] = parts if nc and np else None
+    decomposition = {'method': 'symmetric common-merchant rate and mix; one-period approval contributions; portfolio denominators',
+                     'delta_pp': 100 * (portfolio['auth_rate'] - calc(previous)['auth_rate']) if nc and np else None,
+                     'components_pp': components if nc and np else None}
+    trends = []
+    for offset in range(-5, 1):
+        month = shift(end.replace(day=1), offset)
+        last = month.replace(day=calendar.monthrange(month.year, month.month)[1])
+        if last > end:
+            last = end
+        w = window(month.isoformat(), last.isoformat())
+        trends.append({'start': month.isoformat(), 'end': last.isoformat(), 'coverage': 'observed rows only; continuity unverified',
+                       'layer_observed': {k: bool(v) for k, v in w.items()},
+                       'metrics': calc(w) if any(w.values()) else None})
+    # Use all raw observations for collision detection, including rows excluded by deduplication.
+    order_merchants = defaultdict(set)
+    for layer in ('auth', 'settlement'):
+        for r in raw[layer]:
+            if r.get('Order ID'):
+                order_merchants[r['Order ID']].add(r.get('Sub merchant ID', ''))
+    auth_variants = defaultdict(set)
+    for r in raw['auth']:
+        if r.get('Transaction ID') and r.get('Attempt'):
+            auth_variants[r['Transaction ID'], r['Attempt']].add(row_variant({k: v for k, v in r.items() if not k.startswith('_')}))
+    conflicted_orders = {r.get('Order ID') for r in raw['auth'] if r.get('Order ID') and
+                         len(auth_variants.get((r.get('Transaction ID'), r.get('Attempt')), ())) > 1}
+    approved = defaultdict(list)
+    for r in layers['auth']:
+        if r.get('Order ID') and r.get('Response type', '').lower() == 'approval' and day(r['Timestamp']) <= end.isoformat():
+            approved[r['Order ID'], r['_merchant']].append(r)
+    def journey(rows):
+        if not present['settlement']:
+            return None
+        result = {'matched_sales': 0, 'unmatched_sales': 0, 'ambiguous_sales': 0, 'linked_prior_period_approval': 0}
+        for r in rows:
+            if r.get('Settlement type') != 'Sale' or r.get('Settlement status') != 'Settled':
+                continue
+            order, mid = r.get('Order ID', ''), r['_merchant']
+            candidates = approved.get((order, mid), [])
+            if not order or not mid or not r.get('Sub merchant ID'):
+                result['unmatched_sales'] += 1
+            elif order in conflicted_orders or len(order_merchants[order]) != 1 or len(candidates) > 1:
+                result['ambiguous_sales'] += 1
+            elif len(candidates) == 1 and day(candidates[0]['Timestamp']) <= day(r['Settlement date']):
+                result['matched_sales'] += 1
+                if day(candidates[0]['Timestamp']) < start.isoformat():
+                    result['linked_prior_period_approval'] += 1
+            else:
+                result['unmatched_sales'] += 1
+        return result
+    portfolio['journey'] = journey(main['settlement'])
+    for m in merchants:
+        m['journey'] = journey([r for r in main['settlement'] if r['_merchant'] == (m['id'] or '')])
+    history = []
+    for offset in range(-11, 1):
+        month = shift(end.replace(day=1), offset)
+        last = min(month.replace(day=calendar.monthrange(month.year, month.month)[1]), end)
+        rows = window(month.isoformat(), last.isoformat())['dispute']
+        history.append({'start': month.isoformat(), 'end': last.isoformat(), 'layer_present': present['dispute'],
+                        'observed': bool(rows), 'snapshot_count': len(rows) if rows else None,
+                        'stage_labels': counts(rows, 'Lifecycle state') if rows else None, 'formal_rate': None})
+    reconciliation = {'sources': sources, 'deduplication': audit,
+                      'reference_cross_merchant_collisions': sum(len(v) > 1 for v in identities.values()),
+                      'reconciliation': {k: {'portfolio_rows': len(main[k]), 'merchant_rows_including_unknown': sum(sum(r['_merchant'] == mid for r in main[k]) for mid in ids)} for k in main},
+                      'excluded': 'Unrecognized files; invalid and future 340FRD files are recorded in risk_history', 'coverage': 'Observed date ranges do not establish continuous coverage',
+                      'fx': fxmeta, 'disputes': 'Distinct observed snapshots, not verified openings; no formal rates',
+                      'timezone': 'Source timestamps are naive; timezone is unknown'}
+    merchant_risk_history, reconciliation['risk_history'] = risk_history.build(
+        root, end, comparison_merchants, layers['settlement'], layers['dispute'])
+    return {'schema_version': 1, 'metadata': {'main_start': start.isoformat(), 'main_end': end.isoformat(),
+              'requested_main_end': requested_end.isoformat(), 'layers_present': present, 'layer_last_dates': last_dates,
+              'comparison_start': previous_start.isoformat(), 'comparison_end': previous_end.isoformat(),
+              'auth_date': 'Timestamp', 'settlement_date': 'Settlement date', 'dispute_denominator_date': 'Timestamp (transaction/capture date)', 'fx_date': 'Timestamp', 'rate_unit': 'fraction'},
+            'merchant_risk_history': merchant_risk_history,
+            'portfolio': portfolio, 'merchants': merchants, 'merchant_weeks': merchant_weeks, 'merchant_breakdowns': merchant_breakdowns, 'merchant_detail_comparison': merchant_detail_comparison, 'trends': trends, 'dispute_history': history,
+            'comparison': calc(previous), 'comparison_merchants': comparison_merchants, 'comparison_decomposition': decomposition, 'quality': reconciliation}, mapping
+
+
+def main():
+    p = argparse.ArgumentParser(description=__doc__)
+    p.add_argument('--input', type=Path, required=True)
+    p.add_argument('--output', type=Path, required=True)
+    p.add_argument('--main-start', type=date.fromisoformat, required=True)
+    p.add_argument('--main-end', type=date.fromisoformat, required=True)
+    p.add_argument('--fx-rates', type=Path, help='JSON date -> currency -> USD per unit; caller provides provenance')
+    a = p.parse_args()
+    if a.main_start > a.main_end:
+        p.error('main-start must not be after main-end')
+    try:
+        facts, mapping = analyze(a.input, a.main_start, a.main_end, a.fx_rates)
+        a.output.mkdir(parents=True, exist_ok=True)
+        for name, data in [('facts.json', facts), ('coverage-and-reconciliation.json', facts['quality'])]:
+            (a.output / name).write_text(json.dumps(data, ensure_ascii=False, indent=2) + '\n', encoding='utf-8')
+        with (a.output / 'merchant-mapping.csv').open('w', encoding='utf-8', newline='') as f:
+            w = safe_writer(f); w.writerow(['layer', 'merchant_id', 'descriptor', 'attribution', 'provider_namespace', 'rows'])
+            for key, n in sorted(mapping.items()): w.writerow([*key, n])
+        with (a.output / 'merchant-metrics.csv').open('w', encoding='utf-8', newline='') as f:
+            w = safe_writer(f, numeric_columns={6, 8}); w.writerow(['merchant_id', 'name', 'attempts', 'approvals', 'auth_rate', 'sale_count', 'sales_usd', 'refund_count', 'refunds_usd', 'dispute_snapshots'])
+            for m in facts['merchants']:
+                w.writerow([m['id'], m['name'], m['attempts'] if m['layer_observed']['auth'] else None, m['approvals'] if m['layer_observed']['auth'] else None, m['auth_rate'], m['sale_count'], m['sales']['usd'] if m['sales'] else None, m['refund_count'], m['refunds']['usd'] if m['refunds'] else None, m['dispute_snapshots']])
+    except (ValueError, OSError, KeyError, json.JSONDecodeError) as e:
+        p.exit(1, f'error: {e}\n')
+
+
+if __name__ == '__main__':
+    main()