harness-everything 0.28.0 → 0.30.0

package/scripts/system-one-perintent-laya.py

@@ -0,0 +1,134 @@
+#!/usr/bin/env python3
+"""Per-intent Bernoulli evaluation for Laya relevance scores (#255 Phase 3).
+
+Each intent is its own binary decision with its own threshold, fit to the
+owner's secondary band (teacher score >= 0.4). No simplex, no single winner:
+this matches how owner intents actually look (about 2.5 secondaries per
+prompt) and how Laya noul heads natively behave (independent P(true)).
+
+Honest estimate via 2-fold CV; the shipped taus artifact is fit on all rows
+and labeled in-sample. Stdlib only; runs in seconds on saved predictions.
+"""
+import argparse
+import hashlib
+import json
+import sys
+from pathlib import Path
+
+INTENTS = ['explain', 'discuss', 'git', 'fix', 'edit', 'feature', 'refactor',
+           'review', 'test', 'docs', 'plan', 'investigate']
+
+GRID = [round(x / 100, 2) for x in range(5, 100, 5)]
+
+
+def binarize(scores, band=0.4):
+    """Intents whose score reaches the positive band."""
+    return {i for i in INTENTS if scores.get(i, 0.0) >= band}
+
+
+def sweep_threshold(scores, truth):
+    """Best F1 threshold on paired lists; ties break toward silence."""
+    best = (0.95, 0.0, 0.0, 0.0)
+    for tau in GRID:
+        tp = sum(1 for s, t in zip(scores, truth) if s >= tau and t)
+        fp = sum(1 for s, t in zip(scores, truth) if s >= tau and not t)
+        fn = sum(1 for s, t in zip(scores, truth) if s < tau and t)
+        p = tp / (tp + fp) if (tp + fp) else 0.0
+        r = tp / (tp + fn) if (tp + fn) else 0.0
+        f = 2 * p * r / (p + r) if (p + r) else 0.0
+        if (f, tau) > (best[3], best[0]):
+            best = (tau, p, r, f)
+    return best
+
+
+def micro(per):
+    """Micro-averaged P/R/F1 over {intent: (tp, fp, fn)}; empty scores zero."""
+    tp = sum(v[0] for v in per.values())
+    fp = sum(v[1] for v in per.values())
+    fn = sum(v[2] for v in per.values())
+    p = tp / (tp + fp) if (tp + fp) else 0.0
+    r = tp / (tp + fn) if (tp + fn) else 0.0
+    return {'precision': p, 'recall': r, 'f1': 2 * p * r / (p + r) if (p + r) else 0.0}
+
+
+def evaluate(ids, preds, truth_sets, taus):
+    """Multi-label metrics for one id list under fixed taus."""
+    per = {i: [0, 0, 0] for i in INTENTS}
+    abstain, fires_total, truth_total, exact = 0, 0, 0, 0
+    for pid in ids:
+        fired = {i for i in INTENTS if preds[pid].get(i, 0.0) >= taus[i]}
+        truth = truth_sets[pid]
+        fires_total += len(fired)
+        truth_total += len(truth)
+        abstain += not fired
+        exact += fired == truth
+        for i in INTENTS:
+            if i in fired and i in truth:
+                per[i][0] += 1
+            elif i in fired:
+                per[i][1] += 1
+            elif i in truth:
+                per[i][2] += 1
+    n = len(ids)
+    return {'n': n, 'micro': micro({i: tuple(v) for i, v in per.items()}),
+            'abstainRate': abstain / n if n else 0.0,
+            'meanFires': fires_total / n if n else 0.0,
+            'meanTruth': truth_total / n if n else 0.0,
+            'exactSetMatch': exact / n if n else 0.0,
+            'perIntent': {i: {'tau': taus[i], 'tp': per[i][0], 'fp': per[i][1], 'fn': per[i][2],
+                              'support': per[i][0] + per[i][2]} for i in INTENTS}}
+
+
+def fit_taus(ids, preds, truth_sets):
+    """Max-F1 threshold per intent on the given ids."""
+    taus = {}
+    for i in INTENTS:
+        taus[i] = sweep_threshold([preds[pid].get(i, 0.0) for pid in ids],
+                                  [i in truth_sets[pid] for pid in ids])[0]
+    return taus
+
+
+def main(argv=None):
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument('--preds', required=True, help='saved predictions json {id: {intent: score}}')
+    parser.add_argument('--teacher', required=True, help='teacher scores jsonl')
+    parser.add_argument('--band', type=float, default=0.4, help='teacher positive band')
+    parser.add_argument('--out', required=True, help='report json path')
+    parser.add_argument('--taus', required=True, help='threshold artifact json path')
+    args = parser.parse_args(argv)
+    preds = json.loads(Path(args.preds).read_text(encoding='utf-8'))
+    teacher = {r['id']: r for r in
+               (json.loads(line) for line in Path(args.teacher).read_text(encoding='utf-8').splitlines() if line.strip())}
+    ids = sorted(set(preds) & set(teacher))
+    truth_sets = {pid: binarize(teacher[pid]['scores'], args.band) for pid in ids}
+    halves = ([i for k, i in enumerate(ids) if k % 2 == 0], [i for k, i in enumerate(ids) if k % 2 == 1])
+    cv = [evaluate(halves[1], preds, truth_sets, fit_taus(halves[0], preds, truth_sets)),
+          evaluate(halves[0], preds, truth_sets, fit_taus(halves[1], preds, truth_sets))]
+    full_taus = fit_taus(ids, preds, truth_sets)
+    in_sample = evaluate(ids, preds, truth_sets, full_taus)
+    report = {'n': len(ids), 'band': args.band,
+              'cvMicroF1': sum(c['micro']['f1'] for c in cv) / 2,
+              'cv': [{'micro': c['micro'], 'abstainRate': c['abstainRate'],
+                      'meanFires': c['meanFires'], 'exactSetMatch': c['exactSetMatch']} for c in cv],
+              'inSample': {'micro': in_sample['micro'], 'abstainRate': in_sample['abstainRate'],
+                           'meanFires': in_sample['meanFires'], 'meanTruth': in_sample['meanTruth'],
+                           'exactSetMatch': in_sample['exactSetMatch'],
+                           'perIntent': in_sample['perIntent']}}
+    Path(args.out).write_text(json.dumps(report, indent=2), encoding='utf-8')
+    Path(args.taus).write_text(json.dumps(
+        {'taus': full_taus, 'band': args.band, 'inSample': True,
+         'predsSha256': hashlib.sha256(Path(args.preds).read_bytes()).hexdigest(),
+         'teacherSha256': hashlib.sha256(Path(args.teacher).read_bytes()).hexdigest()},
+        indent=2), encoding='utf-8')
+    m = report['inSample']['micro']
+    print(json.dumps({'n': len(ids), 'cvF1': round(report['cvMicroF1'], 3),
+                      'inSampleF1': round(m['f1'], 3), 'inSampleP': round(m['precision'], 3),
+                      'inSampleR': round(m['recall'], 3),
+                      'fires': round(in_sample['meanFires'], 2),
+                      'truth': round(in_sample['meanTruth'], 2),
+                      'abstain': round(in_sample['abstainRate'], 3)}))
+    return 0
+
+
+if __name__ == '__main__':
+    sys.exit(main())

package/scripts/system-one-train-laya.py

@@ -0,0 +1,344 @@
+#!/usr/bin/env python3
+"""Single-device RLCD fine-tuning for Laya (#255 Phase 3).
+
+Faithful single-process port of the upstream Kaggle DDP recipe
+(notebooks/laya_finetune_typed_decisions_2xT4_kaggle.ipynb, train_ddp.py):
+same GRPO + soft cross-entropy objective, same calibration holdout seed,
+same temperature fit, same artifact layout. Differences: no NCCL/DDP;
+cuda uses fp16+scaler, mps/cpu train fp32 without a scaler.
+
+Input is the reviewable items jsonl from system-one-export-laya.py.
+Output is a loadable checkpoint dir (model.safetensors + encoder/ +
+tokenizer/ + rl_agent_config.json). Heavy deps import lazily so contract
+tests run on stdlib.
+"""
+import argparse
+import json
+import random
+import sys
+import time
+from pathlib import Path
+
+try:
+    import torch
+    _TORCH = True
+except ImportError:
+    torch = None
+    _TORCH = False
+
+CALIB_SEED = 20260922
+CALIB_MAX = 400
+INTENTS = ['explain', 'discuss', 'git', 'fix', 'edit', 'feature', 'refactor',
+           'review', 'test', 'docs', 'plan', 'investigate']
+
+
+def need_torch():
+    if not _TORCH:
+        raise RuntimeError('torch is required for training')
+
+
+def intent_weights(intents, table):
+    """Per-item loss multipliers by training intent; unlisted intents weigh 1."""
+    import math  # noqa: PLC0415
+    table = dict(table or {})
+    for intent, weight in table.items():
+        if intent not in INTENTS or not isinstance(weight, (int, float)) \
+                or not math.isfinite(weight) or weight <= 0:
+            raise ValueError(f'intent-weights: bad entry {intent!r}:{weight!r}')
+    return [float(table.get(intent, 1.0)) for intent in intents]
+
+
+def calib_split(items, seed=CALIB_SEED, frac=0.1):
+    """Deterministic calibration split, grouped by prompt when promptId exists.
+
+    Calibration keeps one copy per promptId+intent identity. Oversampled copies
+    survive only in training groups, preventing exact-duplicate leakage and
+    calibration reweighting. Callers without promptId retain per-item behavior.
+    """
+    groups = {}
+    group_order = []
+    for index, item in enumerate(items):
+        prompt_id = item.get('promptId')
+        key = ('prompt', str(prompt_id)) if prompt_id is not None else ('item', index)
+        if key not in groups:
+            groups[key] = []
+            group_order.append(key)
+        groups[key].append(item)
+
+    def calib_rows(group):
+        if not group or group[0].get('promptId') is None:
+            return list(group)
+        seen, rows = set(), []
+        for item in group:
+            identity = (item.get('promptId'), item.get('intent'))
+            if identity in seen:
+                continue
+            seen.add(identity)
+            rows.append(item)
+        return rows
+
+    base_groups = {key: calib_rows(group) for key, group in groups.items()}
+    base_total = sum(len(group) for group in base_groups.values())
+    if base_total == 0:
+        return [], []
+    n_calib = max(1, min(CALIB_MAX, base_total // 10 if frac == 0.1 else int(base_total * frac)))
+
+    shuffled = list(group_order)
+    random.Random(seed).shuffle(shuffled)
+    calib_groups, calib_count = set(), 0
+    for key in shuffled:
+        size = len(base_groups[key])
+        if calib_count == 0 and size > n_calib:
+            calib_groups.add(key)
+            break
+        if calib_count + size <= n_calib:
+            calib_groups.add(key)
+            calib_count += size
+            if calib_count == n_calib:
+                break
+
+    train, calib = [], []
+    for key in group_order:
+        if key in calib_groups:
+            calib.extend(base_groups[key])
+        else:
+            train.extend(groups[key])
+    return train, calib
+
+
+def collate(items, pad_id):
+    """Batch upstream-format items into padded tensors."""
+    need_torch()
+    n, max_len = len(items), max(len(it['ids']) for it in items)
+    kmax = max(len(it['markers']) for it in items)
+    ids = torch.full((n, max_len), pad_id, dtype=torch.long)
+    att = torch.zeros((n, max_len), dtype=torch.long)
+    mpos = torch.zeros((n, kmax), dtype=torch.long)
+    mmask = torch.zeros((n, kmax), dtype=torch.bool)
+    target = torch.zeros((n, kmax), dtype=torch.float32)
+    for i, it in enumerate(items):
+        ids[i, :len(it['ids'])] = torch.tensor(it['ids'])
+        att[i, :len(it['ids'])] = 1
+        k = len(it['markers'])
+        mpos[i, :k] = torch.tensor(it['markers'])
+        mmask[i, :k] = True
+        target[i, :len(it['target'])] = torch.tensor(it['target'], dtype=torch.float32)
+    return {'input_ids': ids, 'attention_mask': att, 'marker_pos': mpos, 'marker_mask': mmask,
+            'target': target,
+            'qtype': torch.tensor([it['qtype'] for it in items]),
+            'label': torch.tensor([it['label'] for it in items])}
+
+
+def fit_one_temp(sel):
+    """Fit one temperature per question type with LBFGS (upstream recipe)."""
+    need_torch()
+    if len(sel) < 10:
+        return 1.0
+    kmax = max(len(z) for z, _ in sel)
+    Z = torch.full((len(sel), kmax), -1e4)
+    T = torch.zeros((len(sel), kmax))
+    for i, (z, t) in enumerate(sel):
+        Z[i, :len(z)] = torch.tensor(z)
+        T[i, :len(t)] = torch.tensor(t, dtype=torch.float32)
+    log_t = torch.zeros(1, requires_grad=True)
+    opt = torch.optim.LBFGS([log_t], lr=0.1, max_iter=100)
+
+    def closure():
+        opt.zero_grad()
+        loss = -(T * torch.log_softmax(Z / log_t.exp(), -1)).sum(-1).mean()
+        loss.backward()
+        return loss
+
+    opt.step(closure)
+    return float(torch.clamp(log_t.exp(), 0.1, 10.0).item())
+
+
+def pick_device(name):
+    need_torch()
+    if name == 'auto':
+        if torch.cuda.is_available():
+            return 'cuda'
+        if hasattr(torch, 'backends') and torch.backends.mps.is_available():
+            return 'mps'
+        return 'cpu'
+    return name
+
+
+def train(args):
+    need_torch()
+    from safetensors.torch import load_file, save_file  # noqa: PLC0415
+    from transformers import AutoTokenizer  # noqa: PLC0415
+    from laya.agent import _fix_tokenizer_config  # noqa: PLC0415
+    from laya.common import build_model, proper_reward  # noqa: PLC0415
+
+    device = pick_device(args.device)
+    use_amp = device == 'cuda'
+    out_dir = Path(args.out)
+    out_dir.mkdir(parents=True, exist_ok=True)
+
+    items = [json.loads(line) for line in Path(args.items).read_text(encoding='utf-8').splitlines() if line.strip()]
+    train_items, calib_items = calib_split(items, seed=args.calib_seed, frac=args.calib_frac)
+    weights_table = json.loads(args.intent_weights)
+    intent_weights([], weights_table)  # validate early, before GPU time
+    print(json.dumps({'trainItems': len(train_items), 'calibItems': len(calib_items), 'device': device,
+                      'intentWeights': weights_table}), flush=True)
+
+    from huggingface_hub import snapshot_download  # noqa: PLC0415
+    model_dir = Path(snapshot_download(args.checkpoint))
+    base = model_dir / args.subfolder if args.subfolder else model_dir
+    _fix_tokenizer_config(str(base))
+    with open(base / 'rl_agent_config.json', encoding='utf-8') as fh:
+        cfg = json.load(fh)
+    cfg['max_tokens_per_batch'] = 4096
+    cfg['max_len'] = 1024
+    cfg['head_max_len'] = 256
+    tok = AutoTokenizer.from_pretrained(str(base / 'tokenizer'))
+    model = build_model(cfg, encoder_dir=str(base / 'encoder'))
+    model.load_state_dict(load_file(str(base / 'model.safetensors')), strict=True)
+    model.encoder.gradient_checkpointing_enable(gradient_checkpointing_kwargs={'use_reentrant': False})
+    model.head_checkpointing = True
+    model.to(device)
+    model.train()
+
+    enc_params = [p for n, p in model.named_parameters() if 'encoder.' in n]
+    head_params = [p for n, p in model.named_parameters() if 'encoder.' not in n]
+    optimizer = torch.optim.AdamW([
+        {'params': enc_params, 'lr': args.lr_encoder},
+        {'params': head_params, 'lr': args.lr_head},
+    ], weight_decay=0.01)
+    updates_per_epoch = max(1, len(train_items) // (args.micro_batch * args.grad_accum))
+    scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(
+        optimizer, T_max=max(1, updates_per_epoch * args.epochs), eta_min=1e-6)
+    scaler = torch.amp.GradScaler('cuda', enabled=use_amp)
+
+    t0 = time.time()
+    for epoch in range(args.epochs):
+        rnd = random.Random(42 + epoch)
+        rnd.shuffle(train_items)
+        sigma = args.sigma_start + (args.sigma_end - args.sigma_start) * (epoch / max(1, args.epochs - 1))
+        epoch_loss, n_batches = 0.0, 0
+        optimizer.zero_grad(set_to_none=True)
+        accum = 0
+        for b in range(0, len(train_items), args.micro_batch):
+            chunk = train_items[b:b + args.micro_batch]
+            batch = collate(chunk, tok.pad_token_id)
+            ctx = torch.amp.autocast('cuda', dtype=torch.float16) if use_amp else torch.amp.autocast('cpu', enabled=False)
+            with ctx:
+                logits, act = model(batch['input_ids'].to(device), batch['attention_mask'].to(device),
+                                    batch['marker_pos'].to(device), batch['marker_mask'].to(device),
+                                    batch['qtype'].to(device))
+            logits = logits.float()
+            mask = batch['marker_mask'].to(device)
+            k = mask.sum(-1, keepdim=True).float()
+            target = batch['target'].to(device)
+            eps = torch.randn((args.group_size,) + logits.shape, device=device) * sigma * mask
+            eps = (eps - eps.sum(-1, keepdim=True) / k) * mask
+            z = logits.detach().unsqueeze(0) + eps
+            q = torch.softmax(z.masked_fill(~mask, -1e4), -1)
+            with torch.no_grad():
+                r = proper_reward(q, target.unsqueeze(0), batch['qtype'].to(device), mask,
+                                  w_sph=0.75, w_rps=1.0)
+                adv = r - r.mean(0, keepdim=True)
+                adv = adv / (adv.std() + 1e-6)
+            logp = -(((z - logits.unsqueeze(0)) ** 2) * mask).sum(-1) / (2 * sigma ** 2)
+            sample_w = torch.tensor(
+                intent_weights([it.get('intent') for it in chunk], weights_table),
+                dtype=torch.float32, device=device)
+            loss_rl = (-(adv * logp).mean(0) * sample_w).mean()
+            loss_ce = ((-(target * torch.log_softmax(logits.masked_fill(~mask, -1e4), -1)).sum(-1)) * sample_w).mean()
+            loss = (loss_rl + loss_ce) / args.grad_accum + 0.0 * act.sum()
+            scaler.scale(loss).backward()
+            accum += 1
+            if accum % args.grad_accum == 0 or (b + args.micro_batch) >= len(train_items):
+                scaler.unscale_(optimizer)
+                torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)
+                scaler.step(optimizer)
+                scaler.update()
+                scheduler.step()
+                optimizer.zero_grad(set_to_none=True)
+            epoch_loss += loss.item() * args.grad_accum
+            n_batches += 1
+            if n_batches % 50 == 0:
+                print(json.dumps({'epoch': epoch + 1, 'step': n_batches,
+                                  'loss': round(loss.item() * args.grad_accum, 4),
+                                  'reward': round(r.mean().item(), 3)}), flush=True)
+        ckpt = out_dir / 'checkpoint_latest'
+        ckpt.mkdir(exist_ok=True)
+        ckpt_sd = {k: v.half().contiguous().cpu() for k, v in model.state_dict().items()}
+        save_file(ckpt_sd, str(ckpt / 'model.safetensors'))
+        model.encoder.config.save_pretrained(str(ckpt / 'encoder'))
+        tok.save_pretrained(str(ckpt / 'tokenizer'))
+        (ckpt / 'checkpoint_meta.json').write_text(json.dumps(
+            {'epoch': epoch + 1, 'totalEpochs': args.epochs,
+             'avgLoss': epoch_loss / max(1, n_batches)}), encoding='utf-8')
+        print(json.dumps({'epochDone': epoch + 1, 'avgLoss': round(epoch_loss / max(1, n_batches), 4),
+                          'elapsedS': round(time.time() - t0, 1)}), flush=True)
+        if device in ('cuda', 'mps'):
+            getattr(torch, device).empty_cache()
+
+    print(json.dumps({'fittingTemperatures': True}), flush=True)
+    del optimizer, scaler, scheduler
+    if device == 'cuda':
+        torch.cuda.empty_cache()
+    model.eval()
+    calib_preds = []
+    with torch.no_grad():
+        for c in range(0, len(calib_items), 16):
+            chunk = calib_items[c:c + 16]
+            cb = collate(chunk, tok.pad_token_id)
+            ctx = torch.amp.autocast('cuda', dtype=torch.float16) if use_amp else torch.amp.autocast('cpu', enabled=False)
+            with ctx:
+                l_sub, _ = model(cb['input_ids'].to(device), cb['attention_mask'].to(device),
+                                 cb['marker_pos'].to(device), cb['marker_mask'].to(device),
+                                 cb['qtype'].to(device))
+            l_np = l_sub.float().cpu().numpy()
+            for r, it in enumerate(chunk):
+                k = len(it['markers'])
+                calib_preds.append((it['qtype'], l_np[r, :k], it['target']))
+    fitted = [1.2, 1.2, 1.2]
+    try:
+        for qt in range(3):
+            sel = [(z, t) for q_type, z, t in calib_preds if q_type == qt]
+            if sel:
+                fitted[qt] = fit_one_temp(sel)
+    except Exception as exc:  # noqa: BLE001
+        print(json.dumps({'temperatureFallback': str(exc)}), flush=True)
+    sd = {k: v.half().contiguous().cpu() for k, v in model.state_dict().items()}
+    save_file(sd, str(out_dir / 'model.safetensors'))
+    model.encoder.config.save_pretrained(str(out_dir / 'encoder'))
+    tok.save_pretrained(str(out_dir / 'tokenizer'))
+    cfg['fine_tuned'] = True
+    cfg['model_name'] = args.model_name
+    cfg['temperature'] = fitted
+    cfg.pop('temperature_by_options', None)
+    (out_dir / 'rl_agent_config.json').write_text(json.dumps(cfg, indent=2), encoding='utf-8')
+    print(json.dumps({'done': True, 'out': str(out_dir), 'temperatures': [round(t, 3) for t in fitted],
+                      'elapsedS': round(time.time() - t0, 1)}), flush=True)
+    return 0
+
+
+def main(argv=None):
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument('--items', required=True, help='items jsonl from system-one-export-laya.py')
+    parser.add_argument('--out', required=True, help='checkpoint output dir')
+    parser.add_argument('--checkpoint', default='convaiinnovations/laya')
+    parser.add_argument('--subfolder', default='multilingual')
+    parser.add_argument('--model-name', default='laya-harness-intent-s1')
+    parser.add_argument('--device', default='auto', choices=['auto', 'cuda', 'mps', 'cpu'])
+    parser.add_argument('--epochs', type=int, default=2)
+    parser.add_argument('--micro-batch', type=int, default=2)
+    parser.add_argument('--grad-accum', type=int, default=8)
+    parser.add_argument('--group-size', type=int, default=4)
+    parser.add_argument('--lr-encoder', type=float, default=2.5e-5)
+    parser.add_argument('--lr-head', type=float, default=1e-4)
+    parser.add_argument('--sigma-start', type=float, default=0.4)
+    parser.add_argument('--sigma-end', type=float, default=0.1)
+    parser.add_argument('--calib-seed', type=int, default=CALIB_SEED)
+    parser.add_argument('--calib-frac', type=float, default=0.1)
+    parser.add_argument('--intent-weights', default='{}', help='JSON {intent: multiplier} for loss weighting')
+    args = parser.parse_args(argv)
+    return train(args)
+
+
+if __name__ == '__main__':
+    sys.exit(main())

package/scripts/system-one-train-ngram.py

@@ -0,0 +1,181 @@
+#!/usr/bin/env python3
+"""Train the System One ngram tier scorer (docs/system-one-training.md, docs/system-one-routing.md "N-gram provider").
+
+Featurization is stdlib-only and must match harness-everything/scripts/system-one/ngram.js exactly
+(ci/mechanism-system-one-ngram.test.js compares them). torch is imported only inside train() for the optimizer.
+"""
+import argparse
+import hashlib
+import json
+import math
+from pathlib import Path
+import struct
+import sys
+
+sys.dont_write_bytecode = True
+REPO = Path(__file__).resolve().parents[1]
+DEFAULT_HOLDOUT = REPO / 'benchmarks' / 'fixtures' / 'system-one-holdout.json'
+INTENT_HOLDOUT = REPO / 'benchmarks' / 'fixtures' / 'system-one-intent-holdout.json'
+
+
+def fnv1a32(text):
+    h = 0x811c9dc5
+    for byte in text.encode('utf-8'):
+        h ^= byte
+        h = (h * 0x01000193) & 0xFFFFFFFF
+    return h
+
+
+def featurize(text, nmax, dim):
+    """bucket -> weight: log(1 + count) per code-point n-gram, one length bucket, L2-normalized."""
+    t = ' '.join(text.lower().split())
+    counts = {}
+    for n in range(1, nmax + 1):
+        for i in range(len(t) - n + 1):
+            gram = t[i:i + n]
+            if not gram.strip():
+                continue
+            b = fnv1a32(f'{n}:{gram}') % dim
+            counts[b] = counts.get(b, 0) + 1
+    k = min(8, int(math.floor(math.log2(len(t.encode('utf-8')) + 1))))
+    counts[fnv1a32(f'len:{k}') % dim] = 1
+    feats = {b: math.log1p(c) for b, c in counts.items()}
+    norm = math.sqrt(sum(v * v for v in feats.values()))
+    return {b: v / norm for b, v in feats.items()}
+
+
+def load_jsonl(path):
+    return [json.loads(line) for line in Path(path).read_text(encoding='utf-8').splitlines() if line.strip()]
+
+
+def sha256_file(path):
+    return hashlib.sha256(Path(path).read_bytes()).hexdigest()
+
+
+def train(args):
+    import torch
+
+    torch.manual_seed(args.seed)
+    data = Path(args.data_dir)
+    intent = args.task == 'intent'
+    scored = args.target == 'scores'
+    holdout = args.holdout or (INTENT_HOLDOUT if intent else DEFAULT_HOLDOUT)
+    corpus = json.loads(Path(holdout).read_text(encoding='utf-8'))
+    catalog = [o['id'] for o in corpus['cases'][0]['request']['options']]
+    labels_file = data / ('labels-intent-scores.jsonl' if scored else 'labels-intent.jsonl' if intent else 'labels.jsonl')
+    if not labels_file.exists():
+        sys.exit(f'missing {labels_file.name}')
+    overrides = data / ('owner-overrides-intent.jsonl' if intent else 'owner-overrides.jsonl')
+    labeled = load_jsonl(labels_file)
+    gold = {r['id']: r['gold'] for r in labeled}
+    secondary = {r['id']: r.get('secondary', []) for r in labeled} if intent else {}
+    relevance = {r['id']: r['scores'] for r in labeled} if scored else {}
+    if overrides.exists():
+        for r in load_jsonl(overrides):
+            gold[r['id']] = r['gold']
+            if intent:
+                secondary[r['id']] = r.get('secondary', [])
+            if scored:
+                # An override without scores counts as 0.6 for its primary and 0.4 for its secondary intents.
+                relevance[r['id']] = r.get('scores') or {s: 0.6 if s == r['gold'] else 0.4 for s in [r['gold'], *r.get('secondary', [])] if s is not None}
+    excluded = data / 'owner-excluded.jsonl'
+    skip = {r['id'] for r in load_jsonl(excluded)} if excluded.exists() else set()
+    rows = [p for p in load_jsonl(data / 'prompts.jsonl') if p['id'] in gold and p['id'] not in skip]
+    index = lambda g: catalog.index('unclassified' if g is None else g)
+    split = {name: [r for r in rows if r['split'] == name] for name in ('train', 'validation')}
+
+    def matrix(items):
+        ii, jj, vv = [], [], []
+        for r, item in enumerate(items):
+            for b, v in featurize(item['text'], args.nmax, args.dim).items():
+                ii.append(r); jj.append(b); vv.append(v)
+        return torch.sparse_coo_tensor([ii, jj], vv, (len(items), args.dim), dtype=torch.float32).coalesce()
+
+    x_train, x_val = matrix(split['train']), matrix(split['validation'])
+    y_train = torch.tensor([index(gold[r['id']]) for r in split['train']])
+    # Intent targets are soft: the primary alone gets 1.0; with secondaries it gets 0.6 and they share 0.4.
+    target = y_train
+    if scored:
+        # Scored intent labels: each intent is its own binary target with its relevance score as the soft value.
+        target = torch.tensor([[(0.6 if gold[r['id']] is None else 0.0) if c == 'unclassified' else float(relevance[r['id']].get(c, 0.0))
+                                for c in catalog] for r in split['train']], dtype=torch.float32)
+    elif intent:
+        rows_t = []
+        for r in split['train']:
+            t = [0.0] * len(catalog)
+            sec = [s for s in secondary.get(r['id'], []) if s in catalog]
+            if gold[r['id']] is None or not sec:
+                t[index(gold[r['id']])] = 1.0
+            else:
+                t[index(gold[r['id']])] = 0.6
+                for s in sec:
+                    t[catalog.index(s)] += 0.4 / len(sec)
+            rows_t.append(t)
+        target = torch.tensor(rows_t, dtype=torch.float32)
+    # Class mass counts the soft targets, so an intent seen only as secondary still carries weight.
+    counts = target.sum(0) if intent or scored else torch.bincount(y_train, minlength=len(catalog)).float()
+    inverse = torch.where(counts > 0, counts.sum() / (len(catalog) * counts.clamp_min(1e-6)), torch.zeros_like(counts))
+    weight = {'none': None, 'sqrt': inverse.sqrt(), 'full': inverse}[args.balance]
+    W = torch.zeros(args.dim, len(catalog), requires_grad=True)
+    bias = torch.zeros(len(catalog), requires_grad=True)
+    optimizer = torch.optim.Adam([W, bias], lr=args.lr)
+    for _ in range(args.epochs):
+        optimizer.zero_grad()
+        logits = torch.sparse.mm(x_train, W) + bias
+        if scored:
+            per = torch.nn.functional.binary_cross_entropy_with_logits(logits, target, reduction='none')
+            fit = (per * (weight if weight is not None else 1.0)).mean()
+        else:
+            fit = torch.nn.functional.cross_entropy(logits, target, weight=weight)
+        loss = fit + args.weight_decay * (W ** 2).sum()
+        loss.backward()
+        optimizer.step()
+    with torch.no_grad():
+        # The provider returns the softmax over the logits for either loss (docs/system-one-training.md).
+        probs = torch.softmax(torch.sparse.mm(x_val, W) + bias, 1)
+    out = Path(args.out)
+    out.mkdir(parents=True, exist_ok=True)
+    weights = W.detach().contiguous().view(-1).tolist()
+    (out / 'harness-routing-v1.bin').write_bytes(struct.pack(f'<{len(weights)}f', *weights))
+    datasets = {name: sha256_file(data / name) for name in ('prompts.jsonl', labels_file.name)}
+    if overrides.exists():
+        datasets[overrides.name] = sha256_file(overrides)
+    sidecar = {'format': 'harness-ngram', 'formatVersion': 1, 'dim': args.dim, 'nmax': args.nmax, 'hash': 'fnv1a32', 'catalog': catalog,
+               'bias': [float(v) for v in bias.detach().tolist()],
+               'metadata': {'domain': 'harness-routing-v1', 'task': args.task, 'balance': args.balance, 'weightDecay': args.weight_decay, 'epochs': args.epochs,
+                            'seed': args.seed, 'datasets': datasets, 'counts': {k: len(v) for k, v in split.items()},
+                            **({'target': 'scores', 'loss': 'bce'} if scored else {})}}
+    (out / 'harness-routing-v1.json').write_text(json.dumps(sidecar), encoding='utf-8')
+    with (out / 'val-scores.jsonl').open('w', encoding='utf-8') as fh:
+        for r, p in zip(split['validation'], probs.tolist()):
+            row = {'id': r['id'], 'gold': gold[r['id']], 'probs': p}
+            if intent:
+                row['secondary'] = secondary.get(r['id'], [])
+            fh.write(json.dumps(row) + '\n')
+    y_val = torch.tensor([index(gold[r['id']]) for r in split['validation']])
+    accuracy = (probs.argmax(1) == y_val).float().mean().item() if len(y_val) else 0.0
+    print(json.dumps({'counts': sidecar['metadata']['counts'], 'validationAccuracy': accuracy}))
+
+
+def main(argv=None):
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument('--data-dir', required=True)
+    parser.add_argument('--out', required=True)
+    parser.add_argument('--task', choices=['tier', 'intent'], default='tier')
+    parser.add_argument('--target', choices=['label', 'scores'], default='label', help='scores: per-intent relevance targets from labels-intent-scores.jsonl (intent only)')
+    parser.add_argument('--holdout', default=None, help='source of the fixed catalog only (defaults to the task holdout); holdout rows are never trained on')
+    parser.add_argument('--dim', type=int, default=1 << 16)
+    parser.add_argument('--nmax', type=int, default=3)
+    parser.add_argument('--balance', choices=['none', 'sqrt', 'full'], default='sqrt')
+    parser.add_argument('--weight-decay', type=float, default=1e-4)
+    parser.add_argument('--epochs', type=int, default=60)
+    parser.add_argument('--lr', type=float, default=0.01)
+    parser.add_argument('--seed', type=int, default=7)
+    args = parser.parse_args(argv)
+    if args.target == 'scores' and args.task != 'intent':
+        parser.error('--target scores needs --task intent')
+    train(args)
+
+
+if __name__ == '__main__':
+    main()