harness-everything 0.27.0 → 0.29.0

package/scripts/system-one-label.js

@@ -3,6 +3,7 @@
 // Labels System One training prompts with an LLM against the owner's rules (docs/system-one-corpus.md).
 // See docs/system-one-training.md. Labels stay local; only the holdout agreement report may be committed.
 const fs = require('node:fs');
+const os = require('node:os');
 const path = require('node:path');
 const { spawn } = require('node:child_process');
 const { createHash } = require('node:crypto');
@@ -105,6 +106,32 @@ function commandRunner(model) {
     child.stdin.end(prompt);
   });
 }
+// One stateless `codex exec` per batch: prompt on stdin, schema-checked last message in a temp file,
+// read-only sandbox, no session files and no user config (plugins/hooks), so nothing accumulates across batches.
+function codexRunner(model) {
+  const base = process.env.HARNESS_S1_LABEL_COMMAND ? JSON.parse(process.env.HARNESS_S1_LABEL_COMMAND) : ['codex'];
+  const work = fs.mkdtempSync(path.join(os.tmpdir(), 'harness-s1-codex-'));
+  const schemaFile = path.join(work, 'schema.json');
+  fs.writeFileSync(schemaFile, SCHEMA);
+  let n = 0;
+  return prompt => new Promise((resolve, reject) => {
+    const outFile = path.join(work, `reply-${process.pid}-${n++}.json`);
+    const args = ['exec', '--model', model, '-c', 'model_reasoning_effort=medium', '--sandbox', 'read-only', '--skip-git-repo-check',
+      '--ephemeral', '--ignore-user-config', '--output-schema', schemaFile, '-o', outFile, '-C', work, '-'];
+    const child = spawn(base[0], [...base.slice(1), ...args], { stdio: ['pipe', 'ignore', 'ignore'], windowsHide: true,
+      shell: process.platform === 'win32' && !process.env.HARNESS_S1_LABEL_COMMAND });
+    const timer = setTimeout(() => child.kill(), 600000);
+    child.on('error', err => { clearTimeout(timer); reject(err); });
+    child.on('close', code => {
+      clearTimeout(timer);
+      let reply = null;
+      try { reply = JSON.parse(fs.readFileSync(outFile, 'utf8')); } catch (_) { reject(new Error(`codex-exit-${code}: no reply`)); return; }
+      try { fs.unlinkSync(outFile); } catch (_) { /* temp file */ }
+      resolve(reply);
+    });
+    child.stdin.end(prompt);
+  });
+}
 function outsideRepo(file) {
   const rel = path.relative(REPO, path.resolve(file));
   if (!rel.startsWith('..') && !path.isAbsolute(rel)) throw new Error('refusing to write labels inside the repository');
@@ -113,10 +140,12 @@ function outsideRepo(file) {
 async function main(argv) {
   const [op, ...rest] = argv;
   const opt = (name, fallback) => { const i = rest.indexOf(name); return i >= 0 ? rest[i + 1] : fallback; };
-  const model = opt('--model', 'sonnet');
+  const engine = opt('--engine', 'claude');
+  if (!['claude', 'codex'].includes(engine)) throw new Error('--engine must be claude or codex');
+  const model = opt('--model', engine === 'codex' ? 'gpt-5.6-luna' : 'sonnet');
   const rules = rulesFromDoc(fs.readFileSync(path.join(REPO, 'docs', 'system-one-corpus.md'), 'utf8'));
-  const labeler = { model, rulesSha256: sha(rules).slice(0, 16) };
-  const run = commandRunner(model);
+  const labeler = { model, rulesSha256: sha(rules).slice(0, 16), ...(engine === 'codex' ? { engine } : {}) };
+  const run = engine === 'codex' ? codexRunner(model) : commandRunner(model);
   const concurrency = Number(opt('--concurrency', '4')); const batchSize = Number(opt('--batch', '100'));
   if (op === 'label') {
     const input = opt('--in'); const out = opt('--out');

package/scripts/system-one-train-ngram.py

@@ -0,0 +1,129 @@
+#!/usr/bin/env python3
+"""Train the System One ngram tier scorer (docs/system-one-training.md, docs/system-one-routing.md "N-gram provider").
+
+Featurization is stdlib-only and must match harness-everything/scripts/system-one/ngram.js exactly
+(ci/mechanism-system-one-ngram.test.js compares them). torch is imported only inside train() for the optimizer.
+"""
+import argparse
+import hashlib
+import json
+import math
+from pathlib import Path
+import struct
+import sys
+
+sys.dont_write_bytecode = True
+REPO = Path(__file__).resolve().parents[1]
+DEFAULT_HOLDOUT = REPO / 'benchmarks' / 'fixtures' / 'system-one-holdout.json'
+
+
+def fnv1a32(text):
+    h = 0x811c9dc5
+    for byte in text.encode('utf-8'):
+        h ^= byte
+        h = (h * 0x01000193) & 0xFFFFFFFF
+    return h
+
+
+def featurize(text, nmax, dim):
+    """bucket -> weight: log(1 + count) per code-point n-gram, one length bucket, L2-normalized."""
+    t = ' '.join(text.lower().split())
+    counts = {}
+    for n in range(1, nmax + 1):
+        for i in range(len(t) - n + 1):
+            gram = t[i:i + n]
+            if not gram.strip():
+                continue
+            b = fnv1a32(f'{n}:{gram}') % dim
+            counts[b] = counts.get(b, 0) + 1
+    k = min(8, int(math.floor(math.log2(len(t.encode('utf-8')) + 1))))
+    counts[fnv1a32(f'len:{k}') % dim] = 1
+    feats = {b: math.log1p(c) for b, c in counts.items()}
+    norm = math.sqrt(sum(v * v for v in feats.values()))
+    return {b: v / norm for b, v in feats.items()}
+
+
+def load_jsonl(path):
+    return [json.loads(line) for line in Path(path).read_text(encoding='utf-8').splitlines() if line.strip()]
+
+
+def sha256_file(path):
+    return hashlib.sha256(Path(path).read_bytes()).hexdigest()
+
+
+def train(args):
+    import torch
+
+    torch.manual_seed(args.seed)
+    data = Path(args.data_dir)
+    corpus = json.loads(Path(args.holdout).read_text(encoding='utf-8'))
+    catalog = [o['id'] for o in corpus['cases'][0]['request']['options']]
+    gold = {r['id']: r['gold'] for r in load_jsonl(data / 'labels.jsonl')}
+    overrides = data / 'owner-overrides.jsonl'
+    if overrides.exists():
+        gold.update({r['id']: r['gold'] for r in load_jsonl(overrides)})
+    excluded = data / 'owner-excluded.jsonl'
+    skip = {r['id'] for r in load_jsonl(excluded)} if excluded.exists() else set()
+    rows = [p for p in load_jsonl(data / 'prompts.jsonl') if p['id'] in gold and p['id'] not in skip]
+    index = lambda g: catalog.index('unclassified' if g is None else g)
+    split = {name: [r for r in rows if r['split'] == name] for name in ('train', 'validation')}
+
+    def matrix(items):
+        ii, jj, vv = [], [], []
+        for r, item in enumerate(items):
+            for b, v in featurize(item['text'], args.nmax, args.dim).items():
+                ii.append(r); jj.append(b); vv.append(v)
+        return torch.sparse_coo_tensor([ii, jj], vv, (len(items), args.dim), dtype=torch.float32).coalesce()
+
+    x_train, x_val = matrix(split['train']), matrix(split['validation'])
+    y_train = torch.tensor([index(gold[r['id']]) for r in split['train']])
+    counts = torch.bincount(y_train, minlength=len(catalog)).float()
+    inverse = torch.where(counts > 0, counts.sum() / (len(catalog) * counts.clamp_min(1)), torch.zeros_like(counts))
+    weight = {'none': None, 'sqrt': inverse.sqrt(), 'full': inverse}[args.balance]
+    W = torch.zeros(args.dim, len(catalog), requires_grad=True)
+    bias = torch.zeros(len(catalog), requires_grad=True)
+    optimizer = torch.optim.Adam([W, bias], lr=args.lr)
+    for _ in range(args.epochs):
+        optimizer.zero_grad()
+        loss = torch.nn.functional.cross_entropy(torch.sparse.mm(x_train, W) + bias, y_train, weight=weight) + args.weight_decay * (W ** 2).sum()
+        loss.backward()
+        optimizer.step()
+    with torch.no_grad():
+        probs = torch.softmax(torch.sparse.mm(x_val, W) + bias, 1)
+    out = Path(args.out)
+    out.mkdir(parents=True, exist_ok=True)
+    weights = W.detach().contiguous().view(-1).tolist()
+    (out / 'harness-routing-v1.bin').write_bytes(struct.pack(f'<{len(weights)}f', *weights))
+    datasets = {name: sha256_file(data / name) for name in ('prompts.jsonl', 'labels.jsonl')}
+    if overrides.exists():
+        datasets['owner-overrides.jsonl'] = sha256_file(overrides)
+    sidecar = {'format': 'harness-ngram', 'formatVersion': 1, 'dim': args.dim, 'nmax': args.nmax, 'hash': 'fnv1a32', 'catalog': catalog,
+               'bias': [float(v) for v in bias.detach().tolist()],
+               'metadata': {'domain': 'harness-routing-v1', 'balance': args.balance, 'weightDecay': args.weight_decay, 'epochs': args.epochs,
+                            'seed': args.seed, 'datasets': datasets, 'counts': {k: len(v) for k, v in split.items()}}}
+    (out / 'harness-routing-v1.json').write_text(json.dumps(sidecar), encoding='utf-8')
+    with (out / 'val-scores.jsonl').open('w', encoding='utf-8') as fh:
+        for r, p in zip(split['validation'], probs.tolist()):
+            fh.write(json.dumps({'id': r['id'], 'gold': gold[r['id']], 'probs': p}) + '\n')
+    y_val = torch.tensor([index(gold[r['id']]) for r in split['validation']])
+    accuracy = (probs.argmax(1) == y_val).float().mean().item() if len(y_val) else 0.0
+    print(json.dumps({'counts': sidecar['metadata']['counts'], 'validationAccuracy': accuracy}))
+
+
+def main(argv=None):
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument('--data-dir', required=True)
+    parser.add_argument('--out', required=True)
+    parser.add_argument('--holdout', default=str(DEFAULT_HOLDOUT), help='source of the fixed catalog only; holdout rows are never trained on')
+    parser.add_argument('--dim', type=int, default=1 << 16)
+    parser.add_argument('--nmax', type=int, default=3)
+    parser.add_argument('--balance', choices=['none', 'sqrt', 'full'], default='sqrt')
+    parser.add_argument('--weight-decay', type=float, default=1e-4)
+    parser.add_argument('--epochs', type=int, default=60)
+    parser.add_argument('--lr', type=float, default=0.01)
+    parser.add_argument('--seed', type=int, default=7)
+    train(parser.parse_args(argv))
+
+
+if __name__ == '__main__':
+    main()

package/scripts/system-one-train.py

@@ -50,6 +50,16 @@ def join(prompts, labels, overrides):
     return [{'id': p['id'], 'split': p['split'], 'text': p['text'], 'gold': gold[p['id']]} for p in prompts if p['id'] in gold]
 
 
+def class_weights(label_indices, num_classes):
+    """Inverse-frequency weights that average to 1 over the examples; absent classes get 0."""
+    counts = [0] * num_classes
+    for i in label_indices:
+        counts[i] += 1
+    present = sum(1 for c in counts if c)
+    total = len(label_indices)
+    return [total / (present * c) if c else 0.0 for c in counts]
+
+
 def make_batches(lengths, token_budget):
     """Length-bucketed batches: each batch's size times its longest item stays within the budget."""
     order = sorted(range(len(lengths)), key=lambda i: (lengths[i], i))
@@ -94,6 +104,8 @@ def train(args):
     train_batches = make_batches(lengths(split['train']), args.token_budget)
     val_batches = make_batches(lengths(split['validation']), args.token_budget)
     optimizer = torch.optim.AdamW(model.parameters(), lr=args.lr, weight_decay=0.01)
+    # Class balance counters the tier2-heavy labels; validation NLL stays unweighted so epochs remain comparable.
+    weight = torch.tensor(class_weights([e.label for e in train_x], len(catalog)), dtype=torch.float) if args.balance else None
 
     def evaluate():
         model.eval()
@@ -118,7 +130,7 @@ def train(args):
         total = 0.0
         for b in order:
             tensors = collator([train_x[i] for i in train_batches[b]])
-            loss = torch.nn.functional.cross_entropy(model(tensors), tensors['labels'])
+            loss = torch.nn.functional.cross_entropy(model(tensors), tensors['labels'], weight=weight)
             optimizer.zero_grad()
             loss.backward()
             torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)
@@ -161,6 +173,7 @@ def main(argv=None):
     parser.add_argument('--threads', type=int, default=4)
     parser.add_argument('--context-tokens', type=int, default=1024)
     parser.add_argument('--token-budget', type=int, default=16384)
+    parser.add_argument('--balance', action='store_true', help='inverse-frequency class weights in the training loss')
     train(parser.parse_args(argv))