harness-everything 0.29.0 → 0.30.0

package/scripts/system-one-export-laya.py

@@ -0,0 +1,197 @@
+#!/usr/bin/env python3
+"""Export soft-target RLCD training items for Laya fine-tuning (#255 Phase 3).
+
+Each train prompt with teacher relevance scores becomes 12 `noul` items, one
+per owner intent (catalog order, v1 wording shared with system-one-label-laya).
+Teacher relevance p maps to gold probabilities {true: p, false: 1-p}, matching
+the upstream fine-tune notebook (LocalLLaMA/typed-decisions, noul target
+[false, true], qtype 2, label = argmax).
+
+Output is reviewable jsonl (one item per line); the training host converts it
+to train_items.pt with torch.save (never commit .pt). Sequence building goes
+through laya.common.build_sequence so tokenization matches inference exactly.
+Stdlib only at import time; `laya` is imported lazily inside the default
+builder so contract tests run without model weights.
+"""
+import argparse
+import hashlib
+import importlib.util
+import json
+import random
+import sys
+from pathlib import Path
+
+HERE = Path(__file__).resolve()
+_label_spec = importlib.util.spec_from_file_location(
+    'laya_labeler', HERE.parent / 'system-one-label-laya.py')
+_label = importlib.util.module_from_spec(_label_spec)
+_label_spec.loader.exec_module(_label)
+INTENTS = _label.INTENTS
+QUESTIONS_VERSION = _label.QUESTIONS_VERSION
+build_questions = _label.build_questions
+
+# laya.common.QTYPES = {"choice": 0, "score": 1, "noul": 2}.
+QTYPE_NOUL = 2
+
+
+def build_item(state_text, instructions, probs, builder):
+    """One upstream-format training item from gold {true, false} probabilities."""
+    pf, pt = float(probs['false']), float(probs['true'])
+    if not (pf >= 0 and pt >= 0 and abs(pf + pt - 1.0) < 1e-6):
+        raise ValueError(f'probabilities must sum to 1, got {probs!r}')
+    built = builder(state_text, instructions, {'true': pt, 'false': pf})
+    if built is None:
+        return None
+    target = [built['target'][0], built['target'][1]]
+    if abs(sum(target) - 1.0) >= 1e-6:
+        raise ValueError(f'builder target must sum to 1, got {target!r}')
+    return {'ids': built['ids'], 'markers': built['markers'], 'qtype': QTYPE_NOUL,
+            'target': target, 'label': target.index(max(target))}
+
+
+def parse_oversample(spec):
+    """Parse 'intent:factor,...' into {intent: factor}; factor 1 is rejected."""
+    table = {}
+    for part in (spec or '').split(','):
+        if not part.strip():
+            continue
+        try:
+            intent, factor = part.split(':')
+            factor = int(factor)
+        except ValueError:
+            raise ValueError(f'oversample: bad entry {part!r}, want intent:factor')
+        if intent not in INTENTS or factor < 2:
+            raise ValueError(f'oversample: bad entry {part!r}, want known intent and factor >= 2')
+        table[intent] = factor
+    return table
+
+
+def default_builder_factory(checkpoint='convaiinnovations/laya', subfolder='multilingual'):
+    """Real sequence builder through laya.common (lazy import)."""
+    from huggingface_hub import snapshot_download  # noqa: PLC0415
+    from transformers import AutoTokenizer  # noqa: PLC0415
+    from laya.agent import _fix_tokenizer_config  # noqa: PLC0415
+    from laya.common import build_sequence, render_options  # noqa: PLC0415
+
+    model_dir = Path(snapshot_download(checkpoint))
+    revision = model_dir.name
+    base = model_dir / subfolder if subfolder else model_dir
+    _fix_tokenizer_config(str(base))
+    tok = AutoTokenizer.from_pretrained(str(base / 'tokenizer'))
+    cfg = json.loads((base / 'rl_agent_config.json').read_text(encoding='utf-8'))
+    questions = build_questions()
+
+    def builder(state_text, instructions, probs):
+        question = {'t': 'noul', 'ins': instructions, 'crit': {}}
+        seq, markers = build_sequence(tok, state_text, question, cfg['max_len'], cfg['head_max_len'])
+        if len(markers) != len(render_options(question)):
+            return None
+        return {'ids': seq, 'markers': markers,
+                'target': [probs['false'], probs['true']]}
+
+    provenance = {'checkpoint': checkpoint, 'revision': revision, 'subfolder': subfolder or 'root',
+                  'questions': QUESTIONS_VERSION,
+                  'tokenizerDir': str(base / 'tokenizer')}
+    return builder, provenance
+
+
+def export(data_dir, split, out_path, manifest_path, builder, seed=0, forbid_path=None,
+           teacher_file='labels-intent-scores.jsonl', provenance=None, oversample=None):
+    """Export one prompts.jsonl split to upstream-format items."""
+    if split not in ('train', 'validation'):
+        raise ValueError(f'split must be train or validation, got {split!r}')
+    oversample = dict(oversample or {})
+    for intent, factor in oversample.items():
+        if intent not in INTENTS or not isinstance(factor, int) or factor < 2:
+            raise ValueError(f'oversample: bad entry {intent!r}:{factor!r}')
+    data_dir, out_path, manifest_path = Path(data_dir), Path(out_path), Path(manifest_path)
+    prompts_raw = (data_dir / 'prompts.jsonl').read_text(encoding='utf-8')
+    teacher_raw = (data_dir / teacher_file).read_text(encoding='utf-8')
+    teacher = {r['id']: r for r in
+               (json.loads(line) for line in teacher_raw.splitlines() if line.strip())}
+    rows = sorted((r for r in (json.loads(line) for line in prompts_raw.splitlines() if line.strip())
+                   if r.get('split') == split), key=lambda r: r['id'])
+    if forbid_path is not None:
+        forbidden = set(json.loads(Path(forbid_path).read_text(encoding='utf-8')))
+        clash = sorted({r['id'] for r in rows} & forbidden)
+        if clash:
+            raise ValueError(f'overlap with forbidden ids: {clash[:5]} (n={len(clash)})')
+    if seed:
+        random.Random(seed).shuffle(rows)
+    questions = build_questions()
+    items, skipped_no_teacher, skipped_markers = 0, 0, 0
+    with out_path.open('w', encoding='utf-8') as fh:
+        for prompt in rows:
+            trow = teacher.get(prompt['id'])
+            if trow is None:
+                skipped_no_teacher += 1
+                continue
+            for intent in INTENTS:
+                p = float(trow['scores'].get(intent, 0.0))
+                repeat = oversample.get(intent, 1) if p >= 0.4 else 1
+                item = build_item(prompt['text'], questions[intent]['instructions'],
+                                  {'true': p, 'false': 1.0 - p}, builder)
+                if item is None:
+                    skipped_markers += repeat
+                    continue
+                line = json.dumps({'promptId': prompt['id'], 'intent': intent, **item}) + '\n'
+                for _ in range(repeat):
+                    fh.write(line)
+                items += repeat
+    manifest = {'split': split, 'seed': seed, 'questions': QUESTIONS_VERSION,
+                'builder': provenance or {'builder': 'injected'},
+                'oversample': oversample,
+                'items': items, 'prompts': len(rows),
+                'skippedNoTeacher': skipped_no_teacher, 'skippedMarkers': skipped_markers,
+                'promptsSha256': hashlib.sha256(prompts_raw.encode('utf-8')).hexdigest(),
+                'teacherSha256': hashlib.sha256(teacher_raw.encode('utf-8')).hexdigest(),
+                'teacherFile': teacher_file,
+                'outputSha256': hashlib.sha256(out_path.read_bytes()).hexdigest()}
+    manifest_path.write_text(json.dumps(manifest, indent=2), encoding='utf-8')
+    return {'items': items, 'skippedNoTeacher': skipped_no_teacher,
+            'skippedMarkers': skipped_markers, 'manifest': manifest}
+
+
+def main(argv=None):
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument('--data-dir', required=True, help='private corpus dir')
+    parser.add_argument('--split', choices=['train', 'validation'], default='train')
+    parser.add_argument('--out', required=True, help='output items jsonl')
+    parser.add_argument('--manifest', required=True, help='provenance manifest json')
+    parser.add_argument('--checkpoint', default='convaiinnovations/laya')
+    parser.add_argument('--subfolder', default='multilingual')
+    parser.add_argument('--seed', type=int, default=0, help='0 keeps id order; nonzero shuffles')
+    parser.add_argument('--forbid', default=None, help='json list of forbidden ids (overlap guard)')
+    parser.add_argument('--teacher', default='labels-intent-scores.jsonl')
+    parser.add_argument('--oversample', default='', help='intent:factor,... to repeat positive intent items')
+    parser.add_argument('--limit', type=int, default=0, help='export only the first N prompts (0 = all)')
+    args = parser.parse_args(argv)
+    oversample = parse_oversample(args.oversample)
+    builder, provenance = default_builder_factory(args.checkpoint, args.subfolder)
+    print(json.dumps({'builder': provenance}))
+    # --limit applies before export by staging a filtered view; export() keeps split semantics.
+    if args.limit > 0:
+        import tempfile  # noqa: PLC0415
+        data_dir = Path(args.data_dir)
+        prompts_raw = (data_dir / 'prompts.jsonl').read_text(encoding='utf-8')
+        rows = [json.loads(line) for line in prompts_raw.splitlines() if line.strip()]
+        todo = sorted((r for r in rows if r.get('split') == args.split), key=lambda r: r['id'])[:args.limit]
+        with tempfile.TemporaryDirectory() as tmp:
+            view = Path(tmp) / 'view'
+            view.mkdir()
+            (view / 'prompts.jsonl').write_text(
+                '\n'.join(json.dumps(r) for r in todo) + '\n', encoding='utf-8')
+            (view / args.teacher).write_text((data_dir / args.teacher).read_text(encoding='utf-8'), encoding='utf-8')
+            summary = export(view, args.split, args.out, args.manifest, builder,
+                             seed=args.seed, forbid_path=args.forbid, teacher_file=args.teacher,
+                             provenance=provenance, oversample=oversample)
+    else:
+        summary = export(args.data_dir, args.split, args.out, args.manifest, builder,
+                         seed=args.seed, forbid_path=args.forbid, teacher_file=args.teacher,
+                         provenance=provenance, oversample=oversample)
+    print(json.dumps({k: v for k, v in summary.items() if k != 'manifest'}))
+    return 0
+
+
+if __name__ == '__main__':
+    sys.exit(main())

package/scripts/system-one-label-laya.py

@@ -0,0 +1,243 @@
+#!/usr/bin/env python3
+"""Zero-shot dense intent labeling with Laya (#255 Phase 1+2).
+
+Maps each of the 12 owner intents (catalogs.js order, frozen) to one Laya
+`noul` question and records the yes-probability as the per-intent relevance
+score. Output rows match the `labels-intent-scores.jsonl` trainer contract
+{id, gold, secondary, scores}, but MUST be written to a separate file: the
+sonnet-scored labels are never overwritten. Derived gold/secondary are
+mechanical bootstrap only (labeler.derived=true), never owner truth.
+
+Stdlib only at import time; `laya` / `laya_mlx` are imported lazily inside
+the backend constructors so contract tests run without model weights.
+"""
+import argparse
+import hashlib
+import json
+import math
+import statistics
+import sys
+import time
+from pathlib import Path
+
+# Owner catalog order (harness-everything/scripts/system-one/catalogs.js).
+# This order is the scorer output order; changing it invalidates weights.
+INTENTS = ['explain', 'discuss', 'git', 'fix', 'edit', 'feature', 'refactor',
+           'review', 'test', 'docs', 'plan', 'investigate']
+
+QUESTIONS_VERSION = 'v1'
+
+# The sonnet-scored labels file. The labeler refuses to write to it.
+PROTECTED_OUTPUT = 'labels-intent-scores.jsonl'
+
+_PROPOSITIONS = {
+    'explain': 'The request asks for an explanation of how something works or an answer to a question.',
+    'discuss': 'The request asks to weigh options, give an opinion, or settle a decision.',
+    'git': 'The request asks for Git or GitHub housekeeping: commit, push, branch, or pull request.',
+    'fix': 'The request asks to repair wrong behavior: a bug, crash, failing test, or build.',
+    'edit': 'The request asks to change an existing value, setting, text, or behavior.',
+    'feature': 'The request asks to add new behavior: a command, option, format, or script.',
+    'refactor': 'The request asks to restructure or unify existing code without adding new behavior.',
+    'review': 'The request asks to evaluate an existing artifact against a standard.',
+    'test': 'The request asks to run tests or builds, or to write tests as the main deliverable.',
+    'docs': 'The request asks to write or edit documentation as the main deliverable.',
+    'plan': 'The request asks to produce a plan, specification, or breakdown before implementation.',
+    'investigate': 'The request asks to find facts or a cause and report back without changing anything.',
+}
+
+# v2 adds contrastive exclusions aimed at the measured top-2 co-firing pairs
+# (edit/refactor, fix/investigate, investigate/review/explain, docs/edit,
+# feature/edit, plan/discuss). Same 12 intents, same semantics: exclusions
+# only sharpen boundaries, they never add or remove a category.
+_PROPOSITIONS_V2 = {
+    'explain': 'The request asks for an explanation of how something works or an answer to a question. Not investigating an unknown cause (investigate).',
+    'discuss': 'The request asks to weigh options, give an opinion, or settle a decision. Not producing an implementation plan or breakdown (plan).',
+    'git': 'The request asks for Git or GitHub housekeeping: commit, push, branch, or pull request.',
+    'fix': 'The request asks to repair wrong behavior: a bug, crash, failing test, or build. Not merely finding the cause (investigate); only when a repair is requested.',
+    'edit': 'The request asks to change an existing value, setting, text, or behavior. Not a bug repair (fix), not a restructuring (refactor), not new behavior (feature), not documentation as the deliverable (docs).',
+    'feature': 'The request asks to add new behavior: a command, option, format, or script. Not modifying existing behavior (edit).',
+    'refactor': 'The request asks to restructure or unify existing code without adding new behavior. Not a value change (edit), not a bug repair (fix).',
+    'review': 'The request asks to evaluate an existing artifact against a standard. Not fact-finding (investigate).',
+    'test': 'The request asks to run tests or builds, or to write tests as the main deliverable.',
+    'docs': 'The request asks to write or edit documentation as the main deliverable. Not incidental text edits alongside other work (edit).',
+    'plan': 'The request asks to produce a plan, specification, or breakdown before implementation. Not weighing options in discussion (discuss).',
+    'investigate': 'The request asks to find facts or a cause and report back without changing anything. Not repairing (fix), not judging quality (review), not explaining how things work (explain).',
+}
+
+_QUESTIONS = {'v1': _PROPOSITIONS, 'v2': _PROPOSITIONS_V2}
+
+
+def build_questions(version='v1'):
+    """One `noul` question per intent, in catalog order."""
+    if version not in _QUESTIONS:
+        raise ValueError(f'unknown questions version: {version!r}')
+    table = _QUESTIONS[version]
+    return {intent: {'type': 'noul', 'instructions': table[intent]} for intent in INTENTS}
+
+
+def validate_scores(scores):
+    """True when scores cover exactly the 12 intents with finite 0..1 floats."""
+    if not isinstance(scores, dict) or set(scores) != set(INTENTS):
+        raise ValueError(f'scores keys must be exactly the 12 intents, got {sorted(scores) if isinstance(scores, dict) else type(scores)}')
+    for intent in INTENTS:
+        v = scores[intent]
+        if not isinstance(v, float) or not math.isfinite(v) or not 0.0 <= v <= 1.0:
+            raise ValueError(f'scores[{intent!r}] out of range: {v!r}')
+    return True
+
+
+def derive_label(scores):
+    """Mechanical {gold, secondary} from bands (bootstrap only, derived=true).
+
+    Mirrors the scoredLabel bands: secondary is every other intent at 0.4+;
+    a max at or below 0.2 abstains to null. Ties break by catalog order.
+    """
+    validate_scores(scores)
+    top = max(scores[intent] for intent in INTENTS)
+    if top <= 0.2:
+        return {'gold': None, 'secondary': [], 'derived': True}
+    gold = max(INTENTS, key=lambda i: (scores[i], -INTENTS.index(i)))
+    secondary = sorted((i for i in INTENTS if i != gold and scores[i] >= 0.4),
+                       key=lambda i: (-scores[i], INTENTS.index(i)))
+    return {'gold': gold, 'secondary': secondary, 'derived': True}
+
+
+def output_row(prompt, scores, backend, out_name, routing=None, questions_version='v1'):
+    """One trainer-contract row for a labeled prompt."""
+    if Path(out_name).name == PROTECTED_OUTPUT:
+        raise ValueError(f'protected output: refusing to overwrite {PROTECTED_OUTPUT}')
+    derived = derive_label(scores)
+    labeler = {'task': 'intent', 'target': 'scores', 'backend': backend,
+               'questions': questions_version, 'derived': True}
+    if routing is not None:
+        labeler['routing'] = routing
+    return {'id': prompt['id'], 'gold': derived['gold'], 'secondary': derived['secondary'],
+            'scores': {intent: scores[intent] for intent in INTENTS},
+            'labeler': labeler}
+
+
+def torch_backend(questions_version='v1'):
+    """Official PyTorch Laya backend (lazy import). Returns (fn, provenance)."""
+    import laya  # noqa: PLC0415
+    router = laya.Router()
+    questions = build_questions(questions_version)
+
+    def run(text):
+        started = time.monotonic()
+        result = router.predict(text, questions)
+        elapsed_ms = (time.monotonic() - started) * 1000.0
+        answers = result['answers']
+        scores = {intent: float(answers[intent]['noul']) for intent in INTENTS}
+        return scores, result['routing']['model'], elapsed_ms
+
+    provenance = {'package': 'laya', 'version': laya.__version__, 'questions': questions_version}
+    return run, provenance
+
+
+def mlx_backend(questions_version='v1'):
+    """Native MLX Laya port backend (lazy import, inference only)."""
+    import laya_mlx  # noqa: PLC0415
+    from laya_mlx import Router  # noqa: PLC0415
+    router = Router()
+    questions = build_questions(questions_version)
+
+    def run(text):
+        started = time.monotonic()
+        result = router.predict(text, questions)
+        elapsed_ms = (time.monotonic() - started) * 1000.0
+        answers = result['answers']
+        scores = {intent: float(answers[intent]['noul']) for intent in INTENTS}
+        return scores, result['routing']['model'], elapsed_ms
+
+    provenance = {'package': 'laya-mlx', 'version': laya_mlx.__version__, 'questions': questions_version}
+    return run, provenance
+
+
+def label_rows(rows, split, input_sha, out_path, manifest_path, backend_fn, backend_name, resume=False,
+               questions_version='v1'):
+    """Label explicit rows; returns a summary dict."""
+    out_path, manifest_path = Path(out_path), Path(manifest_path)
+    if out_path.name == PROTECTED_OUTPUT:
+        raise ValueError(f'protected output: refusing to overwrite {PROTECTED_OUTPUT}')
+    done = set()
+    if resume and out_path.exists():
+        for line in out_path.read_text(encoding='utf-8').splitlines():
+            if line.strip():
+                done.add(json.loads(line)['id'])
+        rows = [r for r in rows if r['id'] not in done]
+    routing, latencies, this_run = {}, [], 0
+    with out_path.open('a' if (resume and done) else 'w', encoding='utf-8') as fh:
+        for prompt in rows:
+            scores, routed, elapsed_ms = backend_fn(prompt['text'])
+            fh.write(json.dumps(output_row(prompt, scores, backend_name, out_path.name, routed,
+                                              questions_version)) + '\n')
+            routing[routed] = routing.get(routed, 0) + 1
+            latencies.append(elapsed_ms)
+            this_run += 1
+    total, full_routing = this_run, routing
+    if resume:
+        # The file is the source of truth: recount everything for cumulative totals.
+        full_routing = {}
+        total = 0
+        for line in out_path.read_text(encoding='utf-8').splitlines():
+            if not line.strip():
+                continue
+            row = json.loads(line)
+            routed = (row.get('labeler') or {}).get('routing', 'unknown')
+            full_routing[routed] = full_routing.get(routed, 0) + 1
+            total += 1
+    manifest = {'backend': backend_name, 'split': split, 'questions': questions_version,
+                'labeled': total, 'thisRun': this_run, 'resumed': bool(resume and done),
+                'inputSha256': input_sha, 'routing': full_routing,
+                'latencyMs': {'p50': statistics.median(latencies) if latencies else 0.0,
+                              'p95': statistics.quantiles(latencies, n=100)[94] if len(latencies) >= 100 else (max(latencies) if latencies else 0.0)}}
+    manifest_path.write_text(json.dumps(manifest, indent=2), encoding='utf-8')
+    return manifest
+
+
+def label_split(data_dir, split, out_path, manifest_path, backend_fn, backend_name, questions_version='v1'):
+    """Label one prompts.jsonl split; returns a summary dict."""
+    if split not in ('train', 'validation'):
+        raise ValueError(f'split must be train or validation, got {split!r}')
+    data_dir = Path(data_dir)
+    prompts_raw = (data_dir / 'prompts.jsonl').read_text(encoding='utf-8')
+    rows = [json.loads(line) for line in prompts_raw.splitlines() if line.strip()]
+    todo = [r for r in rows if r.get('split') == split]
+    return label_rows(todo, split, hashlib.sha256(prompts_raw.encode('utf-8')).hexdigest(),
+                      out_path, manifest_path, backend_fn, backend_name,
+                      questions_version=questions_version)
+
+
+def main(argv=None):
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument('--data-dir', required=True, help='private corpus dir (holds prompts.jsonl)')
+    parser.add_argument('--split', choices=['train', 'validation'], default='validation')
+    parser.add_argument('--out', required=True, help='output jsonl (never labels-intent-scores.jsonl)')
+    parser.add_argument('--manifest', required=True, help='provenance manifest json')
+    parser.add_argument('--backend', choices=['torch', 'mlx'], required=True)
+    parser.add_argument('--questions', choices=['v1', 'v2'], default='v1')
+    parser.add_argument('--limit', type=int, default=0, help='label only the first N rows (0 = all)')
+    parser.add_argument('--resume', action='store_true', help='skip ids already present in --out')
+    args = parser.parse_args(argv)
+    run, provenance = (torch_backend(args.questions) if args.backend == 'torch'
+                       else mlx_backend(args.questions))
+    print(json.dumps({'backend': args.backend, 'provenance': provenance}))
+
+    data_dir = Path(args.data_dir)
+    prompts_raw = (data_dir / 'prompts.jsonl').read_text(encoding='utf-8')
+    rows = [json.loads(line) for line in prompts_raw.splitlines() if line.strip()]
+    todo = [r for r in rows if r.get('split') == args.split]
+    if args.limit > 0:
+        todo = todo[:args.limit]
+    summary = label_rows(todo, args.split,
+                         hashlib.sha256(prompts_raw.encode('utf-8')).hexdigest(),
+                         args.out, args.manifest, run, args.backend, resume=args.resume,
+                         questions_version=args.questions)
+    summary['limit'] = args.limit
+    Path(args.manifest).write_text(json.dumps(summary, indent=2), encoding='utf-8')
+    print(json.dumps(summary))
+    return 0
+
+
+if __name__ == '__main__':
+    sys.exit(main())