harness-everything 0.28.0 → 0.30.0

package/ci/system-one-laya-label-test.py

@@ -0,0 +1,201 @@
+"""Stdlib contract tests for the Laya dense intent labeler (#255 Phase 1+2).
+
+No model dependency: backends are injected as stubs. Run with:
+    python3 ci/system-one-laya-label-test.py
+"""
+import importlib.util
+import json
+import sys
+import tempfile
+import unittest
+from pathlib import Path
+
+sys.dont_write_bytecode = True  # never leave __pycache__ in the source tree that packaging copies
+path = Path(__file__).resolve().parents[1] / 'scripts/system-one-label-laya.py'
+spec = importlib.util.spec_from_file_location('laya_labeler', path)
+laya_labeler = importlib.util.module_from_spec(spec)
+spec.loader.exec_module(laya_labeler)
+
+INTENTS = ['explain', 'discuss', 'git', 'fix', 'edit', 'feature', 'refactor',
+           'review', 'test', 'docs', 'plan', 'investigate']
+
+
+class MappingTests(unittest.TestCase):
+    def test_intent_order_matches_owner_catalog(self):
+        # catalogs.js INTENT_OPTIONS order is the scorer output order; it must not drift.
+        self.assertEqual(laya_labeler.INTENTS, INTENTS)
+
+    def test_every_intent_has_one_noul_question(self):
+        questions = laya_labeler.build_questions()
+        self.assertEqual(list(questions), INTENTS, 'question order follows the catalog order')
+        for intent, q in questions.items():
+            self.assertEqual(q['type'], 'noul')
+            self.assertTrue(q['instructions'], intent)
+
+    def test_question_order_change_keeps_semantic_mapping(self):
+        a = laya_labeler.build_questions()
+        b = laya_labeler.build_questions()
+        self.assertEqual(a, b)
+        self.assertEqual(list(a), INTENTS)
+
+    def test_mapping_version_is_pinned(self):
+        self.assertRegex(laya_labeler.QUESTIONS_VERSION, r'^v\d+$')
+
+    def test_v2_keeps_intents_and_order_but_sharpens_text(self):
+        v1 = laya_labeler.build_questions('v1')
+        v2 = laya_labeler.build_questions('v2')
+        self.assertEqual(list(v2), INTENTS)
+        for intent, q in v2.items():
+            self.assertEqual(q['type'], 'noul')
+            self.assertTrue(q['instructions'], intent)
+        self.assertTrue(any(v2[i]['instructions'] != v1[i]['instructions'] for i in INTENTS),
+                        'v2 rewords at least one intent')
+        self.assertEqual(v2['git']['instructions'], v1['git']['instructions'],
+                         'unconfused intents keep v1 text')
+
+    def test_unknown_questions_version_fails(self):
+        with self.assertRaisesRegex(ValueError, 'unknown questions version'):
+            laya_labeler.build_questions('v9')
+
+
+class ValidatorTests(unittest.TestCase):
+    def scores(self, **over):
+        base = {i: 0.0 for i in INTENTS}
+        base.update(over)
+        return base
+
+    def test_valid_scores_pass(self):
+        self.assertTrue(laya_labeler.validate_scores(self.scores(fix=0.9, test=0.4)))
+
+    def test_missing_or_unknown_key_fails(self):
+        bad = self.scores(fix=0.9)
+        del bad['fix']
+        with self.assertRaisesRegex(ValueError, 'keys'):
+            laya_labeler.validate_scores(bad)
+        bad2 = self.scores(fix=0.9)
+        bad2['nope'] = 0.1
+        with self.assertRaisesRegex(ValueError, 'keys'):
+            laya_labeler.validate_scores(bad2)
+
+    def test_out_of_range_or_nan_fails(self):
+        for v in (-0.1, 1.1, float('nan')):
+            with self.assertRaisesRegex(ValueError, 'range', msg=f'value={v}'):
+                laya_labeler.validate_scores(self.scores(fix=v))
+
+    def test_non_float_fails(self):
+        with self.assertRaisesRegex(ValueError, 'range'):
+            laya_labeler.validate_scores(self.scores(fix='0.9'))
+
+
+class DeriveTests(unittest.TestCase):
+    def scores(self, **over):
+        base = {i: 0.0 for i in INTENTS}
+        base.update(over)
+        return base
+
+    def test_primary_and_secondary_from_bands(self):
+        out = laya_labeler.derive_label(self.scores(fix=0.9, test=0.5, docs=0.3))
+        self.assertEqual(out['gold'], 'fix')
+        self.assertEqual(out['secondary'], ['test'])
+
+    def test_low_max_abstains(self):
+        out = laya_labeler.derive_label(self.scores(explain=0.2))
+        self.assertIsNone(out['gold'])
+        self.assertEqual(out['secondary'], [])
+
+    def test_tie_breaks_by_catalog_order(self):
+        out = laya_labeler.derive_label(self.scores(fix=0.7, test=0.7))
+        self.assertEqual(out['gold'], 'fix')
+
+    def test_derived_labels_are_marked_bootstrap(self):
+        out = laya_labeler.derive_label(self.scores(fix=0.9))
+        self.assertTrue(out['derived'])
+
+
+class GuardTests(unittest.TestCase):
+    def test_refuses_to_overwrite_sonnet_scores_file(self):
+        with self.assertRaisesRegex(ValueError, 'protected'):
+            laya_labeler.output_row({'id': 'x', 'text': 'fix it'}, {}, backend='stub',
+                                    out_name='labels-intent-scores.jsonl')
+
+    def test_output_row_shape_matches_trainer_contract(self):
+        scores = {i: (0.9 if i == 'fix' else 0.0) for i in INTENTS}
+        row = laya_labeler.output_row({'id': 'abc', 'text': 'fix it'}, scores,
+                                      backend='stub', out_name='labels-intent-laya-zs.jsonl',
+                                      routing='english')
+        self.assertEqual(row['id'], 'abc')
+        self.assertEqual(row['gold'], 'fix')
+        self.assertEqual(row['secondary'], [])
+        self.assertEqual(list(row['scores']), INTENTS)
+        self.assertEqual(row['labeler']['task'], 'intent')
+        self.assertEqual(row['labeler']['routing'], 'english')
+        self.assertEqual(row['labeler']['questions'], 'v1')
+        self.assertEqual(json.loads(json.dumps(row)), row)
+
+    def test_output_row_records_questions_version(self):
+        scores = {i: 0.1 for i in INTENTS}
+        row = laya_labeler.output_row({'id': 'abc', 'text': 'x'}, scores,
+                                      backend='stub', out_name='o.jsonl', questions_version='v2')
+        self.assertEqual(row['labeler']['questions'], 'v2')
+
+
+class EndToEndTests(unittest.TestCase):
+    def test_stub_backend_labels_a_split(self):
+        prompts = [{'id': 'p1', 'split': 'validation', 'text': 'fix the crash'},
+                   {'id': 'p2', 'split': 'train', 'text': 'commit and push'}]
+        calls = []
+
+        def stub(text):
+            calls.append(text)
+            return ({i: (0.8 if i == 'fix' else 0.05) for i in INTENTS}, 'english', 1.0)
+
+        with tempfile.TemporaryDirectory() as d:
+            data = Path(d) / 'data'
+            data.mkdir()
+            (data / 'prompts.jsonl').write_text(
+                '\n'.join(json.dumps(p) for p in prompts) + '\n', encoding='utf-8')
+            out = Path(d) / 'labels-intent-laya-zs.jsonl'
+            manifest = Path(d) / 'manifest.json'
+            summary = laya_labeler.label_split(
+                data, 'validation', out, manifest, backend_fn=stub, backend_name='stub')
+            self.assertEqual(calls, ['fix the crash'], 'only the requested split is labeled')
+            rows = [json.loads(line) for line in out.read_text(encoding='utf-8').strip().split('\n')]
+            self.assertEqual([r['id'] for r in rows], ['p1'])
+            self.assertEqual(rows[0]['gold'], 'fix')
+            self.assertEqual(summary['labeled'], 1)
+            man = json.loads(manifest.read_text(encoding='utf-8'))
+            self.assertEqual(man['backend'], 'stub')
+            self.assertEqual(man['split'], 'validation')
+
+    def test_resume_skips_done_ids(self):
+        calls = []
+
+        def stub(text):
+            calls.append(text)
+            return ({i: (0.8 if i == 'fix' else 0.05) for i in INTENTS}, 'english', 1.0)
+
+        with tempfile.TemporaryDirectory() as d:
+            data = Path(d) / 'data'
+            data.mkdir()
+            prompts = [{'id': f'p{i}', 'split': 'validation', 'text': f'text {i}'} for i in range(3)]
+            (data / 'prompts.jsonl').write_text(
+                '\n'.join(json.dumps(p) for p in prompts) + '\n', encoding='utf-8')
+            out = Path(d) / 'labels-intent-laya-zs.jsonl'
+            manifest = Path(d) / 'manifest.json'
+            laya_labeler.label_split(data, 'validation', out, manifest,
+                                     backend_fn=stub, backend_name='stub')
+            self.assertEqual(len(calls), 3)
+            laya_labeler.label_rows(
+                prompts, 'validation', 'sha', out, manifest, backend_fn=stub,
+                backend_name='stub', resume=True)
+            self.assertEqual(len(calls), 3, 'resume makes no new backend calls')
+            self.assertEqual(
+                [json.loads(line)['id'] for line in out.read_text(encoding='utf-8').splitlines()],
+                ['p0', 'p1', 'p2'], 'resume appends nothing when all ids are done')
+            man2 = json.loads(manifest.read_text(encoding='utf-8'))
+            self.assertEqual([man2['labeled'], man2['thisRun'], man2['resumed']], [3, 0, True])
+            self.assertEqual(man2['routing'], {'english': 3})
+
+
+if __name__ == '__main__':
+    unittest.main()

package/ci/system-one-laya-perintent-test.py

@@ -0,0 +1,60 @@
+"""Stdlib tests for per-intent Bernoulli evaluation (#255 Phase 3).
+
+Each intent is its own binary decision with its own threshold: no simplex,
+no single winner. Run with:
+    python3 ci/system-one-laya-perintent-test.py
+"""
+import importlib.util
+import sys
+import unittest
+from pathlib import Path
+
+sys.dont_write_bytecode = True  # never leave __pycache__ in the source tree that packaging copies
+path = Path(__file__).resolve().parents[1] / 'scripts/system-one-perintent-laya.py'
+spec = importlib.util.spec_from_file_location('laya_perintent', path)
+laya_perintent = importlib.util.module_from_spec(spec)
+spec.loader.exec_module(laya_perintent)
+
+INTENTS = ['explain', 'discuss', 'git', 'fix', 'edit', 'feature', 'refactor',
+           'review', 'test', 'docs', 'plan', 'investigate']
+
+
+class BinarizeTests(unittest.TestCase):
+    def test_band_boundary(self):
+        self.assertEqual(laya_perintent.binarize({'fix': 0.4, 'test': 0.39}), {'fix', 'test'} - {'test'})
+        self.assertEqual(laya_perintent.binarize({'fix': 0.4}), {'fix'})
+
+    def test_custom_positive_band(self):
+        self.assertEqual(laya_perintent.binarize({'fix': 0.5}, band=0.6), set())
+
+
+class SweepTests(unittest.TestCase):
+    def test_separable_intent_gets_perfect_threshold(self):
+        scores = [0.1, 0.2, 0.8, 0.9]
+        truth = [False, False, True, True]
+        tau, p, r, f = laya_perintent.sweep_threshold(scores, truth)
+        self.assertEqual((p, r, f), (1.0, 1.0, 1.0))
+        self.assertTrue(0.2 < tau <= 0.8)
+
+    def test_all_negative_stays_silent(self):
+        tau, p, r, f = laya_perintent.sweep_threshold([0.1, 0.2, 0.3], [False, False, False])
+        self.assertEqual(r, 0.0)
+        self.assertTrue(tau > 0.3)
+
+
+class MicroTests(unittest.TestCase):
+    def test_hand_computed_micro(self):
+        # totals: tp=1, fp=2, fn=2.
+        per = {'a': (1, 1, 1), 'b': (0, 1, 1)}  # (tp, fp, fn)
+        m = laya_perintent.micro(per)
+        self.assertAlmostEqual(m['precision'], 1 / 3)
+        self.assertAlmostEqual(m['recall'], 1 / 3)
+        self.assertAlmostEqual(m['f1'], 1 / 3)
+
+    def test_empty_predictions_score_zero_not_nan(self):
+        m = laya_perintent.micro({'a': (0, 0, 2)})
+        self.assertEqual((m['precision'], m['recall'], m['f1']), (0.0, 0.0, 0.0))
+
+
+if __name__ == '__main__':
+    unittest.main()

package/ci/system-one-laya-provider-test.py

@@ -0,0 +1,115 @@
+"""Contract tests for the Laya fine-tuned provider adapter (#255 Phase 3).
+
+The 13-option evaluator vector, unclassified ramp, limits and artifact
+rules are all pinned here with a stub predictor: no model weights needed.
+Run with:
+    python3 ci/system-one-laya-provider-test.py
+"""
+import hashlib
+import importlib.util
+import json
+import sys
+import tempfile
+import unittest
+from pathlib import Path
+
+sys.dont_write_bytecode = True  # never leave __pycache__ in the source tree that packaging copies
+path = Path(__file__).resolve().parents[1] / 'harness-everything/scripts/system-one/laya_adapter.py'
+spec = importlib.util.spec_from_file_location('laya_adapter', path)
+laya_adapter = importlib.util.module_from_spec(spec)
+spec.loader.exec_module(laya_adapter)
+
+INTENTS = ['explain', 'discuss', 'git', 'fix', 'edit', 'feature', 'refactor',
+           'review', 'test', 'docs', 'plan', 'investigate']
+OPTIONS = [{'id': i, 'text': f'{i} text'} for i in INTENTS] + [{'id': 'unclassified', 'text': 'unclear'}]
+
+
+def request(options=None, task='intent', context='fix it'):
+    return {'schemaVersion': 1, 'task': task, 'context': context,
+            'options': options if options is not None else OPTIONS,
+            'catalogHash': 'b' * 64, 'requestHash': 'a' * 64}
+
+
+def stub_predict(probs):
+    def run(state, questions):
+        assert state and set(questions) == set(INTENTS)
+        return dict(probs)
+    return run
+
+
+class ArtifactTests(unittest.TestCase):
+    def write_ckpt(self, root, weights=b'fake-weights-1', config=b'{"cfg":1}'):
+        ckpt = Path(root) / 'model.safetensors'
+        ckpt.write_bytes(weights)
+        (Path(root) / 'rl_agent_config.json').write_bytes(config)
+        return {'checkpoint': str(ckpt),
+                'weightsSha256': hashlib.sha256(weights).hexdigest(),
+                'configSha256': hashlib.sha256(config).hexdigest()}
+
+    def test_verify_ok_and_config_sibling_rule(self):
+        with tempfile.TemporaryDirectory() as d:
+            m = self.write_ckpt(d)
+            self.assertEqual(laya_adapter.verify_artifacts(m), Path(m['checkpoint']))
+
+    def test_hash_mismatch_and_bad_format_fail(self):
+        with tempfile.TemporaryDirectory() as d:
+            m = self.write_ckpt(d)
+            bad = dict(m, weightsSha256='0' * 64)
+            with self.assertRaisesRegex(ValueError, 'artifact-hash'):
+                laya_adapter.verify_artifacts(bad)
+            bad2 = dict(m, checkpoint=str(Path(d) / 'model.pt'))
+            with self.assertRaisesRegex(ValueError, 'checkpoint-format'):
+                laya_adapter.verify_artifacts(bad2)
+            bad3 = dict(m, checkpoint='relative/model.safetensors')
+            with self.assertRaisesRegex(ValueError, 'checkpoint-format'):
+                laya_adapter.verify_artifacts(bad3)
+
+
+class ScorerTests(unittest.TestCase):
+    def probs(self, **over):
+        base = {i: 0.05 for i in INTENTS}
+        base.update(over)
+        return base
+
+    def test_scores_follow_request_option_order_raw(self):
+        score, _ = laya_adapter.make_scorer(stub_predict(self.probs(fix=0.8)))
+        shuffled = [OPTIONS[5], OPTIONS[0], *[o for o in OPTIONS if o['id'] not in ('feature', 'explain')]]
+        out = score(request(options=shuffled))
+        self.assertEqual(len(out), 13)
+        by_id = {o['id']: p for o, p in zip(shuffled, out)}
+        self.assertGreater(by_id['fix'], by_id['test'])
+        self.assertTrue(all(0.0 <= p <= 1.0 for p in out))
+        self.assertGreater(sum(out), 1.0, 'independent scores sum past one: no simplex')
+
+    def test_confident_max_leaves_no_unclassified_mass(self):
+        score, _ = laya_adapter.make_scorer(stub_predict(self.probs(fix=0.9)))
+        out = score(request())
+        self.assertEqual(out[-1], 0.0, 'max above the cutoff closes the abstain ramp')
+
+    def test_flat_lows_abstain_through_unclassified(self):
+        score, _ = laya_adapter.make_scorer(stub_predict(self.probs()))
+        out = score(request())
+        self.assertGreater(out[-1], 0.5, 'nothing actionable concentrates on unclassified')
+
+    def test_non_intent_task_and_missing_intent_fail(self):
+        score, _ = laya_adapter.make_scorer(stub_predict(self.probs()))
+        with self.assertRaisesRegex(ValueError, 'task'):
+            score(request(task='tier'))
+        with self.assertRaisesRegex(ValueError, 'options'):
+            score(request(options=[{'id': 'fix', 'text': 'x'}, {'id': 'unclassified', 'text': 'y'}]))
+
+
+class LimitTests(unittest.TestCase):
+    def test_boundary_passes_and_overflow_fails(self):
+        score, config = laya_adapter.make_scorer(stub_predict({i: 0.1 for i in INTENTS}))
+        ok_ctx = 'x' * config['context_tokens']
+        laya_adapter.check_limits(request(context=ok_ctx), config)
+        with self.assertRaisesRegex(ValueError, 'input-too-long'):
+            laya_adapter.check_limits(request(context=ok_ctx + 'x'), config)
+        big_opt = [{'id': 'a', 'text': 'y' * (config['option_tokens'] + 1)}, {'id': 'b', 'text': 'z'}]
+        with self.assertRaisesRegex(ValueError, 'input-too-long'):
+            laya_adapter.check_limits(request(options=big_opt), config)
+
+
+if __name__ == '__main__':
+    unittest.main()

package/ci/system-one-laya-train-test.py

@@ -0,0 +1,93 @@
+"""Contract tests for the single-device Laya trainer (#255 Phase 3).
+
+Pure parts run on stdlib; torch parts skip when torch is missing
+(same convention as S1-I08/S1-I13). Run with:
+    python3 ci/system-one-laya-train-test.py
+"""
+import importlib.util
+import sys
+import unittest
+from pathlib import Path
+
+sys.dont_write_bytecode = True  # never leave __pycache__ in the source tree that packaging copies
+path = Path(__file__).resolve().parents[1] / 'scripts/system-one-train-laya.py'
+spec = importlib.util.spec_from_file_location('laya_trainer', path)
+laya_trainer = importlib.util.module_from_spec(spec)
+spec.loader.exec_module(laya_trainer)
+
+try:
+    import torch  # noqa: F401
+    HAS_TORCH = True
+except ImportError:
+    HAS_TORCH = False
+
+
+class CalibSplitTests(unittest.TestCase):
+    def test_holdout_is_stable_disjoint_and_ten_percent(self):
+        items = [{'id': i} for i in range(1000)]
+        a_train, a_calib = laya_trainer.calib_split(items, seed=20260922, frac=0.1)
+        b_train, b_calib = laya_trainer.calib_split(items, seed=20260922, frac=0.1)
+        self.assertEqual(a_calib, b_calib, 'same seed replays the holdout')
+        self.assertEqual(len(a_calib), 100)
+        self.assertEqual(len(set(map(str, a_train)) & set(map(str, a_calib))), 0, 'disjoint')
+        c_train, c_calib = laya_trainer.calib_split(items, seed=1, frac=0.1)
+        self.assertNotEqual(a_calib, c_calib, 'seed matters')
+
+    def test_small_inputs_keep_at_least_one_calib(self):
+        _, calib = laya_trainer.calib_split([{'id': i} for i in range(5)], seed=7, frac=0.1)
+        self.assertEqual(len(calib), 1)
+
+    def test_prompt_groups_do_not_cross_split_and_calib_deduplicates_oversamples(self):
+        items = []
+        for prompt_id in range(20):
+            items.append({'promptId': str(prompt_id), 'intent': 'fix'})
+            items.append({'promptId': str(prompt_id), 'intent': 'refactor'})
+            items.append({'promptId': str(prompt_id), 'intent': 'refactor'})  # oversampled duplicate
+        train, calib = laya_trainer.calib_split(items, seed=20260922, frac=0.1)
+        train_prompts = {it['promptId'] for it in train}
+        calib_prompts = {it['promptId'] for it in calib}
+        self.assertFalse(train_prompts & calib_prompts, 'a prompt must stay wholly on one side')
+        calib_keys = [(it['promptId'], it['intent']) for it in calib]
+        self.assertEqual(len(calib_keys), len(set(calib_keys)),
+                         'oversampled duplicates must not overweight temperature fitting')
+        self.assertEqual(len(calib), 4, '10% of 40 de-duplicated prompt-intent identities')
+        self.assertEqual(sum(1 for it in train if it['intent'] == 'refactor'), 2 * len(train_prompts),
+                         'training keeps oversampled copies for non-calibration prompt groups')
+
+
+class IntentWeightTests(unittest.TestCase):
+    def test_default_weights_are_one(self):
+        self.assertEqual(laya_trainer.intent_weights(['fix', 'test'], {}), [1.0, 1.0])
+
+    def test_table_applies_per_item_intent(self):
+        self.assertEqual(laya_trainer.intent_weights(['refactor', 'fix', 'refactor'], {'refactor': 2.0}),
+                         [2.0, 1.0, 2.0])
+
+    def test_bad_table_fails(self):
+        with self.assertRaisesRegex(ValueError, 'intent-weights'):
+            laya_trainer.intent_weights(['fix'], {'nope': 2.0})
+        with self.assertRaisesRegex(ValueError, 'intent-weights'):
+            laya_trainer.intent_weights(['fix'], {'fix': 0})
+
+
+@unittest.skipUnless(HAS_TORCH, 'torch not installed')
+class TorchTests(unittest.TestCase):
+    def test_collate_shapes(self):
+        items = [{'ids': [1, 2, 3], 'markers': [0, 2], 'qtype': 2,
+                  'target': [0.3, 0.7], 'label': 1} for _ in range(4)]
+        batch = laya_trainer.collate(items, pad_id=0)
+        self.assertEqual(tuple(batch['input_ids'].shape), (4, 3))
+        self.assertEqual(tuple(batch['marker_pos'].shape), (4, 2))
+        self.assertTrue(bool((batch['target'].sum(-1) - 1.0 < 1e-6).all()))
+
+    def test_temperature_fit_is_finite_and_bounded(self):
+        import random
+        rng = random.Random(0)
+        sel = [([rng.uniform(-1, 1), rng.uniform(-1, 1)],
+                [0.4, 0.6]) for _ in range(30)]
+        t = laya_trainer.fit_one_temp(sel)
+        self.assertTrue(0.1 <= t <= 10.0)
+
+
+if __name__ == '__main__':
+    unittest.main()

package/ci/system-one-ngram-parity.py

@@ -0,0 +1,19 @@
+"""Print the trainer's n-gram features for texts on stdin (JSON list) so the Node provider can be compared exactly."""
+import importlib.util
+import json
+from pathlib import Path
+import sys
+
+sys.dont_write_bytecode = True
+ROOT = Path(__file__).resolve().parents[1]
+spec = importlib.util.spec_from_file_location('ngram_trainer', ROOT / 'scripts' / 'system-one-train-ngram.py')
+trainer = importlib.util.module_from_spec(spec)
+spec.loader.exec_module(trainer)
+assert 'torch' not in sys.modules, 'importing the ngram trainer must not import torch'
+
+request = json.loads(sys.stdin.read())
+out = []
+for text in request['texts']:
+    feats = trainer.featurize(text, request['nmax'], request['dim'])
+    out.append({str(k): v for k, v in sorted(feats.items())})
+print(json.dumps({'hash': [trainer.fnv1a32(s) for s in request['hashes']], 'features': out}))

package/ci/system-one-train-test.py

@@ -46,6 +46,12 @@ assert flat == list(range(len(lengths))), 'every example exactly once'
 assert all(len(b) * max(lengths[i] for i in b) <= 4096 or len(b) == 1 for b in batches), 'budget respected'
 assert batches == trainer.make_batches(lengths, token_budget=4096), 'deterministic'
 
+weights = trainer.class_weights([0, 0, 0, 1, 2, 2], 4)
+assert len(weights) == 4
+assert weights[3] == 0.0, 'an absent class gets no weight'
+assert weights[1] > weights[2] > weights[0], weights
+assert abs(sum(w * n for w, n in zip(weights, [3, 1, 2, 0])) - 6) < 1e-9, 'weights average to 1 over the examples'
+
 with tempfile.TemporaryDirectory() as tmp:
     p = Path(tmp) / 'rows.jsonl'
     p.write_text('\n'.join(json.dumps(r) for r in prompts) + '\n', encoding='utf-8')

package/docs/routing.md

@@ -64,6 +64,31 @@ Keep one short evidence item inline when that reads better; use nested bullets f
 
 This MUST is semantic rather than a hard runtime lock: hosts may observe/remind with different strength, but lack of a blocking hook does not downgrade the contract to optional advice.
 
+### Turn label line
+
+End the final message of **every** turn with one line that rendered Markdown hides. This includes short answers, lookups, and turns that end with a question back to the user; unlike the Harness Status, it is not limited to non-trivial work:
+
+```
+<!-- harness-label {"v":1,"validity":"actionable","contextDependent":false,"tier":"tier2","intents":["fix","test"],"workflow":"iterative-single","skills":["tdd"]} -->
+```
+
+It states the agent's own judgement after the work, not a copy of the router's suggestion: `validity` (`actionable` or `invalid`), `contextDependent` (the prompt text alone would not have been enough), `tier` under tier rules v2, up to three `intents`, the `workflow` strategy and the `skills` actually used. The observation collector stores it as training data ([system-one-observations.md](system-one-observations.md)). A missing or malformed line is recorded with a reason code; it never blocks a turn or changes routing.
+
+Tier values:
+
+| Tier | Meaning |
+| --- | --- |
+| `tier1` | answer, lookup, status or Git operation; no file change |
+| `tier2` | a bounded file change: code, config or docs |
+| `tier3` | a new feature, a refactor, a redefinition, or cross-component work |
+| `null` | only when `validity` is `invalid` |
+
+The router prints this contract as the **last** section of its output, so it
+is the nearest instruction when the turn ends. In live Claude Code runs the
+label appeared on 1 of 4 turns while it sat inside the Harness Status contract
+(which is scoped to non-trivial work), and on 5 of 6 turns as its own final
+section.
+
 ## Suggested skills vs. selected workflow
 
 These are intentionally different concepts.

package/docs/system-one-corpus.md

@@ -16,7 +16,7 @@ to, judged from the prompt alone?** The labels follow the tier model in
 | --- | --- | --- |
 | `tier1` | Something answered or done without changing project behavior: Git/GitHub housekeeping (commit, push, open a PR, sync, tag, post given results as a comment), a status check, reading a log or searching and reporting, the smallest doc fix (a typo, a link, an ignore entry), or a reply the assistant can give directly, including non-engineering chores and discussion questions | "commit these changes", "is issue 88 still open?", "find every repo still using the old logging package", "close the other apps" |
 | `tier2` | A bounded change or investigation: any code, configuration or CI edit (a rename, a value, a revert), running tests or a build, filing an issue, explaining code that must be read first, most bug fixes, a focused review or benchmark, planning or synthesis that produces documents (phase plans, an ADR, an overview), and answers that settle open decisions for work in progress | "rename tmp to buffer", "run npm test", "the CLI fails with this error, fix it", "1. agree 2. keep the old name" |
-| `tier3` | New or changed behavior delivered with tests, and anything larger: a new flag or format, a behavior change across producers and tests, a large refactor, a dependency upgrade with breakage, a migration, a repository-wide change, a multi-phase implementation, or delegation to several agents | "add a --verbose flag with tests", "migrate to ESM", "dispatch agents to audit every repo" |
+| `tier3` | Work that needs several agents (sub-agents included); a new module or subsystem, a refactor, or a change to a framework or interface definition; or code or design work that spans several repositories or several workflows | "add a plugin system to the CLI", "refactor the parser into modules", "change the plugin interface", "dispatch agents to audit every repo" |
 | `null` | No actionable content: a bare continuation, "do what you said", a vague reaction, a question about what the assistant meant, or pasted status with no request | "go", "yes", "照剛剛說的改", "something is off here" |
 
 Rules:
@@ -31,6 +31,30 @@ Rules:
    reformat or extend the assistant's previous answer without changing files is
    `tier1`. A requested change to code or configuration, however small, is at
    least `tier2`. Only the smallest doc fixes listed in the table stay `tier1`.
+   The line between `tier2` and `tier3` is the owner's: `tier3` when the work
+   needs several agents or sub-agents, adds a feature, refactors, changes a
+   framework or interface definition, or spans several repositories or
+   workflows. Fixing, adjusting, investigating or reviewing existing behavior
+   within one workflow is `tier2`.
+   The owner settled four more lines from the third spot-check:
+   - **Git is `tier1` at any width.** Committing sub-repositories and then the
+     main repository is still housekeeping. "Several repositories" applies to
+     code or design work only.
+   - **Scope, not the word "add".** Adding an option, a function or a parser
+     inside one existing tool or component is `tier2`. A new module or
+     subsystem, or an addition that spans several components, is `tier3`.
+     A bug fix, even with a regression test, is `tier2`.
+   - **Repository-wide or cross-component work is `tier3`.** Replacing or
+     unifying a pattern across the repository (every `console.log`, all error
+     handling, every path in a report), upgrading a dependency and fixing what
+     breaks, a data migration, a CI matrix across platforms, or restructuring a
+     documentation tree is `tier3`, even when each edit is small.
+   - **A short reply follows the work in progress.** A reply that asks to
+     change an output (code, a document, an example) is `tier2`. Only a pure
+     question or a bare acknowledgement stays `tier1`.
+   - **Explicit agents or architecture make `tier3`.** A request for several
+     agents, a team discussion or a verifier pass, or one that redefines an
+     architecture, is `tier3` even when it is phrased as discussion.
 4. Explicit workflow words ("use fable", "no subagents") do not change the gold.
    Precedence is a router policy, not a label.
 5. Never derive gold from the lexical router or any model output. A reviewer may
@@ -102,6 +126,10 @@ a decision on its own.
 | `benchmarks/fixtures/system-one-holdout.json` | Evaluation corpus, rebuilt with `node scripts/system-one-corpus.js build <draft> <reviews> <corpus>` |
 
 The reviewed holdout has 215 cases (11 rejected): en 93 and zh-TW 122; gold
-`tier1` 47, `tier2` 97, `tier3` 51 and `null` 20. It meets the `reviewedHoldout`
-gate. It says nothing about model quality until a `harness-routing-v1`
+`tier1` 48, `tier2` 104, `tier3` 43 and `null` 20. It meets the `reviewedHoldout`
+gate. After the third training spot-check the owner moved nine cases to the new
+lines: `h-long-git-ops-1` to `tier1`, and the `h-add-cli-flag-*`,
+`h-hash-algorithm-support-1/2`, `h-fix-cli-error-log-3/4` and
+`h-fix-failing-test-3` cases to `tier2`. Results recorded before that change
+used 47 `tier1`, 97 `tier2` and 51 `tier3`. It says nothing about model quality until a `harness-routing-v1`
 checkpoint is evaluated on it.