harness-everything 0.29.0 → 0.30.1

package/ci/system-one-laya-train-test.py

@@ -0,0 +1,93 @@
+"""Contract tests for the single-device Laya trainer (#255 Phase 3).
+
+Pure parts run on stdlib; torch parts skip when torch is missing
+(same convention as S1-I08/S1-I13). Run with:
+    python3 ci/system-one-laya-train-test.py
+"""
+import importlib.util
+import sys
+import unittest
+from pathlib import Path
+
+sys.dont_write_bytecode = True  # never leave __pycache__ in the source tree that packaging copies
+path = Path(__file__).resolve().parents[1] / 'scripts/system-one-train-laya.py'
+spec = importlib.util.spec_from_file_location('laya_trainer', path)
+laya_trainer = importlib.util.module_from_spec(spec)
+spec.loader.exec_module(laya_trainer)
+
+try:
+    import torch  # noqa: F401
+    HAS_TORCH = True
+except ImportError:
+    HAS_TORCH = False
+
+
+class CalibSplitTests(unittest.TestCase):
+    def test_holdout_is_stable_disjoint_and_ten_percent(self):
+        items = [{'id': i} for i in range(1000)]
+        a_train, a_calib = laya_trainer.calib_split(items, seed=20260922, frac=0.1)
+        b_train, b_calib = laya_trainer.calib_split(items, seed=20260922, frac=0.1)
+        self.assertEqual(a_calib, b_calib, 'same seed replays the holdout')
+        self.assertEqual(len(a_calib), 100)
+        self.assertEqual(len(set(map(str, a_train)) & set(map(str, a_calib))), 0, 'disjoint')
+        c_train, c_calib = laya_trainer.calib_split(items, seed=1, frac=0.1)
+        self.assertNotEqual(a_calib, c_calib, 'seed matters')
+
+    def test_small_inputs_keep_at_least_one_calib(self):
+        _, calib = laya_trainer.calib_split([{'id': i} for i in range(5)], seed=7, frac=0.1)
+        self.assertEqual(len(calib), 1)
+
+    def test_prompt_groups_do_not_cross_split_and_calib_deduplicates_oversamples(self):
+        items = []
+        for prompt_id in range(20):
+            items.append({'promptId': str(prompt_id), 'intent': 'fix'})
+            items.append({'promptId': str(prompt_id), 'intent': 'refactor'})
+            items.append({'promptId': str(prompt_id), 'intent': 'refactor'})  # oversampled duplicate
+        train, calib = laya_trainer.calib_split(items, seed=20260922, frac=0.1)
+        train_prompts = {it['promptId'] for it in train}
+        calib_prompts = {it['promptId'] for it in calib}
+        self.assertFalse(train_prompts & calib_prompts, 'a prompt must stay wholly on one side')
+        calib_keys = [(it['promptId'], it['intent']) for it in calib]
+        self.assertEqual(len(calib_keys), len(set(calib_keys)),
+                         'oversampled duplicates must not overweight temperature fitting')
+        self.assertEqual(len(calib), 4, '10% of 40 de-duplicated prompt-intent identities')
+        self.assertEqual(sum(1 for it in train if it['intent'] == 'refactor'), 2 * len(train_prompts),
+                         'training keeps oversampled copies for non-calibration prompt groups')
+
+
+class IntentWeightTests(unittest.TestCase):
+    def test_default_weights_are_one(self):
+        self.assertEqual(laya_trainer.intent_weights(['fix', 'test'], {}), [1.0, 1.0])
+
+    def test_table_applies_per_item_intent(self):
+        self.assertEqual(laya_trainer.intent_weights(['refactor', 'fix', 'refactor'], {'refactor': 2.0}),
+                         [2.0, 1.0, 2.0])
+
+    def test_bad_table_fails(self):
+        with self.assertRaisesRegex(ValueError, 'intent-weights'):
+            laya_trainer.intent_weights(['fix'], {'nope': 2.0})
+        with self.assertRaisesRegex(ValueError, 'intent-weights'):
+            laya_trainer.intent_weights(['fix'], {'fix': 0})
+
+
+@unittest.skipUnless(HAS_TORCH, 'torch not installed')
+class TorchTests(unittest.TestCase):
+    def test_collate_shapes(self):
+        items = [{'ids': [1, 2, 3], 'markers': [0, 2], 'qtype': 2,
+                  'target': [0.3, 0.7], 'label': 1} for _ in range(4)]
+        batch = laya_trainer.collate(items, pad_id=0)
+        self.assertEqual(tuple(batch['input_ids'].shape), (4, 3))
+        self.assertEqual(tuple(batch['marker_pos'].shape), (4, 2))
+        self.assertTrue(bool((batch['target'].sum(-1) - 1.0 < 1e-6).all()))
+
+    def test_temperature_fit_is_finite_and_bounded(self):
+        import random
+        rng = random.Random(0)
+        sel = [([rng.uniform(-1, 1), rng.uniform(-1, 1)],
+                [0.4, 0.6]) for _ in range(30)]
+        t = laya_trainer.fit_one_temp(sel)
+        self.assertTrue(0.1 <= t <= 10.0)
+
+
+if __name__ == '__main__':
+    unittest.main()

package/docs/routing.md

@@ -64,6 +64,31 @@ Keep one short evidence item inline when that reads better; use nested bullets f
 
 This MUST is semantic rather than a hard runtime lock: hosts may observe/remind with different strength, but lack of a blocking hook does not downgrade the contract to optional advice.
 
+### Turn label line
+
+End the final message of **every** turn with one line that rendered Markdown hides. This includes short answers, lookups, and turns that end with a question back to the user; unlike the Harness Status, it is not limited to non-trivial work:
+
+```
+<!-- harness-label {"v":1,"validity":"actionable","contextDependent":false,"tier":"tier2","intents":["fix","test"],"workflow":"iterative-single","skills":["tdd"]} -->
+```
+
+It states the agent's own judgement after the work, not a copy of the router's suggestion: `validity` (`actionable` or `invalid`), `contextDependent` (the prompt text alone would not have been enough), `tier` under tier rules v2, up to three `intents`, the `workflow` strategy and the `skills` actually used. The observation collector stores it as training data ([system-one-observations.md](system-one-observations.md)). A missing or malformed line is recorded with a reason code; it never blocks a turn or changes routing.
+
+Tier values:
+
+| Tier | Meaning |
+| --- | --- |
+| `tier1` | answer, lookup, status or Git operation; no file change |
+| `tier2` | a bounded file change: code, config or docs |
+| `tier3` | a new feature, a refactor, a redefinition, or cross-component work |
+| `null` | only when `validity` is `invalid` |
+
+The router prints this contract as the **last** section of its output, so it
+is the nearest instruction when the turn ends. In live Claude Code runs the
+label appeared on 1 of 4 turns while it sat inside the Harness Status contract
+(which is scoped to non-trivial work), and on 5 of 6 turns as its own final
+section.
+
 ## Suggested skills vs. selected workflow
 
 These are intentionally different concepts.

package/docs/system-one-intent-gates.md

@@ -0,0 +1,102 @@
+# System One intent promotion gates (relevance-native)
+
+Status: **superseded** by [system-one-suggestion-gates.md](system-one-suggestion-gates.md).
+This table remains the contract for the runs made under it (for example
+`benchmarks/results/system-one/laya-ft-2026-09-25/`); those results are
+never re-scored under the new gates.
+
+Originally decided in
+[#233](https://github.com/dyphn1/Harness-everything/issues/233) (accept
+relevance-native direction, keep production migration gated); evidence in
+[#255](https://github.com/dyphn1/Harness-everything/issues/255). Thresholds
+below are proposals: they take effect only after owner review, and the final
+holdout runs **once** against the reviewed numbers. No threshold, gate, or
+wording change may use the frozen holdout as feedback.
+
+This replaces the single-winner intent gates (acceptedPrecision ≥ 0.85 at
+coverage ≥ 0.8), which a 57%-precision / 66%-recall multi-label system
+cannot clear by construction: owner prompts carry about 2.5 secondaries on
+average, so a single-winner contract caps coverage near 6% no matter how
+good the ranking is.
+
+## Ground truth
+
+- **Training/validation truth.** Teacher relevance scores binarized at the
+  owner secondary band: intent is positive iff score ≥ 0.4
+  ([system-one-intent.md](system-one-intent.md#relevance-scores)).
+- **Holdout truth.** Owner-reviewed `{gold, secondary}` per case. A predicted
+  set is compared as sets; graded agreement still applies where the report
+  cites it, but promotion gates use the binary metrics below.
+
+## Threshold flow (frozen)
+
+1. Train on train rows only.
+2. Fit one threshold per intent (max-F1) on validation, 2-fold CV reported.
+3. Freeze thresholds, code, and this document.
+4. Score the frozen holdout exactly once (two runs for repeatability).
+5. Gate on the table below. A fail is a fail; re-tuning restarts at step 2
+   with a documented reason, never at step 4.
+
+## Metrics
+
+- **micro precision / recall / F1** over all (case, intent) pairs.
+- **macro F1**: mean of per-intent F1.
+- **per-intent precision / recall / support** (support = holdout positives).
+- **predicted cardinality**: mean fires per prompt vs gold mean.
+- **abstain rate**: prompts with no intent fired.
+- **exact-set-match**: all 12 decisions correct (reported, not gated).
+- **weak-intent precision**: minimum precision over intents with support.
+- **family consistency**: model vs gold (same definition as the old gates).
+- **repeatability**: byte-identical decisions across two runs.
+- **latency**: warm p50/p95 per decision batch.
+- **policy evidence**: structural controls untouched (floor, fallback,
+  abstain path, rollback availability).
+
+## Proposed gates
+
+Anchors: per-intent train-majority predicts nothing (micro-F1 0), so gates
+are absolute. Reference measurement is the fine-tuned multilingual model on
+validation teacher proxy (micro-F1 CV 0.60, fires 2.0 vs truth 1.8); holdout
+gold is stricter, so thresholds sit below it.
+
+| Metric | Gate | Rationale |
+| --- | --- | --- |
+| micro-F1 | ≥ 0.55 | CV 0.60 minus holdout strictness margin |
+| macro-F1 | ≥ 0.50 | reference ≈ 0.61 |
+| per-intent F1 (support ≥ 10) | ≥ 0.30 | keeps refactor-class weak spots visible, not fatal |
+| weak-intent precision | ≥ 0.30 | tripwire for silent degradation |
+| mean fires | within gold mean ± 1.0 | blocks spray (4+) and collapse (<1) |
+| abstain rate | ≤ 0.20 | reference 0.09 |
+| family consistency (model) | ≥ gold | same bar as the old gates |
+| repeatability | == 1 | deterministic inference required |
+| warm p95 | ≤ 250 ms | environment-tolerant bar (see latency note) |
+| structural controls | unchanged + rollback path live | migration stays shadow-compatible |
+
+`rolloutReady` is true only when every gate passes. `policyEvidence` and
+`liveHostEvidence` remain false until independently produced; they are
+reported, not bypassed.
+
+## Latency note
+
+The old single-winner gate used warm p95 ≤ 100 ms. That bar flakes on
+developer hardware for model-backed transports: the fine-tuned multilingual
+model measured p50 135 ms / p95 179 ms on a Mac mini M4 over 444 warm
+samples, and thermal throttling, GPU contention, or memory pressure can push
+any single run past 100 ms without a code change. The 250 ms bar still rules
+out pathological transports (per-request model reloads at 7–10 s, cold CPU
+fallbacks in seconds) while tolerating host variance. For reference, the
+MLX port of the same model class measures 7–13 ms; transport choice, not
+the gate, is where that gap belongs. Latency evidence always ships with the
+host record (cpu/os/node/python); a gate pass on one host is not a claim
+about another.
+
+## Non-goals and invariants
+
+- The single-winner path (contract.decide as shipped, n-gram provider,
+  lexical fallback) stays live as rollback until promotion completes.
+- Tier structural floor, explicit workflow constraints, action/security
+  approval, and scope/memory ownership stay outside learned scoring.
+- Taxonomy is frozen: 12 intents in catalog order plus abstain. Wording
+  versions are recorded per artifact but never change categories.
+- Private prompts never enter the public repository; public evidence holds
+  aggregates, hashes, and per-intent metrics only.

package/docs/system-one-intent.md

@@ -0,0 +1,173 @@
+# System One intent stage: catalog and labeling
+
+This document defines the second stage of [System One](system-one-routing.md#staged-classification):
+the **intent** of a prompt, meaning the kind of work it asks for (issue
+[#233](https://github.com/dyphn1/Harness-everything/issues/233)). The tier stage is
+defined in [system-one-corpus.md](system-one-corpus.md) and is unchanged. The intent
+stage reuses the tier pipeline in [system-one-training.md](system-one-training.md) with
+its own catalog, labels, holdout and gates.
+
+## Intent labels
+
+An intent label answers one question: **what kinds of work does this prompt ask the
+assistant to do, judged from the prompt and its session context?** The catalog is the
+repository owner's (confirmed 2026-09-24, with `edit` added the same day) and is fixed.
+Its order is the scorer's option order.
+
+A label has two parts:
+
+- **Primary.** Exactly one intent, or `null`. This is the work that produces the main
+  deliverable.
+- **Secondary.** Zero or more other intents that the request also needs. Changing a
+  default port, for example, is primary `edit` with secondary `test` and `docs` when
+  tests and docs mention the port. Secondary never contains the primary intent or
+  `null`, and it is empty when the primary is `null`.
+
+| Gold | Use when the prompt asks the assistant to | Examples of shape |
+| --- | --- | --- |
+| `explain` | Answer a question or explain how something works, without deciding anything or changing files | "what does this flag do?", "why is this test slow?" (explanation only), "今天這段 code 在做什麼" |
+| `discuss` | Weigh options, give an opinion, or settle a decision for work in progress | "should we keep the old name?", "1. agree 2. use hash", "你覺得哪個方案好" |
+| `git` | Perform Git or GitHub housekeeping: commit, push, branch, merge, rebase, cherry-pick, tag, open or merge a PR, post a comment | "commit and push", "開 PR", "commit the sub-repos, then the main repo" |
+| `fix` | Repair behavior that is wrong: a bug, a crash, a failing or flaky test, a build or CI failure, a dependency conflict | "the CLI panics here, fix it", "CI times out, find the cause and fix it" |
+| `edit` | Change an existing value, setting, text or behavior on request, when it is neither a bug nor new behavior | "change the default port from 8080 to 3000", "把 timeout 改成 30 秒", "rename the button label to Save" |
+| `feature` | Add new behavior: a command, an option, a format, an endpoint, a script, a migration | "add a --verbose flag", "support YAML input", "寫一個資料遷移腳本" |
+| `refactor` | Restructure or unify existing code without new behavior: rename, move, extract, replace a pattern everywhere, upgrade a dependency | "rename component to module everywhere", "replace console.log with the logger", "把常數搬出去" |
+| `review` | Evaluate an existing artifact against a standard: a PR, a diff, a design, a document, test coverage, a benchmark result | "review this PR", "檢查測試是不是只有 happy path", "audit the skill descriptions" |
+| `test` | Run tests or builds, or write tests as the main deliverable | "run npm test", "add tests for the parser" |
+| `docs` | Write or edit documentation as the main deliverable: README, guides, comments, changelogs, a memory note | "update the install guide", "改成全英文文件" |
+| `plan` | Produce a plan, specification, roadmap, ADR or breakdown before implementation | "plan phase 4", "拆分實作階段", "write an ADR for the new cache" |
+| `investigate` | Find facts or a cause and report back, without being asked to change anything | "find every repo still using the old logger", "查一下為什麼變慢，先不要改" |
+| `null` | Nothing actionable: a bare continuation, a vague reaction, pasted status with no request, or a request outside software work | "go", "嗯", "今天股價多少" |
+
+Rules:
+
+1. **One primary intent, then the rest as secondary.** The primary is the work that
+   produces the main deliverable. "Fix the bug and add a regression test" is primary
+   `fix`, secondary `test`. "Investigate and fix" is primary `fix`, secondary
+   `investigate`. "Implement the feature, then commit" is primary `feature`, secondary
+   `git`. Add a secondary intent only when the request asks for it or clearly needs it,
+   not for every step an assistant might take.
+2. **Change beats inquiry.** When the prompt asks for a change, label the change, not
+   the reading that precedes it. `investigate` and `explain` are for prompts that only
+   ask for information.
+3. **`investigate` against `explain`.** `investigate` needs searching or running
+   something to find facts; `explain` can be answered from what is already known or
+   shown.
+4. **`review` against `investigate`.** `review` judges a given artifact against a
+   standard; `investigate` looks for a cause or for facts that are not yet known.
+5. **`fix`, `edit`, `feature` and `refactor`.** Wrong behavior made right is `fix`.
+   A requested change to a value, setting, text or existing behavior that was not wrong
+   is `edit`. New behavior is `feature`. The same behavior in a new structure is
+   `refactor`.
+6. **Short replies follow the work in progress**, as in the tier rules. A reply that
+   asks to change an output takes the intent of that change ("not this format, use the
+   table" in a docs session is `docs`). A reply that settles decisions is `discuss`.
+7. **Intent is independent of tier.** A repository-wide `refactor` is `tier3` and a
+   one-line `refactor` is `tier2`; both are `refactor`. Never infer intent from a tier
+   label or tier from an intent label.
+8. **Never derive gold from the lexical router or any model output.** A reviewer may
+   read a proposed intent, but the recorded gold is the reviewer's decision.
+
+### Relevance scores
+
+A scored label rates every intent of the catalog, except `null`, from 0 to 1 in steps of
+0.05. The owner set this scale on 2026-09-24 (it replaces a four-level draft of the same
+day). System One only suggests skills, so a scored label is meant to rank every candidate,
+not to pick one.
+
+| Score | Band |
+| --- | --- |
+| 0.6 or more | Primary relevance: the prompt asks for this work as a main deliverable |
+| 0.4 to 0.55 | Secondary relevance: the request also needs this work |
+| 0.2 to 0.35 | Related: useful context or a likely follow-up that the prompt does not ask for |
+| 0.05 to 0.15 | Faint: a remote link to the prompt |
+| 0 | Unrelated |
+
+Rules:
+
+1. **Score every intent on its own.** This replaces "add a secondary intent only when the
+   request asks for it" above for scored labels: every intent gets a considered score.
+   The scores are not a budget shared by the intents.
+2. **No ties.** Two intents never share a score other than 0, so the scores rank the
+   whole catalog.
+3. **At most eight zeros.** At least four intents score above 0.
+4. **Name one primary.** It is the intent with the highest score, and it scores 0.6 or
+   more. Rule 1 above decides which work produces the main deliverable.
+5. **`null` means nothing actionable.** With a `null` primary, no intent scores more
+   than 0.2.
+6. **Bands give the label.** The secondary intents are every other intent that scores
+   0.4 or more, highest first. Related intents are kept only as scores.
+
+## Holdout
+
+The intent holdout reuses the prompts of the tier holdout
+(`benchmarks/fixtures/system-one-holdout.json`). Their families are already
+holdout-only, so no training prompt can leak into them. Each case gets an intent gold
+through the same owner review as the tier holdout. An intent review decision also
+records `secondary`, the list of secondary intents. `accept` and `relabel` refer to the
+primary intent only. The review records live in
+`benchmarks/fixtures/system-one-intent-reviews.json`, and the evaluation corpus is
+`benchmarks/fixtures/system-one-intent-holdout.json`. The `reviewedHoldout` gate is the
+same: at least 200 reviewed cases and at least 50 per language. The report also gives
+the count per intent, because a class with only a few holdout cases cannot be measured.
+
+**Scope.** The intent stage runs only on actionable prompts. When the tier stage
+abstains (`unclassified`), no intent is asked. In the first review the owner rejected
+all 25 non-actionable prompts, which left 190 cases, so the holdout gains
+**supplementary intent-only families** (`synthetic:authored`). Those families favor
+the thinnest intents (`test`, `plan`, `discuss`, `review`, `edit`). They are appended
+to the intent draft, reviewed the same way, and added to the training-leakage filter
+like every holdout prompt.
+
+## Scoring
+
+System One gives a probability for every intent. The highest is the predicted
+primary, and every other intent at or above a calibrated threshold is a predicted
+secondary. The skills stage consumes the primary intent together with the accepted
+secondary intents.
+
+Two intent labels are compared by **graded agreement**. Each side is a primary `p`
+plus a secondary set `S`, and its full set is `{p} ∪ S`. The owner set these grades:
+
+| Case | Score |
+| --- | --- |
+| Same primary | 1.0 |
+| Different primaries, and each primary is in the other side's full set (only the order differs) | 0.6 |
+| Different primaries, and only one primary is in the other side's full set; or the order differs but the secondary counts differ by 3 or more | 0.3 |
+| Neither primary is in the other side's full set | 0 |
+
+Both sides agreeing on intent but differing in order or in the number of steps means
+the same goal with a different breakdown, not a different goal. Graded agreement is
+reported next to exact primary agreement, never instead of it. The same grades apply
+to the labeler check and to accepted predictions of a model.
+
+**Family consistency.** A family groups variants and translations of one scenario.
+The report gives the share of families whose members share one primary intent, for the
+owner's gold and for the predictions. This makes a language-dependent intent visible
+without treating every variant as a translation.
+
+## Training data
+
+The collected prompts are labeled a second time, for intent. The labeler returns a
+primary and a secondary list for each prompt. Intent labels are stored beside the tier
+labels, never in place of them. Owner spot-checks for intent show session context, like
+the tier spot-checks. The labeler check against the intent holdout must reach a graded
+agreement of 80% before training data is labeled.
+
+**Scored labels.** The second intent labeling, requested by the owner on 2026-09-24,
+asks for [relevance scores](#relevance-scores) instead of a primary and a short list.
+The labeler returns a primary and a score for every intent. Scored labels are stored in
+`labels-intent-scores.jsonl`, beside the earlier intent labels. Each row keeps the
+primary as `gold`, the secondary intents derived from the bands and all the scores. The
+labeler check grades the derived primary and secondary intents against the holdout, as
+above. Its report also gives the mean number of secondary and related intents per
+prompt, next to the owner's mean number of secondary intents.
+
+## Gates
+
+The intent stage uses the rollout gates of
+[system-one-routing.md](system-one-routing.md#rollout-gates): accepted precision at least
+85% (graded agreement over accepted predictions, with exact primary precision
+reported beside it), coverage at least 80%, repeatability, warm p95 at most 100 ms and
+verified provenance. There is no lexical intent router, so the macro-F1 baseline is the
+majority-class predictor on the holdout.

package/docs/system-one-laya-training.md

@@ -0,0 +1,38 @@
+# System One Laya weak-intent training contract
+
+This document defines the training-data semantics used by the #255 weak-intent experiment.
+
+## Intent-level oversampling
+
+`system-one-export-laya.py --oversample intent:factor` is an **intent-item** intervention.
+
+For a prompt whose teacher score for the selected intent is at least `0.4`:
+
+- only that `promptId + intent` training item is repeated to the requested factor;
+- sibling intents from the same prompt are emitted once;
+- prompts below the positive threshold are not repeated;
+- the manifest records the requested oversample map.
+
+Example: `--oversample refactor:3` emits three copies of the positive `refactor` item, but still emits one copy each of `review`, `explain`, and every other sibling intent.
+
+This distinction is required so a weak-intent intervention does not silently reweight neighboring intents.
+
+## Calibration isolation
+
+The trainer calibration split is **prompt-group disjoint**.
+
+- All items sharing one `promptId` must stay on the same side of the train/calibration boundary.
+- Calibration uses one copy of each `promptId + intent` identity.
+- Oversampled duplicates are retained only for prompt groups assigned to training.
+- Oversampling therefore cannot create an exact duplicate in both training and calibration, and it cannot overweight temperature fitting.
+- Items without `promptId` retain the legacy per-item deterministic split behavior for unit-level callers.
+
+The calibration selection remains deterministic for a fixed seed and continues to respect the configured fraction and `CALIB_MAX` target over the de-duplicated base identities.
+
+## Loss weights
+
+`system-one-train-laya.py --intent-weights '{"refactor": 2.0}'` applies a per-sample multiplier to both RL and CE losses for the named intent. Unlisted intents remain at weight `1.0`. Weights must be finite and greater than zero.
+
+## Evidence rule
+
+Results produced before these isolation rules were enforced must not be used as causal evidence for neighbor-intent regressions without re-running the experiment. In particular, prompt-level duplication and train/calibration duplicate leakage confound interpretation of an intent-specific intervention.

package/docs/system-one-observations.md

@@ -0,0 +1,206 @@
+# System One observations: collecting routing data from real sessions
+
+Status: **accepted**, 2026-09-27. Owner decisions are recorded below.
+Implementation: `hooks/scripts/observation-hook.js`,
+`hooks/scripts/lib/observations.js`, `scripts/system-one-observations-export.js`
+and `scripts/system-one-observations-review.js`; tests in
+`ci/mechanism-34-observations.test.js`. Issue
+[#233](https://github.com/dyphn1/Harness-everything/issues/233).
+
+## Why
+
+Two data problems limit the System One scorers
+([system-one-suggestion-gates.md](system-one-suggestion-gates.md)):
+
+1. There are too few valid, text-decidable examples.
+2. Teacher labels from an offline LLM are thin, and were often assigned with
+   session context the small model never sees.
+
+Every Harness turn already produces better material. The host LLM sees the
+whole conversation, states its view in the Harness Status, and then acts.
+Recording the prompt, that view, and what the agent actually did, turn by
+turn, gives labeled data that grows with daily use. The data covers
+validity, intent, tier, workflow and skills.
+
+## Owner decisions (2026-09-27)
+
+- The host LLM adds one machine-readable label line to its final message
+  each turn. The token cost is accepted.
+- Collection is **on by default**, with no environment variable needed to
+  enable it. `HARNESS_OBSERVATIONS=off` turns it off; this opt-out exists
+  because the plugin ships to other users.
+- Hosts: **Claude Code** and **Codex** first.
+- This document is the specification; implementation follows
+  docs → RED → GREEN.
+
+## One observation per turn
+
+A turn runs from `UserPromptSubmit` to `Stop`. Its record:
+
+| Field | Source | Notes |
+| --- | --- | --- |
+| `id`, `sessionId`, `turn`, `host`, `observedAt` | hooks | `id` = hash of session id + turn |
+| `prompt` | `UserPromptSubmit` | stored in the private text store only (see Storage) |
+| `previous` | the last assistant message before the prompt | truncated to 2 KB; text store only |
+| `router` | kernel-router output | lexical tier, strategy, suggested skills; System One shadow scores when present |
+| `behavior` | `PostToolUse`, `PostToolUseFailure` | counts and categories, never arguments: files written, distinct repositories written, files created, commands by class (git, gh, test, build, package, shell), skills loaded or read, subagents started. Failed attempts are counted separately (`toolFailures`, `failedWrites`, `failedSkillLoads`) and never count as a write or a load |
+| `workflow` | `workflow-run.json` / `workflow-disposition` | selected and confirmed strategy |
+| `selfReport` | label line in the final message | see below; `null` if missing or invalid |
+| `derived` | exporter | labels computed from the fields above |
+
+### Label line
+
+At the end of the final message of each turn the agent writes one line,
+which is invisible in rendered Markdown:
+
+```
+<!-- harness-label {"v":1,"validity":"actionable","contextDependent":false,"tier":"tier2","intents":["fix","test"],"workflow":"iterative-single","skills":["tdd"]} -->
+```
+
+| Key | Values |
+| --- | --- |
+| `validity` | `actionable` or `invalid` (continuation, feedback on earlier work, pointer to earlier content, paste or chatter) |
+| `contextDependent` | `true` when the prompt text alone would not have been enough to know the task |
+| `tier` | `tier1` (no file change), `tier2` (bounded file change: code, config or docs), `tier3` (feature, refactor, redefinition, cross-component) or `null` (only for `invalid`), under tier rules v2 |
+| `intents` | up to 3 of the 12 intent ids, strongest first |
+| `workflow` | a router strategy id or `null` |
+| `skills` | skill ids actually used this turn |
+
+The line states the agent's own judgement **after** the work, not a copy of
+the router's suggestion. A line that does not parse, or has an invalid
+validity, tier or workflow, is recorded as `selfReport: null` with a reason
+code; it never blocks the turn. Two slips seen in live runs are tolerated
+rather than losing the whole label: unknown intent ids are dropped and kept
+in `unknownIntents`, and a final label line missing its closing `-->` is
+accepted when it ends the message.
+
+## Labels and how much to trust them
+
+The label is the host LLM's own judgement. It saw the whole conversation and
+did the work, so no fixed rule replaces it (owner decision, 2026-09-27: no
+file-count threshold for tier).
+
+| Label | Source | Override |
+| --- | --- | --- |
+| validity, contextDependent | self-report | owner review |
+| intents | self-report | — |
+| tier | self-report (tier rules v2) | owner review |
+| workflow | self-report, else the confirmed strategy | — |
+| skills | skills actually loaded or read, else the self-report | — |
+
+Behavior counters (files, repositories, command classes, skills,
+subagents) are kept as **evidence**, never as a label rule. They serve two
+purposes:
+
+- **Contradictions.** Some records contradict themselves. A `tier1` label
+  after files were written contradicts the tier1 definition (no code
+  change). These records go to the owner's review page.
+- **Future features.** The counters can train or check later models
+  without re-reading transcripts.
+
+**Owner review.** Missing labels, contradictions, and turns where the
+router's tier and the self-reported tier differ go to a blind review page.
+The owner decides validity, context dependence and tier; each decision
+overrides the self-report, an explicit `false` included, and also labels a
+turn that has no self-report.
+
+**Bias controls.** The agent sees the router's suggestion before it
+answers, so it may copy it. Router output and self-report are both stored.
+Each export reports how often they agree, over turns where the router chose
+a tier (`unclassified` is not a tier); a rate close to 1 over many turns is a
+warning sign. The review page samples the turns where they differ.
+
+## Storage, index and privacy
+
+- **Location**: `~/.agents/harness-everything/system-one/observations/`,
+  outside every workspace. It is never written under `memories/repo`, which
+  can be committed.
+- **Text store**: `text/<sha256>.json` holds `prompt` and `previous`, with
+  the collector's redaction applied (email addresses and home paths).
+- **Index**: `observations-index.json` uses the `index_memory.js` record
+  shape: `schemaVersion: 1` and `records[]`. Each record has `id`,
+  `status`, `validUntil`, `contentSha256`, `scope`, `writer.sessionId` and
+  `source: "observation"`, plus the non-text fields above. `scope.taskTerms`
+  hold normalized terms, so `retrieveMemoryRecords`-style lookups can find
+  similar past turns.
+- **Retention**: `validUntil` is 180 days after `observedAt`. An expired
+  record keeps its labels and counts, but its text is deleted. Expiry is
+  enforced on read as well: the exporter and the review page skip the text
+  of any record past `validUntil`, whether or not a later turn has swept it.
+  A text blob is content-addressed, so it is deleted only when no active
+  record references it. Session state that holds text (the previous-message
+  tail and an unfinished turn's prompt) follows the same 180 days, counted
+  from when the text was captured. Finishing or recovering an old turn never
+  renews that deadline: its record's `validUntil` comes from the capture
+  time, and a turn whose text has already expired is recorded with its
+  labels and counts but no text.
+- **Concurrency**: tool hooks may run in parallel. Each tool event is written
+  as its own immutable file, and `Stop` adds them up, so no event is lost to
+  a concurrent read-modify-write.
+- **Telemetry stays content-free.** The telemetry channel keeps its
+  `FORBIDDEN_KEYS` rule. Observations are a separate, local-only channel and
+  are never uploaded by Harness.
+- `PRIVACY.md` is updated to state that prompts and the previous assistant
+  message are stored locally by default, where they are stored, how long
+  they are kept, and how to turn this off.
+
+## Hosts
+
+- **Claude Code**: plugin hooks `UserPromptSubmit`, `PostToolUse` and
+  `Stop`. The final message is read from the Stop payload or from the
+  transcript the payload points to.
+- **Codex**: the same hooks. Verified live on codex-cli 0.157.1
+  (2026-09-28):
+  - Codex adds `UserPromptSubmit` context to the model only from JSON
+    `hookSpecificOutput.additionalContext`. With the router's plain stdout
+    the model never saw the routing contract or the label instruction, so
+    it wrote no label; the same text as JSON reached the model and all five
+    test turns carried a label. The Codex manifest therefore runs the router
+    through `hooks/scripts/codex-user-prompt.js`, which wraps its output.
+  - Plugin-bundled hooks do not run on this version (`plugin_hooks` is
+    `removed`); the explicit fallback `harness codex-hooks install` is
+    needed, and Codex runs the hooks only after they are trusted in
+    `/hooks`.
+  - The final message comes from the rollout's `task_complete` event.
+- The final message must belong to the current turn. A transcript scan
+  stops at the start of the current turn (Codex `task_started` or a user
+  message; a Claude user prompt) and a Codex `task_complete` must carry the
+  current `turn_id` when both are known. With no final message for the
+  current turn, `selfReport` is `null` with `final-message-unavailable`; the
+  previous turn's label is never reused.
+- Failure follows each host's convention: the `PostToolUseFailure` event
+  (Claude Code), or an error flag or non-zero exit code in the tool response
+  (Codex).
+- A missing field records `null` with a reason code. Collection must never
+  block, slow down, or fail a turn; each hook has a short time budget.
+
+## Export
+
+`scripts/system-one-observations-export.js` writes to the private data
+repository:
+
+- `prompts-observed.jsonl`: `{id, family, source: "observation", split, text}`
+- `labels-observed.jsonl`: `{id, validity, contextDependent, tier, tierSource, intents, workflow, skills}`
+
+`family` groups near-duplicate prompts. The split is by time: the newest 15%
+is validation. The existing holdouts stay frozen and separate. Observed data
+may later seed a new holdout, but only after owner review.
+
+## Phases
+
+1. **Contract.** Define the label-line format in `AGENTS.md` rule 11 and in
+   the Harness Status docs. The parser and validator come with tests.
+2. **Collector.** Hook-side recording for Claude Code: the turn record,
+   behavior counters, the text store, the index and the opt-out. Tests cover
+   redaction, the time budget, fail-open behavior and index shape.
+3. **Codex.** Verify the payload fields, then add the adapter and its tests.
+4. **Exporter.** Derived labels, the family grouping, the time split, and
+   bias reports.
+5. **Review page.** A blind review of disagreements; decisions feed back as
+   owner overrides.
+
+## Non-goals
+
+- Nothing is uploaded and there is no remote collection.
+- The label line never gates or changes routing.
+- Observed data does not replace the owner-reviewed holdouts.