harness-everything 0.29.0 → 0.30.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (108) hide show
  1. package/.claude-plugin/marketplace.json +1 -1
  2. package/.claude-plugin/plugin.json +1 -1
  3. package/AGENTS.md +1 -0
  4. package/CHANGELOG.md +42 -0
  5. package/PRIVACY.md +9 -0
  6. package/benchmarks/fixtures/system-one-intent-holdout-draft.json +2228 -0
  7. package/benchmarks/fixtures/system-one-intent-holdout.json +16509 -0
  8. package/benchmarks/fixtures/system-one-intent-reviews.json +2723 -0
  9. package/benchmarks/results/system-one/intent-ngram-v1-2026-09-24/README.md +64 -0
  10. package/benchmarks/results/system-one/intent-ngram-v1-2026-09-24/report.json +15177 -0
  11. package/benchmarks/results/system-one/labeler-holdout-check-intent-2026-09-24/report.json +2976 -0
  12. package/benchmarks/results/system-one/labeler-holdout-check-intent-scores-haiku-2026-09-24/report-005-steps.json +6125 -0
  13. package/benchmarks/results/system-one/labeler-holdout-check-intent-scores-haiku-2026-09-24/report-haiku-sonnet-escalation.json +6344 -0
  14. package/benchmarks/results/system-one/labeler-holdout-check-intent-scores-haiku-2026-09-24/report-scaled-margin-escalation.json +6206 -0
  15. package/benchmarks/results/system-one/labeler-holdout-check-intent-scores-haiku-2026-09-24/report.json +6103 -0
  16. package/benchmarks/results/system-one/laya-ft-2026-09-25/README.md +83 -0
  17. package/benchmarks/results/system-one/laya-ft-2026-09-25/report.json +71 -0
  18. package/benchmarks/results/system-one/laya-ft-2026-09-25/shadow-report.json +44 -0
  19. package/benchmarks/results/system-one/laya-zs-2026-09-25/README.md +68 -0
  20. package/benchmarks/results/system-one/laya-zs-2026-09-25/report.json +173 -0
  21. package/ci/fable-mode-test.js +16 -0
  22. package/ci/mechanism-34-observations.test.js +550 -0
  23. package/ci/mechanism-system-one-collect.test.js +16 -0
  24. package/ci/mechanism-system-one-intent.test.js +406 -0
  25. package/ci/mechanism-system-one-laya.test.js +45 -0
  26. package/ci/mechanism-system-one-relevance.test.js +112 -0
  27. package/ci/mechanism-system-one-resident.test.js +16 -0
  28. package/ci/system-one-laya-calibrate-test.py +65 -0
  29. package/ci/system-one-laya-export-test.py +155 -0
  30. package/ci/system-one-laya-label-test.py +201 -0
  31. package/ci/system-one-laya-perintent-test.py +60 -0
  32. package/ci/system-one-laya-provider-test.py +115 -0
  33. package/ci/system-one-laya-train-test.py +93 -0
  34. package/docs/routing.md +25 -0
  35. package/docs/system-one-intent-gates.md +102 -0
  36. package/docs/system-one-intent.md +173 -0
  37. package/docs/system-one-laya-training.md +38 -0
  38. package/docs/system-one-observations.md +206 -0
  39. package/docs/system-one-routing.md +25 -10
  40. package/docs/system-one-suggestion-gates.md +292 -0
  41. package/docs/system-one-training.md +56 -6
  42. package/docs/workflows/fable-mode.md +7 -0
  43. package/fable-mode/SKILL.md +1 -1
  44. package/fable-mode/agents/fable-orchestrator.md +4 -4
  45. package/fable-mode/agents/fable-worker-haiku.md +4 -4
  46. package/fable-mode/agents/fable-worker-sonnet.md +4 -4
  47. package/fable-mode/execution-guardrails/SKILL.md +5 -3
  48. package/fable-mode/execution-guardrails/references/guardrail-rules.md +12 -8
  49. package/fable-mode/fable-haiku/SKILL.md +1 -1
  50. package/fable-mode/fable-opus/SKILL.md +1 -1
  51. package/fable-mode/fable-sonnet/SKILL.md +1 -1
  52. package/harness-everything/SKILL.md +1 -1
  53. package/harness-everything/scripts/kernel-router.js +10 -0
  54. package/harness-everything/scripts/system-one/catalogs.js +35 -0
  55. package/harness-everything/scripts/system-one/contract.js +45 -1
  56. package/harness-everything/scripts/system-one/evaluate.js +115 -19
  57. package/harness-everything/scripts/system-one/intent.js +22 -0
  58. package/harness-everything/scripts/system-one/laya_adapter.py +141 -0
  59. package/harness-everything/scripts/system-one/provider.js +10 -3
  60. package/harness-everything/scripts/system-one/resident.js +8 -3
  61. package/harness-everything/scripts/system-one/router.js +1 -1
  62. package/hooks/hooks.json +40 -0
  63. package/hooks/scripts/codex-user-prompt.js +39 -0
  64. package/hooks/scripts/lib/observations.js +581 -0
  65. package/hooks/scripts/observation-hook.js +25 -0
  66. package/opencode-plugin/plugin.json +1 -1
  67. package/package.json +1 -1
  68. package/plugins/harness-everything/.codex-plugin/plugin.json +1 -1
  69. package/plugins/harness-everything/hooks/hooks.json +35 -2
  70. package/plugins/harness-everything/hooks/scripts/codex-user-prompt.js +39 -0
  71. package/plugins/harness-everything/hooks/scripts/lib/observations.js +581 -0
  72. package/plugins/harness-everything/hooks/scripts/observation-hook.js +25 -0
  73. package/plugins/harness-everything/plugin.json +1 -1
  74. package/plugins/harness-everything/skills/fable-mode/SKILL.md +1 -1
  75. package/plugins/harness-everything/skills/fable-mode/agents/fable-orchestrator.md +4 -4
  76. package/plugins/harness-everything/skills/fable-mode/agents/fable-worker-haiku.md +4 -4
  77. package/plugins/harness-everything/skills/fable-mode/agents/fable-worker-sonnet.md +4 -4
  78. package/plugins/harness-everything/skills/fable-mode/execution-guardrails/SKILL.md +5 -3
  79. package/plugins/harness-everything/skills/fable-mode/execution-guardrails/references/guardrail-rules.md +12 -8
  80. package/plugins/harness-everything/skills/fable-mode/fable-haiku/SKILL.md +1 -1
  81. package/plugins/harness-everything/skills/fable-mode/fable-opus/SKILL.md +1 -1
  82. package/plugins/harness-everything/skills/fable-mode/fable-sonnet/SKILL.md +1 -1
  83. package/plugins/harness-everything/skills/harness-everything/SKILL.md +1 -1
  84. package/plugins/harness-everything/skills/harness-everything/scripts/kernel-router.js +10 -0
  85. package/plugins/harness-everything/skills/harness-everything/scripts/system-one/catalogs.js +35 -0
  86. package/plugins/harness-everything/skills/harness-everything/scripts/system-one/contract.js +45 -1
  87. package/plugins/harness-everything/skills/harness-everything/scripts/system-one/evaluate.js +115 -19
  88. package/plugins/harness-everything/skills/harness-everything/scripts/system-one/intent.js +22 -0
  89. package/plugins/harness-everything/skills/harness-everything/scripts/system-one/laya_adapter.py +141 -0
  90. package/plugins/harness-everything/skills/harness-everything/scripts/system-one/provider.js +10 -3
  91. package/plugins/harness-everything/skills/harness-everything/scripts/system-one/resident.js +8 -3
  92. package/plugins/harness-everything/skills/harness-everything/scripts/system-one/router.js +1 -1
  93. package/sbom.cdx.json +7 -7
  94. package/scripts/evaluate-system-one.js +17 -5
  95. package/scripts/system-one-calibrate-laya.py +165 -0
  96. package/scripts/system-one-calibrate.js +42 -13
  97. package/scripts/system-one-collect.js +7 -3
  98. package/scripts/system-one-compare-laya.py +156 -0
  99. package/scripts/system-one-corpus.js +22 -15
  100. package/scripts/system-one-eval-laya.js +112 -0
  101. package/scripts/system-one-export-laya.py +197 -0
  102. package/scripts/system-one-label-laya.py +243 -0
  103. package/scripts/system-one-label.js +163 -37
  104. package/scripts/system-one-observations-export.js +106 -0
  105. package/scripts/system-one-observations-review.js +86 -0
  106. package/scripts/system-one-perintent-laya.py +134 -0
  107. package/scripts/system-one-train-laya.py +344 -0
  108. package/scripts/system-one-train-ngram.py +66 -14
@@ -105,6 +105,15 @@ function persistWorkflow(contract, payload, prompt) {
105
105
  return { ...context, hooksRoot, retained: false, memoryCapability };
106
106
  }
107
107
 
108
+ // Printed last so it is the nearest instruction when the turn ends (docs/system-one-observations.md#label-line).
109
+ function printLabelContract() {
110
+ console.log('\n=> TURN LABEL LINE (MUST, EVERY TURN — short answers, lookups and trivial edits included):');
111
+ console.log(' - The last line of your final message MUST be one hidden label line with your own post-work judgement, not a copy of the router tier above:');
112
+ console.log(' <!-- harness-label {"v":1,"validity":"actionable|invalid","contextDependent":true|false,"tier":"tier1|tier2|tier3|null","intents":["up to 3 of explain,discuss,git,fix,edit,feature,refactor,review,test,docs,plan,investigate"],"workflow":"strategy id or null","skills":["skills used"]} -->');
113
+ console.log(' - tier1: answer, lookup, status or git operation with no code change; tier2: bounded change; tier3: new feature, refactor, redefinition or cross-component work. Use null only when validity is invalid.');
114
+ console.log(' - It is invisible in rendered Markdown and records local training data; it never gates routing.');
115
+ }
116
+
108
117
  function run(raw) {
109
118
  const promptArg = process.argv.slice(2).join(' ');
110
119
  let payload = null;
@@ -162,6 +171,7 @@ function run(raw) {
162
171
  } else {
163
172
  console.log(' - Runtime state was not persisted; reminders may be less precise, but execution remains available.');
164
173
  }
174
+ printLabelContract();
165
175
  if (failure) console.error('[Workflow Reminder] routing state was not persisted: ' + failure.message);
166
176
  if (result && result.status !== 0) console.error('[Workflow Reminder] router returned status ' + result.status + '; contract observation is degraded, not blocking.');
167
177
  }
@@ -0,0 +1,35 @@
1
+ 'use strict';
2
+ // Fixed System One catalogs, one per stage (docs/system-one-routing.md#staged-classification).
3
+ // Option order is the scorer's output order; changing it invalidates trained weights.
4
+ const { TIER_OPTIONS } = require('./router');
5
+ const INTENT_OPTIONS = Object.freeze([
6
+ { id: 'explain', text: 'Answer a question or explain how something works.' },
7
+ { id: 'discuss', text: 'Weigh options, give an opinion or settle a decision.' },
8
+ { id: 'git', text: 'Git or GitHub housekeeping: commit, push, branch, pull request.' },
9
+ { id: 'fix', text: 'Repair wrong behavior: a bug, crash, failing test or build.' },
10
+ { id: 'edit', text: 'Change an existing value, setting, text or behavior on request.' },
11
+ { id: 'feature', text: 'Add new behavior: a command, option, format or script.' },
12
+ { id: 'refactor', text: 'Restructure or unify existing code without new behavior.' },
13
+ { id: 'review', text: 'Evaluate an existing artifact against a standard.' },
14
+ { id: 'test', text: 'Run tests or builds, or write tests as the main deliverable.' },
15
+ { id: 'docs', text: 'Write or edit documentation as the main deliverable.' },
16
+ { id: 'plan', text: 'Produce a plan, specification or breakdown before implementation.' },
17
+ { id: 'investigate', text: 'Find facts or a cause and report back without changing anything.' },
18
+ { id: 'unclassified', text: 'Unclear intent, insufficient context or outside these categories.' },
19
+ ].map(Object.freeze));
20
+ const CATALOGS = Object.freeze({ tier: TIER_OPTIONS, intent: INTENT_OPTIONS });
21
+ // Gold labels of a stage: its catalog ids, with the abstaining option recorded as null.
22
+ function goldLabels(task) {
23
+ if (!Object.hasOwn(CATALOGS, task)) throw new TypeError('unknown-task');
24
+ return CATALOGS[task].map(o => (o.id === 'unclassified' ? null : o.id));
25
+ }
26
+ // Stages whose labels carry secondary intents beside the primary choice (docs/system-one-intent.md).
27
+ const MULTI = Object.freeze(['intent']);
28
+ // Secondary labels: distinct catalog ids other than the primary; none when the primary abstains.
29
+ function validSecondary(task, primary, secondary) {
30
+ if (!MULTI.includes(task)) return secondary === undefined;
31
+ const ids = goldLabels(task).filter(g => g !== null);
32
+ return Array.isArray(secondary) && new Set(secondary).size === secondary.length
33
+ && secondary.every(s => ids.includes(s) && s !== primary) && (primary !== null || secondary.length === 0);
34
+ }
35
+ module.exports = { INTENT_OPTIONS, CATALOGS, MULTI, goldLabels, validSecondary };
@@ -42,4 +42,48 @@ function decide(req, res, policy = {}) {
42
42
  return { status: reason === 'scored' ? 'accepted' : 'abstain', reason,
43
43
  selectedId: reason === 'scored' ? ranked[0].id : null, confidence, margin, model: { ...res.model } };
44
44
  }
45
- module.exports = { createRequest, validateRequest, decide };
45
+ // Relevance-native decisions (#233): every option except unclassified is an
46
+ // independent Bernoulli against its own threshold. No simplex rule applies:
47
+ // the raw probabilities must each stand in 0..1 but need not sum to one.
48
+ function validThresholds(req, thresholds) {
49
+ const ids = req.options.map(o => o.id).filter(id => id !== 'unclassified');
50
+ if (!thresholds || typeof thresholds !== 'object' || Array.isArray(thresholds)) return false;
51
+ const keys = Object.keys(thresholds);
52
+ return keys.length === ids.length && ids.every(id => probability(thresholds[id]))
53
+ && keys.every(k => ids.includes(k));
54
+ }
55
+ function decideIntents(req, res, policy = {}) {
56
+ validateRequest(req);
57
+ if (!policy || typeof policy !== 'object' || Array.isArray(policy)
58
+ || Object.keys(policy).some(k => !['domain', 'thresholds'].includes(k))
59
+ || !validThresholds(req, policy.thresholds)) fail('invalid-policy');
60
+ const { domain = 'harness-routing-v1', thresholds } = policy;
61
+ if (!text(domain)) fail('invalid-policy');
62
+ const empty = { status: 'invalid-output', reason: 'invalid-response', selectedId: null,
63
+ confidence: null, margin: null, model: null, intents: null };
64
+ if (!exact(res, ['schemaVersion', 'requestHash', 'catalogHash', 'model', 'scores'])
65
+ || res.schemaVersion !== 1 || res.requestHash !== req.requestHash || res.catalogHash !== req.catalogHash
66
+ || !exact(res.model, ['id', 'revision', 'domain']) || !Object.values(res.model).every(v => text(v))
67
+ || !Array.isArray(res.scores) || res.scores.length !== req.options.length
68
+ || res.scores.some((s, i) => !exact(s, ['id', 'probability']) || s.id !== req.options[i].id || !probability(s.probability))) return empty;
69
+ const model = { ...res.model };
70
+ if (model.domain !== domain) {
71
+ return { status: 'abstain', reason: 'domain-mismatch', selectedId: null,
72
+ confidence: null, margin: null, model, intents: null };
73
+ }
74
+ const intents = {};
75
+ for (const s of res.scores) {
76
+ if (s.id === 'unclassified') continue;
77
+ intents[s.id] = { probability: s.probability, fired: s.probability >= thresholds[s.id] };
78
+ }
79
+ const fired = Object.entries(intents).filter(([, v]) => v.fired).sort((a, b) => b[1].probability - a[1].probability);
80
+ if (!fired.length) {
81
+ return { status: 'abstain', reason: 'no-intent-fired', selectedId: null,
82
+ confidence: null, margin: null, model, intents };
83
+ }
84
+ const rest = Object.values(intents).filter(v => v !== fired[0][1]).map(v => v.probability);
85
+ const confidence = fired[0][1].probability;
86
+ return { status: 'accepted', reason: 'scored', selectedId: fired[0][0], confidence,
87
+ margin: confidence - Math.max(...rest, 0), model, intents };
88
+ }
89
+ module.exports = { createRequest, validateRequest, decide, decideIntents };
@@ -1,26 +1,32 @@
1
1
  'use strict';
2
2
  const { validateRequest } = require('./contract');
3
- const { TIER_OPTIONS } = require('./router');
3
+ const { CATALOGS, MULTI, goldLabels, validSecondary } = require('./catalogs');
4
+ const { gradedAgreement, familyConsistency } = require('./intent');
4
5
  const exact = (obj, keys) => obj && typeof obj === 'object' && !Array.isArray(obj) && Object.keys(obj).length === keys.length && keys.every(k => Object.hasOwn(obj, k));
5
6
  const nonempty = v => typeof v === 'string' && v.trim().length > 0;
6
- const labels = ['tier1', 'tier2', 'tier3', null];
7
7
  const finite = v => typeof v === 'number' && Number.isFinite(v);
8
8
  const fail = () => { throw new TypeError('invalid-evaluation-input'); };
9
- function validateCorpus(corpus) {
9
+ // A corpus holds one stage: every request uses that stage's fixed catalog. Returns the stage's gold labels.
10
+ function corpusLabels(corpus) {
10
11
  if (!exact(corpus, ['schemaVersion', 'cases']) || corpus.schemaVersion !== 1 || !Array.isArray(corpus.cases) || !corpus.cases.length) fail();
12
+ const task = corpus.cases[0]?.request?.task;
13
+ if (!Object.hasOwn(CATALOGS, task)) fail();
14
+ const labels = goldLabels(task); const catalog = JSON.stringify(CATALOGS[task]);
11
15
  const ids = new Set(); const hashes = new Set(); const families = new Map();
12
16
  for (const c of corpus.cases) {
13
- if (!exact(c, ['id', 'family', 'split', 'language', 'source', 'reviewed', 'request', 'gold'])
17
+ if (!exact(c, ['id', 'family', 'split', 'language', 'source', 'reviewed', 'request', 'gold', ...(MULTI.includes(task) ? ['secondary'] : [])])
14
18
  || ![c.id, c.family, c.source].every(nonempty) || !['train', 'validation', 'holdout'].includes(c.split)
15
- || !['en', 'zh-TW'].includes(c.language) || typeof c.reviewed !== 'boolean' || !labels.includes(c.gold)) fail();
19
+ || !['en', 'zh-TW'].includes(c.language) || typeof c.reviewed !== 'boolean' || !labels.includes(c.gold)
20
+ || !validSecondary(task, c.gold, c.secondary)) fail();
16
21
  validateRequest(c.request);
17
- if (c.request.task !== 'tier' || JSON.stringify(c.request.options) !== JSON.stringify(TIER_OPTIONS)
22
+ if (c.request.task !== task || JSON.stringify(c.request.options) !== catalog
18
23
  || ids.has(c.id) || hashes.has(c.request.requestHash) || (families.has(c.family) && families.get(c.family) !== c.split)) fail();
19
24
  ids.add(c.id); hashes.add(c.request.requestHash); families.set(c.family, c.split);
20
25
  }
21
- return true;
26
+ return labels;
22
27
  }
23
- function prediction(decision) {
28
+ function validateCorpus(corpus) { corpusLabels(corpus); return true; }
29
+ function prediction(decision, labels) {
24
30
  if (exact(decision, ['status', 'reason']) && decision.status === 'unavailable' && nonempty(decision.reason)) return null;
25
31
  if (!exact(decision, ['status', 'reason', 'selectedId', 'confidence', 'margin', 'model'])
26
32
  || !['accepted', 'abstain', 'invalid-output'].includes(decision.status) || !nonempty(decision.reason)
@@ -31,15 +37,15 @@ function prediction(decision) {
31
37
  return decision.selectedId;
32
38
  }
33
39
  // Scored decisions carry the full catalog-order probability vector; unavailable/invalid outputs carry null.
34
- function checkScores(decision, scores) {
40
+ function checkScores(decision, scores, labels) {
35
41
  const scored = ['accepted', 'abstain'].includes(decision.status);
36
42
  if (!scored) { if (scores !== null) fail(); return; }
37
- if (!Array.isArray(scores) || scores.length !== TIER_OPTIONS.length
43
+ if (!Array.isArray(scores) || scores.length !== labels.length
38
44
  || scores.some(p => !finite(p) || p < 0 || p > 1) || Math.abs(scores.reduce((s, p) => s + p, 0) - 1) > 1e-6) fail();
39
45
  const ranked = [...scores].sort((a, b) => b - a);
40
46
  if (decision.confidence !== ranked[0] || decision.margin !== ranked[0] - ranked[1]) fail();
41
47
  }
42
- function metrics(gold, predicted) {
48
+ function metrics(gold, predicted, labels) {
43
49
  const count = gold.length;
44
50
  const accepted = predicted.filter(x => x !== null).length;
45
51
  const correct = predicted.filter((x, i) => x === gold[i]).length;
@@ -55,9 +61,95 @@ function metrics(gold, predicted) {
55
61
  acceptedErrorRate: accepted ? (accepted - acceptedCorrect) / accepted : null };
56
62
  }
57
63
  function p95(values) { return values.length ? [...values].sort((a, b) => a - b)[Math.ceil(values.length * 0.95) - 1] : null; }
64
+ // Intent corpora: accepted predictions are graded against { gold, secondary }; predicted secondaries are the
65
+ // other intents whose probability reaches the checkpoint's secondary threshold (docs/system-one-intent.md#scoring).
66
+ function gradeIntents(cases, byId, labels, threshold) {
67
+ const ids = labels.map(l => (l === null ? 'unclassified' : l));
68
+ let accepted = 0; let points = 0; let exact = 0;
69
+ for (const c of cases) {
70
+ const run = byId.get(c.id).runs[0]; const selected = prediction(run.decision, labels);
71
+ if (selected === null) continue;
72
+ const secondary = ids.filter((id, i) => id !== selected && id !== 'unclassified' && run.scores[i] >= threshold);
73
+ accepted++; points += gradedAgreement({ gold: c.gold, secondary: c.secondary }, { gold: selected, secondary }); if (selected === c.gold) exact++;
74
+ }
75
+ return { acceptedPrecision: accepted ? points / accepted : null, exactPrecision: accepted ? exact / accepted : null };
76
+ }
77
+ // Relevance-native grading (#233): every label except unclassified is an
78
+ // independent Bernoulli against its own threshold. No simplex rule applies
79
+ // to the score vector. Gates live in docs/system-one-intent-gates.md;
80
+ // this function reports metrics only.
81
+ function gradeRelevance(cases, byId, labels, thresholds) {
82
+ const ids = labels.map(l => (l === null ? 'unclassified' : l));
83
+ const per = new Map(ids.filter(id => id !== 'unclassified').map(id => [id, { tp: 0, fp: 0, fn: 0, support: 0 }]));
84
+ let abstain = 0; let firesTotal = 0; let truthTotal = 0; let exact = 0;
85
+ for (const c of cases) {
86
+ const scores = byId.get(c.id).scores;
87
+ const truth = new Set(c.gold === null ? [] : [c.gold, ...(c.secondary || [])]);
88
+ const fired = new Set(ids.filter((id, i) => id !== 'unclassified' && scores[i] >= thresholds[id]));
89
+ firesTotal += fired.size; truthTotal += truth.size;
90
+ abstain += fired.size === 0 ? 1 : 0;
91
+ exact += fired.size === truth.size && [...fired].every(id => truth.has(id)) ? 1 : 0;
92
+ for (const [id, row] of per) {
93
+ if (fired.has(id) && truth.has(id)) row.tp++;
94
+ else if (fired.has(id)) row.fp++;
95
+ else if (truth.has(id)) row.fn++;
96
+ if (truth.has(id)) row.support++;
97
+ }
98
+ }
99
+ const table = {};
100
+ for (const [id, row] of per) {
101
+ const p = row.tp + row.fp ? row.tp / (row.tp + row.fp) : 0;
102
+ const r = row.tp + row.fn ? row.tp / (row.tp + row.fn) : 0;
103
+ table[id] = { ...row, tau: thresholds[id], precision: p, recall: r, f1: p + r ? 2 * p * r / (p + r) : 0 };
104
+ }
105
+ const tp = [...per.values()].reduce((s, r) => s + r.tp, 0);
106
+ const fp = [...per.values()].reduce((s, r) => s + r.fp, 0);
107
+ const fn = [...per.values()].reduce((s, r) => s + r.fn, 0);
108
+ const precision = tp + fp ? tp / (tp + fp) : 0;
109
+ const recall = tp + fn ? tp / (tp + fn) : 0;
110
+ const tableEntries = [...per.entries()].map(([id, row]) => [id, table[id]]);
111
+ const scored = tableEntries.filter(([, t]) => t.support > 0 || t.tp + t.fp > 0);
112
+ const supported = tableEntries.filter(([, t]) => t.support > 0);
113
+ const n = cases.length;
114
+ return {
115
+ micro: { precision, recall, f1: precision + recall ? 2 * precision * recall / (precision + recall) : 0 },
116
+ macroF1: scored.length ? scored.reduce((s, [, t]) => s + t.f1, 0) / scored.length : 0,
117
+ perIntent: table,
118
+ cardinality: { predicted: n ? firesTotal / n : 0, gold: n ? truthTotal / n : 0 },
119
+ abstainRate: n ? abstain / n : 0,
120
+ exactSetMatch: n ? exact / n : 0,
121
+ weakPrecision: supported.length ? Math.min(...supported.map(([, t]) => t.precision)) : 0,
122
+ };
123
+ }
124
+ // Relevance-native promotion gates. Thresholds are the reviewed numbers in
125
+ // docs/system-one-intent-gates.md; policy/live-host evidence stays reported
126
+ // (false) until independently produced, so rolloutReady is red by
127
+ // construction pre-production.
128
+ function relevanceGates(multi, evidence) {
129
+ const supported = Object.values(multi.perIntent).filter(t => t.support >= 10);
130
+ const gates = {
131
+ microF1: multi.micro.f1 >= 0.55,
132
+ macroF1: multi.macroF1 >= 0.50,
133
+ perIntentFloors: supported.every(t => t.f1 >= 0.30),
134
+ weakPrecision: multi.weakPrecision >= 0.30,
135
+ cardinality: Math.abs(multi.cardinality.predicted - multi.cardinality.gold) <= 1.0,
136
+ abstainRate: multi.abstainRate <= 0.20,
137
+ familyConsistency: evidence.family.model.rate >= evidence.family.gold.rate,
138
+ repeatability: evidence.agreement === 1,
139
+ warmLatency: evidence.warmP95Ms <= 250,
140
+ structuralControls: !!(evidence.structural && evidence.structural.unchanged && evidence.structural.rollbackLive),
141
+ policyEvidence: false,
142
+ liveHostEvidence: false,
143
+ };
144
+ // rolloutReady stays false until policy/live-host evidence exists; the
145
+ // gates above are necessary but not sufficient pre-production.
146
+ return { ...gates, rolloutReady: false };
147
+ }
58
148
  // sourceEvidence is the provider provenance result; only the pinned cua_s1 revision satisfies the gate.
59
- function evaluate(corpus, records, sourceEvidence = null) {
60
- validateCorpus(corpus);
149
+ function evaluate(corpus, records, sourceEvidence = null, { secondaryThreshold } = {}) {
150
+ const labels = corpusLabels(corpus);
151
+ const multi = MULTI.includes(corpus.cases[0].request.task);
152
+ if (multi && !(finite(secondaryThreshold) && secondaryThreshold >= 0.05 && secondaryThreshold <= 0.5)) fail();
61
153
  const cases = corpus.cases.filter(c => c.split === 'holdout');
62
154
  if (!cases.length || !Array.isArray(records) || records.length !== cases.length) fail();
63
155
  const byId = new Map(); const models = new Set(); const cold = []; const warm = [];
@@ -66,8 +158,8 @@ function evaluate(corpus, records, sourceEvidence = null) {
66
158
  || !labels.includes(record.baseline) || !Array.isArray(record.runs) || record.runs.length !== 2) fail();
67
159
  for (const run of record.runs) {
68
160
  if (!exact(run, ['decision', 'scores', 'latencyMs', 'coldStart']) || !finite(run.latencyMs) || run.latencyMs < 0 || typeof run.coldStart !== 'boolean') fail();
69
- prediction(run.decision);
70
- checkScores(run.decision, run.scores);
161
+ prediction(run.decision, labels);
162
+ checkScores(run.decision, run.scores, labels);
71
163
  if (run.decision.model) models.add(JSON.stringify([run.decision.model.id, run.decision.model.revision, run.decision.model.domain]));
72
164
  (run.coldStart ? cold : warm).push(run.latencyMs);
73
165
  }
@@ -75,8 +167,12 @@ function evaluate(corpus, records, sourceEvidence = null) {
75
167
  }
76
168
  if (models.size > 1) fail();
77
169
  const gold = cases.map(c => c.gold);
78
- const baseline = metrics(gold, cases.map(c => byId.get(c.id).baseline));
79
- const model = metrics(gold, cases.map(c => prediction(byId.get(c.id).runs[0].decision)));
170
+ const baseline = metrics(gold, cases.map(c => byId.get(c.id).baseline), labels);
171
+ const predicted = cases.map(c => prediction(byId.get(c.id).runs[0].decision, labels));
172
+ const model = { ...metrics(gold, predicted, labels), ...(multi ? gradeIntents(cases, byId, labels, secondaryThreshold) : {}) };
173
+ const families = multi ? { familyConsistency: {
174
+ gold: familyConsistency(cases.map(c => ({ family: c.family, primary: c.gold }))),
175
+ model: familyConsistency(cases.map((c, i) => ({ family: c.family, primary: predicted[i] }))) } } : {};
80
176
  const canonicalRun = ({ decision: d, scores }) => JSON.stringify([d.status, d.reason, d.selectedId ?? null, d.confidence ?? null, d.margin ?? null,
81
177
  d.model ? [d.model.id, d.model.revision, d.model.domain] : null, scores]);
82
178
  const agreement = records.filter(r => canonicalRun(r.runs[0]) === canonicalRun(r.runs[1])).length / cases.length;
@@ -94,6 +190,6 @@ function evaluate(corpus, records, sourceEvidence = null) {
94
190
  policyEvidence: false,
95
191
  liveHostEvidence: false,
96
192
  };
97
- return { schemaVersion: 1, cases: cases.length, baseline, model, agreement, latency, gates, rolloutReady: Object.values(gates).every(Boolean) };
193
+ return { schemaVersion: 1, cases: cases.length, baseline, model, ...families, agreement, latency, gates, rolloutReady: Object.values(gates).every(Boolean) };
98
194
  }
99
- module.exports = { validateCorpus, evaluate };
195
+ module.exports = { validateCorpus, evaluate, gradeRelevance, relevanceGates };
@@ -0,0 +1,22 @@
1
+ 'use strict';
2
+ // Intent-stage comparisons (docs/system-one-intent.md#scoring). A label is { gold, secondary }.
3
+ const full = l => new Set([l.gold, ...(l.secondary || [])]);
4
+ // The owner's grades: same primary 1.0; swapped order 0.6; one-sided, or swapped with a
5
+ // secondary-count gap of 3 or more, 0.3; otherwise 0.
6
+ function gradedAgreement(a, b) {
7
+ if (a.gold === b.gold) return 1;
8
+ if (a.gold === null || b.gold === null) return 0;
9
+ const aInB = full(b).has(a.gold); const bInA = full(a).has(b.gold);
10
+ if (aInB && bInA) return Math.abs((a.secondary || []).length - (b.secondary || []).length) >= 3 ? 0.3 : 0.6;
11
+ return aInB || bInA ? 0.3 : 0;
12
+ }
13
+ // Share of families with two or more members whose members share one primary intent.
14
+ function familyConsistency(rows) {
15
+ const byFamily = new Map();
16
+ for (const r of rows) { if (!byFamily.has(r.family)) byFamily.set(r.family, new Set()); byFamily.get(r.family).add(r.primary); }
17
+ const sizes = new Map(); for (const r of rows) sizes.set(r.family, (sizes.get(r.family) || 0) + 1);
18
+ const multi = [...byFamily].filter(([f]) => sizes.get(f) >= 2);
19
+ const consistent = multi.filter(([, set]) => set.size === 1).length;
20
+ return { families: multi.length, consistent, rate: multi.length ? consistent / multi.length : null };
21
+ }
22
+ module.exports = { gradedAgreement, familyConsistency };
@@ -0,0 +1,141 @@
1
+ """Fine-tuned Laya intent provider bridge (#233 shadow, research use).
2
+
3
+ Speaks the same adapter protocol as cua_adapter.py (oneshot stdin/stdout,
4
+ resident --serve, --provenance). It emits RAW independent relevance scores
5
+ (one per intent plus the unclassified ramp): they deliberately do NOT sum to
6
+ one, so the single-winner contract.decide fails closed on them while
7
+ contract.decideIntents consumes them directly. Do not re-normalize into a
8
+ simplex anywhere in this path.
9
+ """
10
+ import hashlib
11
+ import importlib.util
12
+ import json
13
+ import sys
14
+ from pathlib import Path
15
+
16
+ _HERE = Path(__file__).resolve()
17
+ if str(_HERE.parent) not in sys.path:
18
+ sys.path.insert(0, str(_HERE.parent))
19
+ import cua_adapter # noqa: E402 (resident serve machinery: socket server, token, phases)
20
+
21
+ _label_spec = importlib.util.spec_from_file_location(
22
+ 'laya_labeler', _HERE.parents[3] / 'scripts' / 'system-one-label-laya.py')
23
+ _label = importlib.util.module_from_spec(_label_spec)
24
+ _label_spec.loader.exec_module(_label)
25
+ INTENTS = _label.INTENTS
26
+ QUESTIONS_VERSION = _label.QUESTIONS_VERSION
27
+ build_questions = _label.build_questions
28
+
29
+ # B1 evidence (full-model validation sweep): cutoff 0.45 maximizes graded
30
+ # agreement with null rate near the teacher's 14%. Pinned, not tuned per run.
31
+ NULL_CUTOFF = 0.45
32
+ # Byte limits: holdout contexts peak at 368 B, validation at 1005 B; the
33
+ # multilingual window is 1024 tokens (~4 KB). Rejection is fail-safe lexical.
34
+ LIMITS = {'context_tokens': 4096, 'option_tokens': 1024}
35
+
36
+
37
+ def verify_artifacts(manifest):
38
+ """Pin weights plus the training config beside them (rl_agent_config.json)."""
39
+ weights = Path(manifest['checkpoint'])
40
+ if not weights.is_absolute() or weights.suffix != '.safetensors':
41
+ raise ValueError('checkpoint-format')
42
+ config = weights.parent / 'rl_agent_config.json'
43
+ for file, key in [(weights, 'weightsSha256'), (config, 'configSha256')]:
44
+ with file.open('rb') as handle:
45
+ digest = hashlib.file_digest(handle, 'sha256').hexdigest()
46
+ if digest != manifest[key]:
47
+ raise ValueError('artifact-hash')
48
+ return weights
49
+
50
+
51
+ def check_limits(request, config):
52
+ if len(request['context'].encode('utf-8')) > config['context_tokens'] or any(
53
+ len(option['text'].encode('utf-8')) > config['option_tokens'] for option in request['options']
54
+ ):
55
+ raise ValueError('input-too-long')
56
+
57
+
58
+ def make_scorer(predict_fn, cutoff=NULL_CUTOFF):
59
+ """Build score(request) -> 13 raw probabilities in request-option order.
60
+
61
+ Raw means independent: 12 noul answers plus the unclassified ramp, each
62
+ in 0..1, with no sum-to-one normalization.
63
+ """
64
+ questions = build_questions()
65
+
66
+ def score(request):
67
+ if request.get('task') != 'intent':
68
+ raise ValueError('task: laya intent bridge only')
69
+ ids = [o['id'] for o in request['options']]
70
+ if set(INTENTS) - set(ids):
71
+ raise ValueError('options: all 12 intents required')
72
+ raw = predict_fn(request['context'], questions)
73
+ top = max(float(raw[i]) for i in INTENTS)
74
+ ramp = max(0.0, min(1.0, (cutoff - top) / cutoff)) if cutoff > 0 else 0.0
75
+ by_id = {i: float(raw[i]) for i in INTENTS}
76
+ by_id['unclassified'] = ramp
77
+ return [by_id[i] for i in ids]
78
+
79
+ return score, dict(LIMITS)
80
+
81
+
82
+ def load_scorer(weights, manifest):
83
+ """Load the fine-tuned checkpoint once; return (score, limits)."""
84
+ import laya # noqa: PLC0415
85
+ agent = laya.load(str(weights.parent))
86
+ questions = build_questions()
87
+
88
+ def predict_fn(state, _questions):
89
+ answers = agent.predict(state, questions)['answers']
90
+ return {intent: float(answers[intent]['noul']) for intent in INTENTS}
91
+
92
+ return make_scorer(predict_fn)
93
+
94
+
95
+ def provenance():
96
+ """Installed laya facts only; never loads a checkpoint."""
97
+ import importlib.metadata as metadata # noqa: PLC0415
98
+ import platform # noqa: PLC0415
99
+
100
+ def version(name):
101
+ try:
102
+ return metadata.version(name)
103
+ except metadata.PackageNotFoundError:
104
+ return None
105
+
106
+ return {'schemaVersion': 1, 'python': {'implementation': platform.python_implementation(),
107
+ 'version': platform.python_version()},
108
+ 'laya': {'distribution': 'laya', 'version': version('laya'),
109
+ 'checkpoint': 'convaiinnovations/laya:multilingual',
110
+ 'questions': QUESTIONS_VERSION, 'nullCutoff': NULL_CUTOFF},
111
+ 'torch': version('torch')}
112
+
113
+
114
+ def infer(request, manifest):
115
+ weights = verify_artifacts(manifest)
116
+ score, config = load_scorer(weights, manifest)
117
+ check_limits(request, config)
118
+ return cua_adapter.build_response(request, manifest, score(request))
119
+
120
+
121
+ def main():
122
+ try:
123
+ if sys.argv[1:] == ['--provenance']:
124
+ print(json.dumps(provenance(), allow_nan=False))
125
+ return 0
126
+ if sys.argv[1:2] == ['--serve']:
127
+ return cua_adapter.serve_main(sys.argv[2:], scorer_factory=load_scorer)
128
+ manifest = json.loads(sys.argv[1])
129
+ raw = sys.stdin.buffer.read(2 * 1024 * 1024 + 1)
130
+ if len(raw) > 2 * 1024 * 1024:
131
+ raise ValueError('request-limit')
132
+ request = json.loads(raw)
133
+ print(json.dumps(infer(request, manifest), allow_nan=False))
134
+ except Exception:
135
+ print('laya-provider-unavailable', file=sys.stderr)
136
+ return 1
137
+ return 0
138
+
139
+
140
+ if __name__ == '__main__':
141
+ sys.exit(main())
@@ -10,9 +10,15 @@ const unavailable = reason => ({ status: 'unavailable', reason });
10
10
  const exact = (value, keys) => value && typeof value === 'object' && !Array.isArray(value)
11
11
  && Object.keys(value).length === keys.length && keys.every(k => Object.hasOwn(value, k));
12
12
  const label = value => typeof value === 'string' && value.length > 0 && value.length <= 128 && !/[\u0000-\u001f]/.test(value);
13
+ // Resident adapter selection: a basename inside the system-one scripts dir, never a path.
14
+ const adapterName = value => typeof value === 'string' && /^[a-z][a-z0-9_-]{0,63}\.py$/.test(value);
13
15
  // Thresholds calibrated on validation for this checkpoint (docs/system-one-training.md).
14
16
  const inRange = (v, lo, hi) => typeof v === 'number' && Number.isFinite(v) && v >= lo && v <= hi;
15
- const validAcceptance = a => exact(a, ['minConfidence', 'minMargin']) && inRange(a.minConfidence, 0.5, 0.99) && inRange(a.minMargin, 0, 0.9);
17
+ // secondaryThreshold is optional and belongs to intent checkpoints (docs/system-one-intent.md#scoring).
18
+ const validAcceptance = a => (exact(a, ['minConfidence', 'minMargin']) || (exact(a, ['minConfidence', 'minMargin', 'secondaryThreshold']) && inRange(a.secondaryThreshold, 0.05, 0.5)))
19
+ && inRange(a.minConfidence, 0.5, 0.99) && inRange(a.minMargin, 0, 0.9);
20
+ // The part of the acceptance thresholds that contract.decide takes.
21
+ const decisionPolicy = acceptance => (acceptance ? { minConfidence: acceptance.minConfidence, minMargin: acceptance.minMargin } : {});
16
22
  // A scored result carries the manifest's thresholds so every caller decides with them.
17
23
  const withAcceptance = (result, manifest) => (result && result.status === 'scored' && manifest.acceptance
18
24
  ? { ...result, acceptance: { ...manifest.acceptance } } : result);
@@ -22,8 +28,9 @@ function validateManifest(m) {
22
28
  const required = ['schemaVersion', ...(ngram ? [] : ['python']), 'checkpoint', 'weightsSha256', 'configSha256', 'modelId', 'revision', 'domain'];
23
29
  const strings = ['modelId', 'revision', 'domain', ...(ngram && !Object.hasOwn(m, 'python') ? [] : ['python'])];
24
30
  if (!m || typeof m !== 'object' || Array.isArray(m) || required.some(k => !Object.hasOwn(m, k))
25
- || Object.keys(m).some(k => ![...required, 'python', 'timeoutMs', 'transport', 'idleTimeoutMs', 'acceptance'].includes(k)) || m.schemaVersion !== 1
31
+ || Object.keys(m).some(k => ![...required, 'python', 'timeoutMs', 'transport', 'idleTimeoutMs', 'acceptance', 'adapter'].includes(k)) || m.schemaVersion !== 1
26
32
  || (m.acceptance !== undefined && !validAcceptance(m.acceptance))
33
+ || (m.adapter !== undefined && !adapterName(m.adapter))
27
34
  || strings.some(k => typeof m[k] !== 'string' || !m[k].trim() || m[k].length > 4096 || m[k].includes('\0'))
28
35
  || typeof m.checkpoint !== 'string' || !path.isAbsolute(m.checkpoint) || path.extname(m.checkpoint) !== (ngram ? '.bin' : '.safetensors')
29
36
  || ['weightsSha256', 'configSha256'].some(k => typeof m[k] !== 'string' || !/^[a-f0-9]{64}$/.test(m[k]))
@@ -105,4 +112,4 @@ function provenance(manifestPath) {
105
112
  try { return { status: 'recorded', provenance: validateProvenance(result.value) }; }
106
113
  catch (_) { return unavailable('provider-json'); }
107
114
  }
108
- module.exports = { score, scoreAsync, runProvider, validateManifest, readManifest, provenance, validateProvenance, PINNED_CUA_S1_REVISION };
115
+ module.exports = { score, scoreAsync, runProvider, validateManifest, readManifest, provenance, validateProvenance, decisionPolicy, PINNED_CUA_S1_REVISION };
@@ -10,7 +10,12 @@ const { spawn } = require('node:child_process');
10
10
  const { Worker } = require('node:worker_threads');
11
11
  const { validateRequest } = require('./contract');
12
12
 
13
- const ADAPTER = path.join(__dirname, 'cua_adapter.py');
13
+ const ADAPTER_PATTERN = /^[a-z][a-z0-9_-]{0,63}\.py$/;
14
+ const adapterPath = manifest => {
15
+ const name = manifest.adapter || 'cua_adapter.py';
16
+ if (!ADAPTER_PATTERN.test(name)) throw new TypeError('invalid-adapter');
17
+ return path.join(__dirname, name);
18
+ };
14
19
  const LOCK_TTL_MS = 60000;
15
20
  const CALL_TIMEOUT_MS = 1000;
16
21
  const MAX_REPLY_BYTES = 1024 * 1024;
@@ -18,7 +23,7 @@ const unavailable = reason => ({ status: 'unavailable', reason });
18
23
  const sleep = ms => Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms);
19
24
  const exact = (value, keys) => value && typeof value === 'object' && !Array.isArray(value)
20
25
  && Object.keys(value).length === keys.length && keys.every(k => Object.hasOwn(value, k));
21
- const defaultCommand = manifest => [manifest.python, [ADAPTER, '--serve']];
26
+ const defaultCommand = manifest => [manifest.python, [adapterPath(manifest), '--serve']];
22
27
 
23
28
  function stateFiles(manifest, manifestPath) {
24
29
  const digest = createHash('sha256').update(JSON.stringify(manifest)).digest('hex').slice(0, 16);
@@ -197,4 +202,4 @@ if (require.main === module) {
197
202
  if (op === 'stop') stop(manifest, manifestPath);
198
203
  process.stdout.write(`${JSON.stringify(status(manifest, manifestPath))}\n`);
199
204
  }
200
- module.exports = { stateFiles, readState, callSync, callAsync, score, scoreAsync, ensureReady, status, stop, LOCK_TTL_MS, CALL_TIMEOUT_MS };
205
+ module.exports = { stateFiles, readState, callSync, callAsync, score, scoreAsync, ensureReady, status, stop, defaultCommand, LOCK_TTL_MS, CALL_TIMEOUT_MS };
@@ -26,7 +26,7 @@ function selectTier(input, env = process.env, scorer = provider.score) {
26
26
  const allowed = ['provider-timeout', 'provider-output-limit', 'provider-exit', 'provider-json', 'provider-config', 'provider-unavailable', 'provider-starting'];
27
27
  return fallback(allowed.includes(result?.reason) ? result.reason : 'provider-unavailable');
28
28
  }
29
- const decision = decide(request, result.response, result.acceptance ? { ...result.acceptance } : {});
29
+ const decision = decide(request, result.response, provider.decisionPolicy(result.acceptance));
30
30
  if (decision.selectedId === 'unclassified') Object.assign(decision, { selectedId: null, status: 'abstain', reason: 'unclassified' });
31
31
  const accepted = decision.status === 'accepted';
32
32
  const belowFloor = accepted && input.floor != null && RANK[decision.selectedId] < RANK[input.floor];
package/hooks/hooks.json CHANGED
@@ -23,6 +23,16 @@
23
23
  ],
24
24
  "description": "Establish Harness routing invariants and an active workflow execution contract before host/domain skill selection",
25
25
  "id": "harness:pre:kernel-router"
26
+ },
27
+ {
28
+ "hooks": [
29
+ {
30
+ "type": "command",
31
+ "command": "node \"${CLAUDE_PLUGIN_ROOT}/hooks/scripts/observation-hook.js\""
32
+ }
33
+ ],
34
+ "description": "Record a local, content-private System One turn observation (docs/system-one-observations.md); fail-open and silent",
35
+ "id": "harness:obs:observation-userpromptsubmit"
26
36
  }
27
37
  ],
28
38
  "PreToolUse": [
@@ -155,6 +165,16 @@
155
165
  ],
156
166
  "description": "Close Harness skill active-window telemetry at the turn boundary",
157
167
  "id": "harness:stop:skill-telemetry"
168
+ },
169
+ {
170
+ "hooks": [
171
+ {
172
+ "type": "command",
173
+ "command": "node \"${CLAUDE_PLUGIN_ROOT}/hooks/scripts/observation-hook.js\""
174
+ }
175
+ ],
176
+ "description": "Record a local, content-private System One turn observation (docs/system-one-observations.md); fail-open and silent",
177
+ "id": "harness:obs:observation-stop"
158
178
  }
159
179
  ],
160
180
  "PostToolUse": [
@@ -234,6 +254,16 @@
234
254
  ],
235
255
  "description": "Record privacy-safe Harness skill load completion telemetry",
236
256
  "id": "harness:post:skill-telemetry"
257
+ },
258
+ {
259
+ "hooks": [
260
+ {
261
+ "type": "command",
262
+ "command": "node \"${CLAUDE_PLUGIN_ROOT}/hooks/scripts/observation-hook.js\""
263
+ }
264
+ ],
265
+ "description": "Record a local, content-private System One turn observation (docs/system-one-observations.md); fail-open and silent",
266
+ "id": "harness:obs:observation-posttooluse"
237
267
  }
238
268
  ],
239
269
  "PostToolUseFailure": [
@@ -280,6 +310,16 @@
280
310
  ],
281
311
  "description": "Record failed Harness skill loading without prompt/tool arguments",
282
312
  "id": "harness:post-failure:skill-telemetry"
313
+ },
314
+ {
315
+ "hooks": [
316
+ {
317
+ "type": "command",
318
+ "command": "node \"${CLAUDE_PLUGIN_ROOT}/hooks/scripts/observation-hook.js\""
319
+ }
320
+ ],
321
+ "description": "Record a local, content-private System One turn observation (docs/system-one-observations.md); fail-open and silent",
322
+ "id": "harness:obs:observation-posttoolusefailure"
283
323
  }
284
324
  ]
285
325
  }