harness-everything 0.29.0 → 0.30.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +1 -1
- package/.claude-plugin/plugin.json +1 -1
- package/AGENTS.md +1 -0
- package/CHANGELOG.md +42 -0
- package/PRIVACY.md +9 -0
- package/benchmarks/fixtures/system-one-intent-holdout-draft.json +2228 -0
- package/benchmarks/fixtures/system-one-intent-holdout.json +16509 -0
- package/benchmarks/fixtures/system-one-intent-reviews.json +2723 -0
- package/benchmarks/results/system-one/intent-ngram-v1-2026-09-24/README.md +64 -0
- package/benchmarks/results/system-one/intent-ngram-v1-2026-09-24/report.json +15177 -0
- package/benchmarks/results/system-one/labeler-holdout-check-intent-2026-09-24/report.json +2976 -0
- package/benchmarks/results/system-one/labeler-holdout-check-intent-scores-haiku-2026-09-24/report-005-steps.json +6125 -0
- package/benchmarks/results/system-one/labeler-holdout-check-intent-scores-haiku-2026-09-24/report-haiku-sonnet-escalation.json +6344 -0
- package/benchmarks/results/system-one/labeler-holdout-check-intent-scores-haiku-2026-09-24/report-scaled-margin-escalation.json +6206 -0
- package/benchmarks/results/system-one/labeler-holdout-check-intent-scores-haiku-2026-09-24/report.json +6103 -0
- package/benchmarks/results/system-one/laya-ft-2026-09-25/README.md +83 -0
- package/benchmarks/results/system-one/laya-ft-2026-09-25/report.json +71 -0
- package/benchmarks/results/system-one/laya-ft-2026-09-25/shadow-report.json +44 -0
- package/benchmarks/results/system-one/laya-zs-2026-09-25/README.md +68 -0
- package/benchmarks/results/system-one/laya-zs-2026-09-25/report.json +173 -0
- package/ci/fable-mode-test.js +16 -0
- package/ci/mechanism-34-observations.test.js +550 -0
- package/ci/mechanism-system-one-collect.test.js +16 -0
- package/ci/mechanism-system-one-intent.test.js +406 -0
- package/ci/mechanism-system-one-laya.test.js +45 -0
- package/ci/mechanism-system-one-relevance.test.js +112 -0
- package/ci/mechanism-system-one-resident.test.js +16 -0
- package/ci/system-one-laya-calibrate-test.py +65 -0
- package/ci/system-one-laya-export-test.py +155 -0
- package/ci/system-one-laya-label-test.py +201 -0
- package/ci/system-one-laya-perintent-test.py +60 -0
- package/ci/system-one-laya-provider-test.py +115 -0
- package/ci/system-one-laya-train-test.py +93 -0
- package/docs/routing.md +25 -0
- package/docs/system-one-intent-gates.md +102 -0
- package/docs/system-one-intent.md +173 -0
- package/docs/system-one-laya-training.md +38 -0
- package/docs/system-one-observations.md +206 -0
- package/docs/system-one-routing.md +25 -10
- package/docs/system-one-suggestion-gates.md +292 -0
- package/docs/system-one-training.md +56 -6
- package/docs/workflows/fable-mode.md +7 -0
- package/fable-mode/SKILL.md +1 -1
- package/fable-mode/agents/fable-orchestrator.md +4 -4
- package/fable-mode/agents/fable-worker-haiku.md +4 -4
- package/fable-mode/agents/fable-worker-sonnet.md +4 -4
- package/fable-mode/execution-guardrails/SKILL.md +5 -3
- package/fable-mode/execution-guardrails/references/guardrail-rules.md +12 -8
- package/fable-mode/fable-haiku/SKILL.md +1 -1
- package/fable-mode/fable-opus/SKILL.md +1 -1
- package/fable-mode/fable-sonnet/SKILL.md +1 -1
- package/harness-everything/SKILL.md +1 -1
- package/harness-everything/scripts/kernel-router.js +10 -0
- package/harness-everything/scripts/system-one/catalogs.js +35 -0
- package/harness-everything/scripts/system-one/contract.js +45 -1
- package/harness-everything/scripts/system-one/evaluate.js +115 -19
- package/harness-everything/scripts/system-one/intent.js +22 -0
- package/harness-everything/scripts/system-one/laya_adapter.py +141 -0
- package/harness-everything/scripts/system-one/provider.js +10 -3
- package/harness-everything/scripts/system-one/resident.js +8 -3
- package/harness-everything/scripts/system-one/router.js +1 -1
- package/hooks/hooks.json +40 -0
- package/hooks/scripts/codex-user-prompt.js +39 -0
- package/hooks/scripts/lib/observations.js +581 -0
- package/hooks/scripts/observation-hook.js +25 -0
- package/opencode-plugin/plugin.json +1 -1
- package/package.json +1 -1
- package/plugins/harness-everything/.codex-plugin/plugin.json +1 -1
- package/plugins/harness-everything/hooks/hooks.json +35 -2
- package/plugins/harness-everything/hooks/scripts/codex-user-prompt.js +39 -0
- package/plugins/harness-everything/hooks/scripts/lib/observations.js +581 -0
- package/plugins/harness-everything/hooks/scripts/observation-hook.js +25 -0
- package/plugins/harness-everything/plugin.json +1 -1
- package/plugins/harness-everything/skills/fable-mode/SKILL.md +1 -1
- package/plugins/harness-everything/skills/fable-mode/agents/fable-orchestrator.md +4 -4
- package/plugins/harness-everything/skills/fable-mode/agents/fable-worker-haiku.md +4 -4
- package/plugins/harness-everything/skills/fable-mode/agents/fable-worker-sonnet.md +4 -4
- package/plugins/harness-everything/skills/fable-mode/execution-guardrails/SKILL.md +5 -3
- package/plugins/harness-everything/skills/fable-mode/execution-guardrails/references/guardrail-rules.md +12 -8
- package/plugins/harness-everything/skills/fable-mode/fable-haiku/SKILL.md +1 -1
- package/plugins/harness-everything/skills/fable-mode/fable-opus/SKILL.md +1 -1
- package/plugins/harness-everything/skills/fable-mode/fable-sonnet/SKILL.md +1 -1
- package/plugins/harness-everything/skills/harness-everything/SKILL.md +1 -1
- package/plugins/harness-everything/skills/harness-everything/scripts/kernel-router.js +10 -0
- package/plugins/harness-everything/skills/harness-everything/scripts/system-one/catalogs.js +35 -0
- package/plugins/harness-everything/skills/harness-everything/scripts/system-one/contract.js +45 -1
- package/plugins/harness-everything/skills/harness-everything/scripts/system-one/evaluate.js +115 -19
- package/plugins/harness-everything/skills/harness-everything/scripts/system-one/intent.js +22 -0
- package/plugins/harness-everything/skills/harness-everything/scripts/system-one/laya_adapter.py +141 -0
- package/plugins/harness-everything/skills/harness-everything/scripts/system-one/provider.js +10 -3
- package/plugins/harness-everything/skills/harness-everything/scripts/system-one/resident.js +8 -3
- package/plugins/harness-everything/skills/harness-everything/scripts/system-one/router.js +1 -1
- package/sbom.cdx.json +7 -7
- package/scripts/evaluate-system-one.js +17 -5
- package/scripts/system-one-calibrate-laya.py +165 -0
- package/scripts/system-one-calibrate.js +42 -13
- package/scripts/system-one-collect.js +7 -3
- package/scripts/system-one-compare-laya.py +156 -0
- package/scripts/system-one-corpus.js +22 -15
- package/scripts/system-one-eval-laya.js +112 -0
- package/scripts/system-one-export-laya.py +197 -0
- package/scripts/system-one-label-laya.py +243 -0
- package/scripts/system-one-label.js +163 -37
- package/scripts/system-one-observations-export.js +106 -0
- package/scripts/system-one-observations-review.js +86 -0
- package/scripts/system-one-perintent-laya.py +134 -0
- package/scripts/system-one-train-laya.py +344 -0
- package/scripts/system-one-train-ngram.py +66 -14
|
@@ -105,6 +105,15 @@ function persistWorkflow(contract, payload, prompt) {
|
|
|
105
105
|
return { ...context, hooksRoot, retained: false, memoryCapability };
|
|
106
106
|
}
|
|
107
107
|
|
|
108
|
+
// Printed last so it is the nearest instruction when the turn ends (docs/system-one-observations.md#label-line).
|
|
109
|
+
function printLabelContract() {
|
|
110
|
+
console.log('\n=> TURN LABEL LINE (MUST, EVERY TURN — short answers, lookups and trivial edits included):');
|
|
111
|
+
console.log(' - The last line of your final message MUST be one hidden label line with your own post-work judgement, not a copy of the router tier above:');
|
|
112
|
+
console.log(' <!-- harness-label {"v":1,"validity":"actionable|invalid","contextDependent":true|false,"tier":"tier1|tier2|tier3|null","intents":["up to 3 of explain,discuss,git,fix,edit,feature,refactor,review,test,docs,plan,investigate"],"workflow":"strategy id or null","skills":["skills used"]} -->');
|
|
113
|
+
console.log(' - tier1: answer, lookup, status or git operation with no code change; tier2: bounded change; tier3: new feature, refactor, redefinition or cross-component work. Use null only when validity is invalid.');
|
|
114
|
+
console.log(' - It is invisible in rendered Markdown and records local training data; it never gates routing.');
|
|
115
|
+
}
|
|
116
|
+
|
|
108
117
|
function run(raw) {
|
|
109
118
|
const promptArg = process.argv.slice(2).join(' ');
|
|
110
119
|
let payload = null;
|
|
@@ -162,6 +171,7 @@ function run(raw) {
|
|
|
162
171
|
} else {
|
|
163
172
|
console.log(' - Runtime state was not persisted; reminders may be less precise, but execution remains available.');
|
|
164
173
|
}
|
|
174
|
+
printLabelContract();
|
|
165
175
|
if (failure) console.error('[Workflow Reminder] routing state was not persisted: ' + failure.message);
|
|
166
176
|
if (result && result.status !== 0) console.error('[Workflow Reminder] router returned status ' + result.status + '; contract observation is degraded, not blocking.');
|
|
167
177
|
}
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
// Fixed System One catalogs, one per stage (docs/system-one-routing.md#staged-classification).
|
|
3
|
+
// Option order is the scorer's output order; changing it invalidates trained weights.
|
|
4
|
+
const { TIER_OPTIONS } = require('./router');
|
|
5
|
+
const INTENT_OPTIONS = Object.freeze([
|
|
6
|
+
{ id: 'explain', text: 'Answer a question or explain how something works.' },
|
|
7
|
+
{ id: 'discuss', text: 'Weigh options, give an opinion or settle a decision.' },
|
|
8
|
+
{ id: 'git', text: 'Git or GitHub housekeeping: commit, push, branch, pull request.' },
|
|
9
|
+
{ id: 'fix', text: 'Repair wrong behavior: a bug, crash, failing test or build.' },
|
|
10
|
+
{ id: 'edit', text: 'Change an existing value, setting, text or behavior on request.' },
|
|
11
|
+
{ id: 'feature', text: 'Add new behavior: a command, option, format or script.' },
|
|
12
|
+
{ id: 'refactor', text: 'Restructure or unify existing code without new behavior.' },
|
|
13
|
+
{ id: 'review', text: 'Evaluate an existing artifact against a standard.' },
|
|
14
|
+
{ id: 'test', text: 'Run tests or builds, or write tests as the main deliverable.' },
|
|
15
|
+
{ id: 'docs', text: 'Write or edit documentation as the main deliverable.' },
|
|
16
|
+
{ id: 'plan', text: 'Produce a plan, specification or breakdown before implementation.' },
|
|
17
|
+
{ id: 'investigate', text: 'Find facts or a cause and report back without changing anything.' },
|
|
18
|
+
{ id: 'unclassified', text: 'Unclear intent, insufficient context or outside these categories.' },
|
|
19
|
+
].map(Object.freeze));
|
|
20
|
+
const CATALOGS = Object.freeze({ tier: TIER_OPTIONS, intent: INTENT_OPTIONS });
|
|
21
|
+
// Gold labels of a stage: its catalog ids, with the abstaining option recorded as null.
|
|
22
|
+
function goldLabels(task) {
|
|
23
|
+
if (!Object.hasOwn(CATALOGS, task)) throw new TypeError('unknown-task');
|
|
24
|
+
return CATALOGS[task].map(o => (o.id === 'unclassified' ? null : o.id));
|
|
25
|
+
}
|
|
26
|
+
// Stages whose labels carry secondary intents beside the primary choice (docs/system-one-intent.md).
|
|
27
|
+
const MULTI = Object.freeze(['intent']);
|
|
28
|
+
// Secondary labels: distinct catalog ids other than the primary; none when the primary abstains.
|
|
29
|
+
function validSecondary(task, primary, secondary) {
|
|
30
|
+
if (!MULTI.includes(task)) return secondary === undefined;
|
|
31
|
+
const ids = goldLabels(task).filter(g => g !== null);
|
|
32
|
+
return Array.isArray(secondary) && new Set(secondary).size === secondary.length
|
|
33
|
+
&& secondary.every(s => ids.includes(s) && s !== primary) && (primary !== null || secondary.length === 0);
|
|
34
|
+
}
|
|
35
|
+
module.exports = { INTENT_OPTIONS, CATALOGS, MULTI, goldLabels, validSecondary };
|
|
@@ -42,4 +42,48 @@ function decide(req, res, policy = {}) {
|
|
|
42
42
|
return { status: reason === 'scored' ? 'accepted' : 'abstain', reason,
|
|
43
43
|
selectedId: reason === 'scored' ? ranked[0].id : null, confidence, margin, model: { ...res.model } };
|
|
44
44
|
}
|
|
45
|
-
|
|
45
|
+
// Relevance-native decisions (#233): every option except unclassified is an
|
|
46
|
+
// independent Bernoulli against its own threshold. No simplex rule applies:
|
|
47
|
+
// the raw probabilities must each stand in 0..1 but need not sum to one.
|
|
48
|
+
function validThresholds(req, thresholds) {
|
|
49
|
+
const ids = req.options.map(o => o.id).filter(id => id !== 'unclassified');
|
|
50
|
+
if (!thresholds || typeof thresholds !== 'object' || Array.isArray(thresholds)) return false;
|
|
51
|
+
const keys = Object.keys(thresholds);
|
|
52
|
+
return keys.length === ids.length && ids.every(id => probability(thresholds[id]))
|
|
53
|
+
&& keys.every(k => ids.includes(k));
|
|
54
|
+
}
|
|
55
|
+
function decideIntents(req, res, policy = {}) {
|
|
56
|
+
validateRequest(req);
|
|
57
|
+
if (!policy || typeof policy !== 'object' || Array.isArray(policy)
|
|
58
|
+
|| Object.keys(policy).some(k => !['domain', 'thresholds'].includes(k))
|
|
59
|
+
|| !validThresholds(req, policy.thresholds)) fail('invalid-policy');
|
|
60
|
+
const { domain = 'harness-routing-v1', thresholds } = policy;
|
|
61
|
+
if (!text(domain)) fail('invalid-policy');
|
|
62
|
+
const empty = { status: 'invalid-output', reason: 'invalid-response', selectedId: null,
|
|
63
|
+
confidence: null, margin: null, model: null, intents: null };
|
|
64
|
+
if (!exact(res, ['schemaVersion', 'requestHash', 'catalogHash', 'model', 'scores'])
|
|
65
|
+
|| res.schemaVersion !== 1 || res.requestHash !== req.requestHash || res.catalogHash !== req.catalogHash
|
|
66
|
+
|| !exact(res.model, ['id', 'revision', 'domain']) || !Object.values(res.model).every(v => text(v))
|
|
67
|
+
|| !Array.isArray(res.scores) || res.scores.length !== req.options.length
|
|
68
|
+
|| res.scores.some((s, i) => !exact(s, ['id', 'probability']) || s.id !== req.options[i].id || !probability(s.probability))) return empty;
|
|
69
|
+
const model = { ...res.model };
|
|
70
|
+
if (model.domain !== domain) {
|
|
71
|
+
return { status: 'abstain', reason: 'domain-mismatch', selectedId: null,
|
|
72
|
+
confidence: null, margin: null, model, intents: null };
|
|
73
|
+
}
|
|
74
|
+
const intents = {};
|
|
75
|
+
for (const s of res.scores) {
|
|
76
|
+
if (s.id === 'unclassified') continue;
|
|
77
|
+
intents[s.id] = { probability: s.probability, fired: s.probability >= thresholds[s.id] };
|
|
78
|
+
}
|
|
79
|
+
const fired = Object.entries(intents).filter(([, v]) => v.fired).sort((a, b) => b[1].probability - a[1].probability);
|
|
80
|
+
if (!fired.length) {
|
|
81
|
+
return { status: 'abstain', reason: 'no-intent-fired', selectedId: null,
|
|
82
|
+
confidence: null, margin: null, model, intents };
|
|
83
|
+
}
|
|
84
|
+
const rest = Object.values(intents).filter(v => v !== fired[0][1]).map(v => v.probability);
|
|
85
|
+
const confidence = fired[0][1].probability;
|
|
86
|
+
return { status: 'accepted', reason: 'scored', selectedId: fired[0][0], confidence,
|
|
87
|
+
margin: confidence - Math.max(...rest, 0), model, intents };
|
|
88
|
+
}
|
|
89
|
+
module.exports = { createRequest, validateRequest, decide, decideIntents };
|
|
@@ -1,26 +1,32 @@
|
|
|
1
1
|
'use strict';
|
|
2
2
|
const { validateRequest } = require('./contract');
|
|
3
|
-
const {
|
|
3
|
+
const { CATALOGS, MULTI, goldLabels, validSecondary } = require('./catalogs');
|
|
4
|
+
const { gradedAgreement, familyConsistency } = require('./intent');
|
|
4
5
|
const exact = (obj, keys) => obj && typeof obj === 'object' && !Array.isArray(obj) && Object.keys(obj).length === keys.length && keys.every(k => Object.hasOwn(obj, k));
|
|
5
6
|
const nonempty = v => typeof v === 'string' && v.trim().length > 0;
|
|
6
|
-
const labels = ['tier1', 'tier2', 'tier3', null];
|
|
7
7
|
const finite = v => typeof v === 'number' && Number.isFinite(v);
|
|
8
8
|
const fail = () => { throw new TypeError('invalid-evaluation-input'); };
|
|
9
|
-
|
|
9
|
+
// A corpus holds one stage: every request uses that stage's fixed catalog. Returns the stage's gold labels.
|
|
10
|
+
function corpusLabels(corpus) {
|
|
10
11
|
if (!exact(corpus, ['schemaVersion', 'cases']) || corpus.schemaVersion !== 1 || !Array.isArray(corpus.cases) || !corpus.cases.length) fail();
|
|
12
|
+
const task = corpus.cases[0]?.request?.task;
|
|
13
|
+
if (!Object.hasOwn(CATALOGS, task)) fail();
|
|
14
|
+
const labels = goldLabels(task); const catalog = JSON.stringify(CATALOGS[task]);
|
|
11
15
|
const ids = new Set(); const hashes = new Set(); const families = new Map();
|
|
12
16
|
for (const c of corpus.cases) {
|
|
13
|
-
if (!exact(c, ['id', 'family', 'split', 'language', 'source', 'reviewed', 'request', 'gold'])
|
|
17
|
+
if (!exact(c, ['id', 'family', 'split', 'language', 'source', 'reviewed', 'request', 'gold', ...(MULTI.includes(task) ? ['secondary'] : [])])
|
|
14
18
|
|| ![c.id, c.family, c.source].every(nonempty) || !['train', 'validation', 'holdout'].includes(c.split)
|
|
15
|
-
|| !['en', 'zh-TW'].includes(c.language) || typeof c.reviewed !== 'boolean' || !labels.includes(c.gold)
|
|
19
|
+
|| !['en', 'zh-TW'].includes(c.language) || typeof c.reviewed !== 'boolean' || !labels.includes(c.gold)
|
|
20
|
+
|| !validSecondary(task, c.gold, c.secondary)) fail();
|
|
16
21
|
validateRequest(c.request);
|
|
17
|
-
if (c.request.task !==
|
|
22
|
+
if (c.request.task !== task || JSON.stringify(c.request.options) !== catalog
|
|
18
23
|
|| ids.has(c.id) || hashes.has(c.request.requestHash) || (families.has(c.family) && families.get(c.family) !== c.split)) fail();
|
|
19
24
|
ids.add(c.id); hashes.add(c.request.requestHash); families.set(c.family, c.split);
|
|
20
25
|
}
|
|
21
|
-
return
|
|
26
|
+
return labels;
|
|
22
27
|
}
|
|
23
|
-
function
|
|
28
|
+
function validateCorpus(corpus) { corpusLabels(corpus); return true; }
|
|
29
|
+
function prediction(decision, labels) {
|
|
24
30
|
if (exact(decision, ['status', 'reason']) && decision.status === 'unavailable' && nonempty(decision.reason)) return null;
|
|
25
31
|
if (!exact(decision, ['status', 'reason', 'selectedId', 'confidence', 'margin', 'model'])
|
|
26
32
|
|| !['accepted', 'abstain', 'invalid-output'].includes(decision.status) || !nonempty(decision.reason)
|
|
@@ -31,15 +37,15 @@ function prediction(decision) {
|
|
|
31
37
|
return decision.selectedId;
|
|
32
38
|
}
|
|
33
39
|
// Scored decisions carry the full catalog-order probability vector; unavailable/invalid outputs carry null.
|
|
34
|
-
function checkScores(decision, scores) {
|
|
40
|
+
function checkScores(decision, scores, labels) {
|
|
35
41
|
const scored = ['accepted', 'abstain'].includes(decision.status);
|
|
36
42
|
if (!scored) { if (scores !== null) fail(); return; }
|
|
37
|
-
if (!Array.isArray(scores) || scores.length !==
|
|
43
|
+
if (!Array.isArray(scores) || scores.length !== labels.length
|
|
38
44
|
|| scores.some(p => !finite(p) || p < 0 || p > 1) || Math.abs(scores.reduce((s, p) => s + p, 0) - 1) > 1e-6) fail();
|
|
39
45
|
const ranked = [...scores].sort((a, b) => b - a);
|
|
40
46
|
if (decision.confidence !== ranked[0] || decision.margin !== ranked[0] - ranked[1]) fail();
|
|
41
47
|
}
|
|
42
|
-
function metrics(gold, predicted) {
|
|
48
|
+
function metrics(gold, predicted, labels) {
|
|
43
49
|
const count = gold.length;
|
|
44
50
|
const accepted = predicted.filter(x => x !== null).length;
|
|
45
51
|
const correct = predicted.filter((x, i) => x === gold[i]).length;
|
|
@@ -55,9 +61,95 @@ function metrics(gold, predicted) {
|
|
|
55
61
|
acceptedErrorRate: accepted ? (accepted - acceptedCorrect) / accepted : null };
|
|
56
62
|
}
|
|
57
63
|
function p95(values) { return values.length ? [...values].sort((a, b) => a - b)[Math.ceil(values.length * 0.95) - 1] : null; }
|
|
64
|
+
// Intent corpora: accepted predictions are graded against { gold, secondary }; predicted secondaries are the
|
|
65
|
+
// other intents whose probability reaches the checkpoint's secondary threshold (docs/system-one-intent.md#scoring).
|
|
66
|
+
function gradeIntents(cases, byId, labels, threshold) {
|
|
67
|
+
const ids = labels.map(l => (l === null ? 'unclassified' : l));
|
|
68
|
+
let accepted = 0; let points = 0; let exact = 0;
|
|
69
|
+
for (const c of cases) {
|
|
70
|
+
const run = byId.get(c.id).runs[0]; const selected = prediction(run.decision, labels);
|
|
71
|
+
if (selected === null) continue;
|
|
72
|
+
const secondary = ids.filter((id, i) => id !== selected && id !== 'unclassified' && run.scores[i] >= threshold);
|
|
73
|
+
accepted++; points += gradedAgreement({ gold: c.gold, secondary: c.secondary }, { gold: selected, secondary }); if (selected === c.gold) exact++;
|
|
74
|
+
}
|
|
75
|
+
return { acceptedPrecision: accepted ? points / accepted : null, exactPrecision: accepted ? exact / accepted : null };
|
|
76
|
+
}
|
|
77
|
+
// Relevance-native grading (#233): every label except unclassified is an
|
|
78
|
+
// independent Bernoulli against its own threshold. No simplex rule applies
|
|
79
|
+
// to the score vector. Gates live in docs/system-one-intent-gates.md;
|
|
80
|
+
// this function reports metrics only.
|
|
81
|
+
function gradeRelevance(cases, byId, labels, thresholds) {
|
|
82
|
+
const ids = labels.map(l => (l === null ? 'unclassified' : l));
|
|
83
|
+
const per = new Map(ids.filter(id => id !== 'unclassified').map(id => [id, { tp: 0, fp: 0, fn: 0, support: 0 }]));
|
|
84
|
+
let abstain = 0; let firesTotal = 0; let truthTotal = 0; let exact = 0;
|
|
85
|
+
for (const c of cases) {
|
|
86
|
+
const scores = byId.get(c.id).scores;
|
|
87
|
+
const truth = new Set(c.gold === null ? [] : [c.gold, ...(c.secondary || [])]);
|
|
88
|
+
const fired = new Set(ids.filter((id, i) => id !== 'unclassified' && scores[i] >= thresholds[id]));
|
|
89
|
+
firesTotal += fired.size; truthTotal += truth.size;
|
|
90
|
+
abstain += fired.size === 0 ? 1 : 0;
|
|
91
|
+
exact += fired.size === truth.size && [...fired].every(id => truth.has(id)) ? 1 : 0;
|
|
92
|
+
for (const [id, row] of per) {
|
|
93
|
+
if (fired.has(id) && truth.has(id)) row.tp++;
|
|
94
|
+
else if (fired.has(id)) row.fp++;
|
|
95
|
+
else if (truth.has(id)) row.fn++;
|
|
96
|
+
if (truth.has(id)) row.support++;
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
const table = {};
|
|
100
|
+
for (const [id, row] of per) {
|
|
101
|
+
const p = row.tp + row.fp ? row.tp / (row.tp + row.fp) : 0;
|
|
102
|
+
const r = row.tp + row.fn ? row.tp / (row.tp + row.fn) : 0;
|
|
103
|
+
table[id] = { ...row, tau: thresholds[id], precision: p, recall: r, f1: p + r ? 2 * p * r / (p + r) : 0 };
|
|
104
|
+
}
|
|
105
|
+
const tp = [...per.values()].reduce((s, r) => s + r.tp, 0);
|
|
106
|
+
const fp = [...per.values()].reduce((s, r) => s + r.fp, 0);
|
|
107
|
+
const fn = [...per.values()].reduce((s, r) => s + r.fn, 0);
|
|
108
|
+
const precision = tp + fp ? tp / (tp + fp) : 0;
|
|
109
|
+
const recall = tp + fn ? tp / (tp + fn) : 0;
|
|
110
|
+
const tableEntries = [...per.entries()].map(([id, row]) => [id, table[id]]);
|
|
111
|
+
const scored = tableEntries.filter(([, t]) => t.support > 0 || t.tp + t.fp > 0);
|
|
112
|
+
const supported = tableEntries.filter(([, t]) => t.support > 0);
|
|
113
|
+
const n = cases.length;
|
|
114
|
+
return {
|
|
115
|
+
micro: { precision, recall, f1: precision + recall ? 2 * precision * recall / (precision + recall) : 0 },
|
|
116
|
+
macroF1: scored.length ? scored.reduce((s, [, t]) => s + t.f1, 0) / scored.length : 0,
|
|
117
|
+
perIntent: table,
|
|
118
|
+
cardinality: { predicted: n ? firesTotal / n : 0, gold: n ? truthTotal / n : 0 },
|
|
119
|
+
abstainRate: n ? abstain / n : 0,
|
|
120
|
+
exactSetMatch: n ? exact / n : 0,
|
|
121
|
+
weakPrecision: supported.length ? Math.min(...supported.map(([, t]) => t.precision)) : 0,
|
|
122
|
+
};
|
|
123
|
+
}
|
|
124
|
+
// Relevance-native promotion gates. Thresholds are the reviewed numbers in
|
|
125
|
+
// docs/system-one-intent-gates.md; policy/live-host evidence stays reported
|
|
126
|
+
// (false) until independently produced, so rolloutReady is red by
|
|
127
|
+
// construction pre-production.
|
|
128
|
+
function relevanceGates(multi, evidence) {
|
|
129
|
+
const supported = Object.values(multi.perIntent).filter(t => t.support >= 10);
|
|
130
|
+
const gates = {
|
|
131
|
+
microF1: multi.micro.f1 >= 0.55,
|
|
132
|
+
macroF1: multi.macroF1 >= 0.50,
|
|
133
|
+
perIntentFloors: supported.every(t => t.f1 >= 0.30),
|
|
134
|
+
weakPrecision: multi.weakPrecision >= 0.30,
|
|
135
|
+
cardinality: Math.abs(multi.cardinality.predicted - multi.cardinality.gold) <= 1.0,
|
|
136
|
+
abstainRate: multi.abstainRate <= 0.20,
|
|
137
|
+
familyConsistency: evidence.family.model.rate >= evidence.family.gold.rate,
|
|
138
|
+
repeatability: evidence.agreement === 1,
|
|
139
|
+
warmLatency: evidence.warmP95Ms <= 250,
|
|
140
|
+
structuralControls: !!(evidence.structural && evidence.structural.unchanged && evidence.structural.rollbackLive),
|
|
141
|
+
policyEvidence: false,
|
|
142
|
+
liveHostEvidence: false,
|
|
143
|
+
};
|
|
144
|
+
// rolloutReady stays false until policy/live-host evidence exists; the
|
|
145
|
+
// gates above are necessary but not sufficient pre-production.
|
|
146
|
+
return { ...gates, rolloutReady: false };
|
|
147
|
+
}
|
|
58
148
|
// sourceEvidence is the provider provenance result; only the pinned cua_s1 revision satisfies the gate.
|
|
59
|
-
function evaluate(corpus, records, sourceEvidence = null) {
|
|
60
|
-
|
|
149
|
+
function evaluate(corpus, records, sourceEvidence = null, { secondaryThreshold } = {}) {
|
|
150
|
+
const labels = corpusLabels(corpus);
|
|
151
|
+
const multi = MULTI.includes(corpus.cases[0].request.task);
|
|
152
|
+
if (multi && !(finite(secondaryThreshold) && secondaryThreshold >= 0.05 && secondaryThreshold <= 0.5)) fail();
|
|
61
153
|
const cases = corpus.cases.filter(c => c.split === 'holdout');
|
|
62
154
|
if (!cases.length || !Array.isArray(records) || records.length !== cases.length) fail();
|
|
63
155
|
const byId = new Map(); const models = new Set(); const cold = []; const warm = [];
|
|
@@ -66,8 +158,8 @@ function evaluate(corpus, records, sourceEvidence = null) {
|
|
|
66
158
|
|| !labels.includes(record.baseline) || !Array.isArray(record.runs) || record.runs.length !== 2) fail();
|
|
67
159
|
for (const run of record.runs) {
|
|
68
160
|
if (!exact(run, ['decision', 'scores', 'latencyMs', 'coldStart']) || !finite(run.latencyMs) || run.latencyMs < 0 || typeof run.coldStart !== 'boolean') fail();
|
|
69
|
-
prediction(run.decision);
|
|
70
|
-
checkScores(run.decision, run.scores);
|
|
161
|
+
prediction(run.decision, labels);
|
|
162
|
+
checkScores(run.decision, run.scores, labels);
|
|
71
163
|
if (run.decision.model) models.add(JSON.stringify([run.decision.model.id, run.decision.model.revision, run.decision.model.domain]));
|
|
72
164
|
(run.coldStart ? cold : warm).push(run.latencyMs);
|
|
73
165
|
}
|
|
@@ -75,8 +167,12 @@ function evaluate(corpus, records, sourceEvidence = null) {
|
|
|
75
167
|
}
|
|
76
168
|
if (models.size > 1) fail();
|
|
77
169
|
const gold = cases.map(c => c.gold);
|
|
78
|
-
const baseline = metrics(gold, cases.map(c => byId.get(c.id).baseline));
|
|
79
|
-
const
|
|
170
|
+
const baseline = metrics(gold, cases.map(c => byId.get(c.id).baseline), labels);
|
|
171
|
+
const predicted = cases.map(c => prediction(byId.get(c.id).runs[0].decision, labels));
|
|
172
|
+
const model = { ...metrics(gold, predicted, labels), ...(multi ? gradeIntents(cases, byId, labels, secondaryThreshold) : {}) };
|
|
173
|
+
const families = multi ? { familyConsistency: {
|
|
174
|
+
gold: familyConsistency(cases.map(c => ({ family: c.family, primary: c.gold }))),
|
|
175
|
+
model: familyConsistency(cases.map((c, i) => ({ family: c.family, primary: predicted[i] }))) } } : {};
|
|
80
176
|
const canonicalRun = ({ decision: d, scores }) => JSON.stringify([d.status, d.reason, d.selectedId ?? null, d.confidence ?? null, d.margin ?? null,
|
|
81
177
|
d.model ? [d.model.id, d.model.revision, d.model.domain] : null, scores]);
|
|
82
178
|
const agreement = records.filter(r => canonicalRun(r.runs[0]) === canonicalRun(r.runs[1])).length / cases.length;
|
|
@@ -94,6 +190,6 @@ function evaluate(corpus, records, sourceEvidence = null) {
|
|
|
94
190
|
policyEvidence: false,
|
|
95
191
|
liveHostEvidence: false,
|
|
96
192
|
};
|
|
97
|
-
return { schemaVersion: 1, cases: cases.length, baseline, model, agreement, latency, gates, rolloutReady: Object.values(gates).every(Boolean) };
|
|
193
|
+
return { schemaVersion: 1, cases: cases.length, baseline, model, ...families, agreement, latency, gates, rolloutReady: Object.values(gates).every(Boolean) };
|
|
98
194
|
}
|
|
99
|
-
module.exports = { validateCorpus, evaluate };
|
|
195
|
+
module.exports = { validateCorpus, evaluate, gradeRelevance, relevanceGates };
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
// Intent-stage comparisons (docs/system-one-intent.md#scoring). A label is { gold, secondary }.
|
|
3
|
+
const full = l => new Set([l.gold, ...(l.secondary || [])]);
|
|
4
|
+
// The owner's grades: same primary 1.0; swapped order 0.6; one-sided, or swapped with a
|
|
5
|
+
// secondary-count gap of 3 or more, 0.3; otherwise 0.
|
|
6
|
+
function gradedAgreement(a, b) {
|
|
7
|
+
if (a.gold === b.gold) return 1;
|
|
8
|
+
if (a.gold === null || b.gold === null) return 0;
|
|
9
|
+
const aInB = full(b).has(a.gold); const bInA = full(a).has(b.gold);
|
|
10
|
+
if (aInB && bInA) return Math.abs((a.secondary || []).length - (b.secondary || []).length) >= 3 ? 0.3 : 0.6;
|
|
11
|
+
return aInB || bInA ? 0.3 : 0;
|
|
12
|
+
}
|
|
13
|
+
// Share of families with two or more members whose members share one primary intent.
|
|
14
|
+
function familyConsistency(rows) {
|
|
15
|
+
const byFamily = new Map();
|
|
16
|
+
for (const r of rows) { if (!byFamily.has(r.family)) byFamily.set(r.family, new Set()); byFamily.get(r.family).add(r.primary); }
|
|
17
|
+
const sizes = new Map(); for (const r of rows) sizes.set(r.family, (sizes.get(r.family) || 0) + 1);
|
|
18
|
+
const multi = [...byFamily].filter(([f]) => sizes.get(f) >= 2);
|
|
19
|
+
const consistent = multi.filter(([, set]) => set.size === 1).length;
|
|
20
|
+
return { families: multi.length, consistent, rate: multi.length ? consistent / multi.length : null };
|
|
21
|
+
}
|
|
22
|
+
module.exports = { gradedAgreement, familyConsistency };
|
|
@@ -0,0 +1,141 @@
|
|
|
1
|
+
"""Fine-tuned Laya intent provider bridge (#233 shadow, research use).
|
|
2
|
+
|
|
3
|
+
Speaks the same adapter protocol as cua_adapter.py (oneshot stdin/stdout,
|
|
4
|
+
resident --serve, --provenance). It emits RAW independent relevance scores
|
|
5
|
+
(one per intent plus the unclassified ramp): they deliberately do NOT sum to
|
|
6
|
+
one, so the single-winner contract.decide fails closed on them while
|
|
7
|
+
contract.decideIntents consumes them directly. Do not re-normalize into a
|
|
8
|
+
simplex anywhere in this path.
|
|
9
|
+
"""
|
|
10
|
+
import hashlib
|
|
11
|
+
import importlib.util
|
|
12
|
+
import json
|
|
13
|
+
import sys
|
|
14
|
+
from pathlib import Path
|
|
15
|
+
|
|
16
|
+
_HERE = Path(__file__).resolve()
|
|
17
|
+
if str(_HERE.parent) not in sys.path:
|
|
18
|
+
sys.path.insert(0, str(_HERE.parent))
|
|
19
|
+
import cua_adapter # noqa: E402 (resident serve machinery: socket server, token, phases)
|
|
20
|
+
|
|
21
|
+
_label_spec = importlib.util.spec_from_file_location(
|
|
22
|
+
'laya_labeler', _HERE.parents[3] / 'scripts' / 'system-one-label-laya.py')
|
|
23
|
+
_label = importlib.util.module_from_spec(_label_spec)
|
|
24
|
+
_label_spec.loader.exec_module(_label)
|
|
25
|
+
INTENTS = _label.INTENTS
|
|
26
|
+
QUESTIONS_VERSION = _label.QUESTIONS_VERSION
|
|
27
|
+
build_questions = _label.build_questions
|
|
28
|
+
|
|
29
|
+
# B1 evidence (full-model validation sweep): cutoff 0.45 maximizes graded
|
|
30
|
+
# agreement with null rate near the teacher's 14%. Pinned, not tuned per run.
|
|
31
|
+
NULL_CUTOFF = 0.45
|
|
32
|
+
# Byte limits: holdout contexts peak at 368 B, validation at 1005 B; the
|
|
33
|
+
# multilingual window is 1024 tokens (~4 KB). Rejection is fail-safe lexical.
|
|
34
|
+
LIMITS = {'context_tokens': 4096, 'option_tokens': 1024}
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def verify_artifacts(manifest):
|
|
38
|
+
"""Pin weights plus the training config beside them (rl_agent_config.json)."""
|
|
39
|
+
weights = Path(manifest['checkpoint'])
|
|
40
|
+
if not weights.is_absolute() or weights.suffix != '.safetensors':
|
|
41
|
+
raise ValueError('checkpoint-format')
|
|
42
|
+
config = weights.parent / 'rl_agent_config.json'
|
|
43
|
+
for file, key in [(weights, 'weightsSha256'), (config, 'configSha256')]:
|
|
44
|
+
with file.open('rb') as handle:
|
|
45
|
+
digest = hashlib.file_digest(handle, 'sha256').hexdigest()
|
|
46
|
+
if digest != manifest[key]:
|
|
47
|
+
raise ValueError('artifact-hash')
|
|
48
|
+
return weights
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def check_limits(request, config):
|
|
52
|
+
if len(request['context'].encode('utf-8')) > config['context_tokens'] or any(
|
|
53
|
+
len(option['text'].encode('utf-8')) > config['option_tokens'] for option in request['options']
|
|
54
|
+
):
|
|
55
|
+
raise ValueError('input-too-long')
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def make_scorer(predict_fn, cutoff=NULL_CUTOFF):
|
|
59
|
+
"""Build score(request) -> 13 raw probabilities in request-option order.
|
|
60
|
+
|
|
61
|
+
Raw means independent: 12 noul answers plus the unclassified ramp, each
|
|
62
|
+
in 0..1, with no sum-to-one normalization.
|
|
63
|
+
"""
|
|
64
|
+
questions = build_questions()
|
|
65
|
+
|
|
66
|
+
def score(request):
|
|
67
|
+
if request.get('task') != 'intent':
|
|
68
|
+
raise ValueError('task: laya intent bridge only')
|
|
69
|
+
ids = [o['id'] for o in request['options']]
|
|
70
|
+
if set(INTENTS) - set(ids):
|
|
71
|
+
raise ValueError('options: all 12 intents required')
|
|
72
|
+
raw = predict_fn(request['context'], questions)
|
|
73
|
+
top = max(float(raw[i]) for i in INTENTS)
|
|
74
|
+
ramp = max(0.0, min(1.0, (cutoff - top) / cutoff)) if cutoff > 0 else 0.0
|
|
75
|
+
by_id = {i: float(raw[i]) for i in INTENTS}
|
|
76
|
+
by_id['unclassified'] = ramp
|
|
77
|
+
return [by_id[i] for i in ids]
|
|
78
|
+
|
|
79
|
+
return score, dict(LIMITS)
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def load_scorer(weights, manifest):
|
|
83
|
+
"""Load the fine-tuned checkpoint once; return (score, limits)."""
|
|
84
|
+
import laya # noqa: PLC0415
|
|
85
|
+
agent = laya.load(str(weights.parent))
|
|
86
|
+
questions = build_questions()
|
|
87
|
+
|
|
88
|
+
def predict_fn(state, _questions):
|
|
89
|
+
answers = agent.predict(state, questions)['answers']
|
|
90
|
+
return {intent: float(answers[intent]['noul']) for intent in INTENTS}
|
|
91
|
+
|
|
92
|
+
return make_scorer(predict_fn)
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
def provenance():
|
|
96
|
+
"""Installed laya facts only; never loads a checkpoint."""
|
|
97
|
+
import importlib.metadata as metadata # noqa: PLC0415
|
|
98
|
+
import platform # noqa: PLC0415
|
|
99
|
+
|
|
100
|
+
def version(name):
|
|
101
|
+
try:
|
|
102
|
+
return metadata.version(name)
|
|
103
|
+
except metadata.PackageNotFoundError:
|
|
104
|
+
return None
|
|
105
|
+
|
|
106
|
+
return {'schemaVersion': 1, 'python': {'implementation': platform.python_implementation(),
|
|
107
|
+
'version': platform.python_version()},
|
|
108
|
+
'laya': {'distribution': 'laya', 'version': version('laya'),
|
|
109
|
+
'checkpoint': 'convaiinnovations/laya:multilingual',
|
|
110
|
+
'questions': QUESTIONS_VERSION, 'nullCutoff': NULL_CUTOFF},
|
|
111
|
+
'torch': version('torch')}
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
def infer(request, manifest):
|
|
115
|
+
weights = verify_artifacts(manifest)
|
|
116
|
+
score, config = load_scorer(weights, manifest)
|
|
117
|
+
check_limits(request, config)
|
|
118
|
+
return cua_adapter.build_response(request, manifest, score(request))
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
def main():
|
|
122
|
+
try:
|
|
123
|
+
if sys.argv[1:] == ['--provenance']:
|
|
124
|
+
print(json.dumps(provenance(), allow_nan=False))
|
|
125
|
+
return 0
|
|
126
|
+
if sys.argv[1:2] == ['--serve']:
|
|
127
|
+
return cua_adapter.serve_main(sys.argv[2:], scorer_factory=load_scorer)
|
|
128
|
+
manifest = json.loads(sys.argv[1])
|
|
129
|
+
raw = sys.stdin.buffer.read(2 * 1024 * 1024 + 1)
|
|
130
|
+
if len(raw) > 2 * 1024 * 1024:
|
|
131
|
+
raise ValueError('request-limit')
|
|
132
|
+
request = json.loads(raw)
|
|
133
|
+
print(json.dumps(infer(request, manifest), allow_nan=False))
|
|
134
|
+
except Exception:
|
|
135
|
+
print('laya-provider-unavailable', file=sys.stderr)
|
|
136
|
+
return 1
|
|
137
|
+
return 0
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
if __name__ == '__main__':
|
|
141
|
+
sys.exit(main())
|
|
@@ -10,9 +10,15 @@ const unavailable = reason => ({ status: 'unavailable', reason });
|
|
|
10
10
|
const exact = (value, keys) => value && typeof value === 'object' && !Array.isArray(value)
|
|
11
11
|
&& Object.keys(value).length === keys.length && keys.every(k => Object.hasOwn(value, k));
|
|
12
12
|
const label = value => typeof value === 'string' && value.length > 0 && value.length <= 128 && !/[\u0000-\u001f]/.test(value);
|
|
13
|
+
// Resident adapter selection: a basename inside the system-one scripts dir, never a path.
|
|
14
|
+
const adapterName = value => typeof value === 'string' && /^[a-z][a-z0-9_-]{0,63}\.py$/.test(value);
|
|
13
15
|
// Thresholds calibrated on validation for this checkpoint (docs/system-one-training.md).
|
|
14
16
|
const inRange = (v, lo, hi) => typeof v === 'number' && Number.isFinite(v) && v >= lo && v <= hi;
|
|
15
|
-
|
|
17
|
+
// secondaryThreshold is optional and belongs to intent checkpoints (docs/system-one-intent.md#scoring).
|
|
18
|
+
const validAcceptance = a => (exact(a, ['minConfidence', 'minMargin']) || (exact(a, ['minConfidence', 'minMargin', 'secondaryThreshold']) && inRange(a.secondaryThreshold, 0.05, 0.5)))
|
|
19
|
+
&& inRange(a.minConfidence, 0.5, 0.99) && inRange(a.minMargin, 0, 0.9);
|
|
20
|
+
// The part of the acceptance thresholds that contract.decide takes.
|
|
21
|
+
const decisionPolicy = acceptance => (acceptance ? { minConfidence: acceptance.minConfidence, minMargin: acceptance.minMargin } : {});
|
|
16
22
|
// A scored result carries the manifest's thresholds so every caller decides with them.
|
|
17
23
|
const withAcceptance = (result, manifest) => (result && result.status === 'scored' && manifest.acceptance
|
|
18
24
|
? { ...result, acceptance: { ...manifest.acceptance } } : result);
|
|
@@ -22,8 +28,9 @@ function validateManifest(m) {
|
|
|
22
28
|
const required = ['schemaVersion', ...(ngram ? [] : ['python']), 'checkpoint', 'weightsSha256', 'configSha256', 'modelId', 'revision', 'domain'];
|
|
23
29
|
const strings = ['modelId', 'revision', 'domain', ...(ngram && !Object.hasOwn(m, 'python') ? [] : ['python'])];
|
|
24
30
|
if (!m || typeof m !== 'object' || Array.isArray(m) || required.some(k => !Object.hasOwn(m, k))
|
|
25
|
-
|| Object.keys(m).some(k => ![...required, 'python', 'timeoutMs', 'transport', 'idleTimeoutMs', 'acceptance'].includes(k)) || m.schemaVersion !== 1
|
|
31
|
+
|| Object.keys(m).some(k => ![...required, 'python', 'timeoutMs', 'transport', 'idleTimeoutMs', 'acceptance', 'adapter'].includes(k)) || m.schemaVersion !== 1
|
|
26
32
|
|| (m.acceptance !== undefined && !validAcceptance(m.acceptance))
|
|
33
|
+
|| (m.adapter !== undefined && !adapterName(m.adapter))
|
|
27
34
|
|| strings.some(k => typeof m[k] !== 'string' || !m[k].trim() || m[k].length > 4096 || m[k].includes('\0'))
|
|
28
35
|
|| typeof m.checkpoint !== 'string' || !path.isAbsolute(m.checkpoint) || path.extname(m.checkpoint) !== (ngram ? '.bin' : '.safetensors')
|
|
29
36
|
|| ['weightsSha256', 'configSha256'].some(k => typeof m[k] !== 'string' || !/^[a-f0-9]{64}$/.test(m[k]))
|
|
@@ -105,4 +112,4 @@ function provenance(manifestPath) {
|
|
|
105
112
|
try { return { status: 'recorded', provenance: validateProvenance(result.value) }; }
|
|
106
113
|
catch (_) { return unavailable('provider-json'); }
|
|
107
114
|
}
|
|
108
|
-
module.exports = { score, scoreAsync, runProvider, validateManifest, readManifest, provenance, validateProvenance, PINNED_CUA_S1_REVISION };
|
|
115
|
+
module.exports = { score, scoreAsync, runProvider, validateManifest, readManifest, provenance, validateProvenance, decisionPolicy, PINNED_CUA_S1_REVISION };
|
|
@@ -10,7 +10,12 @@ const { spawn } = require('node:child_process');
|
|
|
10
10
|
const { Worker } = require('node:worker_threads');
|
|
11
11
|
const { validateRequest } = require('./contract');
|
|
12
12
|
|
|
13
|
-
const
|
|
13
|
+
const ADAPTER_PATTERN = /^[a-z][a-z0-9_-]{0,63}\.py$/;
|
|
14
|
+
const adapterPath = manifest => {
|
|
15
|
+
const name = manifest.adapter || 'cua_adapter.py';
|
|
16
|
+
if (!ADAPTER_PATTERN.test(name)) throw new TypeError('invalid-adapter');
|
|
17
|
+
return path.join(__dirname, name);
|
|
18
|
+
};
|
|
14
19
|
const LOCK_TTL_MS = 60000;
|
|
15
20
|
const CALL_TIMEOUT_MS = 1000;
|
|
16
21
|
const MAX_REPLY_BYTES = 1024 * 1024;
|
|
@@ -18,7 +23,7 @@ const unavailable = reason => ({ status: 'unavailable', reason });
|
|
|
18
23
|
const sleep = ms => Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms);
|
|
19
24
|
const exact = (value, keys) => value && typeof value === 'object' && !Array.isArray(value)
|
|
20
25
|
&& Object.keys(value).length === keys.length && keys.every(k => Object.hasOwn(value, k));
|
|
21
|
-
const defaultCommand = manifest => [manifest.python, [
|
|
26
|
+
const defaultCommand = manifest => [manifest.python, [adapterPath(manifest), '--serve']];
|
|
22
27
|
|
|
23
28
|
function stateFiles(manifest, manifestPath) {
|
|
24
29
|
const digest = createHash('sha256').update(JSON.stringify(manifest)).digest('hex').slice(0, 16);
|
|
@@ -197,4 +202,4 @@ if (require.main === module) {
|
|
|
197
202
|
if (op === 'stop') stop(manifest, manifestPath);
|
|
198
203
|
process.stdout.write(`${JSON.stringify(status(manifest, manifestPath))}\n`);
|
|
199
204
|
}
|
|
200
|
-
module.exports = { stateFiles, readState, callSync, callAsync, score, scoreAsync, ensureReady, status, stop, LOCK_TTL_MS, CALL_TIMEOUT_MS };
|
|
205
|
+
module.exports = { stateFiles, readState, callSync, callAsync, score, scoreAsync, ensureReady, status, stop, defaultCommand, LOCK_TTL_MS, CALL_TIMEOUT_MS };
|
|
@@ -26,7 +26,7 @@ function selectTier(input, env = process.env, scorer = provider.score) {
|
|
|
26
26
|
const allowed = ['provider-timeout', 'provider-output-limit', 'provider-exit', 'provider-json', 'provider-config', 'provider-unavailable', 'provider-starting'];
|
|
27
27
|
return fallback(allowed.includes(result?.reason) ? result.reason : 'provider-unavailable');
|
|
28
28
|
}
|
|
29
|
-
const decision = decide(request, result.response,
|
|
29
|
+
const decision = decide(request, result.response, provider.decisionPolicy(result.acceptance));
|
|
30
30
|
if (decision.selectedId === 'unclassified') Object.assign(decision, { selectedId: null, status: 'abstain', reason: 'unclassified' });
|
|
31
31
|
const accepted = decision.status === 'accepted';
|
|
32
32
|
const belowFloor = accepted && input.floor != null && RANK[decision.selectedId] < RANK[input.floor];
|
package/hooks/hooks.json
CHANGED
|
@@ -23,6 +23,16 @@
|
|
|
23
23
|
],
|
|
24
24
|
"description": "Establish Harness routing invariants and an active workflow execution contract before host/domain skill selection",
|
|
25
25
|
"id": "harness:pre:kernel-router"
|
|
26
|
+
},
|
|
27
|
+
{
|
|
28
|
+
"hooks": [
|
|
29
|
+
{
|
|
30
|
+
"type": "command",
|
|
31
|
+
"command": "node \"${CLAUDE_PLUGIN_ROOT}/hooks/scripts/observation-hook.js\""
|
|
32
|
+
}
|
|
33
|
+
],
|
|
34
|
+
"description": "Record a local, content-private System One turn observation (docs/system-one-observations.md); fail-open and silent",
|
|
35
|
+
"id": "harness:obs:observation-userpromptsubmit"
|
|
26
36
|
}
|
|
27
37
|
],
|
|
28
38
|
"PreToolUse": [
|
|
@@ -155,6 +165,16 @@
|
|
|
155
165
|
],
|
|
156
166
|
"description": "Close Harness skill active-window telemetry at the turn boundary",
|
|
157
167
|
"id": "harness:stop:skill-telemetry"
|
|
168
|
+
},
|
|
169
|
+
{
|
|
170
|
+
"hooks": [
|
|
171
|
+
{
|
|
172
|
+
"type": "command",
|
|
173
|
+
"command": "node \"${CLAUDE_PLUGIN_ROOT}/hooks/scripts/observation-hook.js\""
|
|
174
|
+
}
|
|
175
|
+
],
|
|
176
|
+
"description": "Record a local, content-private System One turn observation (docs/system-one-observations.md); fail-open and silent",
|
|
177
|
+
"id": "harness:obs:observation-stop"
|
|
158
178
|
}
|
|
159
179
|
],
|
|
160
180
|
"PostToolUse": [
|
|
@@ -234,6 +254,16 @@
|
|
|
234
254
|
],
|
|
235
255
|
"description": "Record privacy-safe Harness skill load completion telemetry",
|
|
236
256
|
"id": "harness:post:skill-telemetry"
|
|
257
|
+
},
|
|
258
|
+
{
|
|
259
|
+
"hooks": [
|
|
260
|
+
{
|
|
261
|
+
"type": "command",
|
|
262
|
+
"command": "node \"${CLAUDE_PLUGIN_ROOT}/hooks/scripts/observation-hook.js\""
|
|
263
|
+
}
|
|
264
|
+
],
|
|
265
|
+
"description": "Record a local, content-private System One turn observation (docs/system-one-observations.md); fail-open and silent",
|
|
266
|
+
"id": "harness:obs:observation-posttooluse"
|
|
237
267
|
}
|
|
238
268
|
],
|
|
239
269
|
"PostToolUseFailure": [
|
|
@@ -280,6 +310,16 @@
|
|
|
280
310
|
],
|
|
281
311
|
"description": "Record failed Harness skill loading without prompt/tool arguments",
|
|
282
312
|
"id": "harness:post-failure:skill-telemetry"
|
|
313
|
+
},
|
|
314
|
+
{
|
|
315
|
+
"hooks": [
|
|
316
|
+
{
|
|
317
|
+
"type": "command",
|
|
318
|
+
"command": "node \"${CLAUDE_PLUGIN_ROOT}/hooks/scripts/observation-hook.js\""
|
|
319
|
+
}
|
|
320
|
+
],
|
|
321
|
+
"description": "Record a local, content-private System One turn observation (docs/system-one-observations.md); fail-open and silent",
|
|
322
|
+
"id": "harness:obs:observation-posttoolusefailure"
|
|
283
323
|
}
|
|
284
324
|
]
|
|
285
325
|
}
|