harness-everything 0.28.0 → 0.30.0

package/harness-everything/scripts/system-one/evaluate.js

@@ -1,26 +1,32 @@
 'use strict';
 const { validateRequest } = require('./contract');
-const { TIER_OPTIONS } = require('./router');
+const { CATALOGS, MULTI, goldLabels, validSecondary } = require('./catalogs');
+const { gradedAgreement, familyConsistency } = require('./intent');
 const exact = (obj, keys) => obj && typeof obj === 'object' && !Array.isArray(obj) && Object.keys(obj).length === keys.length && keys.every(k => Object.hasOwn(obj, k));
 const nonempty = v => typeof v === 'string' && v.trim().length > 0;
-const labels = ['tier1', 'tier2', 'tier3', null];
 const finite = v => typeof v === 'number' && Number.isFinite(v);
 const fail = () => { throw new TypeError('invalid-evaluation-input'); };
-function validateCorpus(corpus) {
+// A corpus holds one stage: every request uses that stage's fixed catalog. Returns the stage's gold labels.
+function corpusLabels(corpus) {
   if (!exact(corpus, ['schemaVersion', 'cases']) || corpus.schemaVersion !== 1 || !Array.isArray(corpus.cases) || !corpus.cases.length) fail();
+  const task = corpus.cases[0]?.request?.task;
+  if (!Object.hasOwn(CATALOGS, task)) fail();
+  const labels = goldLabels(task); const catalog = JSON.stringify(CATALOGS[task]);
   const ids = new Set(); const hashes = new Set(); const families = new Map();
   for (const c of corpus.cases) {
-    if (!exact(c, ['id', 'family', 'split', 'language', 'source', 'reviewed', 'request', 'gold'])
+    if (!exact(c, ['id', 'family', 'split', 'language', 'source', 'reviewed', 'request', 'gold', ...(MULTI.includes(task) ? ['secondary'] : [])])
       || ![c.id, c.family, c.source].every(nonempty) || !['train', 'validation', 'holdout'].includes(c.split)
-      || !['en', 'zh-TW'].includes(c.language) || typeof c.reviewed !== 'boolean' || !labels.includes(c.gold)) fail();
+      || !['en', 'zh-TW'].includes(c.language) || typeof c.reviewed !== 'boolean' || !labels.includes(c.gold)
+      || !validSecondary(task, c.gold, c.secondary)) fail();
     validateRequest(c.request);
-    if (c.request.task !== 'tier' || JSON.stringify(c.request.options) !== JSON.stringify(TIER_OPTIONS)
+    if (c.request.task !== task || JSON.stringify(c.request.options) !== catalog
       || ids.has(c.id) || hashes.has(c.request.requestHash) || (families.has(c.family) && families.get(c.family) !== c.split)) fail();
     ids.add(c.id); hashes.add(c.request.requestHash); families.set(c.family, c.split);
   }
-  return true;
+  return labels;
 }
-function prediction(decision) {
+function validateCorpus(corpus) { corpusLabels(corpus); return true; }
+function prediction(decision, labels) {
   if (exact(decision, ['status', 'reason']) && decision.status === 'unavailable' && nonempty(decision.reason)) return null;
   if (!exact(decision, ['status', 'reason', 'selectedId', 'confidence', 'margin', 'model'])
     || !['accepted', 'abstain', 'invalid-output'].includes(decision.status) || !nonempty(decision.reason)
@@ -31,15 +37,15 @@ function prediction(decision) {
   return decision.selectedId;
 }
 // Scored decisions carry the full catalog-order probability vector; unavailable/invalid outputs carry null.
-function checkScores(decision, scores) {
+function checkScores(decision, scores, labels) {
   const scored = ['accepted', 'abstain'].includes(decision.status);
   if (!scored) { if (scores !== null) fail(); return; }
-  if (!Array.isArray(scores) || scores.length !== TIER_OPTIONS.length
+  if (!Array.isArray(scores) || scores.length !== labels.length
     || scores.some(p => !finite(p) || p < 0 || p > 1) || Math.abs(scores.reduce((s, p) => s + p, 0) - 1) > 1e-6) fail();
   const ranked = [...scores].sort((a, b) => b - a);
   if (decision.confidence !== ranked[0] || decision.margin !== ranked[0] - ranked[1]) fail();
 }
-function metrics(gold, predicted) {
+function metrics(gold, predicted, labels) {
   const count = gold.length;
   const accepted = predicted.filter(x => x !== null).length;
   const correct = predicted.filter((x, i) => x === gold[i]).length;
@@ -55,9 +61,95 @@ function metrics(gold, predicted) {
     acceptedErrorRate: accepted ? (accepted - acceptedCorrect) / accepted : null };
 }
 function p95(values) { return values.length ? [...values].sort((a, b) => a - b)[Math.ceil(values.length * 0.95) - 1] : null; }
+// Intent corpora: accepted predictions are graded against { gold, secondary }; predicted secondaries are the
+// other intents whose probability reaches the checkpoint's secondary threshold (docs/system-one-intent.md#scoring).
+function gradeIntents(cases, byId, labels, threshold) {
+  const ids = labels.map(l => (l === null ? 'unclassified' : l));
+  let accepted = 0; let points = 0; let exact = 0;
+  for (const c of cases) {
+    const run = byId.get(c.id).runs[0]; const selected = prediction(run.decision, labels);
+    if (selected === null) continue;
+    const secondary = ids.filter((id, i) => id !== selected && id !== 'unclassified' && run.scores[i] >= threshold);
+    accepted++; points += gradedAgreement({ gold: c.gold, secondary: c.secondary }, { gold: selected, secondary }); if (selected === c.gold) exact++;
+  }
+  return { acceptedPrecision: accepted ? points / accepted : null, exactPrecision: accepted ? exact / accepted : null };
+}
+// Relevance-native grading (#233): every label except unclassified is an
+// independent Bernoulli against its own threshold. No simplex rule applies
+// to the score vector. Gates live in docs/system-one-intent-gates.md;
+// this function reports metrics only.
+function gradeRelevance(cases, byId, labels, thresholds) {
+  const ids = labels.map(l => (l === null ? 'unclassified' : l));
+  const per = new Map(ids.filter(id => id !== 'unclassified').map(id => [id, { tp: 0, fp: 0, fn: 0, support: 0 }]));
+  let abstain = 0; let firesTotal = 0; let truthTotal = 0; let exact = 0;
+  for (const c of cases) {
+    const scores = byId.get(c.id).scores;
+    const truth = new Set(c.gold === null ? [] : [c.gold, ...(c.secondary || [])]);
+    const fired = new Set(ids.filter((id, i) => id !== 'unclassified' && scores[i] >= thresholds[id]));
+    firesTotal += fired.size; truthTotal += truth.size;
+    abstain += fired.size === 0 ? 1 : 0;
+    exact += fired.size === truth.size && [...fired].every(id => truth.has(id)) ? 1 : 0;
+    for (const [id, row] of per) {
+      if (fired.has(id) && truth.has(id)) row.tp++;
+      else if (fired.has(id)) row.fp++;
+      else if (truth.has(id)) row.fn++;
+      if (truth.has(id)) row.support++;
+    }
+  }
+  const table = {};
+  for (const [id, row] of per) {
+    const p = row.tp + row.fp ? row.tp / (row.tp + row.fp) : 0;
+    const r = row.tp + row.fn ? row.tp / (row.tp + row.fn) : 0;
+    table[id] = { ...row, tau: thresholds[id], precision: p, recall: r, f1: p + r ? 2 * p * r / (p + r) : 0 };
+  }
+  const tp = [...per.values()].reduce((s, r) => s + r.tp, 0);
+  const fp = [...per.values()].reduce((s, r) => s + r.fp, 0);
+  const fn = [...per.values()].reduce((s, r) => s + r.fn, 0);
+  const precision = tp + fp ? tp / (tp + fp) : 0;
+  const recall = tp + fn ? tp / (tp + fn) : 0;
+  const tableEntries = [...per.entries()].map(([id, row]) => [id, table[id]]);
+  const scored = tableEntries.filter(([, t]) => t.support > 0 || t.tp + t.fp > 0);
+  const supported = tableEntries.filter(([, t]) => t.support > 0);
+  const n = cases.length;
+  return {
+    micro: { precision, recall, f1: precision + recall ? 2 * precision * recall / (precision + recall) : 0 },
+    macroF1: scored.length ? scored.reduce((s, [, t]) => s + t.f1, 0) / scored.length : 0,
+    perIntent: table,
+    cardinality: { predicted: n ? firesTotal / n : 0, gold: n ? truthTotal / n : 0 },
+    abstainRate: n ? abstain / n : 0,
+    exactSetMatch: n ? exact / n : 0,
+    weakPrecision: supported.length ? Math.min(...supported.map(([, t]) => t.precision)) : 0,
+  };
+}
+// Relevance-native promotion gates. Thresholds are the reviewed numbers in
+// docs/system-one-intent-gates.md; policy/live-host evidence stays reported
+// (false) until independently produced, so rolloutReady is red by
+// construction pre-production.
+function relevanceGates(multi, evidence) {
+  const supported = Object.values(multi.perIntent).filter(t => t.support >= 10);
+  const gates = {
+    microF1: multi.micro.f1 >= 0.55,
+    macroF1: multi.macroF1 >= 0.50,
+    perIntentFloors: supported.every(t => t.f1 >= 0.30),
+    weakPrecision: multi.weakPrecision >= 0.30,
+    cardinality: Math.abs(multi.cardinality.predicted - multi.cardinality.gold) <= 1.0,
+    abstainRate: multi.abstainRate <= 0.20,
+    familyConsistency: evidence.family.model.rate >= evidence.family.gold.rate,
+    repeatability: evidence.agreement === 1,
+    warmLatency: evidence.warmP95Ms <= 250,
+    structuralControls: !!(evidence.structural && evidence.structural.unchanged && evidence.structural.rollbackLive),
+    policyEvidence: false,
+    liveHostEvidence: false,
+  };
+  // rolloutReady stays false until policy/live-host evidence exists; the
+  // gates above are necessary but not sufficient pre-production.
+  return { ...gates, rolloutReady: false };
+}
 // sourceEvidence is the provider provenance result; only the pinned cua_s1 revision satisfies the gate.
-function evaluate(corpus, records, sourceEvidence = null) {
-  validateCorpus(corpus);
+function evaluate(corpus, records, sourceEvidence = null, { secondaryThreshold } = {}) {
+  const labels = corpusLabels(corpus);
+  const multi = MULTI.includes(corpus.cases[0].request.task);
+  if (multi && !(finite(secondaryThreshold) && secondaryThreshold >= 0.05 && secondaryThreshold <= 0.5)) fail();
   const cases = corpus.cases.filter(c => c.split === 'holdout');
   if (!cases.length || !Array.isArray(records) || records.length !== cases.length) fail();
   const byId = new Map(); const models = new Set(); const cold = []; const warm = [];
@@ -66,8 +158,8 @@ function evaluate(corpus, records, sourceEvidence = null) {
       || !labels.includes(record.baseline) || !Array.isArray(record.runs) || record.runs.length !== 2) fail();
     for (const run of record.runs) {
       if (!exact(run, ['decision', 'scores', 'latencyMs', 'coldStart']) || !finite(run.latencyMs) || run.latencyMs < 0 || typeof run.coldStart !== 'boolean') fail();
-      prediction(run.decision);
-      checkScores(run.decision, run.scores);
+      prediction(run.decision, labels);
+      checkScores(run.decision, run.scores, labels);
       if (run.decision.model) models.add(JSON.stringify([run.decision.model.id, run.decision.model.revision, run.decision.model.domain]));
       (run.coldStart ? cold : warm).push(run.latencyMs);
     }
@@ -75,8 +167,12 @@ function evaluate(corpus, records, sourceEvidence = null) {
   }
   if (models.size > 1) fail();
   const gold = cases.map(c => c.gold);
-  const baseline = metrics(gold, cases.map(c => byId.get(c.id).baseline));
-  const model = metrics(gold, cases.map(c => prediction(byId.get(c.id).runs[0].decision)));
+  const baseline = metrics(gold, cases.map(c => byId.get(c.id).baseline), labels);
+  const predicted = cases.map(c => prediction(byId.get(c.id).runs[0].decision, labels));
+  const model = { ...metrics(gold, predicted, labels), ...(multi ? gradeIntents(cases, byId, labels, secondaryThreshold) : {}) };
+  const families = multi ? { familyConsistency: {
+    gold: familyConsistency(cases.map(c => ({ family: c.family, primary: c.gold }))),
+    model: familyConsistency(cases.map((c, i) => ({ family: c.family, primary: predicted[i] }))) } } : {};
   const canonicalRun = ({ decision: d, scores }) => JSON.stringify([d.status, d.reason, d.selectedId ?? null, d.confidence ?? null, d.margin ?? null,
     d.model ? [d.model.id, d.model.revision, d.model.domain] : null, scores]);
   const agreement = records.filter(r => canonicalRun(r.runs[0]) === canonicalRun(r.runs[1])).length / cases.length;
@@ -84,15 +180,16 @@ function evaluate(corpus, records, sourceEvidence = null) {
   const gates = {
     reviewedHoldout: cases.length >= 200 && cases.every(c => c.reviewed)
       && ['en', 'zh-TW'].every(lang => cases.filter(c => c.language === lang).length >= 50),
-    acceptedPrecision: model.acceptedPrecision !== null && model.acceptedPrecision >= 0.98,
+    acceptedPrecision: model.acceptedPrecision !== null && model.acceptedPrecision >= 0.85,
     coverage: model.coverage >= 0.8,
     macroF1: model.macroF1 >= baseline.macroF1,
     repeatability: agreement === 1,
     warmLatency: warm.length >= cases.length && latency.warmP95Ms <= 100,
-    sourceProvenance: sourceEvidence?.status === 'recorded' && sourceEvidence.provenance?.cuaS1?.sourceRevisionStatus === 'pinned',
+    sourceProvenance: sourceEvidence?.status === 'recorded' && (sourceEvidence.provenance?.cuaS1?.sourceRevisionStatus === 'pinned'
+      || (sourceEvidence.transport === 'ngram' && sourceEvidence.artifactVerified === true)),
     policyEvidence: false,
     liveHostEvidence: false,
   };
-  return { schemaVersion: 1, cases: cases.length, baseline, model, agreement, latency, gates, rolloutReady: Object.values(gates).every(Boolean) };
+  return { schemaVersion: 1, cases: cases.length, baseline, model, ...families, agreement, latency, gates, rolloutReady: Object.values(gates).every(Boolean) };
 }
-module.exports = { validateCorpus, evaluate };
+module.exports = { validateCorpus, evaluate, gradeRelevance, relevanceGates };

package/harness-everything/scripts/system-one/intent.js

@@ -0,0 +1,22 @@
+'use strict';
+// Intent-stage comparisons (docs/system-one-intent.md#scoring). A label is { gold, secondary }.
+const full = l => new Set([l.gold, ...(l.secondary || [])]);
+// The owner's grades: same primary 1.0; swapped order 0.6; one-sided, or swapped with a
+// secondary-count gap of 3 or more, 0.3; otherwise 0.
+function gradedAgreement(a, b) {
+  if (a.gold === b.gold) return 1;
+  if (a.gold === null || b.gold === null) return 0;
+  const aInB = full(b).has(a.gold); const bInA = full(a).has(b.gold);
+  if (aInB && bInA) return Math.abs((a.secondary || []).length - (b.secondary || []).length) >= 3 ? 0.3 : 0.6;
+  return aInB || bInA ? 0.3 : 0;
+}
+// Share of families with two or more members whose members share one primary intent.
+function familyConsistency(rows) {
+  const byFamily = new Map();
+  for (const r of rows) { if (!byFamily.has(r.family)) byFamily.set(r.family, new Set()); byFamily.get(r.family).add(r.primary); }
+  const sizes = new Map(); for (const r of rows) sizes.set(r.family, (sizes.get(r.family) || 0) + 1);
+  const multi = [...byFamily].filter(([f]) => sizes.get(f) >= 2);
+  const consistent = multi.filter(([, set]) => set.size === 1).length;
+  return { families: multi.length, consistent, rate: multi.length ? consistent / multi.length : null };
+}
+module.exports = { gradedAgreement, familyConsistency };

package/harness-everything/scripts/system-one/laya_adapter.py

@@ -0,0 +1,141 @@
+"""Fine-tuned Laya intent provider bridge (#233 shadow, research use).
+
+Speaks the same adapter protocol as cua_adapter.py (oneshot stdin/stdout,
+resident --serve, --provenance). It emits RAW independent relevance scores
+(one per intent plus the unclassified ramp): they deliberately do NOT sum to
+one, so the single-winner contract.decide fails closed on them while
+contract.decideIntents consumes them directly. Do not re-normalize into a
+simplex anywhere in this path.
+"""
+import hashlib
+import importlib.util
+import json
+import sys
+from pathlib import Path
+
+_HERE = Path(__file__).resolve()
+if str(_HERE.parent) not in sys.path:
+    sys.path.insert(0, str(_HERE.parent))
+import cua_adapter  # noqa: E402  (resident serve machinery: socket server, token, phases)
+
+_label_spec = importlib.util.spec_from_file_location(
+    'laya_labeler', _HERE.parents[3] / 'scripts' / 'system-one-label-laya.py')
+_label = importlib.util.module_from_spec(_label_spec)
+_label_spec.loader.exec_module(_label)
+INTENTS = _label.INTENTS
+QUESTIONS_VERSION = _label.QUESTIONS_VERSION
+build_questions = _label.build_questions
+
+# B1 evidence (full-model validation sweep): cutoff 0.45 maximizes graded
+# agreement with null rate near the teacher's 14%. Pinned, not tuned per run.
+NULL_CUTOFF = 0.45
+# Byte limits: holdout contexts peak at 368 B, validation at 1005 B; the
+# multilingual window is 1024 tokens (~4 KB). Rejection is fail-safe lexical.
+LIMITS = {'context_tokens': 4096, 'option_tokens': 1024}
+
+
+def verify_artifacts(manifest):
+    """Pin weights plus the training config beside them (rl_agent_config.json)."""
+    weights = Path(manifest['checkpoint'])
+    if not weights.is_absolute() or weights.suffix != '.safetensors':
+        raise ValueError('checkpoint-format')
+    config = weights.parent / 'rl_agent_config.json'
+    for file, key in [(weights, 'weightsSha256'), (config, 'configSha256')]:
+        with file.open('rb') as handle:
+            digest = hashlib.file_digest(handle, 'sha256').hexdigest()
+        if digest != manifest[key]:
+            raise ValueError('artifact-hash')
+    return weights
+
+
+def check_limits(request, config):
+    if len(request['context'].encode('utf-8')) > config['context_tokens'] or any(
+        len(option['text'].encode('utf-8')) > config['option_tokens'] for option in request['options']
+    ):
+        raise ValueError('input-too-long')
+
+
+def make_scorer(predict_fn, cutoff=NULL_CUTOFF):
+    """Build score(request) -> 13 raw probabilities in request-option order.
+
+    Raw means independent: 12 noul answers plus the unclassified ramp, each
+    in 0..1, with no sum-to-one normalization.
+    """
+    questions = build_questions()
+
+    def score(request):
+        if request.get('task') != 'intent':
+            raise ValueError('task: laya intent bridge only')
+        ids = [o['id'] for o in request['options']]
+        if set(INTENTS) - set(ids):
+            raise ValueError('options: all 12 intents required')
+        raw = predict_fn(request['context'], questions)
+        top = max(float(raw[i]) for i in INTENTS)
+        ramp = max(0.0, min(1.0, (cutoff - top) / cutoff)) if cutoff > 0 else 0.0
+        by_id = {i: float(raw[i]) for i in INTENTS}
+        by_id['unclassified'] = ramp
+        return [by_id[i] for i in ids]
+
+    return score, dict(LIMITS)
+
+
+def load_scorer(weights, manifest):
+    """Load the fine-tuned checkpoint once; return (score, limits)."""
+    import laya  # noqa: PLC0415
+    agent = laya.load(str(weights.parent))
+    questions = build_questions()
+
+    def predict_fn(state, _questions):
+        answers = agent.predict(state, questions)['answers']
+        return {intent: float(answers[intent]['noul']) for intent in INTENTS}
+
+    return make_scorer(predict_fn)
+
+
+def provenance():
+    """Installed laya facts only; never loads a checkpoint."""
+    import importlib.metadata as metadata  # noqa: PLC0415
+    import platform  # noqa: PLC0415
+
+    def version(name):
+        try:
+            return metadata.version(name)
+        except metadata.PackageNotFoundError:
+            return None
+
+    return {'schemaVersion': 1, 'python': {'implementation': platform.python_implementation(),
+                                           'version': platform.python_version()},
+            'laya': {'distribution': 'laya', 'version': version('laya'),
+                     'checkpoint': 'convaiinnovations/laya:multilingual',
+                     'questions': QUESTIONS_VERSION, 'nullCutoff': NULL_CUTOFF},
+            'torch': version('torch')}
+
+
+def infer(request, manifest):
+    weights = verify_artifacts(manifest)
+    score, config = load_scorer(weights, manifest)
+    check_limits(request, config)
+    return cua_adapter.build_response(request, manifest, score(request))
+
+
+def main():
+    try:
+        if sys.argv[1:] == ['--provenance']:
+            print(json.dumps(provenance(), allow_nan=False))
+            return 0
+        if sys.argv[1:2] == ['--serve']:
+            return cua_adapter.serve_main(sys.argv[2:], scorer_factory=load_scorer)
+        manifest = json.loads(sys.argv[1])
+        raw = sys.stdin.buffer.read(2 * 1024 * 1024 + 1)
+        if len(raw) > 2 * 1024 * 1024:
+            raise ValueError('request-limit')
+        request = json.loads(raw)
+        print(json.dumps(infer(request, manifest), allow_nan=False))
+    except Exception:
+        print('laya-provider-unavailable', file=sys.stderr)
+        return 1
+    return 0
+
+
+if __name__ == '__main__':
+    sys.exit(main())

package/harness-everything/scripts/system-one/ngram.js

@@ -0,0 +1,84 @@
+'use strict';
+// In-process System One scorer: hashed character n-grams + a linear layer (docs/system-one-routing.md, "N-gram provider").
+// Featurization must match scripts/system-one-train-ngram.py exactly; ci/mechanism-system-one-ngram.test.js proves it.
+const fs = require('node:fs');
+const { createHash } = require('node:crypto');
+
+const SIDE_KEYS = ['format', 'formatVersion', 'dim', 'nmax', 'hash', 'catalog', 'bias', 'metadata'];
+const cache = new Map();
+const configError = () => Object.assign(new Error('provider-config'), { code: 'provider-config' });
+const sha = buf => createHash('sha256').update(buf).digest('hex');
+
+function fnv1a32(text) {
+  let h = 0x811c9dc5;
+  for (const byte of Buffer.from(text, 'utf8')) { h ^= byte; h = Math.imul(h, 0x01000193) >>> 0; }
+  return h >>> 0;
+}
+// Map of bucket -> weight: log(1 + count) per code-point n-gram, one length bucket, L2-normalized.
+function featurize(text, nmax, dim) {
+  const t = text.toLowerCase().split(/\s+/).filter(Boolean).join(' ');
+  const cps = Array.from(t);
+  const counts = new Map();
+  for (let n = 1; n <= nmax; n++) {
+    for (let i = 0; i + n <= cps.length; i++) {
+      const gram = cps.slice(i, i + n).join('');
+      if (!gram.trim()) continue;
+      const b = fnv1a32(`${n}:${gram}`) % dim;
+      counts.set(b, (counts.get(b) || 0) + 1);
+    }
+  }
+  const k = Math.min(8, Math.floor(Math.log2(Buffer.byteLength(t, 'utf8') + 1)));
+  counts.set(fnv1a32(`len:${k}`) % dim, 1);
+  const feats = new Map();
+  let norm = 0;
+  for (const [b, c] of counts) { const v = Math.log1p(c); feats.set(b, v); norm += v * v; }
+  norm = Math.sqrt(norm);
+  for (const [b, v] of feats) feats.set(b, v / norm);
+  return feats;
+}
+function validSidecar(s) {
+  const isPow2 = n => Number.isInteger(n) && n >= 1024 && n <= 1 << 20 && (n & (n - 1)) === 0;
+  return s && typeof s === 'object' && !Array.isArray(s) && Object.keys(s).length === SIDE_KEYS.length && SIDE_KEYS.every(k => Object.hasOwn(s, k))
+    && s.format === 'harness-ngram' && s.formatVersion === 1 && isPow2(s.dim) && Number.isInteger(s.nmax) && s.nmax >= 1 && s.nmax <= 4
+    && s.hash === 'fnv1a32' && Array.isArray(s.catalog) && s.catalog.length >= 2 && new Set(s.catalog).size === s.catalog.length
+    && s.catalog.every(id => typeof id === 'string' && /^[a-z0-9._-]{1,80}$/.test(id))
+    && Array.isArray(s.bias) && s.bias.length === s.catalog.length && s.bias.every(Number.isFinite)
+    && s.metadata && typeof s.metadata === 'object' && !Array.isArray(s.metadata);
+}
+// Loads and verifies the artifact once per process; any mismatch throws provider-config.
+function loadModel(manifest) {
+  const key = `${manifest.checkpoint}|${manifest.weightsSha256}|${manifest.configSha256}`;
+  if (cache.has(key)) return cache.get(key);
+  const sidecarPath = manifest.checkpoint.replace(/\.bin$/, '.json');
+  let side; let bin;
+  try {
+    const raw = fs.readFileSync(sidecarPath);
+    if (raw.length > 65536 || sha(raw) !== manifest.configSha256) throw configError();
+    side = JSON.parse(raw.toString('utf8'));
+    bin = fs.readFileSync(manifest.checkpoint);
+  } catch (_) { throw configError(); }
+  if (sha(bin) !== manifest.weightsSha256 || !validSidecar(side) || bin.length !== side.dim * side.catalog.length * 4) throw configError();
+  const W = new Float32Array(side.dim * side.catalog.length);
+  for (let i = 0; i < W.length; i++) W[i] = bin.readFloatLE(i * 4);
+  const model = { dim: side.dim, nmax: side.nmax, catalog: side.catalog, bias: side.bias, W };
+  cache.set(key, model);
+  return model;
+}
+function score(request, manifest) {
+  const model = loadModel(manifest);
+  const C = model.catalog.length;
+  if (request.options.length !== C || request.options.some((o, i) => o.id !== model.catalog[i])) throw configError();
+  const logits = model.bias.slice();
+  for (const [b, v] of featurize(request.context, model.nmax, model.dim)) {
+    for (let c = 0; c < C; c++) logits[c] += v * model.W[b * C + c];
+  }
+  const max = Math.max(...logits);
+  const exps = logits.map(z => Math.exp(z - max));
+  const sum = exps.reduce((s, x) => s + x, 0);
+  return {
+    schemaVersion: 1, requestHash: request.requestHash, catalogHash: request.catalogHash,
+    model: { id: manifest.modelId, revision: manifest.revision, domain: manifest.domain },
+    scores: request.options.map((o, i) => ({ id: o.id, probability: exps[i] / sum })),
+  };
+}
+module.exports = { fnv1a32, featurize, loadModel, score };

package/harness-everything/scripts/system-one/provider.js

@@ -10,22 +10,32 @@ const unavailable = reason => ({ status: 'unavailable', reason });
 const exact = (value, keys) => value && typeof value === 'object' && !Array.isArray(value)
   && Object.keys(value).length === keys.length && keys.every(k => Object.hasOwn(value, k));
 const label = value => typeof value === 'string' && value.length > 0 && value.length <= 128 && !/[\u0000-\u001f]/.test(value);
+// Resident adapter selection: a basename inside the system-one scripts dir, never a path.
+const adapterName = value => typeof value === 'string' && /^[a-z][a-z0-9_-]{0,63}\.py$/.test(value);
 // Thresholds calibrated on validation for this checkpoint (docs/system-one-training.md).
 const inRange = (v, lo, hi) => typeof v === 'number' && Number.isFinite(v) && v >= lo && v <= hi;
-const validAcceptance = a => exact(a, ['minConfidence', 'minMargin']) && inRange(a.minConfidence, 0.5, 0.99) && inRange(a.minMargin, 0, 0.9);
+// secondaryThreshold is optional and belongs to intent checkpoints (docs/system-one-intent.md#scoring).
+const validAcceptance = a => (exact(a, ['minConfidence', 'minMargin']) || (exact(a, ['minConfidence', 'minMargin', 'secondaryThreshold']) && inRange(a.secondaryThreshold, 0.05, 0.5)))
+  && inRange(a.minConfidence, 0.5, 0.99) && inRange(a.minMargin, 0, 0.9);
+// The part of the acceptance thresholds that contract.decide takes.
+const decisionPolicy = acceptance => (acceptance ? { minConfidence: acceptance.minConfidence, minMargin: acceptance.minMargin } : {});
 // A scored result carries the manifest's thresholds so every caller decides with them.
 const withAcceptance = (result, manifest) => (result && result.status === 'scored' && manifest.acceptance
   ? { ...result, acceptance: { ...manifest.acceptance } } : result);
 function validateManifest(m) {
-  const required = ['schemaVersion', 'python', 'checkpoint', 'weightsSha256', 'configSha256', 'modelId', 'revision', 'domain'];
+  // The in-process ngram transport needs no Python and loads a .bin; the others run the CUA-S1 adapter.
+  const ngram = !!m && m.transport === 'ngram';
+  const required = ['schemaVersion', ...(ngram ? [] : ['python']), 'checkpoint', 'weightsSha256', 'configSha256', 'modelId', 'revision', 'domain'];
+  const strings = ['modelId', 'revision', 'domain', ...(ngram && !Object.hasOwn(m, 'python') ? [] : ['python'])];
   if (!m || typeof m !== 'object' || Array.isArray(m) || required.some(k => !Object.hasOwn(m, k))
-    || Object.keys(m).some(k => ![...required, 'timeoutMs', 'transport', 'idleTimeoutMs', 'acceptance'].includes(k)) || m.schemaVersion !== 1
+    || Object.keys(m).some(k => ![...required, 'python', 'timeoutMs', 'transport', 'idleTimeoutMs', 'acceptance', 'adapter'].includes(k)) || m.schemaVersion !== 1
     || (m.acceptance !== undefined && !validAcceptance(m.acceptance))
-    || ['python', 'modelId', 'revision', 'domain'].some(k => typeof m[k] !== 'string' || !m[k].trim() || m[k].length > 4096 || m[k].includes('\0'))
-    || typeof m.checkpoint !== 'string' || !path.isAbsolute(m.checkpoint) || path.extname(m.checkpoint) !== '.safetensors'
+    || (m.adapter !== undefined && !adapterName(m.adapter))
+    || strings.some(k => typeof m[k] !== 'string' || !m[k].trim() || m[k].length > 4096 || m[k].includes('\0'))
+    || typeof m.checkpoint !== 'string' || !path.isAbsolute(m.checkpoint) || path.extname(m.checkpoint) !== (ngram ? '.bin' : '.safetensors')
     || ['weightsSha256', 'configSha256'].some(k => typeof m[k] !== 'string' || !/^[a-f0-9]{64}$/.test(m[k]))
     || (m.timeoutMs !== undefined && (!Number.isInteger(m.timeoutMs) || m.timeoutMs < 1 || m.timeoutMs > 10000))
-    || (m.transport !== undefined && !['oneshot', 'resident'].includes(m.transport))
+    || (m.transport !== undefined && !['oneshot', 'resident', 'ngram'].includes(m.transport))
     || (m.idleTimeoutMs !== undefined && (!Number.isInteger(m.idleTimeoutMs) || m.idleTimeoutMs < 60000 || m.idleTimeoutMs > 14400000))
     || (m.modelId === 'cua-ai/cua-s1-forms' && m.domain !== 'forms-v1')) throw new TypeError('invalid-manifest');
   return true;
@@ -57,15 +67,22 @@ function score(request, manifestPath) {
   validateRequest(request);
   let manifest;
   try { manifest = readManifest(manifestPath); } catch (_) { return unavailable('provider-config'); }
+  if (manifest.transport === 'ngram') return withAcceptance(inProcess(request, manifest), manifest);
   if (manifest.transport === 'resident') return withAcceptance(require('./resident').score({ ...request }, manifest, manifestPath), manifest);
   return withAcceptance(oneShot(request, manifest), manifest);
 }
+// The ngram transport scores in this process; an artifact mismatch is a configuration error, never a score.
+function inProcess(request, manifest) {
+  try { return { status: 'scored', response: require('./ngram').score({ ...request }, manifest) }; }
+  catch (_) { return unavailable('provider-config'); }
+}
 // Async callers (the hook entry) reach a resident server without the sync worker bridge.
 // One-shot has no server to overlap with, so it runs the same child synchronously.
 async function scoreAsync(request, manifestPath) {
   validateRequest(request);
   let manifest;
   try { manifest = readManifest(manifestPath); } catch (_) { return unavailable('provider-config'); }
+  if (manifest.transport === 'ngram') return withAcceptance(inProcess(request, manifest), manifest);
   if (manifest.transport === 'resident') return withAcceptance(await require('./resident').scoreAsync({ ...request }, manifest, manifestPath), manifest);
   return withAcceptance(oneShot(request, manifest), manifest);
 }
@@ -95,4 +112,4 @@ function provenance(manifestPath) {
   try { return { status: 'recorded', provenance: validateProvenance(result.value) }; }
   catch (_) { return unavailable('provider-json'); }
 }
-module.exports = { score, scoreAsync, runProvider, validateManifest, readManifest, provenance, validateProvenance, PINNED_CUA_S1_REVISION };
+module.exports = { score, scoreAsync, runProvider, validateManifest, readManifest, provenance, validateProvenance, decisionPolicy, PINNED_CUA_S1_REVISION };

package/harness-everything/scripts/system-one/resident.js

@@ -10,7 +10,12 @@ const { spawn } = require('node:child_process');
 const { Worker } = require('node:worker_threads');
 const { validateRequest } = require('./contract');
 
-const ADAPTER = path.join(__dirname, 'cua_adapter.py');
+const ADAPTER_PATTERN = /^[a-z][a-z0-9_-]{0,63}\.py$/;
+const adapterPath = manifest => {
+  const name = manifest.adapter || 'cua_adapter.py';
+  if (!ADAPTER_PATTERN.test(name)) throw new TypeError('invalid-adapter');
+  return path.join(__dirname, name);
+};
 const LOCK_TTL_MS = 60000;
 const CALL_TIMEOUT_MS = 1000;
 const MAX_REPLY_BYTES = 1024 * 1024;
@@ -18,7 +23,7 @@ const unavailable = reason => ({ status: 'unavailable', reason });
 const sleep = ms => Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms);
 const exact = (value, keys) => value && typeof value === 'object' && !Array.isArray(value)
   && Object.keys(value).length === keys.length && keys.every(k => Object.hasOwn(value, k));
-const defaultCommand = manifest => [manifest.python, [ADAPTER, '--serve']];
+const defaultCommand = manifest => [manifest.python, [adapterPath(manifest), '--serve']];
 
 function stateFiles(manifest, manifestPath) {
   const digest = createHash('sha256').update(JSON.stringify(manifest)).digest('hex').slice(0, 16);
@@ -197,4 +202,4 @@ if (require.main === module) {
   if (op === 'stop') stop(manifest, manifestPath);
   process.stdout.write(`${JSON.stringify(status(manifest, manifestPath))}\n`);
 }
-module.exports = { stateFiles, readState, callSync, callAsync, score, scoreAsync, ensureReady, status, stop, LOCK_TTL_MS, CALL_TIMEOUT_MS };
+module.exports = { stateFiles, readState, callSync, callAsync, score, scoreAsync, ensureReady, status, stop, defaultCommand, LOCK_TTL_MS, CALL_TIMEOUT_MS };

package/harness-everything/scripts/system-one/router.js

@@ -26,7 +26,7 @@ function selectTier(input, env = process.env, scorer = provider.score) {
     const allowed = ['provider-timeout', 'provider-output-limit', 'provider-exit', 'provider-json', 'provider-config', 'provider-unavailable', 'provider-starting'];
     return fallback(allowed.includes(result?.reason) ? result.reason : 'provider-unavailable');
   }
-  const decision = decide(request, result.response, result.acceptance ? { ...result.acceptance } : {});
+  const decision = decide(request, result.response, provider.decisionPolicy(result.acceptance));
   if (decision.selectedId === 'unclassified') Object.assign(decision, { selectedId: null, status: 'abstain', reason: 'unclassified' });
   const accepted = decision.status === 'accepted';
   const belowFloor = accepted && input.floor != null && RANK[decision.selectedId] < RANK[input.floor];

package/hooks/hooks.json

@@ -23,6 +23,16 @@
         ],
         "description": "Establish Harness routing invariants and an active workflow execution contract before host/domain skill selection",
         "id": "harness:pre:kernel-router"
+      },
+      {
+        "hooks": [
+          {
+            "type": "command",
+            "command": "node \"${CLAUDE_PLUGIN_ROOT}/hooks/scripts/observation-hook.js\""
+          }
+        ],
+        "description": "Record a local, content-private System One turn observation (docs/system-one-observations.md); fail-open and silent",
+        "id": "harness:obs:observation-userpromptsubmit"
       }
     ],
     "PreToolUse": [
@@ -155,6 +165,16 @@
         ],
         "description": "Close Harness skill active-window telemetry at the turn boundary",
         "id": "harness:stop:skill-telemetry"
+      },
+      {
+        "hooks": [
+          {
+            "type": "command",
+            "command": "node \"${CLAUDE_PLUGIN_ROOT}/hooks/scripts/observation-hook.js\""
+          }
+        ],
+        "description": "Record a local, content-private System One turn observation (docs/system-one-observations.md); fail-open and silent",
+        "id": "harness:obs:observation-stop"
       }
     ],
     "PostToolUse": [
@@ -234,6 +254,16 @@
         ],
         "description": "Record privacy-safe Harness skill load completion telemetry",
         "id": "harness:post:skill-telemetry"
+      },
+      {
+        "hooks": [
+          {
+            "type": "command",
+            "command": "node \"${CLAUDE_PLUGIN_ROOT}/hooks/scripts/observation-hook.js\""
+          }
+        ],
+        "description": "Record a local, content-private System One turn observation (docs/system-one-observations.md); fail-open and silent",
+        "id": "harness:obs:observation-posttooluse"
       }
     ],
     "PostToolUseFailure": [
@@ -280,6 +310,16 @@
         ],
         "description": "Record failed Harness skill loading without prompt/tool arguments",
         "id": "harness:post-failure:skill-telemetry"
+      },
+      {
+        "hooks": [
+          {
+            "type": "command",
+            "command": "node \"${CLAUDE_PLUGIN_ROOT}/hooks/scripts/observation-hook.js\""
+          }
+        ],
+        "description": "Record a local, content-private System One turn observation (docs/system-one-observations.md); fail-open and silent",
+        "id": "harness:obs:observation-posttoolusefailure"
       }
     ]
   }