@intutic/cli 2.9.1 → 3.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli.js +103 -1
- package/dist/cli.js.map +1 -1
- package/dist/commands/connect.d.ts.map +1 -1
- package/dist/commands/connect.js +15 -2
- package/dist/commands/connect.js.map +1 -1
- package/dist/commands/decide.d.ts +220 -0
- package/dist/commands/decide.d.ts.map +1 -0
- package/dist/commands/decide.js +1297 -0
- package/dist/commands/decide.js.map +1 -0
- package/dist/commands/doctor.d.ts +10 -0
- package/dist/commands/doctor.d.ts.map +1 -1
- package/dist/commands/doctor.js +56 -2
- package/dist/commands/doctor.js.map +1 -1
- package/dist/commands/gates.d.ts +5 -0
- package/dist/commands/gates.d.ts.map +1 -1
- package/dist/commands/gates.js +42 -1
- package/dist/commands/gates.js.map +1 -1
- package/dist/commands/judge.d.ts +21 -1
- package/dist/commands/judge.d.ts.map +1 -1
- package/dist/commands/judge.js +71 -5
- package/dist/commands/judge.js.map +1 -1
- package/dist/commands/rollback.d.ts.map +1 -1
- package/dist/commands/rollback.js +11 -0
- package/dist/commands/rollback.js.map +1 -1
- package/dist/commands/skill.d.ts +17 -0
- package/dist/commands/skill.d.ts.map +1 -1
- package/dist/commands/skill.js +172 -7
- package/dist/commands/skill.js.map +1 -1
- package/dist/commands/start.d.ts +6 -0
- package/dist/commands/start.d.ts.map +1 -1
- package/dist/commands/start.js +31 -1
- package/dist/commands/start.js.map +1 -1
- package/dist/lib/decideBudget.d.ts +40 -0
- package/dist/lib/decideBudget.d.ts.map +1 -0
- package/dist/lib/decideBudget.js +46 -0
- package/dist/lib/decideBudget.js.map +1 -0
- package/dist/lib/decideEval.d.ts +126 -0
- package/dist/lib/decideEval.d.ts.map +1 -0
- package/dist/lib/decideEval.js +412 -0
- package/dist/lib/decideEval.js.map +1 -0
- package/dist/lib/decideLocal.d.ts +84 -0
- package/dist/lib/decideLocal.d.ts.map +1 -0
- package/dist/lib/decideLocal.js +178 -0
- package/dist/lib/decideLocal.js.map +1 -0
- package/dist/lib/decideRules.d.ts +43 -0
- package/dist/lib/decideRules.d.ts.map +1 -0
- package/dist/lib/decideRules.js +104 -0
- package/dist/lib/decideRules.js.map +1 -0
- package/dist/lib/localGates.d.ts +7 -7
- package/dist/lib/localGates.d.ts.map +1 -1
- package/dist/lib/localGates.js +94 -11
- package/dist/lib/localGates.js.map +1 -1
- package/dist/lib/policySnapshot.d.ts +27 -0
- package/dist/lib/policySnapshot.d.ts.map +1 -1
- package/dist/lib/policySnapshot.js +64 -2
- package/dist/lib/policySnapshot.js.map +1 -1
- package/package.json +5 -5
|
@@ -0,0 +1,1297 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `intutic decide` — decide rules on this machine: the engine, the rules'
|
|
3
|
+
* evidence, and the entry the shell gates call.
|
|
4
|
+
*
|
|
5
|
+
* test <rule> --state <file> ask a rule's questions about one call
|
|
6
|
+
* eval <rule> --labels <file> measure a rule on labelled calls; with a
|
|
7
|
+
* [--preregister <file>] pre-registration, a pass pins the model
|
|
8
|
+
* [--adversarial] add the adversarial case families
|
|
9
|
+
* calibrate <rule> --labels <file> fit a combiner and band to labels
|
|
10
|
+
* models what the bake-off validated
|
|
11
|
+
* check --stdin one call, for the shell and n8n gates
|
|
12
|
+
* configure write ~/.intutic/decide/engine.json
|
|
13
|
+
* disable / enable every rule to shadow, and back
|
|
14
|
+
* lint read every local decide: line
|
|
15
|
+
*
|
|
16
|
+
* A rule enforces only under a pin for the engine's model digest, and a pin
|
|
17
|
+
* comes only from a passing pre-registered `decide eval`: the band and pass
|
|
18
|
+
* marks are written down before the labels are read, the set is not one an
|
|
19
|
+
* earlier run used, and planted approvals are caught.
|
|
20
|
+
*
|
|
21
|
+
* `check` reads stdin and writes one JSON line, exit 0:
|
|
22
|
+
* in {"v":1,"harness","tool","toolInput","cwd","sessionId","budgetMs","lastUserTurn"}
|
|
23
|
+
* out {"v":1,"outcome":"allow|refuse|escalate","code":null|"DECIDE_REFUSED"|"DECIDE_ESCALATED"|"GOVERNANCE_UNAVAILABLE","ruleId":"decide.<ref>"|null,"reason":"…"}
|
|
24
|
+
* It never refuses for a shadow rule, and it refuses with
|
|
25
|
+
* GOVERNANCE_UNAVAILABLE for an enforcing rule it cannot evaluate, a record
|
|
26
|
+
* that does not decode, or input that does not read. Its start-up time is
|
|
27
|
+
* part of the gate's budget, so it imports nothing it does not use.
|
|
28
|
+
*
|
|
29
|
+
* @module
|
|
30
|
+
*/
|
|
31
|
+
import * as node_fs from 'node:fs';
|
|
32
|
+
import { performance } from 'node:perf_hooks';
|
|
33
|
+
import { dirname, join, resolve } from 'node:path';
|
|
34
|
+
import { appendDecideLogLine, buildDecideResponseState, buildDecideToolCallState, checkSelfHostedDecisionHost, createDecideCanaryFileStore, DECIDE_ADAPTERS, DECIDE_DEFAULT_ENGINE_URL, DECIDE_RECORD_TAG, decideEffectiveRollout, decideLogEngine, decideLogFileName, decideLogRedactionSources, decideOptions, decideQuestionBand, decideRuleHash, decideRuleOutcome, decideRuleRef, DecideEngineError, decodeDecideRecord, evaluateDecideCall, parseDecideLine, resolveDecideEngineConfig, runDecideCanary, validateDecideRule, writeDecideCallLog, } from '@intutic/shared-types';
|
|
35
|
+
import { log } from '../lib/logger.js';
|
|
36
|
+
import { parsePolicySnapshot, resolveSnapshotRulesPath } from '../lib/policySnapshot.js';
|
|
37
|
+
import { candidatesDir, canaryFile, digestFileName, disabledFile, engineFile, engineHost, evalsDir, loadEngine, logsDir, readEngineFile, decideRuleNotes, readEvalRunBaseline, readJson, runsDir, sha256Hex, statsDir, writeEngineFile, writeJson, } from '../lib/decideLocal.js';
|
|
38
|
+
import { adversarialCases, adversarialReport, auc, fitBand, fitCombiner, metrics, parseLabels, PASS_MARKS, quantile, splitOf, trackedP, withBand, } from '../lib/decideEval.js';
|
|
39
|
+
import { decideBudgets, decideRuleEstimateMs } from '../lib/decideBudget.js';
|
|
40
|
+
const nodeFs = {
|
|
41
|
+
readFileSync: node_fs.readFileSync,
|
|
42
|
+
writeFileSync: node_fs.writeFileSync,
|
|
43
|
+
mkdirSync: node_fs.mkdirSync,
|
|
44
|
+
renameSync: node_fs.renameSync,
|
|
45
|
+
openSync: node_fs.openSync,
|
|
46
|
+
writeSync: (fd, data) => node_fs.writeSync(fd, data),
|
|
47
|
+
closeSync: node_fs.closeSync,
|
|
48
|
+
};
|
|
49
|
+
function fail(message) {
|
|
50
|
+
log.error(message);
|
|
51
|
+
process.exitCode = 1;
|
|
52
|
+
}
|
|
53
|
+
const pct = (x) => (x === null ? '—' : `${(x * 100).toFixed(1)}%`);
|
|
54
|
+
const UNAVAILABLE = (reason, ruleId = null) => ({ v: 1, outcome: 'refuse', code: 'GOVERNANCE_UNAVAILABLE', ruleId, reason });
|
|
55
|
+
const ALLOW = (reason) => ({ v: 1, outcome: 'allow', code: null, ruleId: null, reason });
|
|
56
|
+
const MAX_BUDGET_MS = 60_000;
|
|
57
|
+
function readCheckInput(raw) {
|
|
58
|
+
let v;
|
|
59
|
+
try {
|
|
60
|
+
v = JSON.parse(raw);
|
|
61
|
+
}
|
|
62
|
+
catch {
|
|
63
|
+
return null;
|
|
64
|
+
}
|
|
65
|
+
if (typeof v !== 'object' || v === null || Array.isArray(v) || v.v !== 1)
|
|
66
|
+
return null;
|
|
67
|
+
if (typeof v.tool !== 'string' || !v.tool)
|
|
68
|
+
return null;
|
|
69
|
+
if (typeof v.budgetMs !== 'number' || !Number.isFinite(v.budgetMs))
|
|
70
|
+
return null;
|
|
71
|
+
return {
|
|
72
|
+
harness: typeof v.harness === 'string' ? v.harness : '',
|
|
73
|
+
tool: v.tool,
|
|
74
|
+
toolInput: v.toolInput ?? {},
|
|
75
|
+
cwd: typeof v.cwd === 'string' ? v.cwd : '',
|
|
76
|
+
sessionId: typeof v.sessionId === 'string' ? v.sessionId : '',
|
|
77
|
+
budgetMs: Math.min(Math.max(v.budgetMs, 0), MAX_BUDGET_MS),
|
|
78
|
+
lastUserTurn: typeof v.lastUserTurn === 'string' ? v.lastUserTurn : null,
|
|
79
|
+
};
|
|
80
|
+
}
|
|
81
|
+
const checkRule = (ir) => {
|
|
82
|
+
const v = validateDecideRule(ir);
|
|
83
|
+
return v.ok ? decideRuleHash(v.rule, sha256Hex) : null;
|
|
84
|
+
};
|
|
85
|
+
/** The questions whose answers were unknown, for a person to look at. */
|
|
86
|
+
function unknownQuestions(entry, answers) {
|
|
87
|
+
const v = validateDecideRule(entry.ir);
|
|
88
|
+
if (!v.ok || !answers)
|
|
89
|
+
return Object.keys(entry.ir.questions);
|
|
90
|
+
const o = decideRuleOutcome(v.rule, v.expr, Object.fromEntries(Object.entries(answers).map(([n, a]) => [n, { p: a.p, perOrder: a.perOrder }])));
|
|
91
|
+
// A choice question is unknown when an option `when` names is unknown.
|
|
92
|
+
const named = new Map();
|
|
93
|
+
const walk = (e) => {
|
|
94
|
+
if (e.op === 'eq')
|
|
95
|
+
named.set(e.q, [...(named.get(e.q) ?? []), e.o]);
|
|
96
|
+
else if (e.op === 'not')
|
|
97
|
+
walk(e.a);
|
|
98
|
+
else if (e.op === 'and' || e.op === 'or') {
|
|
99
|
+
walk(e.a);
|
|
100
|
+
walk(e.b);
|
|
101
|
+
}
|
|
102
|
+
};
|
|
103
|
+
walk(v.expr);
|
|
104
|
+
const out = [];
|
|
105
|
+
for (const name of Object.keys(v.rule.questions).sort()) {
|
|
106
|
+
if (o.atoms.yes[name] === 'U' || (named.get(name) ?? []).some((opt) => o.atoms.eq[name]?.[opt] === 'U'))
|
|
107
|
+
out.push(name);
|
|
108
|
+
}
|
|
109
|
+
return out;
|
|
110
|
+
}
|
|
111
|
+
function ruleName(entry) {
|
|
112
|
+
return `${entry.ir.id} (${entry.sopTitle})`;
|
|
113
|
+
}
|
|
114
|
+
/**
|
|
115
|
+
* One call's decide rules, for a gate that cannot evaluate them itself.
|
|
116
|
+
* Never throws. `startMs` is when this process started, which the gate's
|
|
117
|
+
* budget was computed against.
|
|
118
|
+
*/
|
|
119
|
+
export async function decideCheck(raw, opts = {}) {
|
|
120
|
+
const startMs = opts.startMs ?? performance.timeOrigin;
|
|
121
|
+
const input = readCheckInput(raw);
|
|
122
|
+
if (!input)
|
|
123
|
+
return UNAVAILABLE('decide check: the input is not a version 1 request');
|
|
124
|
+
const deadlineMs = startMs + input.budgetMs;
|
|
125
|
+
let text;
|
|
126
|
+
const path = resolveSnapshotRulesPath();
|
|
127
|
+
try {
|
|
128
|
+
text = node_fs.readFileSync(path, 'utf8');
|
|
129
|
+
}
|
|
130
|
+
catch {
|
|
131
|
+
return ALLOW('no policy snapshot on this machine');
|
|
132
|
+
}
|
|
133
|
+
const health = parsePolicySnapshot(text, { path });
|
|
134
|
+
if (health.state === 'invalid') {
|
|
135
|
+
// Every gate drops an invalid snapshot's dynamic tier, its decide records and fallback lines with it.
|
|
136
|
+
return ALLOW('the policy snapshot does not verify; its decide rules are not in force');
|
|
137
|
+
}
|
|
138
|
+
const records = [];
|
|
139
|
+
for (const line of text.split('\n')) {
|
|
140
|
+
if (!line.startsWith(`${DECIDE_RECORD_TAG}\t`))
|
|
141
|
+
continue;
|
|
142
|
+
const f = line.split('\t');
|
|
143
|
+
if (!(f[3] ?? '').split(',').includes(input.tool))
|
|
144
|
+
continue;
|
|
145
|
+
const record = decodeDecideRecord(line, checkRule);
|
|
146
|
+
if (record) {
|
|
147
|
+
records.push(record);
|
|
148
|
+
continue;
|
|
149
|
+
}
|
|
150
|
+
// A record a gate would leave its fallback line in force for refuses
|
|
151
|
+
// here too; one stamped shadow with no fallback line refuses nowhere.
|
|
152
|
+
if (f[1] !== '-' || f[2] !== 'shadow')
|
|
153
|
+
return UNAVAILABLE(`a decide rule for ${input.tool} does not decode, so the call is refused until the snapshot is rewritten`);
|
|
154
|
+
}
|
|
155
|
+
if (records.length === 0)
|
|
156
|
+
return ALLOW(`no decide rule names ${input.tool}`);
|
|
157
|
+
const loaded = loadEngine();
|
|
158
|
+
// A rule the engine cannot answer inside the budget (the p95 `doctor`
|
|
159
|
+
// stored) is not asked: an enforcing one escalates, a shadow one is logged
|
|
160
|
+
// and allowed. The gates' own decide tiers do the same.
|
|
161
|
+
const merged = await evaluateDecideCall({ deterministic: 'allow', tool: input.tool, toolInput: input.toolInput, cwd: input.cwd, lastUserTurn: input.lastUserTurn, rules: records, deadlineMs }, {
|
|
162
|
+
engine: loaded.engine,
|
|
163
|
+
engineUrl: loaded.cfg?.url ?? '',
|
|
164
|
+
canary: createDecideCanaryFileStore(canaryFile(), nodeFs),
|
|
165
|
+
sha256Hex,
|
|
166
|
+
q95Ms: loaded.q95Ms,
|
|
167
|
+
parallel: loaded.cfg?.parallel ?? 1,
|
|
168
|
+
});
|
|
169
|
+
try {
|
|
170
|
+
writeDecideCallLog(merged, records, {
|
|
171
|
+
atMs: Date.now(),
|
|
172
|
+
harness: input.harness,
|
|
173
|
+
workspaceId: health.workspaceId,
|
|
174
|
+
sessionId: input.sessionId,
|
|
175
|
+
tool: input.tool,
|
|
176
|
+
toolInput: input.toolInput,
|
|
177
|
+
cwd: input.cwd,
|
|
178
|
+
lastUserTurn: input.lastUserTurn,
|
|
179
|
+
budgetMs: input.budgetMs,
|
|
180
|
+
latencyMs: Math.round(Date.now() - startMs),
|
|
181
|
+
// As every gate names it (decideLog.ts).
|
|
182
|
+
engine: decideLogEngine(loaded.cfg, merged),
|
|
183
|
+
redact: decideLogRedactionSources(),
|
|
184
|
+
}, { fs: nodeFs, logsDir: logsDir(), statsDir: statsDir(), sha256Hex, evalRun: readEvalRunBaseline });
|
|
185
|
+
}
|
|
186
|
+
catch {
|
|
187
|
+
// A log that cannot be written never changes the decision.
|
|
188
|
+
}
|
|
189
|
+
const call = merged.call;
|
|
190
|
+
const entry = call.ruleRef ? records.find((r) => r.ref === call.ruleRef) ?? null : null;
|
|
191
|
+
const ruleId = call.ruleRef ? `decide.${call.ruleRef}` : null;
|
|
192
|
+
if (call.outcome === 'allow')
|
|
193
|
+
return { v: 1, outcome: 'allow', code: null, ruleId: null, reason: '' };
|
|
194
|
+
if (call.code === 'GOVERNANCE_UNAVAILABLE') {
|
|
195
|
+
const report = merged.rules.find((r) => r.ref === call.ruleRef);
|
|
196
|
+
return UNAVAILABLE(entry ? `Decide rule ${ruleName(entry)} could not be evaluated: the decide engine is unavailable (${report?.error ?? 'unknown'})` : (call.reason ?? 'the decide engine is unavailable'), ruleId);
|
|
197
|
+
}
|
|
198
|
+
const report = merged.rules.find((r) => r.ref === call.ruleRef);
|
|
199
|
+
if (call.code === 'DECIDE_REFUSED') {
|
|
200
|
+
return { v: 1, outcome: 'refuse', code: 'DECIDE_REFUSED', ruleId, reason: `Decide rule ${entry ? ruleName(entry) : call.ruleRef} refused this call` };
|
|
201
|
+
}
|
|
202
|
+
const title = entry ? entry.ir.title ?? entry.sopTitle : (call.ruleRef ?? '');
|
|
203
|
+
const unknown = report?.error === 'budget_skip' ? `the engine cannot answer within this harness's ${input.budgetMs} ms` : entry ? unknownQuestions(entry, report?.answers ?? null).join(', ') || 'the combined answer' : '';
|
|
204
|
+
return { v: 1, outcome: call.outcome, code: 'DECIDE_ESCALATED', ruleId, reason: `Needs a person: ${title} — ${unknown}` };
|
|
205
|
+
}
|
|
206
|
+
async function readStdin() {
|
|
207
|
+
const chunks = [];
|
|
208
|
+
for await (const c of process.stdin)
|
|
209
|
+
chunks.push(c);
|
|
210
|
+
return Buffer.concat(chunks).toString('utf8');
|
|
211
|
+
}
|
|
212
|
+
export async function runDecideCheck(opts) {
|
|
213
|
+
let out;
|
|
214
|
+
if (!opts.stdin) {
|
|
215
|
+
out = UNAVAILABLE('decide check reads its request from stdin: pass --stdin');
|
|
216
|
+
}
|
|
217
|
+
else {
|
|
218
|
+
try {
|
|
219
|
+
out = await decideCheck(await readStdin());
|
|
220
|
+
}
|
|
221
|
+
catch (err) {
|
|
222
|
+
out = UNAVAILABLE(`decide check failed: ${err instanceof Error ? err.message : String(err)}`);
|
|
223
|
+
}
|
|
224
|
+
}
|
|
225
|
+
process.stdout.write(JSON.stringify(out) + '\n');
|
|
226
|
+
}
|
|
227
|
+
// ===========================================================================
|
|
228
|
+
// Shared by test / eval / calibrate
|
|
229
|
+
// ===========================================================================
|
|
230
|
+
async function localRule(spec, workspace) {
|
|
231
|
+
const { decideWorkspaceRoot, findDecideRule, loadLocalDecideLines } = await import('../lib/decideRules.js');
|
|
232
|
+
return findDecideRule(await loadLocalDecideLines(decideWorkspaceRoot(workspace)), spec);
|
|
233
|
+
}
|
|
234
|
+
function engineOrFail(loaded) {
|
|
235
|
+
if (loaded.engine instanceof DecideEngineError) {
|
|
236
|
+
fail(`The decide engine is not usable: ${loaded.engine.message} (${loaded.engine.code}). Configure it with \`intutic decide configure\`.`);
|
|
237
|
+
return null;
|
|
238
|
+
}
|
|
239
|
+
return loaded.engine;
|
|
240
|
+
}
|
|
241
|
+
function questionCalls(rule) {
|
|
242
|
+
return Object.keys(rule.questions).map((name) => {
|
|
243
|
+
const q = rule.questions[name];
|
|
244
|
+
return q.type === 'choice' ? { name, type: 'choice', ask: q.ask, options: decideOptions(q) } : { name, type: 'yes_no', ask: q.ask };
|
|
245
|
+
});
|
|
246
|
+
}
|
|
247
|
+
/**
|
|
248
|
+
* The state a gate sends for this call, or the state the gateway sends for
|
|
249
|
+
* this response: for an `on: response` rule, the SOP's text and the
|
|
250
|
+
* response, held to the engine's response limit (`maxResponseStateBytes`,
|
|
251
|
+
* 32,768 bytes by default). `context: ["last_user_turn"]` is reserved in
|
|
252
|
+
* 3.0: no gate supplies the user's last turn, so a case's `lastUserTurn` is
|
|
253
|
+
* never sent, and `test` and `eval` measure what a gate asks.
|
|
254
|
+
*/
|
|
255
|
+
function stateFor(rule, sopTitle, ruleText, c, maxResponseBytes) {
|
|
256
|
+
if (rule.on === 'response') {
|
|
257
|
+
return buildDecideResponseState({ ruleText, content: c.response ?? '', ...(maxResponseBytes !== undefined ? { maxBytes: maxResponseBytes } : {}) });
|
|
258
|
+
}
|
|
259
|
+
return buildDecideToolCallState({
|
|
260
|
+
ruleId: rule.id,
|
|
261
|
+
sopTitle,
|
|
262
|
+
ruleText,
|
|
263
|
+
tool: c.tool,
|
|
264
|
+
cwd: c.cwd,
|
|
265
|
+
toolInput: c.toolInput,
|
|
266
|
+
...((rule.context ?? []).includes('last_user_turn') ? { lastUserTurn: null } : {}),
|
|
267
|
+
});
|
|
268
|
+
}
|
|
269
|
+
/** Says, once, that a `lastUserTurn` a state or labels file carries is not sent. */
|
|
270
|
+
function warnUnsentTurn(count, json) {
|
|
271
|
+
if (count === 0)
|
|
272
|
+
return;
|
|
273
|
+
const message = `${count === 1 ? 'The call carries' : `${count} calls carry`} a lastUserTurn, which is not sent: context: ["last_user_turn"] is reserved in 3.0, and no gate supplies the user's last turn.`;
|
|
274
|
+
if (json)
|
|
275
|
+
process.stderr.write(message + '\n');
|
|
276
|
+
else
|
|
277
|
+
log.warn(message);
|
|
278
|
+
}
|
|
279
|
+
function logState(ref, ruleHash, state) {
|
|
280
|
+
appendDecideLogLine(nodeFs, logsDir(), decideLogFileName(Date.now(), 'decide-state'), JSON.stringify({ v: 1, at: new Date().toISOString(), ref, ruleHash, state }) + '\n');
|
|
281
|
+
}
|
|
282
|
+
/** Asks every case, one after another (the engine runs each case's questions in parallel). */
|
|
283
|
+
async function askCases(engine, rule, sopTitle, ruleText, cases, timeoutMs, opts) {
|
|
284
|
+
const qs = questionCalls(rule);
|
|
285
|
+
const out = [];
|
|
286
|
+
for (const c of cases) {
|
|
287
|
+
let answers;
|
|
288
|
+
try {
|
|
289
|
+
const state = stateFor(rule, sopTitle, ruleText, c, opts.maxResponseBytes);
|
|
290
|
+
if (opts.logState)
|
|
291
|
+
logState(opts.ref, opts.ruleHash, state);
|
|
292
|
+
answers = (await engine.ask(state, qs, Date.now() + timeoutMs * (qs.length * 3 + 1))).answers;
|
|
293
|
+
}
|
|
294
|
+
catch {
|
|
295
|
+
answers = null;
|
|
296
|
+
}
|
|
297
|
+
out.push({ c, answers });
|
|
298
|
+
}
|
|
299
|
+
return out;
|
|
300
|
+
}
|
|
301
|
+
// ===========================================================================
|
|
302
|
+
// test
|
|
303
|
+
// ===========================================================================
|
|
304
|
+
export async function runDecideTest(spec, opts) {
|
|
305
|
+
let hit;
|
|
306
|
+
try {
|
|
307
|
+
hit = await localRule(spec, opts.workspace);
|
|
308
|
+
}
|
|
309
|
+
catch (err) {
|
|
310
|
+
return fail(err instanceof Error ? err.message : String(err));
|
|
311
|
+
}
|
|
312
|
+
const rule = hit.decl.rule;
|
|
313
|
+
let c;
|
|
314
|
+
try {
|
|
315
|
+
const v = JSON.parse(node_fs.readFileSync(resolve(opts.state), 'utf8'));
|
|
316
|
+
if (rule.on === 'response') {
|
|
317
|
+
if (typeof v.response !== 'string')
|
|
318
|
+
throw new Error('it has no "response": an on: response rule is asked about a response\'s text');
|
|
319
|
+
c = { tool: '', toolInput: {}, cwd: '', lastUserTurn: null, response: v.response };
|
|
320
|
+
}
|
|
321
|
+
else {
|
|
322
|
+
if (typeof v.tool !== 'string')
|
|
323
|
+
throw new Error('it has no "tool"');
|
|
324
|
+
c = { tool: v.tool, toolInput: v.toolInput ?? {}, cwd: typeof v.cwd === 'string' ? v.cwd : process.cwd(), lastUserTurn: typeof v.lastUserTurn === 'string' ? v.lastUserTurn : null };
|
|
325
|
+
}
|
|
326
|
+
}
|
|
327
|
+
catch (err) {
|
|
328
|
+
return fail(`Could not read the state file ${opts.state}: ${err instanceof Error ? err.message : String(err)}`);
|
|
329
|
+
}
|
|
330
|
+
const loaded = loadEngine();
|
|
331
|
+
const engine = engineOrFail(loaded);
|
|
332
|
+
if (!engine)
|
|
333
|
+
return;
|
|
334
|
+
let state;
|
|
335
|
+
try {
|
|
336
|
+
state = stateFor(rule, hit.sopTitle, hit.ruleText, c, loaded.cfg?.maxResponseStateBytes);
|
|
337
|
+
}
|
|
338
|
+
catch (err) {
|
|
339
|
+
return fail(err instanceof Error ? err.message : String(err));
|
|
340
|
+
}
|
|
341
|
+
if (opts.logState)
|
|
342
|
+
logState(hit.ref, hit.ruleHash, state);
|
|
343
|
+
warnUnsentTurn(c.lastUserTurn === null ? 0 : 1, opts.json);
|
|
344
|
+
let result;
|
|
345
|
+
try {
|
|
346
|
+
result = await engine.ask(state, questionCalls(rule), Date.now() + (loaded.cfg?.timeoutMs ?? 5000) * 4);
|
|
347
|
+
}
|
|
348
|
+
catch (err) {
|
|
349
|
+
return fail(`The engine did not answer: ${err instanceof Error ? err.message : String(err)}`);
|
|
350
|
+
}
|
|
351
|
+
const inputs = Object.fromEntries(Object.entries(result.answers).map(([n, a]) => [n, { p: a.p, perOrder: a.perOrder }]));
|
|
352
|
+
const v = validateDecideRule(rule);
|
|
353
|
+
if (!v.ok)
|
|
354
|
+
return fail(v.reason);
|
|
355
|
+
const o = decideRuleOutcome(v.rule, v.expr, inputs);
|
|
356
|
+
const pins = (readPins(hit.ruleHash) ?? []).filter((p) => p.modelDigest === result.modelDigest);
|
|
357
|
+
const report = {
|
|
358
|
+
ref: hit.ref,
|
|
359
|
+
ruleHash: hit.ruleHash,
|
|
360
|
+
model: result.model,
|
|
361
|
+
modelDigest: result.modelDigest,
|
|
362
|
+
digestStrength: result.digestStrength,
|
|
363
|
+
pinned: pins.length > 0,
|
|
364
|
+
when: o.value,
|
|
365
|
+
outcome: o.outcome,
|
|
366
|
+
code: o.code,
|
|
367
|
+
answers: Object.fromEntries(Object.entries(result.answers).map(([n, a]) => {
|
|
368
|
+
const band = decideQuestionBand(v.rule, n);
|
|
369
|
+
return [n, a.type === 'yes_no' ? { p: a.p, band: [band.no_below, band.yes_at], value: o.atoms.yes[n] } : { perOrder: a.perOrder, atoms: o.atoms.eq[n] }];
|
|
370
|
+
})),
|
|
371
|
+
latencyMs: result.latencyMs,
|
|
372
|
+
};
|
|
373
|
+
if (opts.json) {
|
|
374
|
+
console.log(JSON.stringify(report, null, 2));
|
|
375
|
+
return;
|
|
376
|
+
}
|
|
377
|
+
log.header(`intutic decide test ${hit.ref}`);
|
|
378
|
+
log.field('Model', `${result.model} (${result.modelDigest}, ${result.digestStrength} digest)`);
|
|
379
|
+
for (const [n, a] of Object.entries(report.answers)) {
|
|
380
|
+
if ('p' in a)
|
|
381
|
+
log.field(n, `p(yes) ${(a.p ?? 0).toFixed(3)} against [${(a.band ?? []).join(', ')}] → ${a.value}`);
|
|
382
|
+
else
|
|
383
|
+
log.field(n, Object.entries(a.atoms ?? {}).map(([opt, t]) => `${opt}=${t}`).join(' '));
|
|
384
|
+
}
|
|
385
|
+
log.field('when', o.value);
|
|
386
|
+
log.field('Outcome', `${o.outcome}${o.code ? ` (${o.code})` : ''}${pins.length > 0 ? '' : ' — shadow: no pin for this model; run `intutic decide eval`'}`);
|
|
387
|
+
log.field('Latency', `${result.latencyMs} ms`);
|
|
388
|
+
}
|
|
389
|
+
/** How long one request to the control plane may take. */
|
|
390
|
+
const CONTROL_PLANE_TIMEOUT_MS = 10_000;
|
|
391
|
+
/** The control plane this machine is logged in to, or null when it is not. */
|
|
392
|
+
async function controlPlaneLogin() {
|
|
393
|
+
const { loadCredentials } = await import('../config/store.js');
|
|
394
|
+
const creds = await loadCredentials();
|
|
395
|
+
if (!creds || !creds.controlPlaneUrl || !creds.apiKey)
|
|
396
|
+
return null;
|
|
397
|
+
let end = creds.controlPlaneUrl.length;
|
|
398
|
+
while (end > 0 && creds.controlPlaneUrl.charCodeAt(end - 1) === 0x2f)
|
|
399
|
+
end--;
|
|
400
|
+
return { url: creds.controlPlaneUrl.slice(0, end), apiKey: creds.apiKey };
|
|
401
|
+
}
|
|
402
|
+
/** One JSON request to the control plane. Throws when it gets no answer; a non-2xx answer is returned. */
|
|
403
|
+
async function controlPlaneRequest(cp, method, path, body) {
|
|
404
|
+
const res = await fetch(`${cp.url}${path}`, {
|
|
405
|
+
method,
|
|
406
|
+
headers: { 'content-type': 'application/json', authorization: `Bearer ${cp.apiKey}` },
|
|
407
|
+
...(body === undefined ? {} : { body: JSON.stringify(body) }),
|
|
408
|
+
signal: AbortSignal.timeout(CONTROL_PLANE_TIMEOUT_MS),
|
|
409
|
+
});
|
|
410
|
+
const { unwrapToonEnvelope } = await import('../lib/api.js');
|
|
411
|
+
let parsed;
|
|
412
|
+
try {
|
|
413
|
+
parsed = unwrapToonEnvelope(await res.json());
|
|
414
|
+
}
|
|
415
|
+
catch {
|
|
416
|
+
parsed = null;
|
|
417
|
+
}
|
|
418
|
+
return { status: res.status, body: typeof parsed === 'object' && parsed !== null && !Array.isArray(parsed) ? parsed : null };
|
|
419
|
+
}
|
|
420
|
+
function errorText(r) {
|
|
421
|
+
return typeof r.body?.error === 'string' ? `${r.status}: ${r.body.error}` : `status ${r.status}`;
|
|
422
|
+
}
|
|
423
|
+
/**
|
|
424
|
+
* The candidate calibration `calibrationId` of this rule, fitted on
|
|
425
|
+
* `modelDigest`, as the control plane serves it. Throws a message for the
|
|
426
|
+
* user when there is none.
|
|
427
|
+
*/
|
|
428
|
+
async function fetchCalibration(cp, ruleHash, calibrationId, modelDigest) {
|
|
429
|
+
const r = await controlPlaneRequest(cp, 'GET', `/api/v1/decide/calibrations?ruleHash=${ruleHash}&limit=500`);
|
|
430
|
+
if (r.status !== 200)
|
|
431
|
+
throw new Error(`The control plane did not list this rule's calibrations (${errorText(r)})`);
|
|
432
|
+
const list = Array.isArray(r.body?.calibrations) ? r.body.calibrations : [];
|
|
433
|
+
const c = list.find((x) => x.calibrationId === calibrationId);
|
|
434
|
+
if (!c)
|
|
435
|
+
throw new Error(`No calibration ${calibrationId} of this rule (rule hash ${ruleHash.slice(0, 12)}) on the control plane`);
|
|
436
|
+
if (c.status !== 'candidate')
|
|
437
|
+
throw new Error(`Calibration ${calibrationId} is ${String(c.status)}; a companion run is for a candidate`);
|
|
438
|
+
if (c.modelDigest !== modelDigest)
|
|
439
|
+
throw new Error(`Calibration ${calibrationId} was fitted on model ${String(c.modelDigest)}; the engine runs ${modelDigest}`);
|
|
440
|
+
const band = c.band;
|
|
441
|
+
if (!band || typeof band.yes_at !== 'number' || typeof band.no_below !== 'number')
|
|
442
|
+
throw new Error(`Calibration ${calibrationId} has no band`);
|
|
443
|
+
const comb = c.combiner;
|
|
444
|
+
let combiner = null;
|
|
445
|
+
if (comb !== null && comb !== undefined) {
|
|
446
|
+
const w = comb.weights;
|
|
447
|
+
if (typeof comb.bias !== 'number' || typeof w !== 'object' || w === null || Object.values(w).some((x) => typeof x !== 'number')) {
|
|
448
|
+
throw new Error(`Calibration ${calibrationId} has a combiner this CLI does not read`);
|
|
449
|
+
}
|
|
450
|
+
combiner = comb;
|
|
451
|
+
}
|
|
452
|
+
return { calibrationId, band: { yes_at: band.yes_at, no_below: band.no_below }, combiner };
|
|
453
|
+
}
|
|
454
|
+
/**
|
|
455
|
+
* The pre-registration as the control plane registers it: the local
|
|
456
|
+
* document, and for a companion run its calibration's id, band and combiner
|
|
457
|
+
* exactly as served, which is how the control plane recognises the run.
|
|
458
|
+
*/
|
|
459
|
+
export function serverPreregistration(doc) {
|
|
460
|
+
const { serverEvalRunId: _id, calibration, ...rest } = doc;
|
|
461
|
+
return calibration ? { ...rest, calibrationId: calibration.calibrationId, band: calibration.band, combiner: calibration.combiner } : rest;
|
|
462
|
+
}
|
|
463
|
+
/** Registers the pre-registration with the control plane: its eval run's id, or why not. */
|
|
464
|
+
async function registerEvalRun(cp, doc) {
|
|
465
|
+
try {
|
|
466
|
+
const r = await controlPlaneRequest(cp, 'POST', '/api/v1/decide/eval-runs', { ruleHash: doc.ruleHash, modelDigest: doc.modelDigest, prereg: serverPreregistration(doc), source: 'cli' });
|
|
467
|
+
const run = r.body?.evalRun;
|
|
468
|
+
if (r.status !== 201 || typeof run?.evalRunId !== 'string')
|
|
469
|
+
return { error: errorText(r) };
|
|
470
|
+
return { evalRunId: run.evalRunId };
|
|
471
|
+
}
|
|
472
|
+
catch (err) {
|
|
473
|
+
return { error: err instanceof Error ? err.message : String(err) };
|
|
474
|
+
}
|
|
475
|
+
}
|
|
476
|
+
function readPins(ruleHash) {
|
|
477
|
+
try {
|
|
478
|
+
return node_fs
|
|
479
|
+
.readdirSync(evalsDir(ruleHash))
|
|
480
|
+
.filter((n) => n.endsWith('.json'))
|
|
481
|
+
.map((n) => readJson(join(evalsDir(ruleHash), n)))
|
|
482
|
+
.filter((p) => p !== null && typeof p.modelDigest === 'string');
|
|
483
|
+
}
|
|
484
|
+
catch {
|
|
485
|
+
return null;
|
|
486
|
+
}
|
|
487
|
+
}
|
|
488
|
+
function questionBands(rule) {
|
|
489
|
+
return Object.fromEntries(Object.keys(rule.questions).sort().map((n) => [n, decideQuestionBand(rule, n)]));
|
|
490
|
+
}
|
|
491
|
+
function usedSetHashes(ruleHash) {
|
|
492
|
+
const out = new Set();
|
|
493
|
+
try {
|
|
494
|
+
for (const n of node_fs.readdirSync(runsDir(ruleHash))) {
|
|
495
|
+
const r = readJson(join(runsDir(ruleHash), n));
|
|
496
|
+
if (r?.preregistered && typeof r.setHash === 'string')
|
|
497
|
+
out.add(r.setHash);
|
|
498
|
+
}
|
|
499
|
+
}
|
|
500
|
+
catch {
|
|
501
|
+
// no runs yet
|
|
502
|
+
}
|
|
503
|
+
return out;
|
|
504
|
+
}
|
|
505
|
+
/**
|
|
506
|
+
* What a pass on a logged-in machine means. The pin is written here, but a
|
|
507
|
+
* connected machine's snapshot takes its decide rules and pins only from the
|
|
508
|
+
* control plane, so the sync daemon does not carry it into the snapshot.
|
|
509
|
+
*/
|
|
510
|
+
export const DECIDE_EVAL_CONNECTED_PIN_NOTE = 'This machine is logged in, and a connected machine\'s policy snapshot takes its decide rules and pins only from the control plane: ' +
|
|
511
|
+
'this pin is not written into it. It applies to local use, on a machine with no login (`intutic gates install`).';
|
|
512
|
+
export async function runDecideEval(spec, opts) {
|
|
513
|
+
let hit;
|
|
514
|
+
try {
|
|
515
|
+
hit = await localRule(spec, opts.workspace);
|
|
516
|
+
}
|
|
517
|
+
catch (err) {
|
|
518
|
+
return fail(err instanceof Error ? err.message : String(err));
|
|
519
|
+
}
|
|
520
|
+
const rule = hit.decl.rule;
|
|
521
|
+
const ruleHash = hit.ruleHash;
|
|
522
|
+
const loaded = loadEngine();
|
|
523
|
+
const engine = engineOrFail(loaded);
|
|
524
|
+
if (!engine)
|
|
525
|
+
return;
|
|
526
|
+
let identity;
|
|
527
|
+
try {
|
|
528
|
+
identity = await engine.describe(Date.now() + (loaded.cfg?.timeoutMs ?? 5000));
|
|
529
|
+
}
|
|
530
|
+
catch (err) {
|
|
531
|
+
return fail(`The engine did not say which model it runs: ${err instanceof Error ? err.message : String(err)}`);
|
|
532
|
+
}
|
|
533
|
+
if (opts.calibration && !opts.preregister)
|
|
534
|
+
return fail('--calibration pre-registers a companion run: pass --preregister <file> too.');
|
|
535
|
+
// On a logged-in machine a pre-registered run is registered with, and uploaded to, the control plane.
|
|
536
|
+
const cp = opts.preregister ? await controlPlaneLogin() : null;
|
|
537
|
+
// Pre-registration: written before any label is read; a run under it must match it.
|
|
538
|
+
let prereg = null;
|
|
539
|
+
if (opts.preregister) {
|
|
540
|
+
const file = resolve(opts.preregister);
|
|
541
|
+
if (!node_fs.existsSync(file)) {
|
|
542
|
+
let calibration;
|
|
543
|
+
if (opts.calibration) {
|
|
544
|
+
if (!cp)
|
|
545
|
+
return fail('--calibration reads the calibration from the control plane: log in first (`intutic login`).');
|
|
546
|
+
try {
|
|
547
|
+
calibration = await fetchCalibration(cp, ruleHash, opts.calibration, identity.modelDigest);
|
|
548
|
+
}
|
|
549
|
+
catch (err) {
|
|
550
|
+
return fail(err instanceof Error ? err.message : String(err));
|
|
551
|
+
}
|
|
552
|
+
}
|
|
553
|
+
const candidate = readJson(join(candidatesDir(ruleHash), digestFileName(identity.modelDigest)));
|
|
554
|
+
const combiner = calibration
|
|
555
|
+
? calibration.combiner
|
|
556
|
+
? { bias: calibration.combiner.bias, weights: calibration.combiner.weights, band: calibration.band }
|
|
557
|
+
: null
|
|
558
|
+
: candidate?.modelDigest === identity.modelDigest && candidate.combiner && candidate.band
|
|
559
|
+
? { ...candidate.combiner, band: candidate.band }
|
|
560
|
+
: null;
|
|
561
|
+
const doc = {
|
|
562
|
+
v: 1,
|
|
563
|
+
ruleRef: hit.ref,
|
|
564
|
+
ruleHash,
|
|
565
|
+
modelDigest: identity.modelDigest,
|
|
566
|
+
band: questionBands(rule),
|
|
567
|
+
combiner,
|
|
568
|
+
passMarks: PASS_MARKS,
|
|
569
|
+
adversarial: true,
|
|
570
|
+
registeredAt: new Date().toISOString(),
|
|
571
|
+
...(calibration ? { calibration } : {}),
|
|
572
|
+
};
|
|
573
|
+
const registered = cp ? await registerEvalRun(cp, doc) : null;
|
|
574
|
+
if (registered && 'evalRunId' in registered)
|
|
575
|
+
doc.serverEvalRunId = registered.evalRunId;
|
|
576
|
+
node_fs.writeFileSync(file, JSON.stringify(doc, null, 2) + '\n', { mode: 0o600, flag: 'wx' });
|
|
577
|
+
log.success(`Pre-registered ${hit.ref} on ${identity.modelDigest} in ${opts.preregister} (sha256 ${sha256Hex(JSON.stringify(doc)).slice(0, 12)}). No label was read.`);
|
|
578
|
+
if (calibration)
|
|
579
|
+
log.dim(`It is the companion run of calibration ${calibration.calibrationId}: scored at its band${calibration.combiner ? ' and combiner' : ''}.`);
|
|
580
|
+
if (registered && 'evalRunId' in registered)
|
|
581
|
+
log.dim(`Registered with the control plane as eval run ${registered.evalRunId}.`);
|
|
582
|
+
else if (registered)
|
|
583
|
+
log.warn(`Not registered with the control plane (${registered.error}); it is registered when the run is evaluated.`);
|
|
584
|
+
log.dim('Run the same command again to evaluate against it.');
|
|
585
|
+
return;
|
|
586
|
+
}
|
|
587
|
+
try {
|
|
588
|
+
prereg = JSON.parse(node_fs.readFileSync(file, 'utf8'));
|
|
589
|
+
}
|
|
590
|
+
catch {
|
|
591
|
+
return fail(`${opts.preregister} is not a pre-registration`);
|
|
592
|
+
}
|
|
593
|
+
if (prereg.v !== 1 || prereg.ruleHash !== ruleHash)
|
|
594
|
+
return fail(`${opts.preregister} was registered for another version of this rule (rule hash ${String(prereg.ruleHash).slice(0, 12)}); pre-register again`);
|
|
595
|
+
if (prereg.modelDigest !== identity.modelDigest)
|
|
596
|
+
return fail(`${opts.preregister} was registered for model ${prereg.modelDigest}; the engine runs ${identity.modelDigest}`);
|
|
597
|
+
if (!(Date.parse(prereg.registeredAt) < Date.now()))
|
|
598
|
+
return fail(`${opts.preregister} has no registration time before now`);
|
|
599
|
+
if (opts.calibration && prereg.calibration?.calibrationId !== opts.calibration) {
|
|
600
|
+
return fail(`${opts.preregister} does not pre-register calibration ${opts.calibration}; pre-register a new file with --calibration`);
|
|
601
|
+
}
|
|
602
|
+
}
|
|
603
|
+
// Registered before any label is read, when the pre-registration could not be.
|
|
604
|
+
let serverRunId = prereg?.serverEvalRunId ?? null;
|
|
605
|
+
let serverError = null;
|
|
606
|
+
if (prereg && cp && !serverRunId) {
|
|
607
|
+
const r = await registerEvalRun(cp, prereg);
|
|
608
|
+
if ('evalRunId' in r)
|
|
609
|
+
serverRunId = r.evalRunId;
|
|
610
|
+
else
|
|
611
|
+
serverError = r.error;
|
|
612
|
+
}
|
|
613
|
+
if (!opts.labels)
|
|
614
|
+
return fail('Pass --labels <file.jsonl>.');
|
|
615
|
+
let labelsText;
|
|
616
|
+
let cases;
|
|
617
|
+
try {
|
|
618
|
+
labelsText = node_fs.readFileSync(resolve(opts.labels), 'utf8');
|
|
619
|
+
cases = parseLabels(labelsText, rule.on);
|
|
620
|
+
}
|
|
621
|
+
catch (err) {
|
|
622
|
+
return fail(`Could not read ${opts.labels}: ${err instanceof Error ? err.message : String(err)}`);
|
|
623
|
+
}
|
|
624
|
+
const setHash = sha256Hex(labelsText);
|
|
625
|
+
warnUnsentTurn(cases.filter((c) => c.lastUserTurn !== null).length, opts.json);
|
|
626
|
+
const adversarial = Boolean(opts.adversarial) || prereg !== null;
|
|
627
|
+
const extra = adversarial ? adversarialCases(cases) : [];
|
|
628
|
+
const timeoutMs = loaded.cfg?.timeoutMs ?? 5000;
|
|
629
|
+
const askOpts = { logState: opts.logState, ref: hit.ref, ruleHash, maxResponseBytes: loaded.cfg?.maxResponseStateBytes };
|
|
630
|
+
const answered = await askCases(engine, rule, hit.sopTitle, hit.ruleText, cases, timeoutMs, askOpts);
|
|
631
|
+
const answeredAdv = await askCases(engine, rule, hit.sopTitle, hit.ruleText, extra, timeoutMs, askOpts);
|
|
632
|
+
const combiner = prereg?.combiner ?? null;
|
|
633
|
+
let band = null;
|
|
634
|
+
let scored = answered;
|
|
635
|
+
let fit = null;
|
|
636
|
+
let evalRule = rule;
|
|
637
|
+
if (!prereg) {
|
|
638
|
+
const dev = answered.filter((a) => splitOf(a.c.id) === 'dev');
|
|
639
|
+
scored = answered.filter((a) => splitOf(a.c.id) === 'test');
|
|
640
|
+
fit = fitBand(rule, dev);
|
|
641
|
+
if (fit) {
|
|
642
|
+
band = fit.band;
|
|
643
|
+
evalRule = withBand(rule, band);
|
|
644
|
+
}
|
|
645
|
+
}
|
|
646
|
+
else if (prereg.calibration && !prereg.calibration.combiner) {
|
|
647
|
+
// A band-only calibration: the rule's own `when`, at the calibration's band.
|
|
648
|
+
evalRule = withBand(rule, prereg.calibration.band);
|
|
649
|
+
}
|
|
650
|
+
const m = metrics(evalRule, scored, combiner);
|
|
651
|
+
const adv = adversarial ? adversarialReport(evalRule, answeredAdv, combiner) : null;
|
|
652
|
+
const engineErrors = [...answered, ...answeredAdv].filter((a) => !a.answers).length;
|
|
653
|
+
// Per question: AUC against the label, and the p10/p90 drift compares with.
|
|
654
|
+
const questions = {};
|
|
655
|
+
for (const name of Object.keys(rule.questions).sort()) {
|
|
656
|
+
const pairs = answered
|
|
657
|
+
.map((a) => ({ p: a.answers?.[name] ? trackedP(rule, name, a.answers[name]) : null, y: a.c.label === 'refuse' }))
|
|
658
|
+
.filter((x) => x.p !== null);
|
|
659
|
+
questions[name] = {
|
|
660
|
+
auc: auc(pairs),
|
|
661
|
+
p10: pairs.length ? quantile(pairs.map((x) => x.p), 0.1) : null,
|
|
662
|
+
p90: pairs.length ? quantile(pairs.map((x) => x.p), 0.9) : null,
|
|
663
|
+
};
|
|
664
|
+
}
|
|
665
|
+
const reusedSet = prereg !== null && usedSetHashes(ruleHash).has(setHash);
|
|
666
|
+
const marks = prereg
|
|
667
|
+
? {
|
|
668
|
+
preregistered: !reusedSet,
|
|
669
|
+
size: m.n >= PASS_MARKS.minCases && m.refuseLabels >= PASS_MARKS.minRefuseLabels,
|
|
670
|
+
recall: m.recall !== null && m.recall >= PASS_MARKS.recall,
|
|
671
|
+
falseAlarms: m.falseAlarms === null || m.falseAlarms <= PASS_MARKS.falseAlarms,
|
|
672
|
+
escalation: m.escalation !== null && m.escalation <= PASS_MARKS.escalation,
|
|
673
|
+
plantedApprovals: adv !== null && adv.planted_approval.recall !== null && adv.planted_approval.recall >= PASS_MARKS.plantedApprovalRecall,
|
|
674
|
+
}
|
|
675
|
+
: null;
|
|
676
|
+
const passed = marks !== null && Object.values(marks).every(Boolean);
|
|
677
|
+
// The results, uploaded to the eval run the control plane registered. Never the labels or a state.
|
|
678
|
+
let server = null;
|
|
679
|
+
if (prereg && cp) {
|
|
680
|
+
server = { evalRunId: serverRunId, uploaded: false, error: serverError, marks: null };
|
|
681
|
+
if (serverRunId) {
|
|
682
|
+
try {
|
|
683
|
+
const r = await controlPlaneRequest(cp, 'POST', `/api/v1/decide/eval-runs/${encodeURIComponent(serverRunId)}/results`, { setHash, results: m, adversarial: adv });
|
|
684
|
+
if (r.status === 200) {
|
|
685
|
+
server.uploaded = true;
|
|
686
|
+
server.marks = r.body?.marks ?? null;
|
|
687
|
+
}
|
|
688
|
+
else
|
|
689
|
+
server.error = errorText(r);
|
|
690
|
+
}
|
|
691
|
+
catch (err) {
|
|
692
|
+
server.error = err instanceof Error ? err.message : String(err);
|
|
693
|
+
}
|
|
694
|
+
}
|
|
695
|
+
}
|
|
696
|
+
const evalRunId = `eval_local_${new Date().toISOString().replace(/[-:.TZ]/g, '').slice(0, 14)}_${sha256Hex(setHash + ruleHash + Date.now()).slice(0, 6)}`;
|
|
697
|
+
const evalAll = (xs) => metrics(evalRule, xs, combiner);
|
|
698
|
+
const report = {
|
|
699
|
+
v: 1,
|
|
700
|
+
evalRunId,
|
|
701
|
+
ruleRef: hit.ref,
|
|
702
|
+
ruleHash,
|
|
703
|
+
model: identity.model,
|
|
704
|
+
modelDigest: identity.modelDigest,
|
|
705
|
+
digestStrength: identity.digestStrength,
|
|
706
|
+
setHash,
|
|
707
|
+
preregistered: prereg !== null,
|
|
708
|
+
reusedSet,
|
|
709
|
+
band: prereg ? null : band,
|
|
710
|
+
bandErrorFreeOnDev: fit?.errorFree ?? null,
|
|
711
|
+
dev: fit?.dev ?? null,
|
|
712
|
+
results: m,
|
|
713
|
+
adversarial: adv,
|
|
714
|
+
engineErrors,
|
|
715
|
+
questions,
|
|
716
|
+
marks,
|
|
717
|
+
passed,
|
|
718
|
+
completedAt: new Date().toISOString(),
|
|
719
|
+
calibrationId: prereg?.calibration?.calibrationId ?? null,
|
|
720
|
+
server,
|
|
721
|
+
// What drift compares this rule's live decisions with.
|
|
722
|
+
baseline: {
|
|
723
|
+
evalRunId,
|
|
724
|
+
escalationRate: evalAll(answered).escalation ?? 0,
|
|
725
|
+
questions: Object.fromEntries(Object.entries(questions)
|
|
726
|
+
.filter(([, q]) => q.p10 !== null && q.p90 !== null)
|
|
727
|
+
.map(([n, q]) => [n, { p10: q.p10, p90: q.p90 }])),
|
|
728
|
+
},
|
|
729
|
+
};
|
|
730
|
+
writeJson(join(runsDir(ruleHash), `${evalRunId}.json`), report);
|
|
731
|
+
let snapshot = null;
|
|
732
|
+
if (passed) {
|
|
733
|
+
const pin = {
|
|
734
|
+
modelDigest: identity.modelDigest,
|
|
735
|
+
evalRunId,
|
|
736
|
+
source: 'local',
|
|
737
|
+
...(combiner ? { combiner: { bias: combiner.bias, weights: combiner.weights }, band: combiner.band } : {}),
|
|
738
|
+
};
|
|
739
|
+
writeJson(join(evalsDir(ruleHash), digestFileName(identity.modelDigest)), pin);
|
|
740
|
+
const { rewriteDecideSnapshot } = await import('../lib/decideRules.js');
|
|
741
|
+
snapshot = await rewriteDecideSnapshot();
|
|
742
|
+
}
|
|
743
|
+
// A pre-registered run that did not pass exits non-zero, whatever the output format.
|
|
744
|
+
if (prereg && !passed)
|
|
745
|
+
process.exitCode = 1;
|
|
746
|
+
if (opts.json) {
|
|
747
|
+
console.log(JSON.stringify(report, null, 2));
|
|
748
|
+
return;
|
|
749
|
+
}
|
|
750
|
+
log.header(`intutic decide eval ${hit.ref}`);
|
|
751
|
+
log.field('Model', `${identity.model} (${identity.modelDigest}, ${identity.digestStrength} digest)`);
|
|
752
|
+
log.field('Cases', `${cases.length} labelled (${prereg ? 'all scored' : `${scored.length} test, ${answered.length - scored.length} dev`})${extra.length ? `, ${extra.length} adversarial` : ''}${engineErrors ? `; ${engineErrors} unanswered, counted as escalations` : ''}`);
|
|
753
|
+
if (!prereg) {
|
|
754
|
+
log.field('Band fitted on dev', band ? `yes_at ${band.yes_at}, no_below ${band.no_below}${fit.errorFree ? '' : ' (no error-free band on dev)'}` : 'none: no dev cases');
|
|
755
|
+
}
|
|
756
|
+
const line = (x) => `recall ${pct(x.recall)} · false alarms ${pct(x.falseAlarms)} · escalation ${pct(x.escalation)} · n ${x.n}`;
|
|
757
|
+
log.field(prereg ? 'Results' : 'Test split', line(m));
|
|
758
|
+
for (const [n, q] of Object.entries(questions))
|
|
759
|
+
log.field(`AUC ${n}`, q.auc === null ? '—' : q.auc.toFixed(3));
|
|
760
|
+
if (adv)
|
|
761
|
+
for (const [f, r] of Object.entries(adv))
|
|
762
|
+
log.field(`Adversarial ${f.replace(/_/g, ' ')}`, `recall ${pct(r.recall)} of ${r.n}`);
|
|
763
|
+
if (server?.uploaded)
|
|
764
|
+
log.field('Control plane', `results uploaded to eval run ${server.evalRunId}${prereg?.calibration ? `, the companion of calibration ${prereg.calibration.calibrationId}` : ''}`);
|
|
765
|
+
else if (server)
|
|
766
|
+
log.warn(`The results were not uploaded to the control plane: ${server.error ?? 'no eval run is registered'}.`);
|
|
767
|
+
console.log('');
|
|
768
|
+
if (!prereg) {
|
|
769
|
+
log.info('Not pre-registered, so no pin. Write the band into the rule if it fits, then:');
|
|
770
|
+
log.dim(` intutic decide eval ${hit.ref} --preregister <file> (writes the registration; reads no label)`);
|
|
771
|
+
log.dim(` intutic decide eval ${hit.ref} --labels <fresh set> --preregister <file>`);
|
|
772
|
+
}
|
|
773
|
+
else if (passed) {
|
|
774
|
+
log.success(`Passed. Pinned ${identity.modelDigest} for this rule (${evalRunId}).`);
|
|
775
|
+
log.dim(snapshot === 'local'
|
|
776
|
+
? 'The policy snapshot was rewritten: a rule with rollout: enforce enforces on this model now.'
|
|
777
|
+
: snapshot === 'connected'
|
|
778
|
+
? DECIDE_EVAL_CONNECTED_PIN_NOTE
|
|
779
|
+
: 'Run `intutic gates install` to write it into the policy snapshot.');
|
|
780
|
+
}
|
|
781
|
+
else {
|
|
782
|
+
const failed = Object.entries(marks).filter(([, ok]) => !ok).map(([k]) => k);
|
|
783
|
+
log.warn(`Did not pass (${failed.join(', ')}). No pin was written; the rule stays in shadow on this model.`);
|
|
784
|
+
}
|
|
785
|
+
}
|
|
786
|
+
// ===========================================================================
|
|
787
|
+
// calibrate
|
|
788
|
+
// ===========================================================================
|
|
789
|
+
export async function runDecideCalibrate(spec, opts) {
|
|
790
|
+
let hit;
|
|
791
|
+
try {
|
|
792
|
+
hit = await localRule(spec, opts.workspace);
|
|
793
|
+
}
|
|
794
|
+
catch (err) {
|
|
795
|
+
return fail(err instanceof Error ? err.message : String(err));
|
|
796
|
+
}
|
|
797
|
+
const rule = hit.decl.rule;
|
|
798
|
+
const loaded = loadEngine();
|
|
799
|
+
const engine = engineOrFail(loaded);
|
|
800
|
+
if (!engine)
|
|
801
|
+
return;
|
|
802
|
+
let cases;
|
|
803
|
+
let labelsText;
|
|
804
|
+
try {
|
|
805
|
+
labelsText = node_fs.readFileSync(resolve(opts.labels), 'utf8');
|
|
806
|
+
cases = parseLabels(labelsText, rule.on);
|
|
807
|
+
}
|
|
808
|
+
catch (err) {
|
|
809
|
+
return fail(`Could not read ${opts.labels}: ${err instanceof Error ? err.message : String(err)}`);
|
|
810
|
+
}
|
|
811
|
+
let identity;
|
|
812
|
+
try {
|
|
813
|
+
identity = await engine.describe(Date.now() + (loaded.cfg?.timeoutMs ?? 5000));
|
|
814
|
+
}
|
|
815
|
+
catch (err) {
|
|
816
|
+
return fail(`The engine did not say which model it runs: ${err instanceof Error ? err.message : String(err)}`);
|
|
817
|
+
}
|
|
818
|
+
const answered = await askCases(engine, rule, hit.sopTitle, hit.ruleText, cases, loaded.cfg?.timeoutMs ?? 5000, { ref: hit.ref, ruleHash: hit.ruleHash, maxResponseBytes: loaded.cfg?.maxResponseStateBytes });
|
|
819
|
+
const dev = answered.filter((a) => splitOf(a.c.id) === 'dev');
|
|
820
|
+
const fitted = fitCombiner(rule, dev.length >= 10 ? dev : answered);
|
|
821
|
+
const fit = fitBand(rule, dev.length > 0 ? dev : answered, (band) => metrics(rule, dev.length > 0 ? dev : answered, { ...fitted, band }));
|
|
822
|
+
if (!fit)
|
|
823
|
+
return fail('No labelled case had an answer.');
|
|
824
|
+
const candidate = {
|
|
825
|
+
v: 1,
|
|
826
|
+
ruleRef: hit.ref,
|
|
827
|
+
ruleHash: hit.ruleHash,
|
|
828
|
+
modelDigest: identity.modelDigest,
|
|
829
|
+
combiner: fitted,
|
|
830
|
+
band: fit.band,
|
|
831
|
+
bandErrorFreeOnDev: fit.errorFree,
|
|
832
|
+
labelsN: cases.length,
|
|
833
|
+
setHash: sha256Hex(labelsText),
|
|
834
|
+
fittedAt: new Date().toISOString(),
|
|
835
|
+
};
|
|
836
|
+
writeJson(join(candidatesDir(hit.ruleHash), digestFileName(identity.modelDigest)), candidate);
|
|
837
|
+
if (opts.json) {
|
|
838
|
+
console.log(JSON.stringify(candidate, null, 2));
|
|
839
|
+
return;
|
|
840
|
+
}
|
|
841
|
+
log.header(`intutic decide calibrate ${hit.ref}`);
|
|
842
|
+
log.field('Model', `${identity.model} (${identity.modelDigest})`);
|
|
843
|
+
log.field('Combiner', `bias ${fitted.bias}; ${Object.entries(fitted.weights).map(([n, w]) => `${n} ${w}`).join(', ')}`);
|
|
844
|
+
log.field('Band', `yes_at ${fit.band.yes_at}, no_below ${fit.band.no_below}${fit.errorFree ? '' : ' (no error-free band on dev)'}`);
|
|
845
|
+
log.success('Saved as a candidate. It replaces `when` only once a pre-registered `intutic decide eval` passes with it:');
|
|
846
|
+
log.dim(` intutic decide eval ${hit.ref} --preregister <file>`);
|
|
847
|
+
}
|
|
848
|
+
// ===========================================================================
|
|
849
|
+
// models
|
|
850
|
+
// ===========================================================================
|
|
851
|
+
/** The bake-off's verdicts (toolcall-v1, 2026-10-17). No laptop model passed, so none is a default. */
|
|
852
|
+
export const DECIDE_MODELS = [
|
|
853
|
+
{ id: 'qwen3.6-27b', tier: 'server', validated: true, band: { yes_at: 0.51, no_below: 0.23 }, note: '4-bit; about 3.5 s per rule on a laptop, too slow for a hook gate there' },
|
|
854
|
+
{ id: 'kev-9b', tier: 'server', validated: true, band: null, note: 'fits a 32 GB machine' },
|
|
855
|
+
{ id: 'qwen3.5:4b', tier: 'laptop', validated: false, band: null, note: 'escalated one case too many (40.4%)' },
|
|
856
|
+
{ id: 'qwen3.5:0.8b', tier: 'laptop', validated: false, band: null, note: 'escalated 84.2%' },
|
|
857
|
+
{ id: 'kev-4b', tier: 'laptop', validated: false, band: null, note: 'missed one violation' },
|
|
858
|
+
{ id: 'kev-0.8b', tier: 'laptop', validated: false, band: null, note: 'missed one violation' },
|
|
859
|
+
{ id: 'granite-guardian-4.1-8b', tier: 'server', validated: false, band: null, note: 'escalated 45.6%' },
|
|
860
|
+
];
|
|
861
|
+
export function runDecideModels(opts) {
|
|
862
|
+
if (opts.json) {
|
|
863
|
+
console.log(JSON.stringify({ laptopValidated: [], models: DECIDE_MODELS }, null, 2));
|
|
864
|
+
return;
|
|
865
|
+
}
|
|
866
|
+
log.header('Decide models');
|
|
867
|
+
log.info('No laptop model is validated: none passed the pre-registered bake-off, so there is no default model.');
|
|
868
|
+
log.dim('Bring a self-hosted model and validate it on your own labels with `intutic decide eval`.');
|
|
869
|
+
console.log('');
|
|
870
|
+
for (const m of DECIDE_MODELS) {
|
|
871
|
+
console.log(` ${m.validated ? '✓' : '✗'} ${m.id.padEnd(26)} ${m.tier.padEnd(7)} ${m.validated ? 'passed (server tier)' : 'did not pass'} — ${m.note}`);
|
|
872
|
+
}
|
|
873
|
+
console.log('');
|
|
874
|
+
log.dim('Nothing is downloaded. With Ollama: `ollama pull <model>`, then `intutic decide configure --model <model>`.');
|
|
875
|
+
}
|
|
876
|
+
export function runDecideConfigure(opts) {
|
|
877
|
+
let current;
|
|
878
|
+
try {
|
|
879
|
+
current = readEngineFile() ?? {};
|
|
880
|
+
}
|
|
881
|
+
catch (err) {
|
|
882
|
+
return fail(err instanceof Error ? err.message : String(err));
|
|
883
|
+
}
|
|
884
|
+
const next = { ...current };
|
|
885
|
+
if (opts.url !== undefined)
|
|
886
|
+
next.url = opts.url;
|
|
887
|
+
if (opts.adapter !== undefined) {
|
|
888
|
+
if (!DECIDE_ADAPTERS.includes(opts.adapter))
|
|
889
|
+
return fail(`--adapter must be one of ${DECIDE_ADAPTERS.join(', ')}`);
|
|
890
|
+
next.adapter = opts.adapter;
|
|
891
|
+
}
|
|
892
|
+
if (opts.model !== undefined)
|
|
893
|
+
next.model = opts.model;
|
|
894
|
+
for (const [flag, key, value] of [['--timeout-ms', 'timeoutMs', opts.timeoutMs], ['--parallel', 'parallel', opts.parallel]]) {
|
|
895
|
+
if (value === undefined)
|
|
896
|
+
continue;
|
|
897
|
+
if (!/^[0-9]+$/.test(value))
|
|
898
|
+
return fail(`${flag} must be a whole number`);
|
|
899
|
+
next[key] = Number(value);
|
|
900
|
+
}
|
|
901
|
+
if (opts.structured !== undefined)
|
|
902
|
+
next.structured = opts.structured;
|
|
903
|
+
// A changed URL is a different engine: its p95 is measured again.
|
|
904
|
+
if (opts.url !== undefined && opts.url !== current.url) {
|
|
905
|
+
delete next.q95Ms;
|
|
906
|
+
delete next.q95MeasuredAt;
|
|
907
|
+
delete next.ollama;
|
|
908
|
+
}
|
|
909
|
+
const url = typeof next.url === 'string' ? next.url : DECIDE_DEFAULT_ENGINE_URL;
|
|
910
|
+
const verdict = checkSelfHostedDecisionHost(url, process.env.INTUTIC_SELF_HOSTED_DECISION_HOSTS);
|
|
911
|
+
if (!verdict.ok) {
|
|
912
|
+
return fail(`${verdict.host || url} is not a self-hosted engine (${verdict.reason}). The engine must run on this machine or your own network; ` +
|
|
913
|
+
'list a private host name in INTUTIC_SELF_HOSTED_DECISION_HOSTS.');
|
|
914
|
+
}
|
|
915
|
+
let cfg;
|
|
916
|
+
try {
|
|
917
|
+
cfg = resolveDecideEngineConfig(next, {});
|
|
918
|
+
}
|
|
919
|
+
catch (err) {
|
|
920
|
+
return fail(err instanceof Error ? err.message : String(err));
|
|
921
|
+
}
|
|
922
|
+
writeEngineFile(next);
|
|
923
|
+
const shown = { url: cfg.url, adapter: cfg.adapter, model: cfg.model || null, timeoutMs: cfg.timeoutMs, parallel: cfg.parallel, structured: cfg.structured };
|
|
924
|
+
if (opts.json) {
|
|
925
|
+
console.log(JSON.stringify({ file: engineFile(), ...shown, apiKey: process.env.INTUTIC_DECIDE_API_KEY ? 'from INTUTIC_DECIDE_API_KEY' : null }, null, 2));
|
|
926
|
+
return;
|
|
927
|
+
}
|
|
928
|
+
log.success(`Wrote ${engineFile()}`);
|
|
929
|
+
for (const [k, v] of Object.entries(shown))
|
|
930
|
+
log.field(k, v === null ? '(none: the engine is not configured until you set --model)' : String(v));
|
|
931
|
+
log.dim('The key, if your engine needs one, is read from INTUTIC_DECIDE_API_KEY and never written. Environment variables override this file.');
|
|
932
|
+
log.dim('Check the engine and run its canary with `intutic doctor`.');
|
|
933
|
+
}
|
|
934
|
+
// ===========================================================================
|
|
935
|
+
// disable / enable
|
|
936
|
+
// ===========================================================================
|
|
937
|
+
export async function runDecideDisable(on) {
|
|
938
|
+
const file = disabledFile();
|
|
939
|
+
if (on)
|
|
940
|
+
writeJson(file, { disabledAt: new Date().toISOString() });
|
|
941
|
+
else
|
|
942
|
+
node_fs.rmSync(file, { force: true });
|
|
943
|
+
const { rewriteDecideSnapshot } = await import('../lib/decideRules.js');
|
|
944
|
+
const where = await rewriteDecideSnapshot();
|
|
945
|
+
if (on)
|
|
946
|
+
log.success('Decide is disabled on this machine: every decide rule runs as shadow (logged, never refusing).');
|
|
947
|
+
else
|
|
948
|
+
log.success('Decide is enabled again: rules with rollout: enforce and a pin for the engine\'s model enforce.');
|
|
949
|
+
log.dim(where === 'local'
|
|
950
|
+
? 'The policy snapshot was rewritten.'
|
|
951
|
+
: where === 'connected'
|
|
952
|
+
? 'The sync daemon applies it to the snapshot on its next poll.'
|
|
953
|
+
: 'It applies the next time the policy snapshot is written (`intutic gates install`).');
|
|
954
|
+
}
|
|
955
|
+
// ===========================================================================
|
|
956
|
+
// lint
|
|
957
|
+
// ===========================================================================
|
|
958
|
+
export async function runDecideLint(opts) {
|
|
959
|
+
const { decideWorkspaceRoot, loadLocalDecideLines } = await import('../lib/decideRules.js');
|
|
960
|
+
const lines = await loadLocalDecideLines(decideWorkspaceRoot(opts.workspace));
|
|
961
|
+
const problems = [];
|
|
962
|
+
const byTool = new Map();
|
|
963
|
+
const rules = [];
|
|
964
|
+
// What 3.0 does not do with a rule as written (a reserved context):
|
|
965
|
+
// warnings, which do not fail the lint.
|
|
966
|
+
const notes = [];
|
|
967
|
+
for (const l of lines) {
|
|
968
|
+
if (l.decl.kind === 'unreadable') {
|
|
969
|
+
problems.push({ file: l.file, line: l.line, error: l.decl.error, hint: /empty|not JSON/.test(l.decl.error) ? 'write one rule per line as `decide: {json}`' : null });
|
|
970
|
+
continue;
|
|
971
|
+
}
|
|
972
|
+
const r = l.decl.rule;
|
|
973
|
+
if (r.on === 'tool_call')
|
|
974
|
+
for (const t of r.tools ?? [])
|
|
975
|
+
byTool.set(t, [...(byTool.get(t) ?? []), l.ref]);
|
|
976
|
+
rules.push({ ref: l.ref, file: l.file, line: l.line, ruleHash: l.ruleHash, on: r.on, rollout: r.rollout ?? 'shadow', tools: r.tools ?? [], pins: readPins(l.ruleHash)?.length ?? 0 });
|
|
977
|
+
for (const note of decideRuleNotes(r))
|
|
978
|
+
notes.push({ file: l.file, line: l.line, ref: l.ref, note });
|
|
979
|
+
}
|
|
980
|
+
const crowded = [...byTool].filter(([, refs]) => refs.length > 4).map(([tool, refs]) => ({ tool, refs }));
|
|
981
|
+
if (opts.json) {
|
|
982
|
+
console.log(JSON.stringify({ rules, problems, notes, crowded, disabled: node_fs.existsSync(disabledFile()) }, null, 2));
|
|
983
|
+
}
|
|
984
|
+
else {
|
|
985
|
+
log.header('intutic decide lint');
|
|
986
|
+
if (lines.length === 0)
|
|
987
|
+
log.info('No decide: lines in .intutic/sops.');
|
|
988
|
+
for (const r of rules) {
|
|
989
|
+
const where = r.on === 'response' ? ' (on: response: judged at the gateway)' : '';
|
|
990
|
+
const state = r.rollout === 'enforce' && r.pins === 0 ? ' (no pin yet: runs as shadow until `intutic decide eval` passes)' : '';
|
|
991
|
+
console.log(` ✓ ${r.file}:${r.line} ${r.ref} ${r.rollout}${where}${state} hash ${r.ruleHash.slice(0, 12)}`);
|
|
992
|
+
}
|
|
993
|
+
for (const p of problems) {
|
|
994
|
+
console.log(` ✗ ${p.file}:${p.line} ${p.error}`);
|
|
995
|
+
if (p.hint)
|
|
996
|
+
console.log(` hint: ${p.hint}`);
|
|
997
|
+
}
|
|
998
|
+
for (const n of notes)
|
|
999
|
+
log.warn(`${n.file}:${n.line}: ${n.note}`);
|
|
1000
|
+
for (const c of crowded)
|
|
1001
|
+
log.warn(`${c.refs.length} decide rules name ${c.tool}; a call evaluates at most 4, and an enforcing rule past the fourth refuses the call: ${c.refs.join(', ')}`);
|
|
1002
|
+
if (node_fs.existsSync(disabledFile()))
|
|
1003
|
+
log.warn('Decide is disabled on this machine (`intutic decide enable` turns it back on).');
|
|
1004
|
+
}
|
|
1005
|
+
if (problems.length > 0)
|
|
1006
|
+
process.exitCode = 1;
|
|
1007
|
+
}
|
|
1008
|
+
/** Calls of the canary's size `doctor` times for the p95. */
|
|
1009
|
+
const P95_CALLS = 20;
|
|
1010
|
+
/** The decide records the local policy snapshot carries (`snapshotText`, or the file the gates read) that decode. */
|
|
1011
|
+
function localDecideRecords(snapshotText) {
|
|
1012
|
+
let text = snapshotText;
|
|
1013
|
+
if (text === undefined) {
|
|
1014
|
+
try {
|
|
1015
|
+
text = node_fs.readFileSync(resolveSnapshotRulesPath(), 'utf8');
|
|
1016
|
+
}
|
|
1017
|
+
catch {
|
|
1018
|
+
text = null;
|
|
1019
|
+
}
|
|
1020
|
+
}
|
|
1021
|
+
return (text ?? '')
|
|
1022
|
+
.split('\n')
|
|
1023
|
+
.filter((l) => l.startsWith(`${DECIDE_RECORD_TAG}\t`))
|
|
1024
|
+
.map((l) => decodeDecideRecord(l, checkRule))
|
|
1025
|
+
.filter((r) => r !== null);
|
|
1026
|
+
}
|
|
1027
|
+
/** Runs the known-answer canary and stores its result in canary.json, where the gates read it. */
|
|
1028
|
+
async function canaryCheck(engine, url, modelDigest, timeoutMs) {
|
|
1029
|
+
try {
|
|
1030
|
+
const r = await runDecideCanary(engine.ask, Date.now() + timeoutMs * 3);
|
|
1031
|
+
createDecideCanaryFileStore(canaryFile(), nodeFs).write({ url, modelDigest, ok: r.ok, atMs: Date.now(), pViolation: r.pViolation, pClean: r.pClean, choiceTop: r.choiceTop });
|
|
1032
|
+
return r.ok
|
|
1033
|
+
? { name: 'Decide Canary', passed: true, detail: `passed (p(violation) ${r.pViolation.toFixed(3)}, p(clean) ${r.pClean.toFixed(3)}, choice ${r.choiceTop})` }
|
|
1034
|
+
: {
|
|
1035
|
+
name: 'Decide Canary',
|
|
1036
|
+
passed: false,
|
|
1037
|
+
detail: `failed (p(violation) ${r.pViolation.toFixed(3)}, p(clean) ${r.pClean.toFixed(3)}, choice ${r.choiceTop}): enforcing decide rules refuse until it passes`,
|
|
1038
|
+
remediation: 'The model answers the fixed known-answer questions wrongly: check the model and adapter.',
|
|
1039
|
+
};
|
|
1040
|
+
}
|
|
1041
|
+
catch (err) {
|
|
1042
|
+
return { name: 'Decide Canary', passed: false, detail: `could not run: ${err instanceof Error ? err.message : String(err)}` };
|
|
1043
|
+
}
|
|
1044
|
+
}
|
|
1045
|
+
/** How long Ollama keeps the model loaded after `intutic start` warms it: a working day. */
|
|
1046
|
+
export const DECIDE_WARM_KEEP_ALIVE = '8h';
|
|
1047
|
+
/** Loading a model from disk can take tens of seconds. */
|
|
1048
|
+
const DECIDE_WARM_TIMEOUT_MS = 60_000;
|
|
1049
|
+
/**
|
|
1050
|
+
* Loads the engine's model and keeps it loaded for {@link DECIDE_WARM_KEEP_ALIVE},
|
|
1051
|
+
* with Ollama's native `POST /api/generate {model, keep_alive}`: with no
|
|
1052
|
+
* prompt it only loads. Whether Ollama's OpenAI-compatible route honours
|
|
1053
|
+
* `keep_alive` is unknown, so the warm-up goes through its own route. False
|
|
1054
|
+
* for a server that is not Ollama, which has no such route and is not
|
|
1055
|
+
* warmed. Throws when the request fails. `cfg` is a configuration
|
|
1056
|
+
* `createDecideEngine` accepted, so its host passed the self-hosted guard.
|
|
1057
|
+
*/
|
|
1058
|
+
export async function warmDecideEngine(cfg, timeoutMs = DECIDE_WARM_TIMEOUT_MS) {
|
|
1059
|
+
if (!cfg.ollama || cfg.adapter !== 'logprobs')
|
|
1060
|
+
return false;
|
|
1061
|
+
let end = cfg.url.length;
|
|
1062
|
+
while (end > 0 && cfg.url.charCodeAt(end - 1) === 0x2f)
|
|
1063
|
+
end--;
|
|
1064
|
+
const headers = { 'content-type': 'application/json' };
|
|
1065
|
+
if (cfg.apiKey)
|
|
1066
|
+
headers.authorization = `Bearer ${cfg.apiKey}`;
|
|
1067
|
+
const res = await fetch(`${cfg.url.slice(0, end)}/api/generate`, {
|
|
1068
|
+
method: 'POST',
|
|
1069
|
+
headers,
|
|
1070
|
+
body: JSON.stringify({ model: cfg.model, keep_alive: DECIDE_WARM_KEEP_ALIVE }),
|
|
1071
|
+
signal: AbortSignal.timeout(timeoutMs),
|
|
1072
|
+
});
|
|
1073
|
+
await res.text();
|
|
1074
|
+
if (!res.ok)
|
|
1075
|
+
throw new Error(`/api/generate returned status ${res.status}`);
|
|
1076
|
+
return true;
|
|
1077
|
+
}
|
|
1078
|
+
/**
|
|
1079
|
+
* What `intutic start` does for decide when this machine has decide rules
|
|
1080
|
+
* and an engine is configured (engine.json, or `INTUTIC_DECIDE_MODEL`): warm
|
|
1081
|
+
* the model, then run the known-answer canary and store its result in
|
|
1082
|
+
* canary.json, as `doctor` does, so the first gated call finds a fresh pass.
|
|
1083
|
+
* Nothing (an empty list) otherwise. Never throws: start reports each line
|
|
1084
|
+
* and goes on whatever they say.
|
|
1085
|
+
*/
|
|
1086
|
+
export async function decideStartChecks(opts = {}) {
|
|
1087
|
+
try {
|
|
1088
|
+
if (localDecideRecords(opts.snapshotText).length === 0)
|
|
1089
|
+
return [];
|
|
1090
|
+
let file;
|
|
1091
|
+
try {
|
|
1092
|
+
file = readEngineFile();
|
|
1093
|
+
}
|
|
1094
|
+
catch {
|
|
1095
|
+
file = { broken: true };
|
|
1096
|
+
}
|
|
1097
|
+
if (file === null && !process.env.INTUTIC_DECIDE_MODEL?.trim())
|
|
1098
|
+
return [];
|
|
1099
|
+
const loaded = loadEngine();
|
|
1100
|
+
if (loaded.engine instanceof DecideEngineError) {
|
|
1101
|
+
return [{ name: 'Decide Engine', passed: false, detail: `${loaded.engine.code}: ${loaded.engine.message}`, remediation: 'Check it with `intutic doctor`.' }];
|
|
1102
|
+
}
|
|
1103
|
+
const engine = loaded.engine;
|
|
1104
|
+
const cfg = loaded.cfg;
|
|
1105
|
+
const out = [];
|
|
1106
|
+
try {
|
|
1107
|
+
if (await warmDecideEngine(cfg)) {
|
|
1108
|
+
out.push({ name: 'Decide Warm-up', passed: true, detail: `${cfg.model} loaded on ${engineHost(cfg.url)}, kept for ${DECIDE_WARM_KEEP_ALIVE}` });
|
|
1109
|
+
}
|
|
1110
|
+
}
|
|
1111
|
+
catch (err) {
|
|
1112
|
+
out.push({ name: 'Decide Warm-up', passed: false, detail: `could not load ${cfg.model}: ${err instanceof Error ? err.message : String(err)}` });
|
|
1113
|
+
}
|
|
1114
|
+
let identity;
|
|
1115
|
+
try {
|
|
1116
|
+
identity = await engine.describe(Date.now() + cfg.timeoutMs);
|
|
1117
|
+
}
|
|
1118
|
+
catch (err) {
|
|
1119
|
+
out.push({
|
|
1120
|
+
name: 'Decide Engine',
|
|
1121
|
+
passed: false,
|
|
1122
|
+
detail: `unreachable or the model is missing: ${err instanceof Error ? err.message : String(err)}; enforcing decide rules refuse until it answers`,
|
|
1123
|
+
remediation: 'Start the engine, or pull the model it names.',
|
|
1124
|
+
});
|
|
1125
|
+
return out;
|
|
1126
|
+
}
|
|
1127
|
+
out.push(await canaryCheck(engine, cfg.url, identity.modelDigest, cfg.timeoutMs));
|
|
1128
|
+
return out;
|
|
1129
|
+
}
|
|
1130
|
+
catch (err) {
|
|
1131
|
+
return [{ name: 'Decide', passed: false, detail: `could not check the engine: ${err instanceof Error ? err.message : String(err)}` }];
|
|
1132
|
+
}
|
|
1133
|
+
}
|
|
1134
|
+
/**
|
|
1135
|
+
* The decide rules the policy snapshot was compiled from, as its JSON keeps
|
|
1136
|
+
* them (`decideRules`, every `on` and the local SOPs' or the control plane's
|
|
1137
|
+
* alike), that read. `snapshotJson` is the JSON's text, or the file beside
|
|
1138
|
+
* the rules file the gates read.
|
|
1139
|
+
*/
|
|
1140
|
+
function snapshotDecideRules(snapshotJson) {
|
|
1141
|
+
let text = snapshotJson;
|
|
1142
|
+
if (text === undefined) {
|
|
1143
|
+
try {
|
|
1144
|
+
text = node_fs.readFileSync(join(dirname(resolveSnapshotRulesPath()), 'policy-snapshot.json'), 'utf8');
|
|
1145
|
+
}
|
|
1146
|
+
catch {
|
|
1147
|
+
text = null;
|
|
1148
|
+
}
|
|
1149
|
+
}
|
|
1150
|
+
let served;
|
|
1151
|
+
try {
|
|
1152
|
+
served = JSON.parse(text ?? 'null')?.decideRules;
|
|
1153
|
+
}
|
|
1154
|
+
catch {
|
|
1155
|
+
return [];
|
|
1156
|
+
}
|
|
1157
|
+
if (!Array.isArray(served))
|
|
1158
|
+
return [];
|
|
1159
|
+
const out = [];
|
|
1160
|
+
for (const r of served) {
|
|
1161
|
+
if (typeof r !== 'object' || r === null)
|
|
1162
|
+
continue;
|
|
1163
|
+
const { sopId, ir } = r;
|
|
1164
|
+
const decl = parseDecideLine(typeof ir === 'string' ? ir : JSON.stringify(ir ?? null));
|
|
1165
|
+
if (decl.kind === 'rule')
|
|
1166
|
+
out.push({ ref: decideRuleRef(typeof sopId === 'string' ? sopId : '', decl.rule.id), rule: decl.rule });
|
|
1167
|
+
}
|
|
1168
|
+
return out;
|
|
1169
|
+
}
|
|
1170
|
+
/**
|
|
1171
|
+
* What 3.0 does not do with the snapshot's decide rules as written, as doctor
|
|
1172
|
+
* lines: every rule declaring the reserved `context: ["last_user_turn"]` is
|
|
1173
|
+
* asked without it. (An `on: response` rule is the gateway's, which judges
|
|
1174
|
+
* each complete response by it.)
|
|
1175
|
+
*/
|
|
1176
|
+
function decideLimitChecks(rules) {
|
|
1177
|
+
const out = [];
|
|
1178
|
+
const context = rules.filter((r) => (r.rule.context ?? []).includes('last_user_turn')).map((r) => r.ref);
|
|
1179
|
+
if (context.length > 0) {
|
|
1180
|
+
out.push({
|
|
1181
|
+
name: 'Decide Context',
|
|
1182
|
+
passed: true,
|
|
1183
|
+
detail: `${context.join(', ')} declare(s) context: ["last_user_turn"], which is reserved and not supplied in 3.0: they are asked without the user's last turn`,
|
|
1184
|
+
remediation: 'No gate passes the last turn yet; `decide test` and `decide eval` ask without it too, so the eval measures what a gate asks.',
|
|
1185
|
+
});
|
|
1186
|
+
}
|
|
1187
|
+
return out;
|
|
1188
|
+
}
|
|
1189
|
+
/**
|
|
1190
|
+
* `intutic doctor`'s decide checks: what 3.0 does not do with the snapshot's
|
|
1191
|
+
* rules as written, then the engine and its model, the canary, the per-call
|
|
1192
|
+
* p95 (stored in engine.json), and every harness and rule pair the budget
|
|
1193
|
+
* cannot fit. Skipped (one passing line) when this machine has no decide
|
|
1194
|
+
* rule and no engine configured.
|
|
1195
|
+
*/
|
|
1196
|
+
export async function decideDoctorChecks(opts = {}) {
|
|
1197
|
+
const limits = decideLimitChecks(snapshotDecideRules(opts.snapshotJson));
|
|
1198
|
+
const engine = await decideEngineDoctorChecks(opts);
|
|
1199
|
+
return limits.length > 0 && engine.length === 1 && engine[0].name === 'Decide' && engine[0].passed ? limits : [...limits, ...engine];
|
|
1200
|
+
}
|
|
1201
|
+
async function decideEngineDoctorChecks(opts) {
|
|
1202
|
+
const records = localDecideRecords(opts.snapshotText);
|
|
1203
|
+
let file;
|
|
1204
|
+
try {
|
|
1205
|
+
file = readEngineFile();
|
|
1206
|
+
}
|
|
1207
|
+
catch {
|
|
1208
|
+
file = { broken: true };
|
|
1209
|
+
}
|
|
1210
|
+
if (records.length === 0 && file === null)
|
|
1211
|
+
return [{ name: 'Decide', passed: true, detail: 'no decide rules and no engine configured' }];
|
|
1212
|
+
const out = [];
|
|
1213
|
+
const loaded = loadEngine();
|
|
1214
|
+
if (loaded.engine instanceof DecideEngineError) {
|
|
1215
|
+
out.push({
|
|
1216
|
+
name: 'Decide Engine',
|
|
1217
|
+
passed: false,
|
|
1218
|
+
detail: `${loaded.engine.code}: ${loaded.engine.message}`,
|
|
1219
|
+
remediation: 'Configure a self-hosted engine with `intutic decide configure --url <url> --model <model>`; `intutic decide models` lists what was validated.',
|
|
1220
|
+
});
|
|
1221
|
+
if (records.some((r) => r.rollout === 'enforce'))
|
|
1222
|
+
out.push({ name: 'Decide Rules', passed: false, detail: `${records.length} rule(s); enforcing ones refuse their tools (GOVERNANCE_UNAVAILABLE) until the engine answers` });
|
|
1223
|
+
return out;
|
|
1224
|
+
}
|
|
1225
|
+
const engine = loaded.engine;
|
|
1226
|
+
const timeout = loaded.cfg.timeoutMs;
|
|
1227
|
+
let identity;
|
|
1228
|
+
try {
|
|
1229
|
+
identity = await engine.describe(Date.now() + timeout);
|
|
1230
|
+
out.push({ name: 'Decide Engine', passed: true, detail: `${engineHost(loaded.cfg.url)} (${loaded.cfg.adapter}), model ${identity.model}, ${identity.digestStrength} digest ${identity.modelDigest}` });
|
|
1231
|
+
if (identity.digestStrength === 'weak') {
|
|
1232
|
+
out.push({ name: 'Decide Model Digest', passed: true, detail: 'weak: the server cannot report a weights digest, so a model swapped under the same name is not detected', remediation: 'Set INTUTIC_DECIDE_MODEL_DIGEST to the weights digest.' });
|
|
1233
|
+
}
|
|
1234
|
+
}
|
|
1235
|
+
catch (err) {
|
|
1236
|
+
out.push({ name: 'Decide Engine', passed: false, detail: `unreachable or the model is missing: ${err instanceof Error ? err.message : String(err)}`, remediation: 'Start the engine, or pull the model it names.' });
|
|
1237
|
+
return out;
|
|
1238
|
+
}
|
|
1239
|
+
const canary = await canaryCheck(engine, loaded.cfg.url, identity.modelDigest, timeout);
|
|
1240
|
+
out.push(canary);
|
|
1241
|
+
if (!canary.passed)
|
|
1242
|
+
return out;
|
|
1243
|
+
// p95 of warm calls the size of the canary's yes/no question.
|
|
1244
|
+
const times = [];
|
|
1245
|
+
const state = buildDecideToolCallState({ ruleId: 'p95', sopTitle: 'Latency probe', ruleText: 'Measure only.', tool: 'Bash', cwd: '/', toolInput: { command: 'ls -la' } });
|
|
1246
|
+
for (let i = 0; i < P95_CALLS; i++) {
|
|
1247
|
+
const t = performance.now();
|
|
1248
|
+
try {
|
|
1249
|
+
await engine.ask(state, [{ name: 'probe', type: 'yes_no', ask: 'Does this command delete files recursively from the root of the filesystem?' }], Date.now() + timeout);
|
|
1250
|
+
}
|
|
1251
|
+
catch {
|
|
1252
|
+
break;
|
|
1253
|
+
}
|
|
1254
|
+
times.push(performance.now() - t);
|
|
1255
|
+
}
|
|
1256
|
+
if (times.length === P95_CALLS) {
|
|
1257
|
+
const q95 = Math.round(quantile(times, 0.95));
|
|
1258
|
+
try {
|
|
1259
|
+
writeEngineFile({ ...(readEngineFile() ?? {}), q95Ms: q95, q95MeasuredAt: new Date().toISOString() });
|
|
1260
|
+
}
|
|
1261
|
+
catch {
|
|
1262
|
+
// the measurement is still reported
|
|
1263
|
+
}
|
|
1264
|
+
out.push({ name: 'Decide Latency', passed: true, detail: `per-call p95 ${q95} ms over ${P95_CALLS} warm calls (stored in ${engineFile()})` });
|
|
1265
|
+
const tight = [];
|
|
1266
|
+
for (const r of records) {
|
|
1267
|
+
const v = validateDecideRule(r.ir);
|
|
1268
|
+
if (!v.ok)
|
|
1269
|
+
continue;
|
|
1270
|
+
const est = decideRuleEstimateMs(v.rule, q95, loaded.cfg.parallel);
|
|
1271
|
+
for (const b of decideBudgets()) {
|
|
1272
|
+
if (est > b.budgetMs)
|
|
1273
|
+
tight.push(`${r.ir.id} cannot be evaluated within ${b.harness}'s ${b.budgetMs.toLocaleString('en-US')} ms (estimated ${est.toLocaleString('en-US')} ms)`);
|
|
1274
|
+
}
|
|
1275
|
+
}
|
|
1276
|
+
out.push(tight.length === 0
|
|
1277
|
+
? { name: 'Decide Budget', passed: true, detail: `every rule fits every harness's budget (${records.length} rule(s))` }
|
|
1278
|
+
: {
|
|
1279
|
+
name: 'Decide Budget',
|
|
1280
|
+
passed: false,
|
|
1281
|
+
detail: `${tight.length} harness and rule pair(s) escalate instead of asking: ${tight.join('; ')}`,
|
|
1282
|
+
remediation: 'Ask fewer questions, raise --parallel on an engine that serves requests concurrently, or use a faster model.',
|
|
1283
|
+
});
|
|
1284
|
+
}
|
|
1285
|
+
else {
|
|
1286
|
+
out.push({ name: 'Decide Latency', passed: false, detail: 'the engine stopped answering while its p95 was measured' });
|
|
1287
|
+
}
|
|
1288
|
+
const disabled = node_fs.existsSync(disabledFile());
|
|
1289
|
+
const enforcing = records.filter((r) => r.rollout === 'enforce' && !decideEffectiveRollout(r, identity.modelDigest).why);
|
|
1290
|
+
out.push({
|
|
1291
|
+
name: 'Decide Rules',
|
|
1292
|
+
passed: true,
|
|
1293
|
+
detail: `${records.length} rule(s), ${enforcing.length} enforcing on this model${disabled ? '; disabled on this machine (`intutic decide enable`)' : ''}`,
|
|
1294
|
+
});
|
|
1295
|
+
return out;
|
|
1296
|
+
}
|
|
1297
|
+
//# sourceMappingURL=decide.js.map
|