dsh-decision-layer 0.7.1 → 0.7.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.en.md +14 -14
- package/README.md +16 -16
- package/docs/en/DETAIL.md +2 -2
- package/docs/zh/DETAIL.md +2 -2
- package/lib/client.js +54 -23
- package/lib/host/backend.d.ts +15 -0
- package/lib/host/backend.js +149 -0
- package/lib/host/config-store.d.ts +12 -0
- package/lib/host/config-store.js +254 -0
- package/lib/host/danger-gate.d.ts +45 -0
- package/lib/host/danger-gate.js +196 -0
- package/lib/host/decision-context.d.ts +21 -0
- package/lib/host/decision-context.js +187 -0
- package/lib/host/excerpt.d.ts +26 -0
- package/lib/host/excerpt.js +76 -0
- package/lib/host/index.d.ts +6 -0
- package/lib/host/index.js +287 -0
- package/lib/host/log-analyze.d.ts +121 -0
- package/lib/host/log-analyze.js +254 -0
- package/lib/host/log-read.d.ts +4 -0
- package/lib/host/log-read.js +34 -0
- package/lib/host/log-store.d.ts +15 -0
- package/lib/host/log-store.js +98 -0
- package/lib/host/loop-guard.d.ts +10 -0
- package/lib/host/loop-guard.js +52 -0
- package/lib/host/manager-api.d.ts +20 -0
- package/lib/host/manager-api.js +174 -0
- package/lib/host/narrowing.d.ts +51 -0
- package/lib/host/narrowing.js +214 -0
- package/lib/host/quality-trend.d.ts +14 -0
- package/lib/host/quality-trend.js +22 -0
- package/lib/host/self-check.d.ts +28 -0
- package/lib/host/self-check.js +111 -0
- package/lib/host/session-state.d.ts +19 -0
- package/lib/host/session-state.js +289 -0
- package/lib/host/state-store.d.ts +14 -0
- package/lib/host/state-store.js +109 -0
- package/lib/host/steer-once.d.ts +13 -0
- package/lib/host/steer-once.js +25 -0
- package/lib/host/task-completion.d.ts +30 -0
- package/lib/host/task-completion.js +166 -0
- package/lib/host/types.d.ts +365 -0
- package/lib/host/types.js +11 -0
- package/package.json +17 -10
- package/service/.gitkeep +0 -0
- package/service/backend.mjs +0 -127
- package/service/config-store.mjs +0 -311
- package/service/danger-gate.mjs +0 -182
- package/service/dsh.mjs +0 -288
- package/service/log-analyze.mjs +0 -257
- package/service/log-read.mjs +0 -27
- package/service/log-store.mjs +0 -82
- package/service/loop-guard.mjs +0 -51
- package/service/manager-api.mjs +0 -137
- package/service/narrowing.mjs +0 -226
- package/service/quality-trend.mjs +0 -29
- package/service/self-check.mjs +0 -131
- package/service/session-state.mjs +0 -260
- package/service/state-store.mjs +0 -100
- package/service/steer-once.mjs +0 -25
- package/service/task-completion.mjs +0 -163
package/service/loop-guard.mjs
DELETED
|
@@ -1,51 +0,0 @@
|
|
|
1
|
-
import { createHash } from 'node:crypto';
|
|
2
|
-
|
|
3
|
-
const MAX_FINGERPRINT_INPUT = 16 * 1024;
|
|
4
|
-
|
|
5
|
-
export function fingerprintCall(exec) {
|
|
6
|
-
const name = typeof exec?.name === 'string' ? exec.name : '';
|
|
7
|
-
let args = '';
|
|
8
|
-
try { args = JSON.stringify(exec?.arguments ?? null); } catch { args = ''; }
|
|
9
|
-
return createHash('sha256').update(name).update('\0').update(args.slice(0, MAX_FINGERPRINT_INPUT)).digest('hex');
|
|
10
|
-
}
|
|
11
|
-
|
|
12
|
-
// Deterministic, backend-free loop guard: it denies only a run of the SAME tool
|
|
13
|
-
// call repeated consecutively past a threshold. Any different intervening call
|
|
14
|
-
// resets the run, so a legitimate identical call reached again later is allowed.
|
|
15
|
-
export function createLoopGuard({ threshold = 3, maxSessions = 1024, sessions } = {}) {
|
|
16
|
-
if (!Number.isSafeInteger(threshold) || threshold < 2) throw new Error('Invalid loop threshold');
|
|
17
|
-
const runs = new Map();
|
|
18
|
-
const runFor = id => {
|
|
19
|
-
let run = runs.get(id);
|
|
20
|
-
if (!run) {
|
|
21
|
-
if (runs.size >= maxSessions) runs.delete(runs.keys().next().value);
|
|
22
|
-
run = { key: undefined, count: 0 };
|
|
23
|
-
runs.set(id, run);
|
|
24
|
-
}
|
|
25
|
-
return run;
|
|
26
|
-
};
|
|
27
|
-
return {
|
|
28
|
-
check(exec) {
|
|
29
|
-
const id = exec?.agent?.session?.id;
|
|
30
|
-
if (typeof id !== 'string') return undefined;
|
|
31
|
-
// The master switch pauses automatic intervention (mirrors the danger gate):
|
|
32
|
-
// a deliberate user toggle skips the loop guard too.
|
|
33
|
-
if (sessions) {
|
|
34
|
-
try {
|
|
35
|
-
if (sessions.snapshot(id).enabled === false) return undefined;
|
|
36
|
-
} catch { /* fall through and guard */ }
|
|
37
|
-
}
|
|
38
|
-
const run = runFor(id);
|
|
39
|
-
const key = fingerprintCall(exec);
|
|
40
|
-
run.count = key === run.key ? run.count + 1 : 1;
|
|
41
|
-
run.key = key;
|
|
42
|
-
if (run.count >= threshold) {
|
|
43
|
-
return `Identical tool call repeated ${run.count} times in a row; blocked to break a loop`;
|
|
44
|
-
}
|
|
45
|
-
return undefined;
|
|
46
|
-
},
|
|
47
|
-
runLength(id) {
|
|
48
|
-
return runs.get(id)?.count ?? 0;
|
|
49
|
-
},
|
|
50
|
-
};
|
|
51
|
-
}
|
package/service/manager-api.mjs
DELETED
|
@@ -1,137 +0,0 @@
|
|
|
1
|
-
import { loadConfig, resolveConfig, saveConfig } from './config-store.mjs';
|
|
2
|
-
|
|
3
|
-
export const API_PREFIX = '/plugins/dsh-decision-layer/api';
|
|
4
|
-
const BODY_LIMIT = 32 * 1024;
|
|
5
|
-
|
|
6
|
-
function respond(res, status, value) {
|
|
7
|
-
res.writeHead(status, { 'content-type': 'application/json; charset=utf-8', 'cache-control': 'no-store',
|
|
8
|
-
'x-content-type-options': 'nosniff', 'referrer-policy': 'no-referrer' });
|
|
9
|
-
res.end(JSON.stringify(value));
|
|
10
|
-
}
|
|
11
|
-
|
|
12
|
-
function authorizedBrowser(req) {
|
|
13
|
-
const address = req.socket?.remoteAddress;
|
|
14
|
-
if (!['127.0.0.1', '::1', '::ffff:127.0.0.1'].includes(address)) return false;
|
|
15
|
-
const host = req.headers.host;
|
|
16
|
-
if (typeof host !== 'string' || !/^(localhost|127\.0\.0\.1|\[::1\]):\d+$/.test(host)) return false;
|
|
17
|
-
if (req.headers['sec-fetch-site'] === 'cross-site') return false;
|
|
18
|
-
const origin = req.headers.origin;
|
|
19
|
-
if (origin === undefined) return req.method === 'GET';
|
|
20
|
-
try {
|
|
21
|
-
const parsed = new URL(origin);
|
|
22
|
-
return ['http:', 'https:'].includes(parsed.protocol) && parsed.host === host;
|
|
23
|
-
} catch { return false; }
|
|
24
|
-
}
|
|
25
|
-
|
|
26
|
-
async function readBody(req) {
|
|
27
|
-
if (!req.headers['content-type']?.startsWith('application/json')) throw new Error('Expected JSON');
|
|
28
|
-
let size = 0;
|
|
29
|
-
const chunks = [];
|
|
30
|
-
for await (const chunk of req) {
|
|
31
|
-
size += chunk.length;
|
|
32
|
-
if (size > BODY_LIMIT) throw new Error('Request too large');
|
|
33
|
-
chunks.push(chunk);
|
|
34
|
-
}
|
|
35
|
-
const value = JSON.parse(Buffer.concat(chunks).toString('utf8'));
|
|
36
|
-
if (value === null || typeof value !== 'object' || Array.isArray(value)) throw new Error('Expected object');
|
|
37
|
-
return value;
|
|
38
|
-
}
|
|
39
|
-
|
|
40
|
-
export function createManagerRoutes({ path, sessions, backend, logStore, analyze, listRows, env = process.env, bindHost = '127.0.0.1' }) {
|
|
41
|
-
const route = (suffix, methods) => ({ kind: 'exact', path: `${API_PREFIX}/${suffix}`, handler: async (req, res) => {
|
|
42
|
-
if (bindHost !== '127.0.0.1' || !authorizedBrowser(req)) return respond(res, 403, { ok: false, error: 'Local same-origin browser request required' });
|
|
43
|
-
if (!Object.hasOwn(methods, req.method)) return respond(res, 405, { ok: false, error: 'Method not allowed' });
|
|
44
|
-
try {
|
|
45
|
-
const value = await methods[req.method](req);
|
|
46
|
-
return respond(res, 200, { ok: true, value });
|
|
47
|
-
} catch (error) {
|
|
48
|
-
// A genuine internal fault (log analysis / annotation write) is a 500; a
|
|
49
|
-
// tagged or untagged validation/config error stays a client 400. Neither
|
|
50
|
-
// body carries error internals.
|
|
51
|
-
if (error?.code === 'INTERNAL') return respond(res, 500, { ok: false, error: 'Internal error' });
|
|
52
|
-
if (error?.code === 'BAD_REQUEST') return respond(res, 400, { ok: false, error: error.message });
|
|
53
|
-
return respond(res, 400, { ok: false, error: 'Invalid request or backend unavailable' });
|
|
54
|
-
}
|
|
55
|
-
} });
|
|
56
|
-
const sessionId = req => {
|
|
57
|
-
const id = new URL(req.url ?? '/', 'http://localhost').searchParams.get('sessionId');
|
|
58
|
-
if (!id) { const error = new Error('sessionId is required'); error.code = 'BAD_REQUEST'; throw error; }
|
|
59
|
-
return id;
|
|
60
|
-
};
|
|
61
|
-
return [
|
|
62
|
-
route('config', {
|
|
63
|
-
GET: async () => ({ ...(await loadConfig(path)), effectiveUrl: (await resolveConfig({ path, env })).url }),
|
|
64
|
-
PUT: async req => {
|
|
65
|
-
const saved = await saveConfig(await readBody(req), path);
|
|
66
|
-
return { ...saved, effectiveUrl: (await resolveConfig({ path, env })).url };
|
|
67
|
-
},
|
|
68
|
-
}),
|
|
69
|
-
route('probe', {
|
|
70
|
-
POST: async () => {
|
|
71
|
-
const effectiveUrl = (await resolveConfig({ path, env })).url;
|
|
72
|
-
try {
|
|
73
|
-
const result = await backend.evaluate({ state: 'Connection test', questions: {
|
|
74
|
-
connected: { type: 'noul', instructions: 'Is this a connection test?' },
|
|
75
|
-
} });
|
|
76
|
-
return { connected: true, model: result.model, usage: result.usage, effectiveUrl };
|
|
77
|
-
} catch { return { connected: false, reason: 'unavailable', effectiveUrl }; }
|
|
78
|
-
},
|
|
79
|
-
}),
|
|
80
|
-
route('metrics', { GET: async req => { const id = sessionId(req); await sessions.ensureLoaded?.(id); return sessions.snapshot(id); } }),
|
|
81
|
-
route('log', { GET: async req => { const id = sessionId(req); await sessions.ensureLoaded?.(id); return { entries: sessions.snapshot(id).log ?? [] }; } }),
|
|
82
|
-
// Read-only analysis over the persisted decision logs for one session. Returns
|
|
83
|
-
// only aggregates and annotation counts — never raw log rows — so gate command
|
|
84
|
-
// / path fragments are not echoed back to the browser.
|
|
85
|
-
route('logs', {
|
|
86
|
-
GET: async req => {
|
|
87
|
-
if (typeof analyze !== 'function') { const error = new Error('analysis unavailable'); error.code = 'BAD_REQUEST'; throw error; }
|
|
88
|
-
const id = sessionId(req);
|
|
89
|
-
try { return await analyze(id); }
|
|
90
|
-
catch (error) { if (error?.code === 'BAD_REQUEST') throw error; const wrapped = new Error('analysis failed'); wrapped.code = 'INTERNAL'; throw wrapped; }
|
|
91
|
-
},
|
|
92
|
-
}),
|
|
93
|
-
// One newest-first page of this session's full persisted decision rows, for
|
|
94
|
-
// the analysis tab's paginated log. Rows carry the same panel fields as /log,
|
|
95
|
-
// including the gate's already-redacted and length-bounded command/path, so
|
|
96
|
-
// this exposes nothing the recent-rows route does not already.
|
|
97
|
-
route('logrows', {
|
|
98
|
-
GET: async req => {
|
|
99
|
-
if (typeof listRows !== 'function') { const error = new Error('log rows unavailable'); error.code = 'BAD_REQUEST'; throw error; }
|
|
100
|
-
const id = sessionId(req);
|
|
101
|
-
const params = new URL(req.url ?? '/', 'http://localhost').searchParams;
|
|
102
|
-
const page = Number.parseInt(params.get('page') ?? '1', 10);
|
|
103
|
-
const pageSize = Number.parseInt(params.get('pageSize') ?? '100', 10);
|
|
104
|
-
try { return await listRows(id, { page: Number.isNaN(page) ? 1 : page, pageSize: Number.isNaN(pageSize) ? 100 : pageSize }); }
|
|
105
|
-
catch (error) { if (error?.code === 'BAD_REQUEST') throw error; const wrapped = new Error('log rows failed'); wrapped.code = 'INTERNAL'; throw wrapped; }
|
|
106
|
-
},
|
|
107
|
-
}),
|
|
108
|
-
// Append one annotation event for a decision. Explicit user intent: validated
|
|
109
|
-
// and written immediately; a write failure surfaces as an error, not silence.
|
|
110
|
-
route('annotate', {
|
|
111
|
-
POST: async req => {
|
|
112
|
-
if (!logStore || typeof logStore.appendAnnotation !== 'function') { const error = new Error('annotation unavailable'); error.code = 'BAD_REQUEST'; throw error; }
|
|
113
|
-
const body = await readBody(req);
|
|
114
|
-
const target = typeof body.target === 'string' ? body.target.trim() : '';
|
|
115
|
-
const rating = body.rating;
|
|
116
|
-
const session = typeof body.sessionId === 'string' ? body.sessionId.trim() : '';
|
|
117
|
-
if (!target || target.length > 128 || !['good', 'bad', 'unsure'].includes(rating) || !session || session.length > 512) {
|
|
118
|
-
const error = new Error('Invalid annotation'); error.code = 'BAD_REQUEST'; throw error;
|
|
119
|
-
}
|
|
120
|
-
// A validated write that still fails is an internal fault (disk / IO), not
|
|
121
|
-
// bad input: surface it as 500 so the client does not read it as a 400.
|
|
122
|
-
try { await logStore.appendAnnotation({ kind: 'annotation', target, rating, sessionId: session }); }
|
|
123
|
-
catch { const error = new Error('annotation write failed'); error.code = 'INTERNAL'; throw error; }
|
|
124
|
-
return { ok: true, target, rating };
|
|
125
|
-
},
|
|
126
|
-
}),
|
|
127
|
-
route('session', {
|
|
128
|
-
GET: async req => { const id = sessionId(req); await sessions.ensureLoaded?.(id); return sessions.snapshot(id); },
|
|
129
|
-
PUT: async req => {
|
|
130
|
-
const body = await readBody(req);
|
|
131
|
-
await sessions.ensureLoaded?.(body.sessionId);
|
|
132
|
-
sessions.setEnabled(body.sessionId, body.enabled);
|
|
133
|
-
return sessions.snapshot(body.sessionId);
|
|
134
|
-
},
|
|
135
|
-
}),
|
|
136
|
-
];
|
|
137
|
-
}
|
package/service/narrowing.mjs
DELETED
|
@@ -1,226 +0,0 @@
|
|
|
1
|
-
const isObject = value => value !== null && typeof value === 'object' && !Array.isArray(value);
|
|
2
|
-
const probability = value => typeof value === 'number' && Number.isFinite(value) && value >= 0 && value <= 1;
|
|
3
|
-
|
|
4
|
-
// Whether a narrowing pass actually restricted the tool set. `applied` is the
|
|
5
|
-
// authoritative field; a vetoed pass (low-keep, too many candidates) is never
|
|
6
|
-
// applied, and rows logged before `applied` existed are read from their reason.
|
|
7
|
-
export const narrowApplied = record => {
|
|
8
|
-
if (record?.applied === false) return false;
|
|
9
|
-
if (record?.reason === 'low-keep' || record?.reason === 'too-many-candidates') return false;
|
|
10
|
-
return record?.applied === true || record?.outcome === 'applied';
|
|
11
|
-
};
|
|
12
|
-
const MAX_STATE = 8 * 1024;
|
|
13
|
-
// The adjudication backend hard-rejects a request carrying more than 64 questions
|
|
14
|
-
// (backend.mjs validateRequest). One relevance question is sent per optional tool,
|
|
15
|
-
// so the candidate cap can never exceed this — otherwise evaluate() throws locally
|
|
16
|
-
// before any network call and the failure is mislabeled as a backend error.
|
|
17
|
-
const MAX_QUESTIONS = 64;
|
|
18
|
-
|
|
19
|
-
// Core tools an agent almost always needs. They are never judged for relevance
|
|
20
|
-
// and never dropped, so narrowing cannot strip the agent's fundamental
|
|
21
|
-
// capabilities. Beyond the filesystem/shell/search/todo basics, this includes
|
|
22
|
-
// the DSH agent meta-abilities (ask the user, goals, skills, delivery, images,
|
|
23
|
-
// background jobs, teammates, subagents): their relevance rarely shows in a
|
|
24
|
-
// single turn's task text, so per-tool judging would keep pruning them.
|
|
25
|
-
export const DEFAULT_CORE_TOOLS = Object.freeze([
|
|
26
|
-
'read', 'write', 'edit', 'bash', 'glob', 'grep', 'ls', 'todo_write',
|
|
27
|
-
'ask_user_question', 'create_goal', 'update_goal', 'get_goal', 'exit_plan_mode',
|
|
28
|
-
'skill', 'present', 'read_image', 'workflow',
|
|
29
|
-
'job_list', 'job_output', 'job_kill',
|
|
30
|
-
'spawn_teammate', 'interrupt_agent', 'list_agents', 'wait_agent', 'send_message',
|
|
31
|
-
'subagent', 'subagent_fork',
|
|
32
|
-
]);
|
|
33
|
-
|
|
34
|
-
// Tool-name prefixes that are always kept and never judged, on top of the core
|
|
35
|
-
// tools. Memory/context tools like OpenViking are "keep resident, trigger on
|
|
36
|
-
// demand": their relevance rarely shows in the literal task text, so per-tool
|
|
37
|
-
// relevance judging would keep pruning them. Teamwork tools (`team_task_`) are
|
|
38
|
-
// deliberately NOT in this list: they are ordinary task-board capabilities and
|
|
39
|
-
// should be judged like any other optional tool. Users can override this list.
|
|
40
|
-
export const DEFAULT_KEEP_PREFIXES = Object.freeze(['mcp__openviking']);
|
|
41
|
-
|
|
42
|
-
// Intersect the host-allowed tool set with the model's relevance verdict. Only
|
|
43
|
-
// removes tools; never adds. Any missing or invalid noul keeps the tool, and a
|
|
44
|
-
// verdict that would strip everything is skipped so the agent is never disabled.
|
|
45
|
-
// `keepBase` is the number of always-kept tools outside `allowed` (core tools and
|
|
46
|
-
// keep-prefix matches); the never-strip-everything guard must count them, or
|
|
47
|
-
// dropping every optional tool would be vetoed even though those tools remain.
|
|
48
|
-
// Drop threshold: a tool is dropped only when its relevance probability is below
|
|
49
|
-
// this value (noul < threshold). The default leans aggressive — only tools the
|
|
50
|
-
// model rates as fairly relevant survive, so the surface stays small. A noul near
|
|
51
|
-
// 0.5 means the model is unsure and is dropped under this default; lower the
|
|
52
|
-
// threshold to keep such may-need tools.
|
|
53
|
-
export const DEFAULT_NARROW_THRESHOLD = 0.7;
|
|
54
|
-
export function narrowTools(allowed, result, { threshold = DEFAULT_NARROW_THRESHOLD, keepBase = 0 } = {}) {
|
|
55
|
-
const tools = Array.isArray(allowed) ? allowed.filter(name => typeof name === 'string') : [];
|
|
56
|
-
const answers = isObject(result?.answers) ? result.answers : undefined;
|
|
57
|
-
if (tools.length === 0 || !answers) return { applied: false, keep: [...tools], drop: [] };
|
|
58
|
-
const base = Number.isSafeInteger(keepBase) && keepBase > 0 ? keepBase : 0;
|
|
59
|
-
|
|
60
|
-
const keep = [];
|
|
61
|
-
const drop = [];
|
|
62
|
-
for (const name of tools) {
|
|
63
|
-
const answer = answers[name];
|
|
64
|
-
if (isObject(answer) && answer.type === 'noul' && probability(answer.noul) && answer.noul < threshold) drop.push(name);
|
|
65
|
-
else keep.push(name);
|
|
66
|
-
}
|
|
67
|
-
if (drop.length === 0 || keep.length + base === 0) return { applied: false, keep: [...tools], drop: [] };
|
|
68
|
-
return { applied: true, keep, drop };
|
|
69
|
-
}
|
|
70
|
-
|
|
71
|
-
// Build one Noul relevance question per candidate tool. The state carries the
|
|
72
|
-
// current task; each question references the tool being judged. Noul answers
|
|
73
|
-
// have no confidence field, so relevance is read straight from the probability.
|
|
74
|
-
export function relevanceQuestions(state, tools) {
|
|
75
|
-
const task = typeof state === 'string' ? state.slice(0, MAX_STATE)
|
|
76
|
-
: isObject(state) ? state : '';
|
|
77
|
-
return {
|
|
78
|
-
state: task,
|
|
79
|
-
questions: Object.fromEntries(tools.map(tool => [tool.name, {
|
|
80
|
-
type: 'noul',
|
|
81
|
-
instructions: {
|
|
82
|
-
tool: { name: tool.name, description: typeof tool.description === 'string' ? tool.description.slice(0, 512) : '' },
|
|
83
|
-
question: '完成当前任务是否会用到 `tool` 描述的这个工具?',
|
|
84
|
-
},
|
|
85
|
-
}])),
|
|
86
|
-
};
|
|
87
|
-
}
|
|
88
|
-
|
|
89
|
-
// Orchestrates one relevance pass over the host-allowed tools. Pure of DSH
|
|
90
|
-
// runtime types: the caller supplies the candidate tools and the task state,
|
|
91
|
-
// and owns applying the restriction (only enforce mode should act on `drop`).
|
|
92
|
-
// `settings` resolves { mode: 'observe' | 'enforce', threshold } live.
|
|
93
|
-
//
|
|
94
|
-
// Guardrails against over-narrowing a large tool surface:
|
|
95
|
-
// A. core tools are always kept and never judged (DEFAULT_CORE_TOOLS);
|
|
96
|
-
// B. when the judged (optional) candidate count exceeds `maxCandidates`,
|
|
97
|
-
// enforce is downgraded to observe so a large surface cannot be mass-pruned;
|
|
98
|
-
// C. if narrowing would leave fewer than `minKeep` tools total, the verdict is
|
|
99
|
-
// treated as untrustworthy and no restriction is applied.
|
|
100
|
-
export function createNarrowing({ evaluate, sessions, settings, coreTools = DEFAULT_CORE_TOOLS,
|
|
101
|
-
keepPrefixes = DEFAULT_KEEP_PREFIXES, minCandidates = 2, maxCandidates = 20, minKeep = 5, splitConcurrency = 8 } = {}) {
|
|
102
|
-
if (typeof evaluate !== 'function') throw new Error('Narrowing requires an evaluator');
|
|
103
|
-
const core = new Set(Array.isArray(coreTools) ? coreTools.filter(name => typeof name === 'string') : []);
|
|
104
|
-
const defaultPrefixes = Array.isArray(keepPrefixes) ? keepPrefixes.filter(prefix => typeof prefix === 'string' && prefix) : [];
|
|
105
|
-
const concurrency = Number.isSafeInteger(splitConcurrency) && splitConcurrency >= 1 ? splitConcurrency : 8;
|
|
106
|
-
// Set once a 'single' (whole-batch) pass hits a context-overflow http-error:
|
|
107
|
-
// from then on 'auto' sends one request per tool. A large-context backend
|
|
108
|
-
// (Jev) never trips this and stays at one call; a small-context backend
|
|
109
|
-
// (tev1, ~2050-token prompt) self-corrects after the first failure.
|
|
110
|
-
let adaptiveSplit = false;
|
|
111
|
-
const resolveSettings = async () => {
|
|
112
|
-
let raw = {};
|
|
113
|
-
try { raw = (typeof settings === 'function' ? await settings() : settings) ?? {}; } catch { raw = {}; }
|
|
114
|
-
const mode = raw?.mode === 'observe' ? 'observe' : 'enforce';
|
|
115
|
-
const threshold = typeof raw?.threshold === 'number' && Number.isFinite(raw.threshold) && raw.threshold >= 0 && raw.threshold <= 1 ? raw.threshold : DEFAULT_NARROW_THRESHOLD;
|
|
116
|
-
const batchMode = raw?.batchMode === 'single' || raw?.batchMode === 'split' ? raw.batchMode : 'auto';
|
|
117
|
-
// An explicit array in settings (including empty) overrides the default set.
|
|
118
|
-
const prefixes = Array.isArray(raw?.keepPrefixes)
|
|
119
|
-
? raw.keepPrefixes.filter(prefix => typeof prefix === 'string' && prefix)
|
|
120
|
-
: defaultPrefixes;
|
|
121
|
-
// A configured optional-candidate ceiling overrides the constructor default,
|
|
122
|
-
// clamped to the backend's per-request question limit so the judged set is
|
|
123
|
-
// never large enough to be rejected before it is sent.
|
|
124
|
-
const configuredCap = Number.isSafeInteger(raw?.maxCandidates) && raw.maxCandidates >= 1 ? raw.maxCandidates : maxCandidates;
|
|
125
|
-
const cap = Math.min(configuredCap, MAX_QUESTIONS);
|
|
126
|
-
return { mode, threshold, prefixes, cap, batchMode };
|
|
127
|
-
};
|
|
128
|
-
const record = (turn, outcome, detail = {}) => {
|
|
129
|
-
const id = turn?.agent?.session?.id;
|
|
130
|
-
if (!sessions || typeof id !== 'string') return;
|
|
131
|
-
try { if (typeof sessions.recordNarrow === 'function') sessions.recordNarrow(id, outcome, detail.dropped ?? 0, narrowApplied({ ...detail, outcome })); }
|
|
132
|
-
catch { /* telemetry is best effort */ }
|
|
133
|
-
if (typeof sessions.log !== 'function') return;
|
|
134
|
-
try { sessions.log(id, { kind: 'narrow', outcome, ...detail }); }
|
|
135
|
-
catch { /* logging is best effort */ }
|
|
136
|
-
};
|
|
137
|
-
// Send one relevance request per tool with bounded concurrency, then merge the
|
|
138
|
-
// per-tool answers into one result. Each request's prompt is only the task
|
|
139
|
-
// state plus a single question, so it cannot exceed a small-context backend's
|
|
140
|
-
// prompt limit. A tool whose request fails is omitted from the merged answers,
|
|
141
|
-
// which narrowTools treats as "keep" (best effort). Returns null only when
|
|
142
|
-
// every tool failed, so the caller can still report a backend error.
|
|
143
|
-
const evaluateSplit = async (state, tools, signal) => {
|
|
144
|
-
const answers = Object.create(null);
|
|
145
|
-
let anyOk = false;
|
|
146
|
-
for (let i = 0; i < tools.length; i += concurrency) {
|
|
147
|
-
const group = tools.slice(i, i + concurrency);
|
|
148
|
-
const settled = await Promise.all(group.map(async tool => {
|
|
149
|
-
try {
|
|
150
|
-
const reply = await evaluate(relevanceQuestions(state, [tool]), { signal });
|
|
151
|
-
const answer = reply?.answers?.[tool.name];
|
|
152
|
-
return isObject(answer) ? [tool.name, answer] : undefined;
|
|
153
|
-
} catch { return undefined; }
|
|
154
|
-
}));
|
|
155
|
-
for (const pair of settled) { if (pair) { answers[pair[0]] = pair[1]; anyOk = true; } }
|
|
156
|
-
}
|
|
157
|
-
return anyOk ? { model: 'split', answers } : null;
|
|
158
|
-
};
|
|
159
|
-
const skip = { evaluated: false, applied: false, keep: [], drop: [], mode: 'observe' };
|
|
160
|
-
return {
|
|
161
|
-
async review(turn) {
|
|
162
|
-
const tools = Array.isArray(turn?.tools) ? turn.tools.filter(tool => tool && typeof tool.name === 'string') : [];
|
|
163
|
-
const names = tools.map(tool => tool.name);
|
|
164
|
-
const { mode, threshold, prefixes, cap, batchMode } = await resolveSettings();
|
|
165
|
-
// A: core tools and keep-prefix matches are always kept and never sent to
|
|
166
|
-
// the model for judging (e.g. mcp__openviking memory tools stay resident).
|
|
167
|
-
const alwaysKeep = name => core.has(name) || prefixes.some(prefix => name.startsWith(prefix));
|
|
168
|
-
const keptNames = names.filter(alwaysKeep);
|
|
169
|
-
const optional = tools.filter(tool => !alwaysKeep(tool.name));
|
|
170
|
-
if (optional.length < minCandidates) return { ...skip, keep: names };
|
|
171
|
-
const hasState = typeof turn?.state === 'string' ? turn.state.trim().length > 0
|
|
172
|
-
: isObject(turn?.state) ? Object.keys(turn.state).length > 0 : false;
|
|
173
|
-
if (!hasState) return { ...skip, keep: names };
|
|
174
|
-
const id = turn?.agent?.session?.id;
|
|
175
|
-
if (sessions && typeof id === 'string') {
|
|
176
|
-
try { if (sessions.snapshot(id).enabled === false) return { ...skip, keep: names }; }
|
|
177
|
-
catch { return { ...skip, keep: names }; }
|
|
178
|
-
}
|
|
179
|
-
// B: a large optional surface must not be enforced — only observed. When
|
|
180
|
-
// over the cap we still evaluate a bounded subset (never more than the
|
|
181
|
-
// backend's question limit) so the pass is logged without throwing, but the
|
|
182
|
-
// result is observe-only and applies no restriction.
|
|
183
|
-
const overCap = optional.length > cap;
|
|
184
|
-
const effectiveMode = overCap ? 'observe' : mode;
|
|
185
|
-
const judged = overCap ? optional.slice(0, cap) : optional;
|
|
186
|
-
const judgedNames = judged.map(tool => tool.name);
|
|
187
|
-
// Choose per-request strategy by backend context capacity. 'split' (or an
|
|
188
|
-
// 'auto' pass that already fell back) sends one request per tool so no
|
|
189
|
-
// prompt exceeds a small-context limit. 'single'/'auto' first try one
|
|
190
|
-
// whole-batch request; under 'auto', a context-overflow http-error flips
|
|
191
|
-
// this instance to split from then on and retries the same tools split.
|
|
192
|
-
let result;
|
|
193
|
-
if (batchMode === 'split' || adaptiveSplit) {
|
|
194
|
-
result = await evaluateSplit(turn.state, judged, turn.signal);
|
|
195
|
-
if (!result) { record(turn, 'error', { reason: 'unreachable', mode: effectiveMode }); return { ...skip, keep: names, mode: effectiveMode }; }
|
|
196
|
-
} else {
|
|
197
|
-
try { result = await evaluate(relevanceQuestions(turn.state, judged), { signal: turn.signal }); }
|
|
198
|
-
catch (error) {
|
|
199
|
-
const reason = typeof error?.reason === 'string' ? error.reason : 'unreachable';
|
|
200
|
-
if (batchMode === 'auto' && reason === 'http-error') {
|
|
201
|
-
adaptiveSplit = true;
|
|
202
|
-
result = await evaluateSplit(turn.state, judged, turn.signal);
|
|
203
|
-
if (!result) { record(turn, 'error', { reason, mode: effectiveMode }); return { ...skip, keep: names, mode: effectiveMode }; }
|
|
204
|
-
} else {
|
|
205
|
-
record(turn, 'error', { reason, mode: effectiveMode });
|
|
206
|
-
return { ...skip, keep: names, mode: effectiveMode };
|
|
207
|
-
}
|
|
208
|
-
}
|
|
209
|
-
}
|
|
210
|
-
const narrowed = narrowTools(judgedNames, result, { threshold, keepBase: keptNames.length });
|
|
211
|
-
const keep = [...keptNames, ...narrowed.keep];
|
|
212
|
-
const drop = narrowed.drop;
|
|
213
|
-
// C: too few tools left over is a sign the verdict over-pruned; do not apply.
|
|
214
|
-
const trustworthy = keep.length >= minKeep;
|
|
215
|
-
const applied = narrowed.applied && trustworthy && !overCap && mode === 'enforce';
|
|
216
|
-
const dropped = drop.length;
|
|
217
|
-
const detail = { dropped, kept: keep.length, mode: effectiveMode, tools: drop, applied };
|
|
218
|
-
// The optional-candidate count (excludes core tools and keep-prefix matches),
|
|
219
|
-
// recorded so the "too many candidates" reason shows the number that tripped it.
|
|
220
|
-
if (overCap) { detail.reason = 'too-many-candidates'; detail.candidates = optional.length; }
|
|
221
|
-
else if (dropped > 0 && !trustworthy) detail.reason = 'low-keep';
|
|
222
|
-
record(turn, dropped > 0 ? 'applied' : 'ok', detail);
|
|
223
|
-
return { evaluated: true, applied, keep, drop, mode: effectiveMode, overCap, trustworthy };
|
|
224
|
-
},
|
|
225
|
-
};
|
|
226
|
-
}
|
|
@@ -1,29 +0,0 @@
|
|
|
1
|
-
// Pure helpers for the output-quality trend alarm. A run counts consecutive
|
|
2
|
-
// high-confidence low self-check scores; the run length maps to a severity that
|
|
3
|
-
// the trigger icon renders (normal → warn → severe). All thresholds are
|
|
4
|
-
// unvalidated heuristics, tuned later on real decision logs.
|
|
5
|
-
|
|
6
|
-
export const DEFAULT_TREND_MIN_CONFIDENCE = 0.6;
|
|
7
|
-
export const DEFAULT_TREND_RUN = 3;
|
|
8
|
-
export const DEFAULT_TREND_SEVERE_RUN = 5;
|
|
9
|
-
|
|
10
|
-
// Advance the run over ONE evaluated self-check:
|
|
11
|
-
// - a good score (not low) resets the run to 0;
|
|
12
|
-
// - a high-confidence low score increments it;
|
|
13
|
-
// - a low-confidence low score is skipped (neither resets nor increments);
|
|
14
|
-
// - an unevaluated/absent verdict must not call this at all.
|
|
15
|
-
// `sample` is { lowScore: boolean, confident: boolean }.
|
|
16
|
-
export function advanceRun(run, sample) {
|
|
17
|
-
const current = Number.isSafeInteger(run) && run >= 0 ? run : 0;
|
|
18
|
-
if (!sample || sample.lowScore !== true) return 0;
|
|
19
|
-
if (sample.confident === true) return current + 1;
|
|
20
|
-
return current;
|
|
21
|
-
}
|
|
22
|
-
|
|
23
|
-
// Map a run length to a severity, given the warn/severe thresholds.
|
|
24
|
-
export function severityOf(run, { trendRun = DEFAULT_TREND_RUN, trendSevereRun = DEFAULT_TREND_SEVERE_RUN } = {}) {
|
|
25
|
-
const value = Number.isSafeInteger(run) && run >= 0 ? run : 0;
|
|
26
|
-
if (value >= trendSevereRun) return 'severe';
|
|
27
|
-
if (value >= trendRun) return 'warn';
|
|
28
|
-
return 'normal';
|
|
29
|
-
}
|
package/service/self-check.mjs
DELETED
|
@@ -1,131 +0,0 @@
|
|
|
1
|
-
import { createSteerOnce } from './steer-once.mjs';
|
|
2
|
-
|
|
3
|
-
const MAX_OUTPUT = 24 * 1024;
|
|
4
|
-
|
|
5
|
-
export const DEFAULT_RUBRIC = Object.freeze([
|
|
6
|
-
'明显答非所问,或存在严重遗漏 / 自相矛盾',
|
|
7
|
-
'基本答到问题,但有可察觉的遗漏或不严谨',
|
|
8
|
-
'完整、准确、无自相矛盾地回应了请求',
|
|
9
|
-
]);
|
|
10
|
-
|
|
11
|
-
const STEER_TEXT = '本回合自检得分偏低:请复核是否答到了用户的问题、有无遗漏或自相矛盾,并在需要时补完。';
|
|
12
|
-
|
|
13
|
-
// A short, single-line hint of the request this self-check judged, so a human
|
|
14
|
-
// reviewing the log knows WHICH turn's output was scored (not just "self-check").
|
|
15
|
-
// Best-effort over an untrusted string: injected OpenViking memory context blocks
|
|
16
|
-
// (which are not part of the user's actual request) are stripped first, then
|
|
17
|
-
// whitespace collapses and the length is capped.
|
|
18
|
-
function subjectOf(text) {
|
|
19
|
-
if (typeof text !== 'string') return undefined;
|
|
20
|
-
const cleaned = text
|
|
21
|
-
// Drop any complete injected memory blocks (there can be several, multi-line).
|
|
22
|
-
.replace(/<openviking-context\b[^>]*>[\s\S]*?<\/openviking-context>/gi, ' ')
|
|
23
|
-
// Drop a dangling, unclosed block (e.g. a truncated tail) through to the end.
|
|
24
|
-
.replace(/<openviking-context\b[\s\S]*$/i, ' ')
|
|
25
|
-
.replace(/\s+/g, ' ')
|
|
26
|
-
.trim();
|
|
27
|
-
return cleaned ? cleaned.slice(0, 120) : undefined;
|
|
28
|
-
}
|
|
29
|
-
|
|
30
|
-
function boundedRubric(rubric) {
|
|
31
|
-
if (!Array.isArray(rubric)) return DEFAULT_RUBRIC;
|
|
32
|
-
const levels = rubric.filter(item => typeof item === 'string' && item.trim() && item.length <= 256).slice(0, 10);
|
|
33
|
-
return levels.length >= 2 ? levels : DEFAULT_RUBRIC;
|
|
34
|
-
}
|
|
35
|
-
|
|
36
|
-
const isProbability = value => typeof value === 'number' && Number.isFinite(value) && value >= 0 && value <= 1;
|
|
37
|
-
|
|
38
|
-
export function scoreQuestion(output, rubric = DEFAULT_RUBRIC) {
|
|
39
|
-
return {
|
|
40
|
-
state: { answer: String(output).slice(0, MAX_OUTPUT) },
|
|
41
|
-
questions: { quality: {
|
|
42
|
-
type: 'score',
|
|
43
|
-
instructions: '按下列从低到高的档位,评估这次最终产出对用户请求的完成质量。',
|
|
44
|
-
criteria: boundedRubric(rubric),
|
|
45
|
-
} },
|
|
46
|
-
};
|
|
47
|
-
}
|
|
48
|
-
|
|
49
|
-
// The confidence floor for interrupting the user with a supplemental prompt.
|
|
50
|
-
// Named rather than inlined because config-store's `checkSettings.minConfidence`
|
|
51
|
-
// and the config form both document this value; a bare literal in three places
|
|
52
|
-
// drifts silently on the next tuning pass.
|
|
53
|
-
export const DEFAULT_MIN_CONFIDENCE = 0.4;
|
|
54
|
-
|
|
55
|
-
// `settings` is an async resolver so rubric/mode/threshold follow live config.
|
|
56
|
-
// `steer(agent, text)` performs the host-native steer; the caller owns building
|
|
57
|
-
// the framework UserMessage so this module stays free of DSH message types.
|
|
58
|
-
export function createSelfCheck({ evaluate, sessions, steer, settings, minConfidence = DEFAULT_MIN_CONFIDENCE } = {}) {
|
|
59
|
-
if (typeof evaluate !== 'function') throw new Error('Self-check requires an evaluator');
|
|
60
|
-
const steerOnce = createSteerOnce();
|
|
61
|
-
const resolveSettings = async () => {
|
|
62
|
-
let raw = {};
|
|
63
|
-
try { raw = (typeof settings === 'function' ? await settings() : settings) ?? {}; } catch { raw = {}; }
|
|
64
|
-
const mode = raw?.mode === 'steer' ? 'steer' : 'observe';
|
|
65
|
-
const lowScoreThreshold = Number.isSafeInteger(raw?.lowScoreThreshold) && raw.lowScoreThreshold >= 0 ? raw.lowScoreThreshold : 1;
|
|
66
|
-
// Confidence required before a low score actually opens the supplemental
|
|
67
|
-
// prompt. The constructor argument is the fallback so a direct caller (and a
|
|
68
|
-
// config without the field) keeps its own floor.
|
|
69
|
-
const confidenceFloor = isProbability(raw?.minConfidence) ? raw.minConfidence : minConfidence;
|
|
70
|
-
// Quality-trend thresholds (unvalidated heuristics; see quality-trend.mjs).
|
|
71
|
-
const trendMinConfidence = typeof raw?.trendMinConfidence === 'number' && Number.isFinite(raw.trendMinConfidence) && raw.trendMinConfidence >= 0 && raw.trendMinConfidence <= 1 ? raw.trendMinConfidence : 0.6;
|
|
72
|
-
const trendRun = Number.isSafeInteger(raw?.trendRun) && raw.trendRun >= 2 ? raw.trendRun : 3;
|
|
73
|
-
const trendSevereRun = Number.isSafeInteger(raw?.trendSevereRun) && raw.trendSevereRun > trendRun ? raw.trendSevereRun : Math.max(trendRun + 1, 5);
|
|
74
|
-
return { mode, rubric: boundedRubric(raw?.rubric), lowScoreThreshold, minConfidence: confidenceFloor, trendMinConfidence, trendRun, trendSevereRun };
|
|
75
|
-
};
|
|
76
|
-
const record = (turn, outcome, detail = {}, trend) => {
|
|
77
|
-
const id = turn?.agent?.session?.id;
|
|
78
|
-
if (!sessions || typeof id !== 'string' || typeof sessions.recordCheck !== 'function') return;
|
|
79
|
-
try { sessions.recordCheck(id, outcome, trend); } catch { /* telemetry is best effort */ }
|
|
80
|
-
if (typeof sessions.log !== 'function') return;
|
|
81
|
-
try { sessions.log(id, { kind: 'check', outcome, ...detail }); }
|
|
82
|
-
catch { /* logging is best effort */ }
|
|
83
|
-
};
|
|
84
|
-
return {
|
|
85
|
-
async review(turn) {
|
|
86
|
-
const output = typeof turn?.output === 'string' ? turn.output.trim() : '';
|
|
87
|
-
if (!output) return { evaluated: false, lowScore: false, steered: false };
|
|
88
|
-
const id = turn?.agent?.session?.id;
|
|
89
|
-
if (sessions && typeof id === 'string') {
|
|
90
|
-
try { if (sessions.snapshot(id).enabled === false) return { evaluated: false, lowScore: false, steered: false }; }
|
|
91
|
-
catch { return { evaluated: false, lowScore: false, steered: false }; }
|
|
92
|
-
}
|
|
93
|
-
// The request this turn answered, carried onto every log row so a reviewer
|
|
94
|
-
// can tell which turn's output each self-check score belongs to.
|
|
95
|
-
const subject = subjectOf(turn?.userText);
|
|
96
|
-
const subjectDetail = subject ? { subject } : {};
|
|
97
|
-
const { mode, rubric, lowScoreThreshold, minConfidence: confidenceFloor, trendMinConfidence, trendRun, trendSevereRun } = await resolveSettings();
|
|
98
|
-
let answer;
|
|
99
|
-
try {
|
|
100
|
-
const result = await evaluate(scoreQuestion(output, rubric), { signal: turn.signal });
|
|
101
|
-
answer = result?.answers?.quality;
|
|
102
|
-
} catch (error) { record(turn, 'error', { ...subjectDetail, reason: typeof error?.reason === 'string' ? error.reason : 'unreachable' }); return { evaluated: false, lowScore: false, steered: false }; }
|
|
103
|
-
if (!answer || answer.type !== 'score' || typeof answer.score !== 'number' || !Number.isFinite(answer.score)) {
|
|
104
|
-
record(turn, 'error', { ...subjectDetail, reason: 'invalid-response' });
|
|
105
|
-
return { evaluated: false, lowScore: false, steered: false };
|
|
106
|
-
}
|
|
107
|
-
// A low-confidence Score is a legitimate answer, not a failure: the score
|
|
108
|
-
// stays usable and is recorded as ok/low. Confidence only gates the steer,
|
|
109
|
-
// because only interrupting the user warrants the model being sure.
|
|
110
|
-
const confidence = typeof answer.confidence === 'number' && Number.isFinite(answer.confidence) ? answer.confidence : undefined;
|
|
111
|
-
const detail = confidence === undefined ? { ...subjectDetail } : { ...subjectDetail, confidence };
|
|
112
|
-
const lowScore = answer.score <= lowScoreThreshold;
|
|
113
|
-
// The trend run advances over every evaluated score; a high-confidence low
|
|
114
|
-
// score is the only thing that grows it. Thresholds flow to session-state.
|
|
115
|
-
const trend = { confident: lowScore && confidence !== undefined && confidence >= trendMinConfidence, trendRun, trendSevereRun };
|
|
116
|
-
// A low score the model is not confident about (confidence present but below
|
|
117
|
-
// the trend threshold) does not advance the quality-trend judgment. Flag it
|
|
118
|
-
// so the decision log can mark it as low-signal noise instead of a real dip.
|
|
119
|
-
if (lowScore && confidence !== undefined && !trend.confident) detail.lowConfidence = true;
|
|
120
|
-
record(turn, lowScore ? 'low' : 'ok', { score: answer.score, ...detail }, trend);
|
|
121
|
-
if (!lowScore) return { evaluated: true, lowScore: false, steered: false, score: answer.score };
|
|
122
|
-
const confident = confidence !== undefined && confidence >= confidenceFloor;
|
|
123
|
-
// The turn budget is consumed only on a successful steer (createSteerOnce),
|
|
124
|
-
// so a transient host failure does not permanently suppress this turn.
|
|
125
|
-
const steered = mode === 'steer' && confident && typeof steer === 'function' && turn?.agent
|
|
126
|
-
? steerOnce.attempt(turn, () => steer(turn.agent, STEER_TEXT))
|
|
127
|
-
: false;
|
|
128
|
-
return { evaluated: true, lowScore: true, steered, score: answer.score };
|
|
129
|
-
},
|
|
130
|
-
};
|
|
131
|
-
}
|