dsh-decision-layer 0.7.2 → 0.7.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.en.md +11 -11
- package/README.md +11 -11
- package/docs/en/DETAIL.md +2 -2
- package/docs/zh/DETAIL.md +2 -2
- package/lib/client.js +88 -25
- package/lib/host/decision-context.d.ts +21 -0
- package/lib/host/decision-context.js +206 -0
- package/lib/host/excerpt.d.ts +1 -0
- package/lib/host/excerpt.js +17 -3
- package/lib/host/index.js +88 -91
- package/lib/host/log-analyze.d.ts +2 -1
- package/lib/host/log-analyze.js +3 -2
- package/lib/host/manager-api.d.ts +1 -0
- package/lib/host/manager-api.js +5 -2
- package/lib/host/narrowing.d.ts +3 -0
- package/lib/host/narrowing.js +3 -2
- package/lib/host/self-check.d.ts +3 -2
- package/lib/host/self-check.js +23 -4
- package/lib/host/session-state.js +1 -0
- package/lib/host/task-completion.d.ts +3 -2
- package/lib/host/task-completion.js +26 -8
- package/lib/host/types.d.ts +22 -0
- package/package.json +1 -1
package/lib/host/self-check.js
CHANGED
|
@@ -2,20 +2,38 @@ import { taskExcerpt } from "./excerpt.js";
|
|
|
2
2
|
import { createSteerOnce } from "./steer-once.js";
|
|
3
3
|
import { errorReason, isProbability } from "./types.js";
|
|
4
4
|
const MAX_OUTPUT = 24 * 1024;
|
|
5
|
+
const ANSWER_TAIL = 8 * 1024;
|
|
6
|
+
const OMITTED = "\n\n\u2026\uFF08\u4E2D\u95F4\u5185\u5BB9\u5DF2\u7701\u7565\uFF09\u2026\n\n";
|
|
5
7
|
const DEFAULT_RUBRIC = Object.freeze([
|
|
6
8
|
"\u660E\u663E\u7B54\u975E\u6240\u95EE\uFF0C\u6216\u5B58\u5728\u4E25\u91CD\u9057\u6F0F / \u81EA\u76F8\u77DB\u76FE",
|
|
7
9
|
"\u57FA\u672C\u7B54\u5230\u95EE\u9898\uFF0C\u4F46\u6709\u53EF\u5BDF\u89C9\u7684\u9057\u6F0F\u6216\u4E0D\u4E25\u8C28",
|
|
8
10
|
"\u5B8C\u6574\u3001\u51C6\u786E\u3001\u65E0\u81EA\u76F8\u77DB\u76FE\u5730\u56DE\u5E94\u4E86\u8BF7\u6C42"
|
|
9
11
|
]);
|
|
10
12
|
const STEER_TEXT = "\u672C\u56DE\u5408\u81EA\u68C0\u5F97\u5206\u504F\u4F4E\uFF1A\u8BF7\u590D\u6838\u662F\u5426\u7B54\u5230\u4E86\u7528\u6237\u7684\u95EE\u9898\u3001\u6709\u65E0\u9057\u6F0F\u6216\u81EA\u76F8\u77DB\u76FE\uFF0C\u5E76\u5728\u9700\u8981\u65F6\u8865\u5B8C\u3002";
|
|
13
|
+
function answerExcerpt(output) {
|
|
14
|
+
const text = String(output);
|
|
15
|
+
if (text.length <= MAX_OUTPUT) return text;
|
|
16
|
+
return `${text.slice(0, MAX_OUTPUT - ANSWER_TAIL)}${OMITTED}${text.slice(-ANSWER_TAIL)}`;
|
|
17
|
+
}
|
|
11
18
|
function boundedRubric(rubric) {
|
|
12
19
|
if (!Array.isArray(rubric)) return DEFAULT_RUBRIC;
|
|
13
20
|
const levels = rubric.filter((item) => typeof item === "string" && Boolean(item.trim()) && item.length <= 256).slice(0, 10);
|
|
14
21
|
return levels.length >= 2 ? levels : DEFAULT_RUBRIC;
|
|
15
22
|
}
|
|
16
|
-
function scoreQuestion(output, rubric = DEFAULT_RUBRIC) {
|
|
23
|
+
function scoreQuestion(output, rubric = DEFAULT_RUBRIC, context) {
|
|
17
24
|
return {
|
|
18
|
-
state: {
|
|
25
|
+
state: {
|
|
26
|
+
// What the turn was asked to do. Without it the model scores an answer
|
|
27
|
+
// against nothing and cannot detect an off-topic reply.
|
|
28
|
+
...context?.request ? { request: context.request } : {},
|
|
29
|
+
...context?.background?.length ? { background: context.background } : {},
|
|
30
|
+
answer: answerExcerpt(output),
|
|
31
|
+
// Short follow-ups ("继续", "补上", "按这个标准重新打分") have no standalone
|
|
32
|
+
// meaning; the task they continue is in background. Saying so explicitly
|
|
33
|
+
// stops the model from treating the follow-up as a new, empty request and
|
|
34
|
+
// from borrowing an unrelated earlier task as this turn's goal.
|
|
35
|
+
...context ? { note: "request \u662F\u672C\u56DE\u5408\u7528\u6237\u8BF7\u6C42\uFF1B\u82E5\u5B83\u53EA\u662F\u300C\u7EE7\u7EED\u300D\u300C\u8865\u4E0A\u300D\u8FD9\u7C7B\u77ED\u8FFD\u95EE\uFF0C\u8BF7\u7ED3\u5408 background \u4E2D\u627F\u63A5\u7684\u4EFB\u52A1\u5224\u65AD\uFF0C\u4E0D\u8981\u628A background \u91CC\u7684\u65E7\u4EFB\u52A1\u5F53\u6210\u65B0\u7684\u8981\u6C42\u3002background \u53EA\u662F\u80CC\u666F\uFF0C\u4E0D\u662F\u672C\u8F6E\u7684\u65B0\u8981\u6C42\u3002answer \u662F\u672C\u56DE\u5408\u6700\u7EC8\u56DE\u590D\u3002" } : {}
|
|
36
|
+
},
|
|
19
37
|
questions: { quality: {
|
|
20
38
|
type: "score",
|
|
21
39
|
instructions: "\u6309\u4E0B\u5217\u4ECE\u4F4E\u5230\u9AD8\u7684\u6863\u4F4D\uFF0C\u8BC4\u4F30\u8FD9\u6B21\u6700\u7EC8\u4EA7\u51FA\u5BF9\u7528\u6237\u8BF7\u6C42\u7684\u5B8C\u6210\u8D28\u91CF\u3002",
|
|
@@ -59,6 +77,7 @@ function createSelfCheck({ evaluate, sessions, steer, settings, minConfidence =
|
|
|
59
77
|
async review(turn) {
|
|
60
78
|
const output = typeof turn?.output === "string" ? turn.output.trim() : "";
|
|
61
79
|
if (!output) return { evaluated: false, lowScore: false, steered: false };
|
|
80
|
+
if (turn?.context !== void 0 && !turn.context.request) return { evaluated: false, lowScore: false, steered: false };
|
|
62
81
|
const id = turn?.agent?.session?.id;
|
|
63
82
|
if (sessions && typeof id === "string") {
|
|
64
83
|
try {
|
|
@@ -67,13 +86,13 @@ function createSelfCheck({ evaluate, sessions, steer, settings, minConfidence =
|
|
|
67
86
|
return { evaluated: false, lowScore: false, steered: false };
|
|
68
87
|
}
|
|
69
88
|
}
|
|
70
|
-
const task = taskExcerpt(turn?.userText);
|
|
89
|
+
const task = taskExcerpt(turn?.context?.request ?? turn?.userText);
|
|
71
90
|
const taskDetail = task ? { task } : {};
|
|
72
91
|
if (Number.isSafeInteger(turn?.turn) && turn?.turn >= 0) taskDetail.turn = turn?.turn;
|
|
73
92
|
const { mode, rubric, lowScoreThreshold, minConfidence: confidenceFloor, trendMinConfidence, trendRun, trendSevereRun } = await resolveSettings();
|
|
74
93
|
let answer;
|
|
75
94
|
try {
|
|
76
|
-
const result = await evaluate(scoreQuestion(output, rubric), { signal: turn?.signal });
|
|
95
|
+
const result = await evaluate(scoreQuestion(output, rubric, turn?.context), { signal: turn?.signal });
|
|
77
96
|
answer = result?.answers?.quality;
|
|
78
97
|
} catch (error) {
|
|
79
98
|
record(turn, "error", { ...taskDetail, reason: errorReason(error) });
|
|
@@ -248,6 +248,7 @@ function createSessionState({ maxSessions = 1024, maxLogEntries = 50, onLog, loa
|
|
|
248
248
|
if (typeof entry.path === "string" && entry.path.trim()) record.path = entry.path.replace(/\s+/g, " ").trim().slice(0, 512);
|
|
249
249
|
if (typeof entry.task === "string" && entry.task.trim()) record.task = redactCredentials(entry.task).replace(/\s+/g, " ").trim().slice(0, 512);
|
|
250
250
|
if (Number.isSafeInteger(entry.turn) && entry.turn >= 0) record.turn = entry.turn;
|
|
251
|
+
if (Number.isSafeInteger(entry.step) && entry.step >= 0) record.step = entry.step;
|
|
251
252
|
if (typeof entry.mode === "string" && entry.mode) record.mode = entry.mode.slice(0, 16);
|
|
252
253
|
if (typeof entry.score === "number" && Number.isFinite(entry.score)) record.score = Math.round(entry.score * 100) / 100;
|
|
253
254
|
if (typeof entry.confidence === "number" && Number.isFinite(entry.confidence)) record.confidence = Math.round(entry.confidence * 100) / 100;
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import type { AgentLike, CompleteSettings, EvaluateInput, Evaluator, SessionsFace } from './types.js';
|
|
1
|
+
import type { AgentLike, CompleteSettings, DecisionContext, EvaluateInput, Evaluator, SessionsFace } from './types.js';
|
|
2
2
|
export declare function extractConditions(userText: unknown): string[];
|
|
3
|
-
export declare function completionQuestions(conditions: unknown, evidence: unknown, answer: unknown): EvaluateInput;
|
|
3
|
+
export declare function completionQuestions(conditions: unknown, evidence: unknown, answer: unknown, context?: DecisionContext): EvaluateInput;
|
|
4
4
|
export declare const DEFAULT_MIN_CONFIDENCE = 0.4;
|
|
5
5
|
export type CompletionTurn = {
|
|
6
6
|
agent?: AgentLike;
|
|
@@ -9,6 +9,7 @@ export type CompletionTurn = {
|
|
|
9
9
|
output?: unknown;
|
|
10
10
|
userText?: unknown;
|
|
11
11
|
evidence?: unknown;
|
|
12
|
+
context?: DecisionContext;
|
|
12
13
|
};
|
|
13
14
|
export type CompletionResult = {
|
|
14
15
|
evaluated: boolean;
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { taskExcerpt } from "./excerpt.js";
|
|
1
2
|
import { createSteerOnce } from "./steer-once.js";
|
|
2
3
|
import { errorReason, isObject, isProbability } from "./types.js";
|
|
3
4
|
const MAX_STATE = 8 * 1024;
|
|
@@ -30,19 +31,33 @@ function extractConditions(userText) {
|
|
|
30
31
|
const compact = text.trim();
|
|
31
32
|
if (!/[\r\n]/.test(compact)) {
|
|
32
33
|
const parts = compact.split(/[、,,]/).map((part) => part.trim()).filter(Boolean);
|
|
33
|
-
|
|
34
|
+
const floor = compact.includes("\u3001") ? 2 : 3;
|
|
35
|
+
if (parts.length >= floor) for (const part of parts) push(part);
|
|
34
36
|
}
|
|
35
37
|
}
|
|
36
38
|
return conditions.slice(0, MAX_CONDITIONS);
|
|
37
39
|
}
|
|
38
|
-
function completionQuestions(conditions, evidence, answer) {
|
|
40
|
+
function completionQuestions(conditions, evidence, answer, context) {
|
|
39
41
|
const items = Array.isArray(conditions) ? conditions.slice(0, MAX_CONDITIONS) : [];
|
|
40
|
-
const tools = Array.isArray(evidence) ? evidence.filter((item) => isObject(item) && typeof item.tool === "string").slice(0, MAX_EVIDENCE).map((item) =>
|
|
42
|
+
const tools = Array.isArray(evidence) ? evidence.filter((item) => isObject(item) && typeof item.tool === "string").slice(0, MAX_EVIDENCE).map((item) => {
|
|
43
|
+
const source = item;
|
|
44
|
+
const entry = {
|
|
45
|
+
tool: source.tool.slice(0, 128),
|
|
46
|
+
outcome: source.outcome === "error" ? "error" : "ok"
|
|
47
|
+
};
|
|
48
|
+
for (const key of ["command", "path", "result"]) {
|
|
49
|
+
const value = source[key];
|
|
50
|
+
if (typeof value === "string" && value.trim()) entry[key] = value.slice(0, 256);
|
|
51
|
+
}
|
|
52
|
+
return entry;
|
|
53
|
+
}) : [];
|
|
41
54
|
const state = {
|
|
55
|
+
...context?.request ? { request: context.request } : {},
|
|
56
|
+
...context?.background?.length ? { background: context.background } : {},
|
|
42
57
|
conditions: items.map((text, index) => ({ id: `c${index}`, text })),
|
|
43
58
|
toolEvidence: tools,
|
|
44
59
|
finalReply: String(answer ?? "").slice(0, MAX_STATE),
|
|
45
|
-
note: "\u5DE5\u5177\u8BC1\u636E\uFF08toolEvidence\uFF09\u662F\u5224\u5B9A\u4F9D\u636E\uFF1BfinalReply \u81EA\u79F0\u5B8C\u6210\u4E0D\u4F5C\u4E3A\u8BC1\u636E\u3002\u7F3A\u5C11\u8BC1\u636E\u65F6\u9009 insufficient_evidence\u3002"
|
|
60
|
+
note: "\u5DE5\u5177\u8BC1\u636E\uFF08toolEvidence\uFF09\u662F\u5224\u5B9A\u4F9D\u636E\uFF1BfinalReply \u81EA\u79F0\u5B8C\u6210\u4E0D\u4F5C\u4E3A\u8BC1\u636E\uFF0Cbackground \u53EA\u662F\u80CC\u666F\u800C\u4E0D\u662F\u672C\u8F6E\u8981\u6C42\u3002\u7F3A\u5C11\u8BC1\u636E\u65F6\u9009 insufficient_evidence\u3002"
|
|
46
61
|
};
|
|
47
62
|
const questions = Object.fromEntries(items.map((text, index) => [`c${index}`, {
|
|
48
63
|
type: "choice",
|
|
@@ -91,6 +106,9 @@ function createTaskCompletion({ evaluate, sessions, steer, settings, minConfiden
|
|
|
91
106
|
return {
|
|
92
107
|
async review(turn) {
|
|
93
108
|
const output = typeof turn?.output === "string" ? turn.output.trim() : "";
|
|
109
|
+
const task = taskExcerpt(turn?.context?.request ?? turn?.userText);
|
|
110
|
+
const taskDetail = task ? { task } : {};
|
|
111
|
+
if (Number.isSafeInteger(turn?.turn) && turn?.turn >= 0) taskDetail.turn = turn?.turn;
|
|
94
112
|
const { mode, minConditions, minConfidence: confidenceFloor } = await resolveSettings();
|
|
95
113
|
const conditions = extractConditions(turn?.userText);
|
|
96
114
|
if (conditions.length < minConditions) return { ...skip };
|
|
@@ -106,14 +124,14 @@ function createTaskCompletion({ evaluate, sessions, steer, settings, minConfiden
|
|
|
106
124
|
}
|
|
107
125
|
let answers;
|
|
108
126
|
try {
|
|
109
|
-
const result = await evaluate(completionQuestions(conditions, evidence, output), { signal: turn?.signal });
|
|
127
|
+
const result = await evaluate(completionQuestions(conditions, evidence, output, turn?.context), { signal: turn?.signal });
|
|
110
128
|
answers = isObject(result?.answers) ? result.answers : void 0;
|
|
111
129
|
} catch (error) {
|
|
112
|
-
record(turn, "error", { reason: errorReason(error) });
|
|
130
|
+
record(turn, "error", { ...taskDetail, reason: errorReason(error) });
|
|
113
131
|
return { ...skip };
|
|
114
132
|
}
|
|
115
133
|
if (!answers) {
|
|
116
|
-
record(turn, "error", { reason: "invalid-response" });
|
|
134
|
+
record(turn, "error", { ...taskDetail, reason: "invalid-response" });
|
|
117
135
|
return { ...skip };
|
|
118
136
|
}
|
|
119
137
|
let satisfied = 0, unsatisfied = 0, insufficient = 0;
|
|
@@ -131,7 +149,7 @@ function createTaskCompletion({ evaluate, sessions, steer, settings, minConfiden
|
|
|
131
149
|
} else insufficient++;
|
|
132
150
|
}
|
|
133
151
|
const outcome = unsatisfied > 0 ? "unsatisfied" : "ok";
|
|
134
|
-
const detail = { conditions: conditions.length, satisfied, unsatisfied, insufficient };
|
|
152
|
+
const detail = { ...taskDetail, conditions: conditions.length, satisfied, unsatisfied, insufficient };
|
|
135
153
|
const confident = unsatisfied > 0 && unsatisfiedMinConf >= confidenceFloor;
|
|
136
154
|
const steered = mode === "steer" && confident && typeof steer === "function" && turn?.agent ? steerOnce.attempt(turn, () => steer(turn.agent, STEER_TEXT)) : false;
|
|
137
155
|
if (steered) detail.steered = true;
|
package/lib/host/types.d.ts
CHANGED
|
@@ -166,6 +166,7 @@ export type LogRow = {
|
|
|
166
166
|
subject?: string;
|
|
167
167
|
task?: string;
|
|
168
168
|
turn?: number;
|
|
169
|
+
step?: number;
|
|
169
170
|
command?: string;
|
|
170
171
|
path?: string;
|
|
171
172
|
mode?: string;
|
|
@@ -268,6 +269,27 @@ export type ToolCandidate = {
|
|
|
268
269
|
name: string;
|
|
269
270
|
description?: string;
|
|
270
271
|
};
|
|
272
|
+
/** Bounded, masked data for adjudication; background is never authorization. */
|
|
273
|
+
export type ToolEvidence = {
|
|
274
|
+
tool: string;
|
|
275
|
+
outcome: 'ok' | 'error';
|
|
276
|
+
command?: string;
|
|
277
|
+
path?: string;
|
|
278
|
+
result?: string;
|
|
279
|
+
};
|
|
280
|
+
export type DecisionContext = {
|
|
281
|
+
request: string;
|
|
282
|
+
background: {
|
|
283
|
+
role: 'user' | 'assistant';
|
|
284
|
+
text: string;
|
|
285
|
+
}[];
|
|
286
|
+
progress: ToolEvidence[];
|
|
287
|
+
};
|
|
288
|
+
export type DecisionSnapshot = {
|
|
289
|
+
context: DecisionContext;
|
|
290
|
+
conditionsText: string;
|
|
291
|
+
evidence: ToolEvidence[];
|
|
292
|
+
};
|
|
271
293
|
export type AgentLike = {
|
|
272
294
|
session?: {
|
|
273
295
|
id?: string;
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "dsh-decision-layer",
|
|
3
|
-
"version": "0.7.
|
|
3
|
+
"version": "0.7.4",
|
|
4
4
|
"description": "A structured decision layer for the DeepSeek Harness agent loop: a danger-call gate, output self-check, tool narrowing, and loop guard backed by a pluggable adjudication model.",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"dsh",
|