@evandu/super-pi 0.1.146 → 0.1.148
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/agents/super-pi-task/core.ts +223 -2
- package/agents/super-pi-task/index.ts +50 -0
- package/dist/.super-pi-check-inputs.json +2 -2
- package/dist/client/.super-pi-client-sources.json +8 -8
- package/dist/client/assets/{CodeMirrorEditor-CEwBCVM6.js → CodeMirrorEditor-DQCjEnxG.js} +2 -2
- package/dist/client/assets/{dist-DC1IocrS.js → dist-1Innu17E.js} +1 -1
- package/dist/client/assets/{dist-CSCUT1PQ.js → dist-1lynFOc5.js} +1 -1
- package/dist/client/assets/{dist-Do52MRwm.js → dist-B1qLZVMK.js} +1 -1
- package/dist/client/assets/{dist-CAOcjKAQ.js → dist-B4pz_xdd.js} +1 -1
- package/dist/client/assets/{dist-B0RXZeXQ.js → dist-B6KwDpeI.js} +1 -1
- package/dist/client/assets/{dist-DFMtP3wG.js → dist-BJVI1Unv.js} +1 -1
- package/dist/client/assets/{dist-oLq6HXLG.js → dist-BWvzD8Pe.js} +1 -1
- package/dist/client/assets/{dist-9F1wL-T5.js → dist-BayWssd0.js} +1 -1
- package/dist/client/assets/{dist-DBUqwiWG.js → dist-Bk_pkBVH.js} +1 -1
- package/dist/client/assets/{dist-CbVf2Cjn.js → dist-BpeClAv0.js} +1 -1
- package/dist/client/assets/{dist-DftD4ibj.js → dist-BsmJg5PX.js} +1 -1
- package/dist/client/assets/{dist-CUB_Chdh.js → dist-Bx7s4rni.js} +1 -1
- package/dist/client/assets/{dist-Bh8Q7Y_n.js → dist-CDBKM9tL.js} +1 -1
- package/dist/client/assets/{dist-BoyICDlL.js → dist-CGzXCzCG.js} +1 -1
- package/dist/client/assets/{dist-BCNaDgNW.js → dist-CRxCwn-Z.js} +1 -1
- package/dist/client/assets/{dist-76ArZ5M4.js → dist-CTDgUnBG.js} +1 -1
- package/dist/client/assets/{dist-BXBSIuTD.js → dist-D1VfMWmf.js} +1 -1
- package/dist/client/assets/{dist-DN0GlnnZ.js → dist-DTZLNEci.js} +1 -1
- package/dist/client/assets/{dist-JvQIIdqB.js → dist-DuUi29km.js} +1 -1
- package/dist/client/assets/{dist-CHt8PX8R.js → dist-F3S0t1oM.js} +1 -1
- package/dist/client/assets/{dist-BuR5Iagd.js → dist-QZ-FAVJV.js} +1 -1
- package/dist/client/assets/{dist-Bx6ZWYUW.js → dist-u0nBg0fZ.js} +1 -1
- package/dist/client/assets/index-BMpqviwU.css +1 -0
- package/dist/client/assets/{index-E7yaLa2C.js → index-Dzyp_7sf.js} +43 -43
- package/dist/client/index.html +2 -2
- package/dist/server/active-memory.d.ts +17 -0
- package/dist/server/active-memory.js +39 -0
- package/dist/server/agents.d.ts +1 -1
- package/dist/server/agents.js +1 -1
- package/dist/server/application/conversation-service.js +6 -0
- package/dist/server/application/run-service.d.ts +16 -0
- package/dist/server/application/run-service.js +175 -26
- package/dist/server/application/session-orchestrator-service.d.ts +103 -1
- package/dist/server/application/session-orchestrator-service.js +308 -8
- package/dist/server/application/session-watchdog.d.ts +159 -0
- package/dist/server/application/session-watchdog.js +244 -0
- package/dist/server/http-server.js +136 -76
- package/dist/server/index.js +3 -0
- package/dist/server/knowledge-vectors.d.ts +15 -0
- package/dist/server/knowledge-vectors.js +118 -37
- package/dist/server/knowledge.d.ts +7 -8
- package/dist/server/knowledge.js +133 -55
- package/dist/server/length-continuation.d.ts +50 -0
- package/dist/server/length-continuation.js +55 -0
- package/dist/server/pi-session-events.js +5 -1
- package/dist/server/pi-session-pool.d.ts +2 -0
- package/dist/server/pi-session-pool.js +16 -0
- package/dist/server/runtime.js +5 -4
- package/dist/server/session-tree-scope.d.ts +125 -0
- package/dist/server/session-tree-scope.js +337 -0
- package/dist/server/store.d.ts +13 -0
- package/dist/server/store.js +37 -4
- package/dist/server/task-tool.d.ts +4 -1
- package/dist/server/task-tool.js +25 -1
- package/dist/server/turn-failure.d.ts +26 -0
- package/dist/server/turn-failure.js +51 -0
- package/dist/server/types.d.ts +15 -1
- package/package.json +20 -3
- package/dist/client/assets/index-D-OxlUxN.css +0 -1
|
@@ -27,8 +27,8 @@ export const KNOWLEDGE_MODEL = "Xenova/bge-small-zh-v1.5";
|
|
|
27
27
|
*/
|
|
28
28
|
export const EMBED_PIECE_CHARS = 300;
|
|
29
29
|
const EMBED_PIECE_OVERLAP = 40;
|
|
30
|
-
/** Per query. Cosine similarity of unit vectors;
|
|
31
|
-
const MIN_COSINE = 0.
|
|
30
|
+
/** Per query. Cosine similarity of unit vectors; threshold raised to 0.35 to filter out semantic noise. */
|
|
31
|
+
export const MIN_COSINE = 0.35;
|
|
32
32
|
/**
|
|
33
33
|
* Splits a retrieval chunk into pieces small enough for the model's window.
|
|
34
34
|
*
|
|
@@ -84,6 +84,11 @@ export class VectorIndex {
|
|
|
84
84
|
workerReady = false;
|
|
85
85
|
workerPending = new Map();
|
|
86
86
|
failure;
|
|
87
|
+
consecutiveFailures = 0;
|
|
88
|
+
lastFailureTime = 0;
|
|
89
|
+
static MAX_FAILURES = 3;
|
|
90
|
+
static COOLDOWN_MS = 30000;
|
|
91
|
+
static EMBED_TIMEOUT_MS = 15000;
|
|
87
92
|
cache = new Map();
|
|
88
93
|
pending = new Set();
|
|
89
94
|
queue = [];
|
|
@@ -102,11 +107,29 @@ export class VectorIndex {
|
|
|
102
107
|
bundledModel() { return resolve(import.meta.dirname, "../../assets/knowledge-model"); }
|
|
103
108
|
enabled() { return process.env.SUPER_PI_KNOWLEDGE_EMBEDDINGS !== "0"; }
|
|
104
109
|
status(indexed) {
|
|
110
|
+
const enabled = this.enabled();
|
|
111
|
+
const bundled = existsSync(this.bundledModel());
|
|
112
|
+
const now = Date.now();
|
|
113
|
+
const inCooldown = this.consecutiveFailures >= VectorIndex.MAX_FAILURES && (now - this.lastFailureTime < VectorIndex.COOLDOWN_MS);
|
|
114
|
+
let health = "ready";
|
|
115
|
+
if (!enabled)
|
|
116
|
+
health = "disabled";
|
|
117
|
+
else if (!bundled || inCooldown)
|
|
118
|
+
health = "degraded";
|
|
119
|
+
else if (this.consecutiveFailures > 0)
|
|
120
|
+
health = "restarting";
|
|
121
|
+
else if (this.worker && this.workerReady)
|
|
122
|
+
health = "ready";
|
|
123
|
+
else
|
|
124
|
+
health = "ready";
|
|
105
125
|
return {
|
|
106
|
-
enabled
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
126
|
+
enabled,
|
|
127
|
+
ready: this.workerReady && health === "ready",
|
|
128
|
+
indexed,
|
|
129
|
+
error: this.failure,
|
|
130
|
+
bundled,
|
|
131
|
+
health,
|
|
132
|
+
available: enabled && bundled && health !== "degraded",
|
|
110
133
|
};
|
|
111
134
|
}
|
|
112
135
|
workspace(workspaceId) { return join(this.directory, Buffer.from(workspaceId).toString("hex")); }
|
|
@@ -121,45 +144,81 @@ export class VectorIndex {
|
|
|
121
144
|
* able to reach, on the first search, for every person who installed the package. Shipping the
|
|
122
145
|
* weights turns a failure mode into a file, and makes the feature work on a machine that has never
|
|
123
146
|
* been online since it was installed.
|
|
147
|
+
*
|
|
148
|
+
* RESILIENT WORKER LIFECYCLE:
|
|
149
|
+
* When the worker encounters an error, crash, or unexpected exit, it cleans up all pending requests
|
|
150
|
+
* without hanging and marks the instance for automatic, on-demand respawning on the next embed call.
|
|
151
|
+
* A bounded backoff / cooldown prevents runaway crash loops if the environment is broken.
|
|
124
152
|
*/
|
|
125
153
|
ensureWorker() {
|
|
154
|
+
if (!this.enabled())
|
|
155
|
+
return undefined;
|
|
126
156
|
if (this.worker)
|
|
127
157
|
return this.worker;
|
|
128
|
-
if (this.failure)
|
|
129
|
-
return undefined;
|
|
130
158
|
if (!existsSync(this.bundledModel())) {
|
|
131
159
|
this.failure = `随包分发的模型缺失:${this.bundledModel()}`;
|
|
132
160
|
return undefined;
|
|
133
161
|
}
|
|
134
|
-
const
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
if (!pending)
|
|
139
|
-
return;
|
|
140
|
-
this.workerPending.delete(message.id);
|
|
141
|
-
if (message.error || !message.data || !message.dims) {
|
|
142
|
-
this.failure = message.error ?? "向量 worker 返回了无效结果";
|
|
143
|
-
pending.resolve(null);
|
|
144
|
-
return;
|
|
162
|
+
const now = Date.now();
|
|
163
|
+
if (this.consecutiveFailures >= VectorIndex.MAX_FAILURES) {
|
|
164
|
+
if (now - this.lastFailureTime < VectorIndex.COOLDOWN_MS) {
|
|
165
|
+
return undefined;
|
|
145
166
|
}
|
|
146
|
-
this.
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
pending.
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
167
|
+
this.consecutiveFailures = 0;
|
|
168
|
+
this.failure = undefined;
|
|
169
|
+
}
|
|
170
|
+
try {
|
|
171
|
+
const worker = new Worker(new URL("./knowledge-embedding-worker.js", import.meta.url));
|
|
172
|
+
worker.unref();
|
|
173
|
+
worker.on("message", (message) => {
|
|
174
|
+
const pending = this.workerPending.get(message.id);
|
|
175
|
+
if (!pending)
|
|
176
|
+
return;
|
|
177
|
+
this.workerPending.delete(message.id);
|
|
178
|
+
if (pending.timeout)
|
|
179
|
+
clearTimeout(pending.timeout);
|
|
180
|
+
if (message.error || !message.data || !message.dims) {
|
|
181
|
+
this.failure = message.error ?? "向量 worker 返回了无效结果";
|
|
182
|
+
pending.resolve(null);
|
|
183
|
+
return;
|
|
184
|
+
}
|
|
185
|
+
this.workerReady = true;
|
|
186
|
+
this.failure = undefined;
|
|
187
|
+
this.consecutiveFailures = 0;
|
|
188
|
+
const dimension = message.dims.at(-1) ?? 0;
|
|
189
|
+
pending.resolve(Array.from({ length: message.dims[0] ?? 0 }, (_, row) => Array.from(message.data.slice(row * dimension, (row + 1) * dimension))));
|
|
190
|
+
});
|
|
191
|
+
const fail = (error) => {
|
|
192
|
+
this.consecutiveFailures++;
|
|
193
|
+
this.lastFailureTime = Date.now();
|
|
194
|
+
this.failure = error.message;
|
|
195
|
+
for (const pending of this.workerPending.values()) {
|
|
196
|
+
if (pending.timeout)
|
|
197
|
+
clearTimeout(pending.timeout);
|
|
198
|
+
pending.resolve(null);
|
|
199
|
+
}
|
|
200
|
+
this.workerPending.clear();
|
|
201
|
+
if (this.worker) {
|
|
202
|
+
try {
|
|
203
|
+
this.worker.terminate();
|
|
204
|
+
}
|
|
205
|
+
catch { }
|
|
206
|
+
}
|
|
207
|
+
this.worker = undefined;
|
|
208
|
+
this.workerReady = false;
|
|
209
|
+
};
|
|
210
|
+
worker.on("error", fail);
|
|
211
|
+
worker.on("exit", (code) => { if (code !== 0)
|
|
212
|
+
fail(new Error(`向量 worker 异常退出(${code})`)); });
|
|
213
|
+
this.worker = worker;
|
|
214
|
+
return worker;
|
|
215
|
+
}
|
|
216
|
+
catch (error) {
|
|
217
|
+
this.consecutiveFailures++;
|
|
218
|
+
this.lastFailureTime = Date.now();
|
|
219
|
+
this.failure = error instanceof Error ? error.message : String(error);
|
|
220
|
+
return undefined;
|
|
221
|
+
}
|
|
163
222
|
}
|
|
164
223
|
async embed(texts) {
|
|
165
224
|
if (!texts.length || !this.enabled())
|
|
@@ -169,10 +228,32 @@ export class VectorIndex {
|
|
|
169
228
|
return null;
|
|
170
229
|
const id = ++this.workerSequence;
|
|
171
230
|
return new Promise((resolvePromise) => {
|
|
172
|
-
|
|
231
|
+
const timeout = setTimeout(() => {
|
|
232
|
+
const pending = this.workerPending.get(id);
|
|
233
|
+
if (pending) {
|
|
234
|
+
this.workerPending.delete(id);
|
|
235
|
+
pending.resolve(null);
|
|
236
|
+
}
|
|
237
|
+
}, VectorIndex.EMBED_TIMEOUT_MS);
|
|
238
|
+
timeout.unref();
|
|
239
|
+
this.workerPending.set(id, { resolve: resolvePromise, timeout });
|
|
173
240
|
worker.postMessage({ id, texts, modelRoot: this.bundledModel(), model: KNOWLEDGE_MODEL });
|
|
174
241
|
});
|
|
175
242
|
}
|
|
243
|
+
async terminate() {
|
|
244
|
+
for (const pending of this.workerPending.values()) {
|
|
245
|
+
if (pending.timeout)
|
|
246
|
+
clearTimeout(pending.timeout);
|
|
247
|
+
pending.resolve(null);
|
|
248
|
+
}
|
|
249
|
+
this.workerPending.clear();
|
|
250
|
+
if (this.worker) {
|
|
251
|
+
const w = this.worker;
|
|
252
|
+
this.worker = undefined;
|
|
253
|
+
this.workerReady = false;
|
|
254
|
+
await w.terminate();
|
|
255
|
+
}
|
|
256
|
+
}
|
|
176
257
|
/** Vectors already stored for one workspace, keyed by document then chunk index. */
|
|
177
258
|
async read(workspaceId, documentId) {
|
|
178
259
|
try {
|
|
@@ -44,7 +44,7 @@ export declare const MAX_DOCUMENT_BYTES: number;
|
|
|
44
44
|
export declare const MAX_KNOWLEDGE_REQUEST_BYTES: number;
|
|
45
45
|
/** Per workspace, across every document. A knowledge base is a curated asset, not a file server. */
|
|
46
46
|
export declare const MAX_WORKSPACE_BYTES: number;
|
|
47
|
-
export declare const DEFAULT_RESULT_LIMIT =
|
|
47
|
+
export declare const DEFAULT_RESULT_LIMIT = 3;
|
|
48
48
|
export declare const MAX_RESULT_LIMIT = 20;
|
|
49
49
|
/** What one search may be asked for. `rerank` and `conversation` exist only for the agent's own calls. */
|
|
50
50
|
export type KnowledgeSearch = {
|
|
@@ -72,14 +72,11 @@ export type KnowledgeRuntime = {
|
|
|
72
72
|
}): AsyncIterable<string>;
|
|
73
73
|
};
|
|
74
74
|
/**
|
|
75
|
-
*
|
|
76
|
-
*
|
|
77
|
-
*
|
|
78
|
-
* still sometimes wrap it in a sentence or a code fence. Recovering the array from that is worth doing;
|
|
79
|
-
* guessing when there is no array at all is not, so anything unparseable is null and the caller keeps
|
|
80
|
-
* the lexical order.
|
|
75
|
+
* Extracts a 1-based candidate integer array from the model's answer, or null.
|
|
76
|
+
* Tolerates surrounding markdown, explanatory text, and trailing commas.
|
|
77
|
+
* Returns empty array [] if the model explicitly declares no candidates are relevant.
|
|
81
78
|
*/
|
|
82
|
-
export declare function parseRanking(output: string): number[] | null;
|
|
79
|
+
export declare function parseRanking(output: string, maxIndex?: number): number[] | null;
|
|
83
80
|
/** Duplicated as a literal in `agents/super-pi-knowledge/index.ts`, the way `TASK_TOOL_NAME` is. */
|
|
84
81
|
export declare const KNOWLEDGE_TOOL_NAME = "knowledge_search";
|
|
85
82
|
/** The import half of the same extension. Duplicated as a literal there, for the same reason. */
|
|
@@ -116,6 +113,8 @@ export declare function digestOf(text: string): string;
|
|
|
116
113
|
* repeated at the head of the next chunk to keep that sentence whole in at least one hit.
|
|
117
114
|
*/
|
|
118
115
|
export declare function chunkDocument(text: string): KnowledgeChunk[];
|
|
116
|
+
export declare const MIN_SCORE_ABSOLUTE = 0.28;
|
|
117
|
+
export declare const RELATIVE_CUTOFF_RATIO = 0.5;
|
|
119
118
|
export declare class KnowledgeBase {
|
|
120
119
|
readonly directory: string;
|
|
121
120
|
private store;
|
package/dist/server/knowledge.js
CHANGED
|
@@ -12,12 +12,12 @@ export const MAX_WORKSPACE_BYTES = 16 * 1024 * 1024;
|
|
|
12
12
|
const CHUNK_CHARS = 1200;
|
|
13
13
|
const CHUNK_OVERLAP_CHARS = 150;
|
|
14
14
|
const MAX_CHUNKS_PER_DOCUMENT = 2000;
|
|
15
|
-
export const DEFAULT_RESULT_LIMIT =
|
|
15
|
+
export const DEFAULT_RESULT_LIMIT = 3;
|
|
16
16
|
export const MAX_RESULT_LIMIT = 20;
|
|
17
|
-
/** Per hit. Enough to answer from, short enough that
|
|
17
|
+
/** Per hit. Enough to answer from, short enough that hits stay inside one turn's prompt budget. */
|
|
18
18
|
const RESULT_TEXT_CAP = 1200;
|
|
19
|
-
/** Across all hits of one search.
|
|
20
|
-
const RESULT_TOTAL_CAP =
|
|
19
|
+
/** Across all hits of one search. Lowered to 4800 to bound prompt impact on standard context windows. */
|
|
20
|
+
const RESULT_TOTAL_CAP = 4800;
|
|
21
21
|
/** Below this many candidates there is no ranking to correct, and the model call is not worth it. */
|
|
22
22
|
const RERANK_MIN_CANDIDATES = 3;
|
|
23
23
|
/** How many candidates the model sees. Larger pools cost prompt without improving the top few. */
|
|
@@ -26,26 +26,64 @@ const RERANK_POOL = 12;
|
|
|
26
26
|
const RERANK_SNIPPET_CHARS = 240;
|
|
27
27
|
const RERANK_TIMEOUT_MS = 12000;
|
|
28
28
|
/**
|
|
29
|
-
*
|
|
30
|
-
*
|
|
31
|
-
*
|
|
32
|
-
* still sometimes wrap it in a sentence or a code fence. Recovering the array from that is worth doing;
|
|
33
|
-
* guessing when there is no array at all is not, so anything unparseable is null and the caller keeps
|
|
34
|
-
* the lexical order.
|
|
29
|
+
* Extracts a 1-based candidate integer array from the model's answer, or null.
|
|
30
|
+
* Tolerates surrounding markdown, explanatory text, and trailing commas.
|
|
31
|
+
* Returns empty array [] if the model explicitly declares no candidates are relevant.
|
|
35
32
|
*/
|
|
36
|
-
export function parseRanking(output) {
|
|
37
|
-
|
|
38
|
-
if (!match)
|
|
33
|
+
export function parseRanking(output, maxIndex) {
|
|
34
|
+
if (!output || typeof output !== "string")
|
|
39
35
|
return null;
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
36
|
+
// 1. Prefer markdown code block: ```json ... ``` or ``` ... ```
|
|
37
|
+
const codeBlock = /```(?:json)?\s*(\[[\s\S]*?\])\s*```/i.exec(output);
|
|
38
|
+
const targetText = codeBlock ? codeBlock[1] : output;
|
|
39
|
+
// 2. Find bracketed arrays
|
|
40
|
+
const matches = [...targetText.matchAll(/\[[\s\S]*?\]/g)];
|
|
41
|
+
if (!matches.length)
|
|
42
|
+
return null;
|
|
43
|
+
// Try parsing from the last match backwards (models often put final answer at the end)
|
|
44
|
+
let parsed = null;
|
|
45
|
+
for (let i = matches.length - 1; i >= 0; i--) {
|
|
46
|
+
const candidate = matches[i][0].replace(/,\s*\]/, "]");
|
|
47
|
+
try {
|
|
48
|
+
const p = JSON.parse(candidate);
|
|
49
|
+
if (Array.isArray(p)) {
|
|
50
|
+
parsed = p;
|
|
51
|
+
break;
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
catch { }
|
|
45
55
|
}
|
|
46
|
-
|
|
56
|
+
if (!Array.isArray(parsed))
|
|
47
57
|
return null;
|
|
58
|
+
if (parsed.length === 0)
|
|
59
|
+
return [];
|
|
60
|
+
// Reject negative numbers or non-integers, per existing test: [0, 2, -1, 1.5] -> null
|
|
61
|
+
for (const item of parsed) {
|
|
62
|
+
if (typeof item === "number") {
|
|
63
|
+
if (!Number.isInteger(item) || item < 0)
|
|
64
|
+
return null;
|
|
65
|
+
}
|
|
66
|
+
else if (typeof item === "string") {
|
|
67
|
+
if (!/^\d+$/.test(item.trim()))
|
|
68
|
+
return null;
|
|
69
|
+
}
|
|
70
|
+
else {
|
|
71
|
+
return null;
|
|
72
|
+
}
|
|
48
73
|
}
|
|
74
|
+
// Deduplicate and filter out-of-bounds (<= 0 or > maxIndex)
|
|
75
|
+
const seen = new Set();
|
|
76
|
+
const result = [];
|
|
77
|
+
for (const item of parsed) {
|
|
78
|
+
const val = typeof item === "string" ? parseInt(item.trim(), 10) : item;
|
|
79
|
+
if (val > 0 && (maxIndex === undefined || val <= maxIndex)) {
|
|
80
|
+
if (!seen.has(val)) {
|
|
81
|
+
seen.add(val);
|
|
82
|
+
result.push(val);
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
return result;
|
|
49
87
|
}
|
|
50
88
|
const CJK_RUN = /[\u3040-\u30ff\u3400-\u4dbf\u4e00-\u9fff\uf900-\ufaff]+/gu;
|
|
51
89
|
const LATIN_WORD = /[\p{L}\p{N}_.+#-]{2,}/gu;
|
|
@@ -212,6 +250,9 @@ const B = 0.75;
|
|
|
212
250
|
* the chunk that actually answers may sit in a section whose heading shares nothing with the question.
|
|
213
251
|
*/
|
|
214
252
|
const WEIGHT = { body: 1, phrase: 0.45, head: 0.5, document: 0.7, vector: 0.9 };
|
|
253
|
+
const TOTAL_WEIGHT = WEIGHT.body + WEIGHT.phrase + WEIGHT.head + WEIGHT.document + WEIGHT.vector; // 3.55
|
|
254
|
+
export const MIN_SCORE_ABSOLUTE = 0.28;
|
|
255
|
+
export const RELATIVE_CUTOFF_RATIO = 0.5;
|
|
215
256
|
function countTerms(terms) {
|
|
216
257
|
const counts = new Map();
|
|
217
258
|
for (const term of terms)
|
|
@@ -321,6 +362,10 @@ export class KnowledgeBase {
|
|
|
321
362
|
const channels = [];
|
|
322
363
|
const docTerms = new Map();
|
|
323
364
|
let docLength = 0;
|
|
365
|
+
for (const term of tokenize(document.title)) {
|
|
366
|
+
docTerms.set(term, (docTerms.get(term) ?? 0) + 1);
|
|
367
|
+
docLength++;
|
|
368
|
+
}
|
|
324
369
|
for (const chunk of chunks) {
|
|
325
370
|
// Tokenised ONCE here, at import time, rather than on every search: a workspace with a hundred
|
|
326
371
|
// documents would otherwise re-split the whole corpus per question.
|
|
@@ -359,9 +404,29 @@ export class KnowledgeBase {
|
|
|
359
404
|
if (cached)
|
|
360
405
|
return cached;
|
|
361
406
|
const documents = (await this.readAll(workspaceId)).map(document => this.index(document));
|
|
407
|
+
const chunkDf = new Map(), phraseDf = new Map(), headDf = new Map(), docDf = new Map();
|
|
408
|
+
let chunkCount = 0;
|
|
409
|
+
for (const document of documents) {
|
|
410
|
+
for (const channels of document.channels) {
|
|
411
|
+
chunkCount++;
|
|
412
|
+
for (const [term] of channels.body)
|
|
413
|
+
chunkDf.set(term, (chunkDf.get(term) ?? 0) + 1);
|
|
414
|
+
for (const [term] of channels.head)
|
|
415
|
+
headDf.set(term, (headDf.get(term) ?? 0) + 1);
|
|
416
|
+
for (const [phrase] of channels.phrase)
|
|
417
|
+
phraseDf.set(phrase, (phraseDf.get(phrase) ?? 0) + 1);
|
|
418
|
+
}
|
|
419
|
+
for (const [term] of document.docTerms)
|
|
420
|
+
docDf.set(term, (docDf.get(term) ?? 0) + 1);
|
|
421
|
+
}
|
|
362
422
|
const index = {
|
|
363
423
|
documents,
|
|
364
424
|
averageLength: documents.length ? documents.reduce((sum, document) => sum + document.docLength, 0) / documents.length : 0,
|
|
425
|
+
chunkCount,
|
|
426
|
+
chunkDf,
|
|
427
|
+
phraseDf,
|
|
428
|
+
headDf,
|
|
429
|
+
docDf,
|
|
365
430
|
};
|
|
366
431
|
this.indexes.set(workspaceId, index);
|
|
367
432
|
// Vectors are built in the background on first sight of a workspace, so a restart with an
|
|
@@ -528,29 +593,9 @@ export class KnowledgeBase {
|
|
|
528
593
|
const phrases = [...new Set(tokenizePhrases(query ?? ""))];
|
|
529
594
|
if (!terms.length && !phrases.length)
|
|
530
595
|
return [];
|
|
531
|
-
const { documents, averageLength } = await this.load(workspaceId);
|
|
596
|
+
const { documents, averageLength, chunkCount, chunkDf, phraseDf, headDf, docDf } = await this.load(workspaceId);
|
|
532
597
|
if (!documents.length || !averageLength)
|
|
533
598
|
return [];
|
|
534
|
-
const chunkCount = documents.reduce((sum, document) => sum + document.chunks.length, 0);
|
|
535
|
-
// Document frequency is counted once per channel, over that channel's own unit: chunks for the
|
|
536
|
-
// passage channels, documents for the document channel.
|
|
537
|
-
const chunkDf = new Map(), phraseDf = new Map(), headDf = new Map(), docDf = new Map();
|
|
538
|
-
for (const document of documents) {
|
|
539
|
-
for (const channels of document.channels) {
|
|
540
|
-
for (const term of terms) {
|
|
541
|
-
if (channels.body.has(term))
|
|
542
|
-
chunkDf.set(term, (chunkDf.get(term) ?? 0) + 1);
|
|
543
|
-
if (channels.head.has(term))
|
|
544
|
-
headDf.set(term, (headDf.get(term) ?? 0) + 1);
|
|
545
|
-
}
|
|
546
|
-
for (const phrase of phrases)
|
|
547
|
-
if (channels.phrase.has(phrase))
|
|
548
|
-
phraseDf.set(phrase, (phraseDf.get(phrase) ?? 0) + 1);
|
|
549
|
-
}
|
|
550
|
-
for (const term of terms)
|
|
551
|
-
if (document.docTerms.has(term))
|
|
552
|
-
docDf.set(term, (docDf.get(term) ?? 0) + 1);
|
|
553
|
-
}
|
|
554
599
|
const hits = [];
|
|
555
600
|
let bodyBest = 0, phraseBest = 0, headBest = 0, docBest = 0;
|
|
556
601
|
for (const document of documents) {
|
|
@@ -601,18 +646,29 @@ export class KnowledgeBase {
|
|
|
601
646
|
const similarities = this.vectors
|
|
602
647
|
? await this.vectors.search(workspaceId, query ?? "", lexical.map(entry => [entry.key, entry.hit.documentId]))
|
|
603
648
|
: null;
|
|
604
|
-
const
|
|
605
|
-
.map(entry => {
|
|
649
|
+
const rawScores = lexical.map(entry => {
|
|
606
650
|
const similarity = similarities?.get(entry.key) ?? 0;
|
|
607
|
-
const
|
|
608
|
-
+ (entry.lexical > 0 ? WEIGHT.document * (docBest ? entry.document / docBest : 0) : 0)
|
|
651
|
+
const rawScore = entry.lexical + (similarities ? WEIGHT.vector * similarity : 0)
|
|
652
|
+
+ (entry.lexical > 0 ? WEIGHT.document * (docBest ? entry.document / docBest : 0) : 0);
|
|
653
|
+
const score = Math.min(1, Math.round(1000 * (rawScore / TOTAL_WEIGHT)) / 1000);
|
|
609
654
|
return { ...entry.hit, score };
|
|
610
|
-
})
|
|
611
|
-
|
|
655
|
+
});
|
|
656
|
+
// 1. Absolute threshold filter: drop anything with score < MIN_SCORE_ABSOLUTE (0.28)
|
|
657
|
+
const passedAbsolute = rawScores.filter(hit => hit.score >= MIN_SCORE_ABSOLUTE);
|
|
658
|
+
if (!passedAbsolute.length)
|
|
659
|
+
return [];
|
|
612
660
|
// Best-first, with the tie broken by chunk order so two equal passages of one document come back
|
|
613
661
|
// in the order a reader would meet them.
|
|
614
|
-
|
|
615
|
-
|
|
662
|
+
passedAbsolute.sort((a, b) => b.score - a.score || a.chunk - b.chunk);
|
|
663
|
+
// 2. Relative cutoff: drop anything with score < bestScore * RELATIVE_CUTOFF_RATIO (0.50)
|
|
664
|
+
const bestScore = passedAbsolute[0].score;
|
|
665
|
+
const fused = passedAbsolute.filter(hit => hit.score >= bestScore * RELATIVE_CUTOFF_RATIO);
|
|
666
|
+
if (!fused.length)
|
|
667
|
+
return [];
|
|
668
|
+
const reranked = options.rerank ? await this.rerank(options.conversation, query ?? "", fused, options.signal) : null;
|
|
669
|
+
const ordered = reranked ?? fused;
|
|
670
|
+
if (!ordered.length)
|
|
671
|
+
return [];
|
|
616
672
|
const selected = [];
|
|
617
673
|
let spent = 0;
|
|
618
674
|
for (const hit of ordered) {
|
|
@@ -670,23 +726,45 @@ export class KnowledgeBase {
|
|
|
670
726
|
agentId: undefined, messages: [], status: "idle",
|
|
671
727
|
};
|
|
672
728
|
const listing = pool.map((hit, position) => `[${position + 1}] 《${hit.title}》${hit.heading ? ` > ${hit.heading}` : ""}\n${hit.text.slice(0, RERANK_SNIPPET_CHARS).replace(/\n+/g, " ")}`).join("\n\n");
|
|
673
|
-
const prompt =
|
|
729
|
+
const prompt = `你是检索排序器。用户提出了一个问题,下面是知识库检索出的候选片段。请按「能否回答这个问题」对候选片段进行筛选与重排序。
|
|
730
|
+
评分准则:
|
|
731
|
+
1. 仅保留能够回答问题或与问题高度相关的片段。
|
|
732
|
+
2. 按相关度从高到低排序,输出候选编号的 JSON 数组,例如 [3, 1]。
|
|
733
|
+
3. 若所有候选片段均不相关,输出空数组 []。
|
|
734
|
+
4. 只输出 JSON 数组,不要输出解释文字,不要输出编号以外的内容。
|
|
735
|
+
|
|
736
|
+
问题:${query}
|
|
737
|
+
|
|
738
|
+
候选片段:
|
|
739
|
+
${listing}`;
|
|
674
740
|
let output = "";
|
|
675
741
|
for await (const piece of this.runtime.answer({ conversation: isolated, prompt, cwd: workspace.path, signal: controller.signal, knowledgeRerank: true })) {
|
|
676
742
|
output += piece;
|
|
677
743
|
if (controller.signal.aborted)
|
|
678
744
|
return null;
|
|
679
745
|
}
|
|
680
|
-
const parsed = parseRanking(output);
|
|
681
|
-
if (
|
|
746
|
+
const parsed = parseRanking(output, pool.length);
|
|
747
|
+
if (parsed === null)
|
|
682
748
|
return null;
|
|
749
|
+
if (parsed.length === 0)
|
|
750
|
+
return [];
|
|
683
751
|
const picked = parsed.map((index) => pool[index - 1]).filter((hit) => Boolean(hit));
|
|
684
752
|
if (!picked.length)
|
|
685
|
-
return
|
|
686
|
-
// Anything the model left out keeps its
|
|
687
|
-
// truncated answer still returns the full pool rather than silently losing candidates.
|
|
753
|
+
return [];
|
|
754
|
+
// Anything the model left out keeps its position after the picks
|
|
688
755
|
const chosen = new Set(picked);
|
|
689
|
-
|
|
756
|
+
const remaining = pool.filter(hit => !chosen.has(hit));
|
|
757
|
+
const reordered = [...picked, ...remaining];
|
|
758
|
+
// Re-assign monotonically non-increasing scores based on pool's score distribution
|
|
759
|
+
const poolScores = pool.map(h => h.score).sort((a, b) => b - a);
|
|
760
|
+
let lastScore = poolScores[0];
|
|
761
|
+
return reordered.map((hit, index) => {
|
|
762
|
+
let assigned = poolScores[index];
|
|
763
|
+
if (assigned > lastScore)
|
|
764
|
+
assigned = lastScore;
|
|
765
|
+
lastScore = assigned;
|
|
766
|
+
return { ...hit, score: assigned };
|
|
767
|
+
});
|
|
690
768
|
}
|
|
691
769
|
catch {
|
|
692
770
|
return null;
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* THE CONTINUATION ORDER: what a session is told when its answer was cut off by the output limit.
|
|
3
|
+
*
|
|
4
|
+
* WHY THIS IS ITS OWN MODULE. Three places have to agree on it: the orchestrator that sends it,
|
|
5
|
+
* the prompt builder that tells the model who is speaking, and the browser that renders the row it
|
|
6
|
+
* becomes. When those three each held their own copy of the wording, a change to one of them left
|
|
7
|
+
* the other two describing a message that was no longer being sent.
|
|
8
|
+
*
|
|
9
|
+
* WHY THE ORDER EXISTS AT ALL. A model that filled its single-response output budget did not fail -
|
|
10
|
+
* it answered, and the answer was cut short. The work it was doing is not over. So the session is
|
|
11
|
+
* asked to carry on, in the SAME conversation, where its own partial answer is already in its
|
|
12
|
+
* context. That is what lets the model resume at the sentence it stopped on instead of starting
|
|
13
|
+
* the answer again.
|
|
14
|
+
*
|
|
15
|
+
* WHY IT IS AN ORDER AND NOT A USER MESSAGE. The person never typed it, and a row under their name
|
|
16
|
+
* saying words they never said is the exact bug this ordering was built to avoid.
|
|
17
|
+
*/
|
|
18
|
+
/**
|
|
19
|
+
* THE ENVELOPE A CONTINUATION ORDER TRAVELS IN.
|
|
20
|
+
*
|
|
21
|
+
* The browser renders a `tool` row by looking at the content, and it tells a delivered result
|
|
22
|
+
* (`<task ...>`) from an order (no envelope) by the same inspection. A continuation is neither: it
|
|
23
|
+
* is the system telling the session to keep writing, and it needs a third label - 「续写」 - because
|
|
24
|
+
* calling it a 「智能体下发」 order would say the system sent a task, which is not what happened.
|
|
25
|
+
*/
|
|
26
|
+
export declare const LENGTH_CONTINUATION_ENVELOPE = "continue_output";
|
|
27
|
+
/**
|
|
28
|
+
* THE TEXT THE MODEL IS GIVEN.
|
|
29
|
+
*
|
|
30
|
+
* It says two things, and both are needed. "Continue" alone invites a model to start the answer
|
|
31
|
+
* again from the top, because the model cannot tell that the earlier text was already accepted;
|
|
32
|
+
* "do not repeat what you have written" is what makes the continuation an extension rather than a
|
|
33
|
+
* second attempt.
|
|
34
|
+
*/
|
|
35
|
+
export declare const LENGTH_CONTINUATION_TEXT = "\u4E0A\u4E00\u6761\u56DE\u7B54\u56E0\u4E3A\u8FBE\u5230\u5355\u6B21\u8F93\u51FA\u7684\u957F\u5EA6\u4E0A\u9650\u88AB\u622A\u65AD\u4E86\u3002\u8BF7\u76F4\u63A5\u4ECE\u88AB\u622A\u65AD\u7684\u5730\u65B9\u63A5\u7740\u5199\u5B8C\uFF0C\u4E0D\u8981\u91CD\u590D\u5DF2\u7ECF\u5199\u8FC7\u7684\u5185\u5BB9\u3002";
|
|
36
|
+
/**
|
|
37
|
+
* THE WHOLE ORDER, ENCLOSED.
|
|
38
|
+
*
|
|
39
|
+
* The envelope is what makes the row recognisable, and it wraps the text so the browser can show
|
|
40
|
+
* the instruction rather than the markup.
|
|
41
|
+
*/
|
|
42
|
+
export declare const LENGTH_CONTINUATION_PROMPT = "<continue_output>\u4E0A\u4E00\u6761\u56DE\u7B54\u56E0\u4E3A\u8FBE\u5230\u5355\u6B21\u8F93\u51FA\u7684\u957F\u5EA6\u4E0A\u9650\u88AB\u622A\u65AD\u4E86\u3002\u8BF7\u76F4\u63A5\u4ECE\u88AB\u622A\u65AD\u7684\u5730\u65B9\u63A5\u7740\u5199\u5B8C\uFF0C\u4E0D\u8981\u91CD\u590D\u5DF2\u7ECF\u5199\u8FC7\u7684\u5185\u5BB9\u3002</continue_output>";
|
|
43
|
+
/**
|
|
44
|
+
* Whether this content is a continuation order.
|
|
45
|
+
*/
|
|
46
|
+
export declare function isLengthContinuation(content: string): boolean;
|
|
47
|
+
/**
|
|
48
|
+
* THE TEXT INSIDE THE ENVELOPE, or the content itself when it carries no envelope.
|
|
49
|
+
*/
|
|
50
|
+
export declare function lengthContinuationText(content: string): string;
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* THE CONTINUATION ORDER: what a session is told when its answer was cut off by the output limit.
|
|
3
|
+
*
|
|
4
|
+
* WHY THIS IS ITS OWN MODULE. Three places have to agree on it: the orchestrator that sends it,
|
|
5
|
+
* the prompt builder that tells the model who is speaking, and the browser that renders the row it
|
|
6
|
+
* becomes. When those three each held their own copy of the wording, a change to one of them left
|
|
7
|
+
* the other two describing a message that was no longer being sent.
|
|
8
|
+
*
|
|
9
|
+
* WHY THE ORDER EXISTS AT ALL. A model that filled its single-response output budget did not fail -
|
|
10
|
+
* it answered, and the answer was cut short. The work it was doing is not over. So the session is
|
|
11
|
+
* asked to carry on, in the SAME conversation, where its own partial answer is already in its
|
|
12
|
+
* context. That is what lets the model resume at the sentence it stopped on instead of starting
|
|
13
|
+
* the answer again.
|
|
14
|
+
*
|
|
15
|
+
* WHY IT IS AN ORDER AND NOT A USER MESSAGE. The person never typed it, and a row under their name
|
|
16
|
+
* saying words they never said is the exact bug this ordering was built to avoid.
|
|
17
|
+
*/
|
|
18
|
+
/**
|
|
19
|
+
* THE ENVELOPE A CONTINUATION ORDER TRAVELS IN.
|
|
20
|
+
*
|
|
21
|
+
* The browser renders a `tool` row by looking at the content, and it tells a delivered result
|
|
22
|
+
* (`<task ...>`) from an order (no envelope) by the same inspection. A continuation is neither: it
|
|
23
|
+
* is the system telling the session to keep writing, and it needs a third label - 「续写」 - because
|
|
24
|
+
* calling it a 「智能体下发」 order would say the system sent a task, which is not what happened.
|
|
25
|
+
*/
|
|
26
|
+
export const LENGTH_CONTINUATION_ENVELOPE = "continue_output";
|
|
27
|
+
/**
|
|
28
|
+
* THE TEXT THE MODEL IS GIVEN.
|
|
29
|
+
*
|
|
30
|
+
* It says two things, and both are needed. "Continue" alone invites a model to start the answer
|
|
31
|
+
* again from the top, because the model cannot tell that the earlier text was already accepted;
|
|
32
|
+
* "do not repeat what you have written" is what makes the continuation an extension rather than a
|
|
33
|
+
* second attempt.
|
|
34
|
+
*/
|
|
35
|
+
export const LENGTH_CONTINUATION_TEXT = "上一条回答因为达到单次输出的长度上限被截断了。请直接从被截断的地方接着写完,不要重复已经写过的内容。";
|
|
36
|
+
/**
|
|
37
|
+
* THE WHOLE ORDER, ENCLOSED.
|
|
38
|
+
*
|
|
39
|
+
* The envelope is what makes the row recognisable, and it wraps the text so the browser can show
|
|
40
|
+
* the instruction rather than the markup.
|
|
41
|
+
*/
|
|
42
|
+
export const LENGTH_CONTINUATION_PROMPT = `<${LENGTH_CONTINUATION_ENVELOPE}>${LENGTH_CONTINUATION_TEXT}</${LENGTH_CONTINUATION_ENVELOPE}>`;
|
|
43
|
+
/**
|
|
44
|
+
* Whether this content is a continuation order.
|
|
45
|
+
*/
|
|
46
|
+
export function isLengthContinuation(content) {
|
|
47
|
+
return content.includes(`<${LENGTH_CONTINUATION_ENVELOPE}>`);
|
|
48
|
+
}
|
|
49
|
+
/**
|
|
50
|
+
* THE TEXT INSIDE THE ENVELOPE, or the content itself when it carries no envelope.
|
|
51
|
+
*/
|
|
52
|
+
export function lengthContinuationText(content) {
|
|
53
|
+
const match = new RegExp(`<${LENGTH_CONTINUATION_ENVELOPE}>([\\s\\S]*?)</${LENGTH_CONTINUATION_ENVELOPE}>`).exec(content);
|
|
54
|
+
return match ? match[1].trim() : content.trim();
|
|
55
|
+
}
|
|
@@ -18,6 +18,7 @@
|
|
|
18
18
|
*/
|
|
19
19
|
import { describeToolCall, describeToolCallArgs, extractPartialResultText, readQuestionCall } from "./tool-calls.js";
|
|
20
20
|
import { toolTargetPath } from "./file-changes.js";
|
|
21
|
+
import { LengthExceededError } from "./turn-failure.js";
|
|
21
22
|
/** Keeps only real strings, so a malformed queue snapshot cannot put objects into a broadcast. */
|
|
22
23
|
const textList = (value) => Array.isArray(value) ? value.filter((item) => typeof item === "string") : undefined;
|
|
23
24
|
/**
|
|
@@ -337,7 +338,10 @@ export function handleSessionEvent(event, queue, handlers, report, failure) {
|
|
|
337
338
|
const held = failure?.current;
|
|
338
339
|
if (held) {
|
|
339
340
|
failure.current = undefined;
|
|
340
|
-
|
|
341
|
+
const err = held.stopReason === "length"
|
|
342
|
+
? new LengthExceededError(held.message)
|
|
343
|
+
: new Error(held.message);
|
|
344
|
+
queue.push({ kind: "error", error: err });
|
|
341
345
|
return true;
|
|
342
346
|
}
|
|
343
347
|
queue.push({ kind: "end" });
|
|
@@ -165,6 +165,8 @@ export declare class PiSessionPool {
|
|
|
165
165
|
* tool allowlist has changed.
|
|
166
166
|
*/
|
|
167
167
|
acquire(request: SessionRequest): Promise<PiSessionLike>;
|
|
168
|
+
/** Keep an accidental session fan-out from retaining one full transcript context per conversation. */
|
|
169
|
+
private evictOverflow;
|
|
168
170
|
private build;
|
|
169
171
|
/**
|
|
170
172
|
* Puts a live session on the model and level the conversation is asking for.
|