@evandu/super-pi 0.1.145 → 0.1.147

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (93) hide show
  1. package/agents/super-pi-task/core.ts +74 -2
  2. package/agents/super-pi-task/index.ts +7 -0
  3. package/dist/.super-pi-check-inputs.json +2 -2
  4. package/dist/client/.super-pi-client-sources.json +11 -11
  5. package/dist/client/assets/{CodeMirrorEditor-A9KvbIkm.js → CodeMirrorEditor-DQCjEnxG.js} +2 -2
  6. package/dist/client/assets/{dist-DM4Olwfx.js → dist-1Innu17E.js} +1 -1
  7. package/dist/client/assets/{dist-bIDce5j-.js → dist-1lynFOc5.js} +1 -1
  8. package/dist/client/assets/{dist-BlxwDnWr.js → dist-B1qLZVMK.js} +1 -1
  9. package/dist/client/assets/{dist-BTVbZigv.js → dist-B4pz_xdd.js} +1 -1
  10. package/dist/client/assets/{dist-UJ46aPS3.js → dist-B6KwDpeI.js} +1 -1
  11. package/dist/client/assets/{dist-CK2KFFtT.js → dist-BJVI1Unv.js} +1 -1
  12. package/dist/client/assets/{dist-CRnm1rnO.js → dist-BWvzD8Pe.js} +1 -1
  13. package/dist/client/assets/{dist-DSHM-PAs.js → dist-BayWssd0.js} +1 -1
  14. package/dist/client/assets/{dist-B9DE7sc5.js → dist-Bk_pkBVH.js} +1 -1
  15. package/dist/client/assets/{dist-D-o7iQ8c.js → dist-BpeClAv0.js} +1 -1
  16. package/dist/client/assets/{dist-CQFASxyb.js → dist-BsmJg5PX.js} +1 -1
  17. package/dist/client/assets/{dist-Ds5gfeAi.js → dist-Bx7s4rni.js} +1 -1
  18. package/dist/client/assets/{dist-UPJPMjWv.js → dist-CDBKM9tL.js} +1 -1
  19. package/dist/client/assets/{dist-CIgjifH-.js → dist-CGzXCzCG.js} +1 -1
  20. package/dist/client/assets/{dist-BAiV8I_O.js → dist-CRxCwn-Z.js} +1 -1
  21. package/dist/client/assets/{dist-Br-AuQUG.js → dist-CTDgUnBG.js} +1 -1
  22. package/dist/client/assets/{dist-4JbFTwg2.js → dist-D1VfMWmf.js} +1 -1
  23. package/dist/client/assets/{dist-BYOpvW02.js → dist-DTZLNEci.js} +1 -1
  24. package/dist/client/assets/{dist-CQV5kOCE.js → dist-DuUi29km.js} +1 -1
  25. package/dist/client/assets/{dist-CjszAoXk.js → dist-F3S0t1oM.js} +1 -1
  26. package/dist/client/assets/{dist-2boAf_pN.js → dist-QZ-FAVJV.js} +1 -1
  27. package/dist/client/assets/{dist-B1p4YF9C.js → dist-u0nBg0fZ.js} +1 -1
  28. package/dist/client/assets/index-BMpqviwU.css +1 -0
  29. package/dist/client/assets/index-Dzyp_7sf.js +102 -0
  30. package/dist/client/index.html +2 -2
  31. package/dist/server/active-memory.d.ts +17 -0
  32. package/dist/server/active-memory.js +39 -0
  33. package/dist/server/agents.d.ts +1 -1
  34. package/dist/server/agents.js +3 -1
  35. package/dist/server/application/conversation-service.d.ts +1 -1
  36. package/dist/server/application/conversation-service.js +12 -4
  37. package/dist/server/application/run-service.d.ts +32 -2
  38. package/dist/server/application/run-service.js +296 -40
  39. package/dist/server/application/scheduled-task-service.d.ts +14 -1
  40. package/dist/server/application/scheduled-task-service.js +27 -2
  41. package/dist/server/application/session-orchestrator-service.d.ts +114 -3
  42. package/dist/server/application/session-orchestrator-service.js +406 -35
  43. package/dist/server/application/session-watchdog.d.ts +159 -0
  44. package/dist/server/application/session-watchdog.js +244 -0
  45. package/dist/server/command-handler.js +1 -1
  46. package/dist/server/conversation-usage.d.ts +19 -0
  47. package/dist/server/conversation-usage.js +40 -0
  48. package/dist/server/file-diff.d.ts +3 -0
  49. package/dist/server/file-diff.js +4 -4
  50. package/dist/server/file-index.d.ts +4 -2
  51. package/dist/server/file-index.js +14 -9
  52. package/dist/server/file-name-index.d.ts +5 -2
  53. package/dist/server/file-name-index.js +22 -5
  54. package/dist/server/git.d.ts +3 -3
  55. package/dist/server/git.js +10 -9
  56. package/dist/server/http-server.js +256 -34
  57. package/dist/server/index.js +3 -0
  58. package/dist/server/knowledge-vectors.d.ts +15 -0
  59. package/dist/server/knowledge-vectors.js +118 -37
  60. package/dist/server/knowledge.d.ts +7 -8
  61. package/dist/server/knowledge.js +133 -55
  62. package/dist/server/length-continuation.d.ts +50 -0
  63. package/dist/server/length-continuation.js +55 -0
  64. package/dist/server/model-context-window.d.ts +46 -0
  65. package/dist/server/model-context-window.js +121 -0
  66. package/dist/server/pi-session-events.js +5 -1
  67. package/dist/server/pi-session-pool.d.ts +2 -0
  68. package/dist/server/pi-session-pool.js +16 -0
  69. package/dist/server/pi-session.d.ts +6 -0
  70. package/dist/server/pi-session.js +15 -1
  71. package/dist/server/runtime.d.ts +3 -0
  72. package/dist/server/runtime.js +23 -7
  73. package/dist/server/session-cwd.d.ts +17 -0
  74. package/dist/server/session-cwd.js +43 -0
  75. package/dist/server/session-ledger.d.ts +7 -0
  76. package/dist/server/session-tree-scope.d.ts +125 -0
  77. package/dist/server/session-tree-scope.js +337 -0
  78. package/dist/server/store.d.ts +35 -0
  79. package/dist/server/store.js +87 -2
  80. package/dist/server/task-tool.d.ts +2 -1
  81. package/dist/server/task-tool.js +4 -1
  82. package/dist/server/terminal.d.ts +1 -0
  83. package/dist/server/terminal.js +5 -3
  84. package/dist/server/turn-failure.d.ts +26 -0
  85. package/dist/server/turn-failure.js +67 -1
  86. package/dist/server/types.d.ts +33 -1
  87. package/dist/server/version.d.ts +25 -2
  88. package/dist/server/version.js +98 -26
  89. package/dist/server/ws-server.d.ts +1 -1
  90. package/dist/server/ws-server.js +7 -5
  91. package/package.json +36 -3
  92. package/dist/client/assets/index-BGsP8pt_.js +0 -100
  93. package/dist/client/assets/index-CiIc734l.css +0 -1
@@ -27,8 +27,8 @@ export const KNOWLEDGE_MODEL = "Xenova/bge-small-zh-v1.5";
27
27
  */
28
28
  export const EMBED_PIECE_CHARS = 300;
29
29
  const EMBED_PIECE_OVERLAP = 40;
30
- /** Per query. Cosine similarity of unit vectors; beyond this the passage is noise. */
31
- const MIN_COSINE = 0.15;
30
+ /** Per query. Cosine similarity of unit vectors; threshold raised to 0.35 to filter out semantic noise. */
31
+ export const MIN_COSINE = 0.35;
32
32
  /**
33
33
  * Splits a retrieval chunk into pieces small enough for the model's window.
34
34
  *
@@ -84,6 +84,11 @@ export class VectorIndex {
84
84
  workerReady = false;
85
85
  workerPending = new Map();
86
86
  failure;
87
+ consecutiveFailures = 0;
88
+ lastFailureTime = 0;
89
+ static MAX_FAILURES = 3;
90
+ static COOLDOWN_MS = 30000;
91
+ static EMBED_TIMEOUT_MS = 15000;
87
92
  cache = new Map();
88
93
  pending = new Set();
89
94
  queue = [];
@@ -102,11 +107,29 @@ export class VectorIndex {
102
107
  bundledModel() { return resolve(import.meta.dirname, "../../assets/knowledge-model"); }
103
108
  enabled() { return process.env.SUPER_PI_KNOWLEDGE_EMBEDDINGS !== "0"; }
104
109
  status(indexed) {
110
+ const enabled = this.enabled();
111
+ const bundled = existsSync(this.bundledModel());
112
+ const now = Date.now();
113
+ const inCooldown = this.consecutiveFailures >= VectorIndex.MAX_FAILURES && (now - this.lastFailureTime < VectorIndex.COOLDOWN_MS);
114
+ let health = "ready";
115
+ if (!enabled)
116
+ health = "disabled";
117
+ else if (!bundled || inCooldown)
118
+ health = "degraded";
119
+ else if (this.consecutiveFailures > 0)
120
+ health = "restarting";
121
+ else if (this.worker && this.workerReady)
122
+ health = "ready";
123
+ else
124
+ health = "ready";
105
125
  return {
106
- enabled: this.enabled(), ready: this.workerReady, indexed, error: this.failure,
107
- // Whether the shipped weights are on disk, reported before the model is ever loaded: "the
108
- // package was built without its assets" and "the model failed to load" are different problems.
109
- bundled: existsSync(this.bundledModel()),
126
+ enabled,
127
+ ready: this.workerReady && health === "ready",
128
+ indexed,
129
+ error: this.failure,
130
+ bundled,
131
+ health,
132
+ available: enabled && bundled && health !== "degraded",
110
133
  };
111
134
  }
112
135
  workspace(workspaceId) { return join(this.directory, Buffer.from(workspaceId).toString("hex")); }
@@ -121,45 +144,81 @@ export class VectorIndex {
121
144
  * able to reach, on the first search, for every person who installed the package. Shipping the
122
145
  * weights turns a failure mode into a file, and makes the feature work on a machine that has never
123
146
  * been online since it was installed.
147
+ *
148
+ * RESILIENT WORKER LIFECYCLE:
149
+ * When the worker encounters an error, crash, or unexpected exit, it cleans up all pending requests
150
+ * without hanging and marks the instance for automatic, on-demand respawning on the next embed call.
151
+ * A bounded backoff / cooldown prevents runaway crash loops if the environment is broken.
124
152
  */
125
153
  ensureWorker() {
154
+ if (!this.enabled())
155
+ return undefined;
126
156
  if (this.worker)
127
157
  return this.worker;
128
- if (this.failure)
129
- return undefined;
130
158
  if (!existsSync(this.bundledModel())) {
131
159
  this.failure = `随包分发的模型缺失:${this.bundledModel()}`;
132
160
  return undefined;
133
161
  }
134
- const worker = new Worker(new URL("./knowledge-embedding-worker.js", import.meta.url));
135
- worker.unref();
136
- worker.on("message", (message) => {
137
- const pending = this.workerPending.get(message.id);
138
- if (!pending)
139
- return;
140
- this.workerPending.delete(message.id);
141
- if (message.error || !message.data || !message.dims) {
142
- this.failure = message.error ?? "向量 worker 返回了无效结果";
143
- pending.resolve(null);
144
- return;
162
+ const now = Date.now();
163
+ if (this.consecutiveFailures >= VectorIndex.MAX_FAILURES) {
164
+ if (now - this.lastFailureTime < VectorIndex.COOLDOWN_MS) {
165
+ return undefined;
145
166
  }
146
- this.workerReady = true;
147
- const dimension = message.dims.at(-1) ?? 0;
148
- pending.resolve(Array.from({ length: message.dims[0] ?? 0 }, (_, row) => Array.from(message.data.slice(row * dimension, (row + 1) * dimension))));
149
- });
150
- const fail = (error) => {
151
- this.failure = error.message;
152
- for (const pending of this.workerPending.values())
153
- pending.resolve(null);
154
- this.workerPending.clear();
155
- this.worker = undefined;
156
- this.workerReady = false;
157
- };
158
- worker.on("error", fail);
159
- worker.on("exit", (code) => { if (code !== 0)
160
- fail(new Error(`向量 worker 异常退出(${code})`)); });
161
- this.worker = worker;
162
- return worker;
167
+ this.consecutiveFailures = 0;
168
+ this.failure = undefined;
169
+ }
170
+ try {
171
+ const worker = new Worker(new URL("./knowledge-embedding-worker.js", import.meta.url));
172
+ worker.unref();
173
+ worker.on("message", (message) => {
174
+ const pending = this.workerPending.get(message.id);
175
+ if (!pending)
176
+ return;
177
+ this.workerPending.delete(message.id);
178
+ if (pending.timeout)
179
+ clearTimeout(pending.timeout);
180
+ if (message.error || !message.data || !message.dims) {
181
+ this.failure = message.error ?? "向量 worker 返回了无效结果";
182
+ pending.resolve(null);
183
+ return;
184
+ }
185
+ this.workerReady = true;
186
+ this.failure = undefined;
187
+ this.consecutiveFailures = 0;
188
+ const dimension = message.dims.at(-1) ?? 0;
189
+ pending.resolve(Array.from({ length: message.dims[0] ?? 0 }, (_, row) => Array.from(message.data.slice(row * dimension, (row + 1) * dimension))));
190
+ });
191
+ const fail = (error) => {
192
+ this.consecutiveFailures++;
193
+ this.lastFailureTime = Date.now();
194
+ this.failure = error.message;
195
+ for (const pending of this.workerPending.values()) {
196
+ if (pending.timeout)
197
+ clearTimeout(pending.timeout);
198
+ pending.resolve(null);
199
+ }
200
+ this.workerPending.clear();
201
+ if (this.worker) {
202
+ try {
203
+ this.worker.terminate();
204
+ }
205
+ catch { }
206
+ }
207
+ this.worker = undefined;
208
+ this.workerReady = false;
209
+ };
210
+ worker.on("error", fail);
211
+ worker.on("exit", (code) => { if (code !== 0)
212
+ fail(new Error(`向量 worker 异常退出(${code})`)); });
213
+ this.worker = worker;
214
+ return worker;
215
+ }
216
+ catch (error) {
217
+ this.consecutiveFailures++;
218
+ this.lastFailureTime = Date.now();
219
+ this.failure = error instanceof Error ? error.message : String(error);
220
+ return undefined;
221
+ }
163
222
  }
164
223
  async embed(texts) {
165
224
  if (!texts.length || !this.enabled())
@@ -169,10 +228,32 @@ export class VectorIndex {
169
228
  return null;
170
229
  const id = ++this.workerSequence;
171
230
  return new Promise((resolvePromise) => {
172
- this.workerPending.set(id, { resolve: resolvePromise });
231
+ const timeout = setTimeout(() => {
232
+ const pending = this.workerPending.get(id);
233
+ if (pending) {
234
+ this.workerPending.delete(id);
235
+ pending.resolve(null);
236
+ }
237
+ }, VectorIndex.EMBED_TIMEOUT_MS);
238
+ timeout.unref();
239
+ this.workerPending.set(id, { resolve: resolvePromise, timeout });
173
240
  worker.postMessage({ id, texts, modelRoot: this.bundledModel(), model: KNOWLEDGE_MODEL });
174
241
  });
175
242
  }
243
+ async terminate() {
244
+ for (const pending of this.workerPending.values()) {
245
+ if (pending.timeout)
246
+ clearTimeout(pending.timeout);
247
+ pending.resolve(null);
248
+ }
249
+ this.workerPending.clear();
250
+ if (this.worker) {
251
+ const w = this.worker;
252
+ this.worker = undefined;
253
+ this.workerReady = false;
254
+ await w.terminate();
255
+ }
256
+ }
176
257
  /** Vectors already stored for one workspace, keyed by document then chunk index. */
177
258
  async read(workspaceId, documentId) {
178
259
  try {
@@ -44,7 +44,7 @@ export declare const MAX_DOCUMENT_BYTES: number;
44
44
  export declare const MAX_KNOWLEDGE_REQUEST_BYTES: number;
45
45
  /** Per workspace, across every document. A knowledge base is a curated asset, not a file server. */
46
46
  export declare const MAX_WORKSPACE_BYTES: number;
47
- export declare const DEFAULT_RESULT_LIMIT = 6;
47
+ export declare const DEFAULT_RESULT_LIMIT = 3;
48
48
  export declare const MAX_RESULT_LIMIT = 20;
49
49
  /** What one search may be asked for. `rerank` and `conversation` exist only for the agent's own calls. */
50
50
  export type KnowledgeSearch = {
@@ -72,14 +72,11 @@ export type KnowledgeRuntime = {
72
72
  }): AsyncIterable<string>;
73
73
  };
74
74
  /**
75
- * The first integer array in the model's answer, or null.
76
- *
77
- * WHY TOLERANT PARSING. The prompt asks for a bare JSON array, and a model that mostly complies will
78
- * still sometimes wrap it in a sentence or a code fence. Recovering the array from that is worth doing;
79
- * guessing when there is no array at all is not, so anything unparseable is null and the caller keeps
80
- * the lexical order.
75
+ * Extracts a 1-based candidate integer array from the model's answer, or null.
76
+ * Tolerates surrounding markdown, explanatory text, and trailing commas.
77
+ * Returns empty array [] if the model explicitly declares no candidates are relevant.
81
78
  */
82
- export declare function parseRanking(output: string): number[] | null;
79
+ export declare function parseRanking(output: string, maxIndex?: number): number[] | null;
83
80
  /** Duplicated as a literal in `agents/super-pi-knowledge/index.ts`, the way `TASK_TOOL_NAME` is. */
84
81
  export declare const KNOWLEDGE_TOOL_NAME = "knowledge_search";
85
82
  /** The import half of the same extension. Duplicated as a literal there, for the same reason. */
@@ -116,6 +113,8 @@ export declare function digestOf(text: string): string;
116
113
  * repeated at the head of the next chunk to keep that sentence whole in at least one hit.
117
114
  */
118
115
  export declare function chunkDocument(text: string): KnowledgeChunk[];
116
+ export declare const MIN_SCORE_ABSOLUTE = 0.28;
117
+ export declare const RELATIVE_CUTOFF_RATIO = 0.5;
119
118
  export declare class KnowledgeBase {
120
119
  readonly directory: string;
121
120
  private store;
@@ -12,12 +12,12 @@ export const MAX_WORKSPACE_BYTES = 16 * 1024 * 1024;
12
12
  const CHUNK_CHARS = 1200;
13
13
  const CHUNK_OVERLAP_CHARS = 150;
14
14
  const MAX_CHUNKS_PER_DOCUMENT = 2000;
15
- export const DEFAULT_RESULT_LIMIT = 6;
15
+ export const DEFAULT_RESULT_LIMIT = 3;
16
16
  export const MAX_RESULT_LIMIT = 20;
17
- /** Per hit. Enough to answer from, short enough that six hits stay inside one turn's prompt budget. */
17
+ /** Per hit. Enough to answer from, short enough that hits stay inside one turn's prompt budget. */
18
18
  const RESULT_TEXT_CAP = 1200;
19
- /** Across all hits of one search. A model that asked for 20 passages still gets a bounded prompt. */
20
- const RESULT_TOTAL_CAP = 12000;
19
+ /** Across all hits of one search. Lowered to 4800 to bound prompt impact on standard context windows. */
20
+ const RESULT_TOTAL_CAP = 4800;
21
21
  /** Below this many candidates there is no ranking to correct, and the model call is not worth it. */
22
22
  const RERANK_MIN_CANDIDATES = 3;
23
23
  /** How many candidates the model sees. Larger pools cost prompt without improving the top few. */
@@ -26,26 +26,64 @@ const RERANK_POOL = 12;
26
26
  const RERANK_SNIPPET_CHARS = 240;
27
27
  const RERANK_TIMEOUT_MS = 12000;
28
28
  /**
29
- * The first integer array in the model's answer, or null.
30
- *
31
- * WHY TOLERANT PARSING. The prompt asks for a bare JSON array, and a model that mostly complies will
32
- * still sometimes wrap it in a sentence or a code fence. Recovering the array from that is worth doing;
33
- * guessing when there is no array at all is not, so anything unparseable is null and the caller keeps
34
- * the lexical order.
29
+ * Extracts a 1-based candidate integer array from the model's answer, or null.
30
+ * Tolerates surrounding markdown, explanatory text, and trailing commas.
31
+ * Returns empty array [] if the model explicitly declares no candidates are relevant.
35
32
  */
36
- export function parseRanking(output) {
37
- const match = /\[\s*\d+(?:\s*,\s*\d+)*\s*\]/.exec(output ?? "");
38
- if (!match)
33
+ export function parseRanking(output, maxIndex) {
34
+ if (!output || typeof output !== "string")
39
35
  return null;
40
- try {
41
- const parsed = JSON.parse(match[0]);
42
- if (!Array.isArray(parsed))
43
- return null;
44
- return parsed.filter((value) => Number.isInteger(value) && value > 0);
36
+ // 1. Prefer markdown code block: ```json ... ``` or ``` ... ```
37
+ const codeBlock = /```(?:json)?\s*(\[[\s\S]*?\])\s*```/i.exec(output);
38
+ const targetText = codeBlock ? codeBlock[1] : output;
39
+ // 2. Find bracketed arrays
40
+ const matches = [...targetText.matchAll(/\[[\s\S]*?\]/g)];
41
+ if (!matches.length)
42
+ return null;
43
+ // Try parsing from the last match backwards (models often put final answer at the end)
44
+ let parsed = null;
45
+ for (let i = matches.length - 1; i >= 0; i--) {
46
+ const candidate = matches[i][0].replace(/,\s*\]/, "]");
47
+ try {
48
+ const p = JSON.parse(candidate);
49
+ if (Array.isArray(p)) {
50
+ parsed = p;
51
+ break;
52
+ }
53
+ }
54
+ catch { }
45
55
  }
46
- catch {
56
+ if (!Array.isArray(parsed))
47
57
  return null;
58
+ if (parsed.length === 0)
59
+ return [];
60
+ // Reject negative numbers or non-integers, per existing test: [0, 2, -1, 1.5] -> null
61
+ for (const item of parsed) {
62
+ if (typeof item === "number") {
63
+ if (!Number.isInteger(item) || item < 0)
64
+ return null;
65
+ }
66
+ else if (typeof item === "string") {
67
+ if (!/^\d+$/.test(item.trim()))
68
+ return null;
69
+ }
70
+ else {
71
+ return null;
72
+ }
48
73
  }
74
+ // Deduplicate and filter out-of-bounds (<= 0 or > maxIndex)
75
+ const seen = new Set();
76
+ const result = [];
77
+ for (const item of parsed) {
78
+ const val = typeof item === "string" ? parseInt(item.trim(), 10) : item;
79
+ if (val > 0 && (maxIndex === undefined || val <= maxIndex)) {
80
+ if (!seen.has(val)) {
81
+ seen.add(val);
82
+ result.push(val);
83
+ }
84
+ }
85
+ }
86
+ return result;
49
87
  }
50
88
  const CJK_RUN = /[\u3040-\u30ff\u3400-\u4dbf\u4e00-\u9fff\uf900-\ufaff]+/gu;
51
89
  const LATIN_WORD = /[\p{L}\p{N}_.+#-]{2,}/gu;
@@ -212,6 +250,9 @@ const B = 0.75;
212
250
  * the chunk that actually answers may sit in a section whose heading shares nothing with the question.
213
251
  */
214
252
  const WEIGHT = { body: 1, phrase: 0.45, head: 0.5, document: 0.7, vector: 0.9 };
253
+ const TOTAL_WEIGHT = WEIGHT.body + WEIGHT.phrase + WEIGHT.head + WEIGHT.document + WEIGHT.vector; // 3.55
254
+ export const MIN_SCORE_ABSOLUTE = 0.28;
255
+ export const RELATIVE_CUTOFF_RATIO = 0.5;
215
256
  function countTerms(terms) {
216
257
  const counts = new Map();
217
258
  for (const term of terms)
@@ -321,6 +362,10 @@ export class KnowledgeBase {
321
362
  const channels = [];
322
363
  const docTerms = new Map();
323
364
  let docLength = 0;
365
+ for (const term of tokenize(document.title)) {
366
+ docTerms.set(term, (docTerms.get(term) ?? 0) + 1);
367
+ docLength++;
368
+ }
324
369
  for (const chunk of chunks) {
325
370
  // Tokenised ONCE here, at import time, rather than on every search: a workspace with a hundred
326
371
  // documents would otherwise re-split the whole corpus per question.
@@ -359,9 +404,29 @@ export class KnowledgeBase {
359
404
  if (cached)
360
405
  return cached;
361
406
  const documents = (await this.readAll(workspaceId)).map(document => this.index(document));
407
+ const chunkDf = new Map(), phraseDf = new Map(), headDf = new Map(), docDf = new Map();
408
+ let chunkCount = 0;
409
+ for (const document of documents) {
410
+ for (const channels of document.channels) {
411
+ chunkCount++;
412
+ for (const [term] of channels.body)
413
+ chunkDf.set(term, (chunkDf.get(term) ?? 0) + 1);
414
+ for (const [term] of channels.head)
415
+ headDf.set(term, (headDf.get(term) ?? 0) + 1);
416
+ for (const [phrase] of channels.phrase)
417
+ phraseDf.set(phrase, (phraseDf.get(phrase) ?? 0) + 1);
418
+ }
419
+ for (const [term] of document.docTerms)
420
+ docDf.set(term, (docDf.get(term) ?? 0) + 1);
421
+ }
362
422
  const index = {
363
423
  documents,
364
424
  averageLength: documents.length ? documents.reduce((sum, document) => sum + document.docLength, 0) / documents.length : 0,
425
+ chunkCount,
426
+ chunkDf,
427
+ phraseDf,
428
+ headDf,
429
+ docDf,
365
430
  };
366
431
  this.indexes.set(workspaceId, index);
367
432
  // Vectors are built in the background on first sight of a workspace, so a restart with an
@@ -528,29 +593,9 @@ export class KnowledgeBase {
528
593
  const phrases = [...new Set(tokenizePhrases(query ?? ""))];
529
594
  if (!terms.length && !phrases.length)
530
595
  return [];
531
- const { documents, averageLength } = await this.load(workspaceId);
596
+ const { documents, averageLength, chunkCount, chunkDf, phraseDf, headDf, docDf } = await this.load(workspaceId);
532
597
  if (!documents.length || !averageLength)
533
598
  return [];
534
- const chunkCount = documents.reduce((sum, document) => sum + document.chunks.length, 0);
535
- // Document frequency is counted once per channel, over that channel's own unit: chunks for the
536
- // passage channels, documents for the document channel.
537
- const chunkDf = new Map(), phraseDf = new Map(), headDf = new Map(), docDf = new Map();
538
- for (const document of documents) {
539
- for (const channels of document.channels) {
540
- for (const term of terms) {
541
- if (channels.body.has(term))
542
- chunkDf.set(term, (chunkDf.get(term) ?? 0) + 1);
543
- if (channels.head.has(term))
544
- headDf.set(term, (headDf.get(term) ?? 0) + 1);
545
- }
546
- for (const phrase of phrases)
547
- if (channels.phrase.has(phrase))
548
- phraseDf.set(phrase, (phraseDf.get(phrase) ?? 0) + 1);
549
- }
550
- for (const term of terms)
551
- if (document.docTerms.has(term))
552
- docDf.set(term, (docDf.get(term) ?? 0) + 1);
553
- }
554
599
  const hits = [];
555
600
  let bodyBest = 0, phraseBest = 0, headBest = 0, docBest = 0;
556
601
  for (const document of documents) {
@@ -601,18 +646,29 @@ export class KnowledgeBase {
601
646
  const similarities = this.vectors
602
647
  ? await this.vectors.search(workspaceId, query ?? "", lexical.map(entry => [entry.key, entry.hit.documentId]))
603
648
  : null;
604
- const fused = lexical
605
- .map(entry => {
649
+ const rawScores = lexical.map(entry => {
606
650
  const similarity = similarities?.get(entry.key) ?? 0;
607
- const score = Math.round(1000 * (entry.lexical + (similarities ? WEIGHT.vector * similarity : 0)
608
- + (entry.lexical > 0 ? WEIGHT.document * (docBest ? entry.document / docBest : 0) : 0))) / 1000;
651
+ const rawScore = entry.lexical + (similarities ? WEIGHT.vector * similarity : 0)
652
+ + (entry.lexical > 0 ? WEIGHT.document * (docBest ? entry.document / docBest : 0) : 0);
653
+ const score = Math.min(1, Math.round(1000 * (rawScore / TOTAL_WEIGHT)) / 1000);
609
654
  return { ...entry.hit, score };
610
- })
611
- .filter(hit => hit.score > 0);
655
+ });
656
+ // 1. Absolute threshold filter: drop anything with score < MIN_SCORE_ABSOLUTE (0.28)
657
+ const passedAbsolute = rawScores.filter(hit => hit.score >= MIN_SCORE_ABSOLUTE);
658
+ if (!passedAbsolute.length)
659
+ return [];
612
660
  // Best-first, with the tie broken by chunk order so two equal passages of one document come back
613
661
  // in the order a reader would meet them.
614
- fused.sort((a, b) => b.score - a.score || a.chunk - b.chunk);
615
- const ordered = options.rerank ? await this.rerank(options.conversation, query ?? "", fused, options.signal) ?? fused : fused;
662
+ passedAbsolute.sort((a, b) => b.score - a.score || a.chunk - b.chunk);
663
+ // 2. Relative cutoff: drop anything with score < bestScore * RELATIVE_CUTOFF_RATIO (0.50)
664
+ const bestScore = passedAbsolute[0].score;
665
+ const fused = passedAbsolute.filter(hit => hit.score >= bestScore * RELATIVE_CUTOFF_RATIO);
666
+ if (!fused.length)
667
+ return [];
668
+ const reranked = options.rerank ? await this.rerank(options.conversation, query ?? "", fused, options.signal) : null;
669
+ const ordered = reranked ?? fused;
670
+ if (!ordered.length)
671
+ return [];
616
672
  const selected = [];
617
673
  let spent = 0;
618
674
  for (const hit of ordered) {
@@ -670,23 +726,45 @@ export class KnowledgeBase {
670
726
  agentId: undefined, messages: [], status: "idle",
671
727
  };
672
728
  const listing = pool.map((hit, position) => `[${position + 1}] 《${hit.title}》${hit.heading ? ` > ${hit.heading}` : ""}\n${hit.text.slice(0, RERANK_SNIPPET_CHARS).replace(/\n+/g, " ")}`).join("\n\n");
673
- const prompt = `你是检索排序器。用户提出了一个问题,下面是知识库检索出的候选片段。请按「能否回答这个问题」从高到低排序,只输出最相关的前几个片段编号,输出 JSON 数组,例如 [3,1]。不要输出解释文字,不要输出编号以外的内容。\n\n问题:${query}\n\n候选片段:\n${listing}`;
729
+ const prompt = `你是检索排序器。用户提出了一个问题,下面是知识库检索出的候选片段。请按「能否回答这个问题」对候选片段进行筛选与重排序。
730
+ 评分准则:
731
+ 1. 仅保留能够回答问题或与问题高度相关的片段。
732
+ 2. 按相关度从高到低排序,输出候选编号的 JSON 数组,例如 [3, 1]。
733
+ 3. 若所有候选片段均不相关,输出空数组 []。
734
+ 4. 只输出 JSON 数组,不要输出解释文字,不要输出编号以外的内容。
735
+
736
+ 问题:${query}
737
+
738
+ 候选片段:
739
+ ${listing}`;
674
740
  let output = "";
675
741
  for await (const piece of this.runtime.answer({ conversation: isolated, prompt, cwd: workspace.path, signal: controller.signal, knowledgeRerank: true })) {
676
742
  output += piece;
677
743
  if (controller.signal.aborted)
678
744
  return null;
679
745
  }
680
- const parsed = parseRanking(output);
681
- if (!parsed?.length)
746
+ const parsed = parseRanking(output, pool.length);
747
+ if (parsed === null)
682
748
  return null;
749
+ if (parsed.length === 0)
750
+ return [];
683
751
  const picked = parsed.map((index) => pool[index - 1]).filter((hit) => Boolean(hit));
684
752
  if (!picked.length)
685
- return null;
686
- // Anything the model left out keeps its lexical position after the picks, so a partial or
687
- // truncated answer still returns the full pool rather than silently losing candidates.
753
+ return [];
754
+ // Anything the model left out keeps its position after the picks
688
755
  const chosen = new Set(picked);
689
- return [...picked, ...pool.filter(hit => !chosen.has(hit))];
756
+ const remaining = pool.filter(hit => !chosen.has(hit));
757
+ const reordered = [...picked, ...remaining];
758
+ // Re-assign monotonically non-increasing scores based on pool's score distribution
759
+ const poolScores = pool.map(h => h.score).sort((a, b) => b - a);
760
+ let lastScore = poolScores[0];
761
+ return reordered.map((hit, index) => {
762
+ let assigned = poolScores[index];
763
+ if (assigned > lastScore)
764
+ assigned = lastScore;
765
+ lastScore = assigned;
766
+ return { ...hit, score: assigned };
767
+ });
690
768
  }
691
769
  catch {
692
770
  return null;
@@ -0,0 +1,50 @@
1
+ /**
2
+ * THE CONTINUATION ORDER: what a session is told when its answer was cut off by the output limit.
3
+ *
4
+ * WHY THIS IS ITS OWN MODULE. Three places have to agree on it: the orchestrator that sends it,
5
+ * the prompt builder that tells the model who is speaking, and the browser that renders the row it
6
+ * becomes. When those three each held their own copy of the wording, a change to one of them left
7
+ * the other two describing a message that was no longer being sent.
8
+ *
9
+ * WHY THE ORDER EXISTS AT ALL. A model that filled its single-response output budget did not fail -
10
+ * it answered, and the answer was cut short. The work it was doing is not over. So the session is
11
+ * asked to carry on, in the SAME conversation, where its own partial answer is already in its
12
+ * context. That is what lets the model resume at the sentence it stopped on instead of starting
13
+ * the answer again.
14
+ *
15
+ * WHY IT IS AN ORDER AND NOT A USER MESSAGE. The person never typed it, and a row under their name
16
+ * saying words they never said is the exact bug this ordering was built to avoid.
17
+ */
18
+ /**
19
+ * THE ENVELOPE A CONTINUATION ORDER TRAVELS IN.
20
+ *
21
+ * The browser renders a `tool` row by looking at the content, and it tells a delivered result
22
+ * (`<task ...>`) from an order (no envelope) by the same inspection. A continuation is neither: it
23
+ * is the system telling the session to keep writing, and it needs a third label - 「续写」 - because
24
+ * calling it a 「智能体下发」 order would say the system sent a task, which is not what happened.
25
+ */
26
+ export declare const LENGTH_CONTINUATION_ENVELOPE = "continue_output";
27
+ /**
28
+ * THE TEXT THE MODEL IS GIVEN.
29
+ *
30
+ * It says two things, and both are needed. "Continue" alone invites a model to start the answer
31
+ * again from the top, because the model cannot tell that the earlier text was already accepted;
32
+ * "do not repeat what you have written" is what makes the continuation an extension rather than a
33
+ * second attempt.
34
+ */
35
+ export declare const LENGTH_CONTINUATION_TEXT = "\u4E0A\u4E00\u6761\u56DE\u7B54\u56E0\u4E3A\u8FBE\u5230\u5355\u6B21\u8F93\u51FA\u7684\u957F\u5EA6\u4E0A\u9650\u88AB\u622A\u65AD\u4E86\u3002\u8BF7\u76F4\u63A5\u4ECE\u88AB\u622A\u65AD\u7684\u5730\u65B9\u63A5\u7740\u5199\u5B8C\uFF0C\u4E0D\u8981\u91CD\u590D\u5DF2\u7ECF\u5199\u8FC7\u7684\u5185\u5BB9\u3002";
36
+ /**
37
+ * THE WHOLE ORDER, ENCLOSED.
38
+ *
39
+ * The envelope is what makes the row recognisable, and it wraps the text so the browser can show
40
+ * the instruction rather than the markup.
41
+ */
42
+ export declare const LENGTH_CONTINUATION_PROMPT = "<continue_output>\u4E0A\u4E00\u6761\u56DE\u7B54\u56E0\u4E3A\u8FBE\u5230\u5355\u6B21\u8F93\u51FA\u7684\u957F\u5EA6\u4E0A\u9650\u88AB\u622A\u65AD\u4E86\u3002\u8BF7\u76F4\u63A5\u4ECE\u88AB\u622A\u65AD\u7684\u5730\u65B9\u63A5\u7740\u5199\u5B8C\uFF0C\u4E0D\u8981\u91CD\u590D\u5DF2\u7ECF\u5199\u8FC7\u7684\u5185\u5BB9\u3002</continue_output>";
43
+ /**
44
+ * Whether this content is a continuation order.
45
+ */
46
+ export declare function isLengthContinuation(content: string): boolean;
47
+ /**
48
+ * THE TEXT INSIDE THE ENVELOPE, or the content itself when it carries no envelope.
49
+ */
50
+ export declare function lengthContinuationText(content: string): string;
@@ -0,0 +1,55 @@
1
+ /**
2
+ * THE CONTINUATION ORDER: what a session is told when its answer was cut off by the output limit.
3
+ *
4
+ * WHY THIS IS ITS OWN MODULE. Three places have to agree on it: the orchestrator that sends it,
5
+ * the prompt builder that tells the model who is speaking, and the browser that renders the row it
6
+ * becomes. When those three each held their own copy of the wording, a change to one of them left
7
+ * the other two describing a message that was no longer being sent.
8
+ *
9
+ * WHY THE ORDER EXISTS AT ALL. A model that filled its single-response output budget did not fail -
10
+ * it answered, and the answer was cut short. The work it was doing is not over. So the session is
11
+ * asked to carry on, in the SAME conversation, where its own partial answer is already in its
12
+ * context. That is what lets the model resume at the sentence it stopped on instead of starting
13
+ * the answer again.
14
+ *
15
+ * WHY IT IS AN ORDER AND NOT A USER MESSAGE. The person never typed it, and a row under their name
16
+ * saying words they never said is the exact bug this ordering was built to avoid.
17
+ */
18
+ /**
19
+ * THE ENVELOPE A CONTINUATION ORDER TRAVELS IN.
20
+ *
21
+ * The browser renders a `tool` row by looking at the content, and it tells a delivered result
22
+ * (`<task ...>`) from an order (no envelope) by the same inspection. A continuation is neither: it
23
+ * is the system telling the session to keep writing, and it needs a third label - 「续写」 - because
24
+ * calling it a 「智能体下发」 order would say the system sent a task, which is not what happened.
25
+ */
26
+ export const LENGTH_CONTINUATION_ENVELOPE = "continue_output";
27
+ /**
28
+ * THE TEXT THE MODEL IS GIVEN.
29
+ *
30
+ * It says two things, and both are needed. "Continue" alone invites a model to start the answer
31
+ * again from the top, because the model cannot tell that the earlier text was already accepted;
32
+ * "do not repeat what you have written" is what makes the continuation an extension rather than a
33
+ * second attempt.
34
+ */
35
+ export const LENGTH_CONTINUATION_TEXT = "上一条回答因为达到单次输出的长度上限被截断了。请直接从被截断的地方接着写完,不要重复已经写过的内容。";
36
+ /**
37
+ * THE WHOLE ORDER, ENCLOSED.
38
+ *
39
+ * The envelope is what makes the row recognisable, and it wraps the text so the browser can show
40
+ * the instruction rather than the markup.
41
+ */
42
+ export const LENGTH_CONTINUATION_PROMPT = `<${LENGTH_CONTINUATION_ENVELOPE}>${LENGTH_CONTINUATION_TEXT}</${LENGTH_CONTINUATION_ENVELOPE}>`;
43
+ /**
44
+ * Whether this content is a continuation order.
45
+ */
46
+ export function isLengthContinuation(content) {
47
+ return content.includes(`<${LENGTH_CONTINUATION_ENVELOPE}>`);
48
+ }
49
+ /**
50
+ * THE TEXT INSIDE THE ENVELOPE, or the content itself when it carries no envelope.
51
+ */
52
+ export function lengthContinuationText(content) {
53
+ const match = new RegExp(`<${LENGTH_CONTINUATION_ENVELOPE}>([\\s\\S]*?)</${LENGTH_CONTINUATION_ENVELOPE}>`).exec(content);
54
+ return match ? match[1].trim() : content.trim();
55
+ }