@evandu/super-pi-server 0.0.0-stage → 0.1.259
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +248 -3
- package/agents/code-auditor.md +64 -0
- package/agents/code-designer.md +64 -0
- package/agents/coder.md +24 -0
- package/agents/critic.md +24 -0
- package/agents/document-organizer.md +48 -0
- package/agents/patcher.md +27 -0
- package/agents/planner.md +30 -0
- package/agents/project-leader.md +28 -0
- package/agents/reviewer.md +27 -0
- package/agents/scout.md +27 -0
- package/agents/solution-designer.md +62 -0
- package/agents/super-pi-documents/index.ts +92 -0
- package/agents/super-pi-feishu/index.ts +88 -0
- package/agents/super-pi-knowledge/index.ts +70 -0
- package/agents/super-pi-managed-model/index.ts +9 -0
- package/agents/super-pi-memory/index.ts +75 -0
- package/agents/super-pi-native-tools/index.ts +1 -0
- package/agents/super-pi-task/core.ts +2262 -0
- package/agents/super-pi-task/index.ts +814 -0
- package/agents/super-pi-task/workflow-schema.ts +34 -0
- package/agents/super-pi-web/index.ts +32 -0
- package/agents/verifier.md +25 -0
- package/assets/fonts/LICENSE +92 -0
- package/assets/fonts/NotoSansSC-Bold.otf +0 -0
- package/assets/fonts/NotoSansSC-Regular.otf +0 -0
- package/assets/knowledge-model/README.md +37 -0
- package/assets/knowledge-model/Xenova/bge-small-zh-v1.5/config.json +31 -0
- package/assets/knowledge-model/Xenova/bge-small-zh-v1.5/onnx/model_quantized.onnx +0 -0
- package/assets/knowledge-model/Xenova/bge-small-zh-v1.5/special_tokens_map.json +7 -0
- package/assets/knowledge-model/Xenova/bge-small-zh-v1.5/tokenizer.json +21278 -0
- package/assets/knowledge-model/Xenova/bge-small-zh-v1.5/tokenizer_config.json +15 -0
- package/assets/knowledge-model/Xenova/bge-small-zh-v1.5/vocab.txt +21128 -0
- package/bin/npm-prefix.d.ts +1 -0
- package/bin/npm-prefix.js +10 -0
- package/bin/super-pi.js +349 -0
- package/dist/client/.super-pi-client-sources.json +82 -0
- package/dist/client/assets/CodeMirrorEditor-mi9rGUce.js +2 -0
- package/dist/client/assets/GitDiffEditor-CpoDDQrA.css +1 -0
- package/dist/client/assets/GitDiffEditor-FQBfGxOY.js +29 -0
- package/dist/client/assets/diff-ChtP43wD.js +1 -0
- package/dist/client/assets/dist-AW5lyI3I.js +1 -0
- package/dist/client/assets/dist-B3-a6uF0.js +7 -0
- package/dist/client/assets/dist-BGxkyh6e.js +1 -0
- package/dist/client/assets/dist-BJxfd7yb.js +11 -0
- package/dist/client/assets/dist-BRVQKlcp.js +1 -0
- package/dist/client/assets/dist-BV64A9Ji.js +1 -0
- package/dist/client/assets/dist-BXAOrsjq.js +1 -0
- package/dist/client/assets/dist-CKF2Jksy.js +1 -0
- package/dist/client/assets/dist-CX_Hm4GS.js +1 -0
- package/dist/client/assets/dist-Cus6_oaF.js +2 -0
- package/dist/client/assets/dist-Cx00EZKR.js +23 -0
- package/dist/client/assets/dist-D71y2EI1.js +1 -0
- package/dist/client/assets/dist-D978o95T.js +5 -0
- package/dist/client/assets/dist-Dc1MBGAG.js +1 -0
- package/dist/client/assets/dist-Dds-qLxY.js +9 -0
- package/dist/client/assets/dist-DeD9copn.js +1 -0
- package/dist/client/assets/dist-DuvGReu-.js +6 -0
- package/dist/client/assets/dist-Dvb0SqcK.js +2 -0
- package/dist/client/assets/dist-J6XWqRpB.js +3 -0
- package/dist/client/assets/dist-QzbOCAU6.js +1 -0
- package/dist/client/assets/dist-moiMRpw_.js +1 -0
- package/dist/client/assets/dist-scBkrwv-.js +2 -0
- package/dist/client/assets/dockerfile-BDePf5PZ.js +1 -0
- package/dist/client/assets/index-Cn7hdrDw.js +350 -0
- package/dist/client/assets/index-UfzqdECL.css +1 -0
- package/dist/client/assets/powershell-D8WpQPaX.js +1 -0
- package/dist/client/assets/properties-C357ku5U.js +1 -0
- package/dist/client/assets/rolldown-runtime-XKyaHtl9.js +1 -0
- package/dist/client/assets/ruby-ChfNwlCY.js +1 -0
- package/dist/client/assets/shell-DwuoZtxw.js +1 -0
- package/dist/client/assets/swift-DKB6_1j6.js +1 -0
- package/dist/client/assets/toml-BPTmHmyx.js +1 -0
- package/dist/client/favicon.ico +0 -0
- package/dist/client/favicon.png +0 -0
- package/dist/client/favicon.svg +10 -0
- package/dist/client/index.html +18 -0
- package/dist/extension.d.ts +15 -0
- package/dist/extension.js +126 -0
- package/dist/server/active-memory.d.ts +247 -0
- package/dist/server/active-memory.js +669 -0
- package/dist/server/agents.d.ts +245 -0
- package/dist/server/agents.js +701 -0
- package/dist/server/application/catalog-service.d.ts +98 -0
- package/dist/server/application/catalog-service.js +245 -0
- package/dist/server/application/conversation-service.d.ts +245 -0
- package/dist/server/application/conversation-service.js +654 -0
- package/dist/server/application/document-service.d.ts +68 -0
- package/dist/server/application/document-service.js +120 -0
- package/dist/server/application/extension-ui-service.d.ts +44 -0
- package/dist/server/application/extension-ui-service.js +135 -0
- package/dist/server/application/feishu-service.d.ts +179 -0
- package/dist/server/application/feishu-service.js +358 -0
- package/dist/server/application/group-chat-service.d.ts +75 -0
- package/dist/server/application/group-chat-service.js +953 -0
- package/dist/server/application/group-execution.d.ts +15 -0
- package/dist/server/application/group-execution.js +77 -0
- package/dist/server/application/project-service.d.ts +464 -0
- package/dist/server/application/project-service.js +2523 -0
- package/dist/server/application/provider-service.d.ts +174 -0
- package/dist/server/application/provider-service.js +510 -0
- package/dist/server/application/run-service.d.ts +547 -0
- package/dist/server/application/run-service.js +2683 -0
- package/dist/server/application/scheduled-task-service.d.ts +184 -0
- package/dist/server/application/scheduled-task-service.js +391 -0
- package/dist/server/application/session-orchestrator-service.d.ts +388 -0
- package/dist/server/application/session-orchestrator-service.js +1146 -0
- package/dist/server/application/session-reminder-service.d.ts +59 -0
- package/dist/server/application/session-reminder-service.js +141 -0
- package/dist/server/application/session-watchdog.d.ts +171 -0
- package/dist/server/application/session-watchdog.js +284 -0
- package/dist/server/application/workspace-service.d.ts +52 -0
- package/dist/server/application/workspace-service.js +186 -0
- package/dist/server/application/worktree-service.d.ts +13 -0
- package/dist/server/application/worktree-service.js +76 -0
- package/dist/server/browser-access.d.ts +34 -0
- package/dist/server/browser-access.js +89 -0
- package/dist/server/capability.d.ts +105 -0
- package/dist/server/capability.js +170 -0
- package/dist/server/command-handler.d.ts +93 -0
- package/dist/server/command-handler.js +540 -0
- package/dist/server/content-search.d.ts +16 -0
- package/dist/server/content-search.js +235 -0
- package/dist/server/conversation-usage.d.ts +22 -0
- package/dist/server/conversation-usage.js +43 -0
- package/dist/server/documents.d.ts +152 -0
- package/dist/server/documents.js +998 -0
- package/dist/server/engineering-learning.d.ts +30 -0
- package/dist/server/engineering-learning.js +106 -0
- package/dist/server/exec-service.d.ts +45 -0
- package/dist/server/exec-service.js +167 -0
- package/dist/server/explicit-file-open.d.ts +34 -0
- package/dist/server/explicit-file-open.js +105 -0
- package/dist/server/failure-text.d.ts +18 -0
- package/dist/server/failure-text.js +44 -0
- package/dist/server/feishu-cli.d.ts +223 -0
- package/dist/server/feishu-cli.js +464 -0
- package/dist/server/file-changes.d.ts +119 -0
- package/dist/server/file-changes.js +280 -0
- package/dist/server/file-diff.d.ts +41 -0
- package/dist/server/file-diff.js +97 -0
- package/dist/server/file-index.d.ts +100 -0
- package/dist/server/file-index.js +296 -0
- package/dist/server/file-name-index.d.ts +160 -0
- package/dist/server/file-name-index.js +411 -0
- package/dist/server/files.d.ts +269 -0
- package/dist/server/files.js +748 -0
- package/dist/server/git.d.ts +245 -0
- package/dist/server/git.js +903 -0
- package/dist/server/http-server.d.ts +45 -0
- package/dist/server/http-server.js +3830 -0
- package/dist/server/index.d.ts +1 -0
- package/dist/server/index.js +485 -0
- package/dist/server/inline-thinking.d.ts +90 -0
- package/dist/server/inline-thinking.js +181 -0
- package/dist/server/json-comments.d.ts +14 -0
- package/dist/server/json-comments.js +71 -0
- package/dist/server/knowledge-embedding-worker.d.ts +1 -0
- package/dist/server/knowledge-embedding-worker.js +27 -0
- package/dist/server/knowledge-vectors.d.ts +150 -0
- package/dist/server/knowledge-vectors.js +385 -0
- package/dist/server/knowledge.d.ts +273 -0
- package/dist/server/knowledge.js +848 -0
- package/dist/server/length-continuation.d.ts +83 -0
- package/dist/server/length-continuation.js +118 -0
- package/dist/server/logging.d.ts +32 -0
- package/dist/server/logging.js +185 -0
- package/dist/server/managed-http-tunnel.d.ts +6 -0
- package/dist/server/managed-http-tunnel.js +85 -0
- package/dist/server/managed-http.d.ts +4 -0
- package/dist/server/managed-http.js +85 -0
- package/dist/server/managed-model-adapter.d.ts +17 -0
- package/dist/server/managed-model-adapter.js +75 -0
- package/dist/server/managed-model-gateway.d.ts +49 -0
- package/dist/server/managed-model-gateway.js +73 -0
- package/dist/server/managed-servers.d.ts +55 -0
- package/dist/server/managed-servers.js +392 -0
- package/dist/server/managed-ssh.d.ts +26 -0
- package/dist/server/managed-ssh.js +73 -0
- package/dist/server/manager-vault.d.ts +11 -0
- package/dist/server/manager-vault.js +67 -0
- package/dist/server/memory-recall.d.ts +25 -0
- package/dist/server/memory-recall.js +45 -0
- package/dist/server/memory.d.ts +67 -0
- package/dist/server/memory.js +216 -0
- package/dist/server/message-media.d.ts +7 -0
- package/dist/server/message-media.js +54 -0
- package/dist/server/model-context-window.d.ts +186 -0
- package/dist/server/model-context-window.js +444 -0
- package/dist/server/model-filter.d.ts +23 -0
- package/dist/server/model-filter.js +48 -0
- package/dist/server/models-store.d.ts +36 -0
- package/dist/server/models-store.js +132 -0
- package/dist/server/native-tool-settings.d.ts +25 -0
- package/dist/server/native-tool-settings.js +41 -0
- package/dist/server/native-tools-extension.d.ts +2 -0
- package/dist/server/native-tools-extension.js +39 -0
- package/dist/server/native-tools-service.d.ts +39 -0
- package/dist/server/native-tools-service.js +240 -0
- package/dist/server/persistence/app-state-repository.d.ts +65 -0
- package/dist/server/persistence/app-state-repository.js +1 -0
- package/dist/server/persistence/json-state-repository.d.ts +14 -0
- package/dist/server/persistence/json-state-repository.js +43 -0
- package/dist/server/persistence/open-state-store.d.ts +3 -0
- package/dist/server/persistence/open-state-store.js +35 -0
- package/dist/server/persistence/rename.d.ts +22 -0
- package/dist/server/persistence/rename.js +44 -0
- package/dist/server/persistence/sqlite-state-repository.d.ts +12 -0
- package/dist/server/persistence/sqlite-state-repository.js +264 -0
- package/dist/server/pi-command.d.ts +94 -0
- package/dist/server/pi-command.js +265 -0
- package/dist/server/pi-native.d.ts +12 -0
- package/dist/server/pi-native.js +16 -0
- package/dist/server/pi-session-events.d.ts +158 -0
- package/dist/server/pi-session-events.js +373 -0
- package/dist/server/pi-session-pool.d.ts +233 -0
- package/dist/server/pi-session-pool.js +382 -0
- package/dist/server/pi-session.d.ts +642 -0
- package/dist/server/pi-session.js +655 -0
- package/dist/server/presentations/schema.d.ts +120 -0
- package/dist/server/presentations/schema.js +128 -0
- package/dist/server/provider-network.d.ts +16 -0
- package/dist/server/provider-network.js +106 -0
- package/dist/server/provider-proxy.d.ts +18 -0
- package/dist/server/provider-proxy.js +200 -0
- package/dist/server/providers.d.ts +333 -0
- package/dist/server/providers.js +989 -0
- package/dist/server/remote-bootstrap.d.ts +17 -0
- package/dist/server/remote-bootstrap.js +119 -0
- package/dist/server/rpc-channel.d.ts +131 -0
- package/dist/server/rpc-channel.js +135 -0
- package/dist/server/runtime.d.ts +1393 -0
- package/dist/server/runtime.js +3286 -0
- package/dist/server/scheduled-tasks.d.ts +101 -0
- package/dist/server/scheduled-tasks.js +138 -0
- package/dist/server/server-context.d.ts +119 -0
- package/dist/server/server-context.js +1 -0
- package/dist/server/session-cwd.d.ts +19 -0
- package/dist/server/session-cwd.js +58 -0
- package/dist/server/session-ledger.d.ts +67 -0
- package/dist/server/session-ledger.js +76 -0
- package/dist/server/session-memory-scope.d.ts +4 -0
- package/dist/server/session-memory-scope.js +30 -0
- package/dist/server/session-repository.d.ts +100 -0
- package/dist/server/session-repository.js +246 -0
- package/dist/server/session-tree-scope.d.ts +125 -0
- package/dist/server/session-tree-scope.js +337 -0
- package/dist/server/skills.d.ts +23 -0
- package/dist/server/skills.js +190 -0
- package/dist/server/store.d.ts +1062 -0
- package/dist/server/store.js +3305 -0
- package/dist/server/subagent-runs.d.ts +185 -0
- package/dist/server/subagent-runs.js +383 -0
- package/dist/server/subagents.d.ts +200 -0
- package/dist/server/subagents.js +619 -0
- package/dist/server/task-tool.d.ts +282 -0
- package/dist/server/task-tool.js +507 -0
- package/dist/server/tavily.d.ts +32 -0
- package/dist/server/tavily.js +136 -0
- package/dist/server/team-context.d.ts +12 -0
- package/dist/server/team-context.js +180 -0
- package/dist/server/terminal.d.ts +163 -0
- package/dist/server/terminal.js +557 -0
- package/dist/server/thinking-level-discovery.d.ts +38 -0
- package/dist/server/thinking-level-discovery.js +183 -0
- package/dist/server/time-zone.d.ts +20 -0
- package/dist/server/time-zone.js +32 -0
- package/dist/server/tool-calls.d.ts +130 -0
- package/dist/server/tool-calls.js +341 -0
- package/dist/server/tool-llm.d.ts +72 -0
- package/dist/server/tool-llm.js +146 -0
- package/dist/server/tool-registry.d.ts +129 -0
- package/dist/server/tool-registry.js +249 -0
- package/dist/server/tool-runner.d.ts +112 -0
- package/dist/server/tool-runner.js +306 -0
- package/dist/server/turn-failure.d.ts +48 -0
- package/dist/server/turn-failure.js +148 -0
- package/dist/server/types.d.ts +2315 -0
- package/dist/server/types.js +11 -0
- package/dist/server/update-status.d.ts +14 -0
- package/dist/server/update-status.js +69 -0
- package/dist/server/update.d.ts +46 -0
- package/dist/server/update.js +104 -0
- package/dist/server/upload.d.ts +35 -0
- package/dist/server/upload.js +177 -0
- package/dist/server/version.d.ts +33 -0
- package/dist/server/version.js +111 -0
- package/dist/server/web.d.ts +27 -0
- package/dist/server/web.js +183 -0
- package/dist/server/workflow-engine.d.ts +47 -0
- package/dist/server/workflow-engine.js +255 -0
- package/dist/server/workflow.d.ts +329 -0
- package/dist/server/workflow.js +469 -0
- package/dist/server/workspace-file-indexes.d.ts +2 -0
- package/dist/server/workspace-file-indexes.js +61 -0
- package/dist/server/workspace-file-operations.d.ts +36 -0
- package/dist/server/workspace-file-operations.js +281 -0
- package/dist/server/workspace-file-response.d.ts +7 -0
- package/dist/server/workspace-file-response.js +79 -0
- package/dist/server/ws-server.d.ts +149 -0
- package/dist/server/ws-server.js +640 -0
- package/dist/shared/agent-order.d.ts +44 -0
- package/dist/shared/agent-order.js +50 -0
- package/dist/shared/archive-decisions.d.ts +44 -0
- package/dist/shared/archive-decisions.js +98 -0
- package/dist/shared/background-task.d.ts +25 -0
- package/dist/shared/background-task.js +183 -0
- package/dist/shared/child-session-lock.d.ts +37 -0
- package/dist/shared/child-session-lock.js +59 -0
- package/dist/shared/content-search.d.ts +22 -0
- package/dist/shared/content-search.js +1 -0
- package/dist/shared/conversation-title.d.ts +4 -0
- package/dist/shared/conversation-title.js +4 -0
- package/dist/shared/file-mentions.d.ts +17 -0
- package/dist/shared/file-mentions.js +91 -0
- package/dist/shared/git-commit.d.ts +11 -0
- package/dist/shared/git-commit.js +1 -0
- package/dist/shared/git-graph.d.ts +42 -0
- package/dist/shared/git-graph.js +30 -0
- package/dist/shared/group-chat.d.ts +132 -0
- package/dist/shared/group-chat.js +6 -0
- package/dist/shared/managed-servers.d.ts +36 -0
- package/dist/shared/managed-servers.js +1 -0
- package/dist/shared/message-media.d.ts +12 -0
- package/dist/shared/message-media.js +27 -0
- package/dist/shared/model-window-label.d.ts +39 -0
- package/dist/shared/model-window-label.js +55 -0
- package/dist/shared/native-tools.d.ts +43 -0
- package/dist/shared/native-tools.js +1 -0
- package/dist/shared/pi-cache-warming.d.ts +9 -0
- package/dist/shared/pi-cache-warming.js +4 -0
- package/dist/shared/project-overview.d.ts +107 -0
- package/dist/shared/project-overview.js +92 -0
- package/dist/shared/project-presentation.d.ts +55 -0
- package/dist/shared/project-presentation.js +111 -0
- package/dist/shared/provider-network.d.ts +30 -0
- package/dist/shared/provider-network.js +1 -0
- package/dist/shared/session-reporting.d.ts +40 -0
- package/dist/shared/session-reporting.js +97 -0
- package/dist/shared/session-search.d.ts +14 -0
- package/dist/shared/session-search.js +1 -0
- package/dist/shared/team-context-view.d.ts +33 -0
- package/dist/shared/team-context-view.js +49 -0
- package/dist/shared/team-context.d.ts +59 -0
- package/dist/shared/team-context.js +3 -0
- package/dist/shared/terminal-protocol.d.ts +71 -0
- package/dist/shared/terminal-protocol.js +1 -0
- package/dist/shared/thinking-levels.d.ts +53 -0
- package/dist/shared/thinking-levels.js +85 -0
- package/dist/shared/tool-glyph.d.ts +7 -0
- package/dist/shared/tool-glyph.js +19 -0
- package/dist/shared/transcript-order.d.ts +7 -0
- package/dist/shared/transcript-order.js +12 -0
- package/dist/shared/update-status.d.ts +28 -0
- package/dist/shared/update-status.js +1 -0
- package/dist/shared/workspace-file-preview.d.ts +4 -0
- package/dist/shared/workspace-file-preview.js +28 -0
- package/package.json +448 -4
- package/scripts/diagnose.mjs +603 -0
- package/scripts/patch-brace-expansion.mjs +260 -0
- package/scripts/restart-server.mjs +104 -0
- package/skills/presentation/SKILL.md +30 -0
- package/skills/presentation/references/data.md +10 -0
- package/skills/presentation/references/quality.md +12 -0
- package/skills/presentation/references/story.md +12 -0
- package/skills/presentation/references/templates-and-editing.md +11 -0
- package/skills/presentation/references/visual-system.md +10 -0
- package/skills/super-pi-agent-development/SKILL.md +25 -0
- package/skills/super-pi-development/SKILL.md +31 -0
- package/skills/super-pi-tool-development/SKILL.md +39 -0
|
@@ -0,0 +1,385 @@
|
|
|
1
|
+
import { randomUUID } from "node:crypto";
|
|
2
|
+
import { existsSync } from "node:fs";
|
|
3
|
+
import { mkdir, readFile, readdir, rename, unlink, writeFile } from "node:fs/promises";
|
|
4
|
+
import { join, resolve } from "node:path";
|
|
5
|
+
import { Worker } from "node:worker_threads";
|
|
6
|
+
/**
|
|
7
|
+
* The model, and the only one this feature can use.
|
|
8
|
+
*
|
|
9
|
+
* WHY THIS ONE. `bge-small-zh` scores 0.618 nDCG@10 averaged over the Chinese retrieval tasks it is
|
|
10
|
+
* benchmarked on - against 0.695 for `bge-base-zh`, which is four times the download. `gte-multilingual-base`
|
|
11
|
+
* scores higher on both Chinese and English, at 324 MB against this one's 23 MB. For a workspace-sized
|
|
12
|
+
* corpus the next thing that would help is not a bigger model but fusing this one with the lexical
|
|
13
|
+
* channels, which costs nothing and is what `KnowledgeBase.search` does.
|
|
14
|
+
*/
|
|
15
|
+
export const KNOWLEDGE_MODEL = "Xenova/bge-small-zh-v1.5";
|
|
16
|
+
/**
|
|
17
|
+
* HOW BIG ONE EMBEDDED PIECE MAY BE, in characters.
|
|
18
|
+
*
|
|
19
|
+
* THIS NUMBER IS A MODEL LIMIT, NOT A TUNING KNOB, and getting it wrong is invisible. The model
|
|
20
|
+
* declares `max_position_embeddings: 512`, and a Chinese character costs roughly one token, so a piece
|
|
21
|
+
* is a little under half the window: 300 characters leaves room for the `[CLS]`/`[SEP]` pair and for
|
|
22
|
+
* tokenisation that splits a few characters into two. The retrieval chunks are 1200 characters - more
|
|
23
|
+
* than double this - which is fine for something a MODEL reads and fatal for something a MODEL
|
|
24
|
+
* ENCODES: handed a 1200-character chunk, roughly 60% of it is silently discarded past the window.
|
|
25
|
+
* English would survive it, so the degradation would land only on Chinese: the language this project
|
|
26
|
+
* is written in.
|
|
27
|
+
*/
|
|
28
|
+
export const EMBED_PIECE_CHARS = 300;
|
|
29
|
+
const EMBED_PIECE_OVERLAP = 40;
|
|
30
|
+
/** Per query. Cosine similarity of unit vectors; threshold raised to 0.35 to filter out semantic noise. */
|
|
31
|
+
export const MIN_COSINE = 0.35;
|
|
32
|
+
/**
|
|
33
|
+
* Splits a retrieval chunk into pieces small enough for the model's window.
|
|
34
|
+
*
|
|
35
|
+
* WHY NOT SHRINK THE RETRIEVAL CHUNKS INSTEAD. 1200 characters is the right size for a passage a model
|
|
36
|
+
* READS - it holds enough context to answer from. Encoding is a different constraint from reading, and
|
|
37
|
+
* conflating them would trade retrieval quality for a convenience the encoder does not need.
|
|
38
|
+
*/
|
|
39
|
+
export function embedPieces(text) {
|
|
40
|
+
const pieces = [];
|
|
41
|
+
for (let offset = 0; offset < text.length; offset += EMBED_PIECE_CHARS - EMBED_PIECE_OVERLAP) {
|
|
42
|
+
const piece = text.slice(offset, offset + EMBED_PIECE_CHARS).trim();
|
|
43
|
+
if (piece)
|
|
44
|
+
pieces.push(piece);
|
|
45
|
+
if (offset + EMBED_PIECE_CHARS >= text.length)
|
|
46
|
+
break;
|
|
47
|
+
}
|
|
48
|
+
return pieces.length ? pieces : [text.slice(0, EMBED_PIECE_CHARS)];
|
|
49
|
+
}
|
|
50
|
+
/** Element-wise max over the pieces, then unit length. See `pool` for why max and not mean. */
|
|
51
|
+
export function pool(vectors) {
|
|
52
|
+
if (!vectors.length)
|
|
53
|
+
return [];
|
|
54
|
+
const pooled = new Array(vectors[0].length).fill(Number.NEGATIVE_INFINITY);
|
|
55
|
+
for (const vector of vectors) {
|
|
56
|
+
for (let i = 0; i < vector.length; i++)
|
|
57
|
+
if (vector[i] > pooled[i])
|
|
58
|
+
pooled[i] = vector[i];
|
|
59
|
+
}
|
|
60
|
+
let norm = 0;
|
|
61
|
+
for (const value of pooled)
|
|
62
|
+
norm += value * value;
|
|
63
|
+
norm = Math.sqrt(norm) || 1;
|
|
64
|
+
return pooled.map(value => value / norm);
|
|
65
|
+
}
|
|
66
|
+
export function cosine(a, b) {
|
|
67
|
+
let dot = 0;
|
|
68
|
+
for (let i = 0; i < a.length && i < b.length; i++)
|
|
69
|
+
dot += a[i] * b[i];
|
|
70
|
+
return dot;
|
|
71
|
+
}
|
|
72
|
+
/**
|
|
73
|
+
* The optional vector half of retrieval.
|
|
74
|
+
*
|
|
75
|
+
* EVERY FAILURE HERE IS A FALLBACK TO LEXICAL, NEVER AN ERROR. The model is an optional npm package
|
|
76
|
+
* that a skipped optional install can take away, and it is a 512-token window
|
|
77
|
+
* wrapped around a Chinese corpus. Any of those failing is a reason to answer from the lexical
|
|
78
|
+
* channels and say so - never a reason to fail a search that the lexical channels can already serve.
|
|
79
|
+
*/
|
|
80
|
+
export class VectorIndex {
|
|
81
|
+
directory;
|
|
82
|
+
worker;
|
|
83
|
+
workerSequence = 0;
|
|
84
|
+
workerReady = false;
|
|
85
|
+
workerPending = new Map();
|
|
86
|
+
failure;
|
|
87
|
+
consecutiveFailures = 0;
|
|
88
|
+
lastFailureTime = 0;
|
|
89
|
+
static MAX_FAILURES = 3;
|
|
90
|
+
static COOLDOWN_MS = 30000;
|
|
91
|
+
static EMBED_TIMEOUT_MS = 15000;
|
|
92
|
+
cache = new Map();
|
|
93
|
+
pending = new Set();
|
|
94
|
+
queue = [];
|
|
95
|
+
background;
|
|
96
|
+
constructor(directory) {
|
|
97
|
+
this.directory = directory;
|
|
98
|
+
}
|
|
99
|
+
/**
|
|
100
|
+
* Where the shipped weights live, relative to this module.
|
|
101
|
+
*
|
|
102
|
+
* `dist/server/knowledge-vectors.js` is two levels below the package root both in a published
|
|
103
|
+
* package and in this repository's own build - the same `../../assets/...` shape `runtime.ts` uses to
|
|
104
|
+
* reach `agents/`. The directory keeps the upstream id in its path because that is how the library
|
|
105
|
+
* looks the model up under `localModelPath`.
|
|
106
|
+
*/
|
|
107
|
+
bundledModel() { return resolve(import.meta.dirname, "../../assets/knowledge-model"); }
|
|
108
|
+
enabled() { return process.env.SUPER_PI_KNOWLEDGE_EMBEDDINGS !== "0"; }
|
|
109
|
+
status(indexed) {
|
|
110
|
+
const enabled = this.enabled();
|
|
111
|
+
const bundled = existsSync(this.bundledModel());
|
|
112
|
+
const now = Date.now();
|
|
113
|
+
const inCooldown = this.consecutiveFailures >= VectorIndex.MAX_FAILURES && (now - this.lastFailureTime < VectorIndex.COOLDOWN_MS);
|
|
114
|
+
let health = "ready";
|
|
115
|
+
if (!enabled)
|
|
116
|
+
health = "disabled";
|
|
117
|
+
else if (!bundled || inCooldown)
|
|
118
|
+
health = "degraded";
|
|
119
|
+
else if (this.consecutiveFailures > 0)
|
|
120
|
+
health = "restarting";
|
|
121
|
+
else if (this.worker && this.workerReady)
|
|
122
|
+
health = "ready";
|
|
123
|
+
else
|
|
124
|
+
health = "ready";
|
|
125
|
+
return {
|
|
126
|
+
enabled,
|
|
127
|
+
ready: this.workerReady && health === "ready",
|
|
128
|
+
indexed,
|
|
129
|
+
error: this.failure,
|
|
130
|
+
bundled,
|
|
131
|
+
health,
|
|
132
|
+
available: enabled && bundled && health !== "degraded",
|
|
133
|
+
};
|
|
134
|
+
}
|
|
135
|
+
workspace(workspaceId) { return join(this.directory, Buffer.from(workspaceId).toString("hex")); }
|
|
136
|
+
folder(workspaceId) { return join(this.workspace(workspaceId), "vectors"); }
|
|
137
|
+
file(workspaceId, documentId) { return join(this.folder(workspaceId), `${documentId}.vec.json`); }
|
|
138
|
+
/**
|
|
139
|
+
* Loads the model once, from the copy that ships with this package. The `dtype` is not optional: on
|
|
140
|
+
* Node the library defaults to fp32, which for this model is 90 MB rather than 23.
|
|
141
|
+
*
|
|
142
|
+
* THERE IS NO NETWORK HERE, and that is the whole point of vendoring the weights. The first version
|
|
143
|
+
* downloaded the model on first use, which put a 23 MB fetch behind a host the machine may not be
|
|
144
|
+
* able to reach, on the first search, for every person who installed the package. Shipping the
|
|
145
|
+
* weights turns a failure mode into a file, and makes the feature work on a machine that has never
|
|
146
|
+
* been online since it was installed.
|
|
147
|
+
*
|
|
148
|
+
* RESILIENT WORKER LIFECYCLE:
|
|
149
|
+
* When the worker encounters an error, crash, or unexpected exit, it cleans up all pending requests
|
|
150
|
+
* without hanging and marks the instance for automatic, on-demand respawning on the next embed call.
|
|
151
|
+
* A bounded backoff / cooldown prevents runaway crash loops if the environment is broken.
|
|
152
|
+
*/
|
|
153
|
+
ensureWorker() {
|
|
154
|
+
if (!this.enabled())
|
|
155
|
+
return undefined;
|
|
156
|
+
if (this.worker)
|
|
157
|
+
return this.worker;
|
|
158
|
+
if (!existsSync(this.bundledModel())) {
|
|
159
|
+
this.failure = `随包分发的模型缺失:${this.bundledModel()}`;
|
|
160
|
+
return undefined;
|
|
161
|
+
}
|
|
162
|
+
const now = Date.now();
|
|
163
|
+
if (this.consecutiveFailures >= VectorIndex.MAX_FAILURES) {
|
|
164
|
+
if (now - this.lastFailureTime < VectorIndex.COOLDOWN_MS) {
|
|
165
|
+
return undefined;
|
|
166
|
+
}
|
|
167
|
+
this.consecutiveFailures = 0;
|
|
168
|
+
this.failure = undefined;
|
|
169
|
+
}
|
|
170
|
+
try {
|
|
171
|
+
const worker = new Worker(new URL("./knowledge-embedding-worker.js", import.meta.url));
|
|
172
|
+
worker.unref();
|
|
173
|
+
worker.on("message", (message) => {
|
|
174
|
+
const pending = this.workerPending.get(message.id);
|
|
175
|
+
if (!pending)
|
|
176
|
+
return;
|
|
177
|
+
this.workerPending.delete(message.id);
|
|
178
|
+
if (pending.timeout)
|
|
179
|
+
clearTimeout(pending.timeout);
|
|
180
|
+
if (message.error || !message.data || !message.dims) {
|
|
181
|
+
this.failure = message.error ?? "向量 worker 返回了无效结果";
|
|
182
|
+
pending.resolve(null);
|
|
183
|
+
return;
|
|
184
|
+
}
|
|
185
|
+
this.workerReady = true;
|
|
186
|
+
this.failure = undefined;
|
|
187
|
+
this.consecutiveFailures = 0;
|
|
188
|
+
const dimension = message.dims.at(-1) ?? 0;
|
|
189
|
+
pending.resolve(Array.from({ length: message.dims[0] ?? 0 }, (_, row) => Array.from(message.data.slice(row * dimension, (row + 1) * dimension))));
|
|
190
|
+
});
|
|
191
|
+
const fail = (error) => {
|
|
192
|
+
this.consecutiveFailures++;
|
|
193
|
+
this.lastFailureTime = Date.now();
|
|
194
|
+
this.failure = error.message;
|
|
195
|
+
for (const pending of this.workerPending.values()) {
|
|
196
|
+
if (pending.timeout)
|
|
197
|
+
clearTimeout(pending.timeout);
|
|
198
|
+
pending.resolve(null);
|
|
199
|
+
}
|
|
200
|
+
this.workerPending.clear();
|
|
201
|
+
if (this.worker) {
|
|
202
|
+
try {
|
|
203
|
+
this.worker.terminate();
|
|
204
|
+
}
|
|
205
|
+
catch { }
|
|
206
|
+
}
|
|
207
|
+
this.worker = undefined;
|
|
208
|
+
this.workerReady = false;
|
|
209
|
+
};
|
|
210
|
+
worker.on("error", fail);
|
|
211
|
+
worker.on("exit", (code) => { if (code !== 0)
|
|
212
|
+
fail(new Error(`向量 worker 异常退出(${code})`)); });
|
|
213
|
+
this.worker = worker;
|
|
214
|
+
return worker;
|
|
215
|
+
}
|
|
216
|
+
catch (error) {
|
|
217
|
+
this.consecutiveFailures++;
|
|
218
|
+
this.lastFailureTime = Date.now();
|
|
219
|
+
this.failure = error instanceof Error ? error.message : String(error);
|
|
220
|
+
return undefined;
|
|
221
|
+
}
|
|
222
|
+
}
|
|
223
|
+
async embed(texts) {
|
|
224
|
+
if (!texts.length || !this.enabled())
|
|
225
|
+
return null;
|
|
226
|
+
const worker = this.ensureWorker();
|
|
227
|
+
if (!worker)
|
|
228
|
+
return null;
|
|
229
|
+
const id = ++this.workerSequence;
|
|
230
|
+
return new Promise((resolvePromise) => {
|
|
231
|
+
const timeout = setTimeout(() => {
|
|
232
|
+
const pending = this.workerPending.get(id);
|
|
233
|
+
if (pending) {
|
|
234
|
+
this.workerPending.delete(id);
|
|
235
|
+
pending.resolve(null);
|
|
236
|
+
}
|
|
237
|
+
}, VectorIndex.EMBED_TIMEOUT_MS);
|
|
238
|
+
timeout.unref();
|
|
239
|
+
this.workerPending.set(id, { resolve: resolvePromise, timeout });
|
|
240
|
+
worker.postMessage({ id, texts, modelRoot: this.bundledModel(), model: KNOWLEDGE_MODEL });
|
|
241
|
+
});
|
|
242
|
+
}
|
|
243
|
+
async terminate() {
|
|
244
|
+
for (const pending of this.workerPending.values()) {
|
|
245
|
+
if (pending.timeout)
|
|
246
|
+
clearTimeout(pending.timeout);
|
|
247
|
+
pending.resolve(null);
|
|
248
|
+
}
|
|
249
|
+
this.workerPending.clear();
|
|
250
|
+
if (this.worker) {
|
|
251
|
+
const w = this.worker;
|
|
252
|
+
this.worker = undefined;
|
|
253
|
+
this.workerReady = false;
|
|
254
|
+
await w.terminate();
|
|
255
|
+
}
|
|
256
|
+
}
|
|
257
|
+
/** Vectors already stored for one workspace, keyed by document then chunk index. */
|
|
258
|
+
async read(workspaceId, documentId) {
|
|
259
|
+
try {
|
|
260
|
+
const document = JSON.parse(await readFile(this.file(workspaceId, documentId), "utf8"));
|
|
261
|
+
return Array.isArray(document?.records) ? document : null;
|
|
262
|
+
}
|
|
263
|
+
catch (error) {
|
|
264
|
+
if (error.code === "ENOENT")
|
|
265
|
+
return null;
|
|
266
|
+
return null;
|
|
267
|
+
}
|
|
268
|
+
}
|
|
269
|
+
async count(workspaceId) {
|
|
270
|
+
try {
|
|
271
|
+
const names = (await readdir(this.folder(workspaceId))).filter(name => name.endsWith(".vec.json"));
|
|
272
|
+
let total = 0;
|
|
273
|
+
for (const name of names) {
|
|
274
|
+
const document = await this.read(workspaceId, name.slice(0, -".vec.json".length));
|
|
275
|
+
total += document?.records.length ?? 0;
|
|
276
|
+
}
|
|
277
|
+
return total;
|
|
278
|
+
}
|
|
279
|
+
catch {
|
|
280
|
+
return 0;
|
|
281
|
+
}
|
|
282
|
+
}
|
|
283
|
+
/** Drops a document's vectors. Called whenever its content changes or it is deleted. */
|
|
284
|
+
async forget(workspaceId, documentId) {
|
|
285
|
+
this.cache.delete(`${workspaceId}/${documentId}`);
|
|
286
|
+
await unlink(this.file(workspaceId, documentId)).catch(() => { });
|
|
287
|
+
}
|
|
288
|
+
/**
|
|
289
|
+
* Embeds anything not already stored, WITHOUT waiting for it.
|
|
290
|
+
*
|
|
291
|
+
* WHY IT IS NOT AWAITED. This runs after an import or on the first search of a session, and the
|
|
292
|
+
* first run of it downloads a model. Neither should make a person wait: the import returns as soon
|
|
293
|
+
* as the document is stored and the search returns lexical hits, and a later search is simply better
|
|
294
|
+
* because by then the vectors exist.
|
|
295
|
+
*
|
|
296
|
+
* WHY REQUESTS QUEUE RATHER THAN DROP. One job runs at a time - two would load the same model twice
|
|
297
|
+
* and race each other's writes - but a request arriving mid-job has to be remembered. Dropping it
|
|
298
|
+
* would mean a document imported while a background index was in flight stayed unindexed until the
|
|
299
|
+
* next restart, which is the kind of bug that only shows up on a busy machine weeks later.
|
|
300
|
+
*/
|
|
301
|
+
schedule(workspaceId, documents) {
|
|
302
|
+
if (!this.enabled() || !documents.length)
|
|
303
|
+
return;
|
|
304
|
+
this.queue.push({ workspaceId, documents });
|
|
305
|
+
if (this.background)
|
|
306
|
+
return;
|
|
307
|
+
this.background = (async () => {
|
|
308
|
+
while (this.queue.length) {
|
|
309
|
+
const next = this.queue.shift();
|
|
310
|
+
try {
|
|
311
|
+
for (const document of next.documents) {
|
|
312
|
+
const key = `${next.workspaceId}/${document.id}`;
|
|
313
|
+
if (this.pending.has(key))
|
|
314
|
+
continue;
|
|
315
|
+
this.pending.add(key);
|
|
316
|
+
const stored = await this.read(next.workspaceId, document.id);
|
|
317
|
+
if (stored?.digest === document.digest && stored.records.length === document.chunks.length) {
|
|
318
|
+
this.pending.delete(key);
|
|
319
|
+
continue;
|
|
320
|
+
}
|
|
321
|
+
const vectors = [];
|
|
322
|
+
for (const chunk of document.chunks) {
|
|
323
|
+
const embedded = await this.embed(embedPieces(chunk.text));
|
|
324
|
+
if (!embedded) {
|
|
325
|
+
this.pending.delete(key);
|
|
326
|
+
return;
|
|
327
|
+
}
|
|
328
|
+
vectors.push({ chunk: chunk.index, values: pool(embedded) });
|
|
329
|
+
}
|
|
330
|
+
const folder = this.folder(next.workspaceId);
|
|
331
|
+
await mkdir(folder, { recursive: true });
|
|
332
|
+
const file = this.file(next.workspaceId, document.id), temporary = `${file}.${randomUUID()}.tmp`;
|
|
333
|
+
await writeFile(temporary, JSON.stringify({ digest: document.digest, records: vectors }), { encoding: "utf8", mode: 0o600 });
|
|
334
|
+
await rename(temporary, file);
|
|
335
|
+
this.cache.set(key, new Map(vectors.map(record => [record.chunk, record])));
|
|
336
|
+
this.pending.delete(key);
|
|
337
|
+
}
|
|
338
|
+
}
|
|
339
|
+
catch (error) {
|
|
340
|
+
console.warn(`Super Pi: 语义索引失败,将只用关键词检索:${error instanceof Error ? error.message : String(error)}`);
|
|
341
|
+
}
|
|
342
|
+
}
|
|
343
|
+
})().finally(() => { this.background = undefined; });
|
|
344
|
+
}
|
|
345
|
+
/**
|
|
346
|
+
* Cosine of every stored chunk against the query, best first.
|
|
347
|
+
*
|
|
348
|
+
* WHY THE RAW COSINE IS NOT RE-NORMALISED BY THE BEST HIT. Every lexical channel is divided by its
|
|
349
|
+
* own maximum, and that is right for them because BM25 scores are unbounded and have no absolute
|
|
350
|
+
* meaning. Cosine is the opposite: it is already a bounded, interpretable quantity, and dividing it
|
|
351
|
+
* by the best hit of the query would turn "nothing here is close" into "the best one is 1.0". The
|
|
352
|
+
* knowledge base's most useful property is that it can say it does not know, and rescaling is exactly
|
|
353
|
+
* how that gets spent.
|
|
354
|
+
*/
|
|
355
|
+
async search(workspaceId, query, keys) {
|
|
356
|
+
if (!this.enabled() || !keys.length)
|
|
357
|
+
return null;
|
|
358
|
+
const documentIds = new Set(keys.map(([, documentId]) => documentId));
|
|
359
|
+
// First query of the session: read what was indexed last time rather than assuming nothing is.
|
|
360
|
+
for (const documentId of documentIds) {
|
|
361
|
+
if (this.cache.has(`${workspaceId}/${documentId}`))
|
|
362
|
+
continue;
|
|
363
|
+
const document = await this.read(workspaceId, documentId);
|
|
364
|
+
if (document)
|
|
365
|
+
this.cache.set(`${workspaceId}/${documentId}`, new Map(document.records.map(record => [record.chunk, record])));
|
|
366
|
+
}
|
|
367
|
+
const available = keys.filter(([, documentId]) => this.cache.get(`${workspaceId}/${documentId}`)?.size);
|
|
368
|
+
if (!available.length)
|
|
369
|
+
return null;
|
|
370
|
+
const vector = await this.embed([query]);
|
|
371
|
+
if (!vector?.[0])
|
|
372
|
+
return null;
|
|
373
|
+
const scores = new Map();
|
|
374
|
+
for (const [key, documentId] of available) {
|
|
375
|
+
const chunk = Number(key.split("#")[1]);
|
|
376
|
+
const record = this.cache.get(`${workspaceId}/${documentId}`)?.get(chunk);
|
|
377
|
+
if (!record)
|
|
378
|
+
continue;
|
|
379
|
+
const similarity = cosine(vector[0], record.values);
|
|
380
|
+
if (similarity >= MIN_COSINE)
|
|
381
|
+
scores.set(key, similarity);
|
|
382
|
+
}
|
|
383
|
+
return scores;
|
|
384
|
+
}
|
|
385
|
+
}
|
|
@@ -0,0 +1,273 @@
|
|
|
1
|
+
import { VectorIndex } from "./knowledge-vectors.js";
|
|
2
|
+
import type { Store } from "./store.js";
|
|
3
|
+
import type { Conversation } from "./types.js";
|
|
4
|
+
/** One document as it lives on disk: the ORIGINAL text, never the chunks. */
|
|
5
|
+
export type KnowledgeDocument = {
|
|
6
|
+
id: string;
|
|
7
|
+
workspaceId: string;
|
|
8
|
+
title: string;
|
|
9
|
+
/** Where it came from - a workspace-relative path, or `pasted` for hand-entered text. */
|
|
10
|
+
source: string;
|
|
11
|
+
format: string;
|
|
12
|
+
characters: number;
|
|
13
|
+
createdAt: string;
|
|
14
|
+
updatedAt: string;
|
|
15
|
+
/**
|
|
16
|
+
* Content fingerprint, used to recognise an import the user already made.
|
|
17
|
+
*
|
|
18
|
+
* WHY IT IS STORED RATHER THAN COMPUTED ON COMPARISON. Deduplication runs inside `add`, which already
|
|
19
|
+
* reads every document in the workspace; hashing each one's full text there would be O(corpus) on
|
|
20
|
+
* every import. Storing it turns the check into one string comparison, and a document written by an
|
|
21
|
+
* older build simply hashes on read (`digestOf`) instead.
|
|
22
|
+
*/
|
|
23
|
+
digest: string;
|
|
24
|
+
text: string;
|
|
25
|
+
};
|
|
26
|
+
/** A retrievable slice. `heading` is the nearest preceding markdown heading, so a hit can be cited. */
|
|
27
|
+
export type KnowledgeChunk = {
|
|
28
|
+
index: number;
|
|
29
|
+
heading: string;
|
|
30
|
+
text: string;
|
|
31
|
+
};
|
|
32
|
+
export type KnowledgeHit = {
|
|
33
|
+
documentId: string;
|
|
34
|
+
title: string;
|
|
35
|
+
source: string;
|
|
36
|
+
heading: string;
|
|
37
|
+
chunk: number;
|
|
38
|
+
score: number;
|
|
39
|
+
text: string;
|
|
40
|
+
};
|
|
41
|
+
/** 1 MB. Big enough for a design doc or a spec, small enough that chunking stays instant. */
|
|
42
|
+
export declare const MAX_DOCUMENT_BYTES: number;
|
|
43
|
+
/** A pasted document rides inside a JSON body, so the request cap leaves room for escaping overhead. */
|
|
44
|
+
export declare const MAX_KNOWLEDGE_REQUEST_BYTES: number;
|
|
45
|
+
/** Per workspace, across every document. A knowledge base is a curated asset, not a file server. */
|
|
46
|
+
export declare const MAX_WORKSPACE_BYTES: number;
|
|
47
|
+
export declare const DEFAULT_RESULT_LIMIT = 3;
|
|
48
|
+
export declare const MAX_RESULT_LIMIT = 20;
|
|
49
|
+
/** What one search may be asked for. `rerank` and `conversation` exist only for the agent's own calls. */
|
|
50
|
+
export type KnowledgeSearch = {
|
|
51
|
+
limit?: number;
|
|
52
|
+
/** Re-order with the session's model. Off for the settings preview, which must stay instant. */
|
|
53
|
+
rerank?: boolean;
|
|
54
|
+
/** Required by `rerank`: it answers with this conversation's model and workspace. */
|
|
55
|
+
conversation?: Conversation;
|
|
56
|
+
signal?: AbortSignal;
|
|
57
|
+
};
|
|
58
|
+
/**
|
|
59
|
+
* The slice of the runtime the reranker needs, declared structurally.
|
|
60
|
+
*
|
|
61
|
+
* Declared here rather than imported so this file does not depend on the spawn machinery that depends
|
|
62
|
+
* on it - `runtime.ts` imports `KNOWLEDGE_TOOL_NAME` from here, and a type-only back edge into that
|
|
63
|
+
* module would make the cycle visible in every import graph.
|
|
64
|
+
*/
|
|
65
|
+
export type KnowledgeRuntime = {
|
|
66
|
+
answer(input: {
|
|
67
|
+
conversation: Conversation;
|
|
68
|
+
prompt: string;
|
|
69
|
+
cwd: string;
|
|
70
|
+
signal: AbortSignal;
|
|
71
|
+
knowledgeRerank?: boolean;
|
|
72
|
+
}): AsyncIterable<string>;
|
|
73
|
+
};
|
|
74
|
+
/**
|
|
75
|
+
* Extracts a 1-based candidate integer array from the model's answer, or null.
|
|
76
|
+
* Tolerates surrounding markdown, explanatory text, and trailing commas.
|
|
77
|
+
* Returns empty array [] if the model explicitly declares no candidates are relevant.
|
|
78
|
+
*/
|
|
79
|
+
export declare function parseRanking(output: string, maxIndex?: number): number[] | null;
|
|
80
|
+
/** Duplicated as a literal in `agents/super-pi-knowledge/index.ts`, the way `TASK_TOOL_NAME` is. */
|
|
81
|
+
export declare const KNOWLEDGE_TOOL_NAME = "knowledge_search";
|
|
82
|
+
/** The import half of the same extension. Duplicated as a literal there, for the same reason. */
|
|
83
|
+
export declare const KNOWLEDGE_IMPORT_TOOL_NAME = "knowledge_import";
|
|
84
|
+
/**
|
|
85
|
+
* Index terms for one string: CJK bigrams plus Latin words.
|
|
86
|
+
*
|
|
87
|
+
* WHY BIGRAMS AND NOT WORDS. Chinese has no spaces, so the word-list tokenizer the rest of this
|
|
88
|
+
* server uses (`ActiveMemory.search`) returns one enormous "word" for a whole sentence and matches
|
|
89
|
+
* nothing usefully. Character bigrams need no dictionary and no dependency, and they carry enough
|
|
90
|
+
* precision that 知识库 and 记忆库 share 识库 but not the whole run - which is what makes a two-term
|
|
91
|
+
* overlap meaningful rather than accidental.
|
|
92
|
+
*/
|
|
93
|
+
export declare function tokenize(text: string): string[];
|
|
94
|
+
/**
|
|
95
|
+
* The longer-range channel: character TRIGRAMS, for the passage that repeats a phrase verbatim.
|
|
96
|
+
*
|
|
97
|
+
* WHY A SECOND GRANULARITY AND NOT JUST BIGRAMS. Bigrams answer "do these share any two-character
|
|
98
|
+
* run", which is high recall but low precision - two unrelated Chinese sentences collide constantly on
|
|
99
|
+
* a shared pair. Trigrams only fire on a real three-character overlap, so when the user's question
|
|
100
|
+
* quotes the document's own wording this channel separates it sharply from the rest of the corpus.
|
|
101
|
+
* It is scored on its own and fused with the bigram channel rather than merged into it, because the
|
|
102
|
+
* two answer different questions and a term that is strong for one is usually weak for the other.
|
|
103
|
+
*/
|
|
104
|
+
export declare function tokenizePhrases(text: string): string[];
|
|
105
|
+
/** The identity of a document's CONTENT, independent of its name, its path and when it arrived. */
|
|
106
|
+
export declare function digestOf(text: string): string;
|
|
107
|
+
/**
|
|
108
|
+
* Splits a document into overlapping chunks, remembering the heading each one sits under.
|
|
109
|
+
*
|
|
110
|
+
* WHY OVERLAP. A passage that spans a boundary is the passage most likely to be the answer - the
|
|
111
|
+
* sentence that defines a term usually starts in one chunk and ends in the next - and a retrieval
|
|
112
|
+
* system that cut there would return a fragment ending mid-thought. The 150-character tail is
|
|
113
|
+
* repeated at the head of the next chunk to keep that sentence whole in at least one hit.
|
|
114
|
+
*/
|
|
115
|
+
export declare function chunkDocument(text: string): KnowledgeChunk[];
|
|
116
|
+
export declare const MIN_SCORE_ABSOLUTE = 0.28;
|
|
117
|
+
export declare const RELATIVE_CUTOFF_RATIO = 0.5;
|
|
118
|
+
export declare class KnowledgeBase {
|
|
119
|
+
readonly directory: string;
|
|
120
|
+
private store;
|
|
121
|
+
private runtime?;
|
|
122
|
+
private vectors?;
|
|
123
|
+
private capabilities;
|
|
124
|
+
private indexes;
|
|
125
|
+
private serial;
|
|
126
|
+
constructor(directory: string, store: Store, runtime?: KnowledgeRuntime | undefined, vectors?: VectorIndex | undefined);
|
|
127
|
+
private hasWorkspace;
|
|
128
|
+
/** `SUPER_PI_KNOWLEDGE_ENABLED=0` turns the whole feature off, the same way memory turns off. */
|
|
129
|
+
enabled(): boolean;
|
|
130
|
+
/** The opaque workspace identity on disk, encoded. Paths never come from the model or the request. */
|
|
131
|
+
private folder;
|
|
132
|
+
private file;
|
|
133
|
+
private requireWorkspace;
|
|
134
|
+
private locked;
|
|
135
|
+
/** Metadata only - the caller listing documents must never receive the full text of a workspace. */
|
|
136
|
+
private summarise;
|
|
137
|
+
private readAll;
|
|
138
|
+
private save;
|
|
139
|
+
private index;
|
|
140
|
+
/**
|
|
141
|
+
* Loads (or rebuilds) one workspace's index. Cache miss is the only thing that reads the disk.
|
|
142
|
+
*
|
|
143
|
+
* WHY REBUILT PER WORKSPACE AND NOT ONE GLOBAL INDEX. Isolation is a security property here, not a
|
|
144
|
+
* performance one: a hit may only ever come from the workspace the turn's capability names. A
|
|
145
|
+
* single shared index would need every search to filter by workspace after scoring, which is one
|
|
146
|
+
* forgotten filter away from a cross-tenant leak. Separate caches cannot express that mistake.
|
|
147
|
+
*
|
|
148
|
+
* WHY THERE IS NO ON-DISK INDEX. Measured, not assumed: rebuilding the index of a 16 MB workspace -
|
|
149
|
+
* the feature's own capacity cap - costs about 180 ms, once per process. A serialised index would be
|
|
150
|
+
* larger than the corpus it describes and JSON.parse of it is not reliably cheaper than the
|
|
151
|
+
* tokenisation, so it would trade a correctness-neutral optimisation for a second on-disk artefact
|
|
152
|
+
* that can drift from the documents.
|
|
153
|
+
*/
|
|
154
|
+
private load;
|
|
155
|
+
private invalidate;
|
|
156
|
+
list(workspaceId: string): Promise<Array<Omit<KnowledgeDocument, "text">>>;
|
|
157
|
+
/**
|
|
158
|
+
* Adds a document, either from a workspace file or from pasted text.
|
|
159
|
+
*
|
|
160
|
+
* WHY THE FILE IS READ HERE AND NOT UPLOADED. A knowledge base is made of documents the person
|
|
161
|
+
* ALREADY has in their workspace - re-uploading a copy of the file would give the agent a second,
|
|
162
|
+
* newer version that diverges from the one the workspace serves. The path therefore goes through
|
|
163
|
+
* the same `resolveInsideRoot` guard the file editor uses, so symlink and `..` escapes are refused
|
|
164
|
+
* by construction instead of by a check someone has to remember.
|
|
165
|
+
*
|
|
166
|
+
* WHO CALLS THIS. The settings window (manage, action "add") AND, since the import tool exists,
|
|
167
|
+
* the primary agent mid-turn on the user's say-so - through the same guard, the same whitelist and
|
|
168
|
+
* the same caps, so an agent import can never do more than a person's import could.
|
|
169
|
+
*/
|
|
170
|
+
add(workspaceId: string, body: {
|
|
171
|
+
path?: string;
|
|
172
|
+
title?: string;
|
|
173
|
+
text?: string;
|
|
174
|
+
}): Promise<Omit<KnowledgeDocument, "text">>;
|
|
175
|
+
remove(workspaceId: string, documentId: string): Promise<void>;
|
|
176
|
+
/**
|
|
177
|
+
* Scores this workspace's chunks on four independent lexical channels and fuses them.
|
|
178
|
+
*
|
|
179
|
+
* WHY LEXICAL AND NOT ONLY EMBEDDINGS. See docs/KNOWLEDGE_BASE.md: the vector channel is fused in
|
|
180
|
+
* here rather than replacing this one, because each is wrong in a way the other catches. BM25
|
|
181
|
+
* misses a paraphrase completely; a vector search will happily return the nearest thing at a
|
|
182
|
+
* confident-looking 0.58 when the answer is not in the knowledge base at all. Summing a normalised
|
|
183
|
+
* lexical score with a normalised cosine score keeps both signals - and keeps "no hit" reachable.
|
|
184
|
+
*
|
|
185
|
+
* WHY NORMALISE PER CHANNEL AND SUM, RATHER THAN RRF. Reciprocal rank fusion is the usual way to
|
|
186
|
+
* combine ranked lists, and it would be a regression here: it discards score magnitude, so a
|
|
187
|
+
* passage that matches decisively and one that matches barely would arrive at the same rank and
|
|
188
|
+
* finish within a couple of percent of each other. The gap between a clear winner and a near miss is
|
|
189
|
+
* exactly what BM25 was measuring. Each channel is therefore divided by its own best score for this
|
|
190
|
+
* query, which preserves the shape of that channel's confidence while making the weights comparable.
|
|
191
|
+
*/
|
|
192
|
+
search(workspaceId: string, query: string, options?: KnowledgeSearch): Promise<KnowledgeHit[]>;
|
|
193
|
+
/**
|
|
194
|
+
* Re-orders the candidate pool with the session's own model.
|
|
195
|
+
*
|
|
196
|
+
* WHY THIS EXISTS AND WHAT IT IS WORTH. Lexical retrieval ranks on shared characters, so a passage
|
|
197
|
+
* that answers in different words sits below passages that merely repeat the question's wording -
|
|
198
|
+
* often below the fold, where the character budget has already cut it off. Reading the candidates
|
|
199
|
+
* is something the model can already do, so it is asked to pick.
|
|
200
|
+
*
|
|
201
|
+
* ITS CEILING, STATED PLAINLY. Reranking can only reorder what the first stage returned. A query
|
|
202
|
+
* with NO lexical overlap at all never reaches this function, which is why the vector channel and
|
|
203
|
+
* not this one is what closes the paraphrase gap - see docs/KNOWLEDGE_BASE.md.
|
|
204
|
+
*
|
|
205
|
+
* WHY IT FAILS OPEN. Every outcome that is not a clean parse returns the lexical order. A reranker
|
|
206
|
+
* that made things worse, hung, or answered with prose would degrade the feature invisibly, and a
|
|
207
|
+
* retrieval layer that is allowed to get worse by being clever is worse than one that is not.
|
|
208
|
+
*/
|
|
209
|
+
private rerank;
|
|
210
|
+
/**
|
|
211
|
+
* Mints a token for one turn. The agent can retrieve and IMPORT (a document the user just asked it
|
|
212
|
+
* to add rides through the same `add` guard the settings window uses), but never remove: curation
|
|
213
|
+
* that deletes a person's documents stays a settings-window act.
|
|
214
|
+
*/
|
|
215
|
+
issue(conversationId: string): string;
|
|
216
|
+
release(token: string): void;
|
|
217
|
+
private identity;
|
|
218
|
+
/**
|
|
219
|
+
* The agent-facing endpoint: a title catalogue, a search, and an import. The import is the SAME
|
|
220
|
+
* `add` the settings window calls - path guard, format whitelist, size caps, dedupe - so an agent
|
|
221
|
+
* cannot do anything a person could not, and the one thing it cannot do is `remove`.
|
|
222
|
+
*/
|
|
223
|
+
tool(token: string, body: {
|
|
224
|
+
kind?: string;
|
|
225
|
+
query?: string;
|
|
226
|
+
limit?: number;
|
|
227
|
+
path?: string;
|
|
228
|
+
title?: string;
|
|
229
|
+
text?: string;
|
|
230
|
+
}, signal?: AbortSignal): Promise<{
|
|
231
|
+
documents: {
|
|
232
|
+
title: string;
|
|
233
|
+
source: string;
|
|
234
|
+
characters: number;
|
|
235
|
+
}[];
|
|
236
|
+
imported?: undefined;
|
|
237
|
+
hits?: undefined;
|
|
238
|
+
reranked?: undefined;
|
|
239
|
+
} | {
|
|
240
|
+
documents?: undefined;
|
|
241
|
+
imported: Omit<KnowledgeDocument, "text">;
|
|
242
|
+
hits?: undefined;
|
|
243
|
+
reranked?: undefined;
|
|
244
|
+
} | {
|
|
245
|
+
documents?: undefined;
|
|
246
|
+
imported?: undefined;
|
|
247
|
+
hits: KnowledgeHit[];
|
|
248
|
+
reranked: boolean;
|
|
249
|
+
}>;
|
|
250
|
+
/** The browser-facing endpoint, behind the ordinary access token and same-origin check. */
|
|
251
|
+
manage(workspaceId: string, body: {
|
|
252
|
+
action?: string;
|
|
253
|
+
path?: string;
|
|
254
|
+
title?: string;
|
|
255
|
+
text?: string;
|
|
256
|
+
documentId?: string;
|
|
257
|
+
query?: string;
|
|
258
|
+
limit?: number;
|
|
259
|
+
}): Promise<{
|
|
260
|
+
hits: KnowledgeHit[];
|
|
261
|
+
documents: Omit<KnowledgeDocument, "text">[];
|
|
262
|
+
semantic: import("./knowledge-vectors.js").VectorStatus | null;
|
|
263
|
+
}>;
|
|
264
|
+
/**
|
|
265
|
+
* What the panel shows, including whether semantic retrieval is actually working.
|
|
266
|
+
*
|
|
267
|
+
* WHY THE STATE IS WORTH SHOWING. The vector half depends on an optional package and a download from
|
|
268
|
+
* a host the machine may not reach, and both fail quietly by design. Without this on screen, "the
|
|
269
|
+
* feature is off" is indistinguishable from "the feature is broken here" - which is the difference
|
|
270
|
+
* between not wanting a feature and not being able to have it.
|
|
271
|
+
*/
|
|
272
|
+
private overview;
|
|
273
|
+
}
|