@sanqianx/project-knowledge 4.6.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (218) hide show
  1. package/.agents/plugins/marketplace.json +20 -0
  2. package/.claude-plugin/marketplace.json +17 -0
  3. package/CHANGELOG.md +780 -0
  4. package/LICENSE +201 -0
  5. package/NOTICE +5 -0
  6. package/README.md +266 -0
  7. package/_site/README.md +30 -0
  8. package/_site/_test/ai-profile-resolver-test.js +137 -0
  9. package/_site/_test/ai-profile-test.js +75 -0
  10. package/_site/_test/ai-vendor-presets-test.js +91 -0
  11. package/_site/_test/automation-queue-test.js +14 -0
  12. package/_site/_test/automation-ui-test.js +40 -0
  13. package/_site/_test/background-task-registry-test.js +43 -0
  14. package/_site/_test/baseline-schema-test.js +91 -0
  15. package/_site/_test/bin-cli-test.js +218 -0
  16. package/_site/_test/bridge-adapter-test.js +95 -0
  17. package/_site/_test/bridge-consumer-service-test.js +310 -0
  18. package/_site/_test/cdp-browser-failure-test.js +100 -0
  19. package/_site/_test/chat-claudecodeui-match-test.js +92 -0
  20. package/_site/_test/claude-executable-discovery-test.js +90 -0
  21. package/_site/_test/claude-session-lifecycle-sweep-test.js +252 -0
  22. package/_site/_test/claude-workbench-test.js +170 -0
  23. package/_site/_test/client-capture-restart-e2e-test.js +189 -0
  24. package/_site/_test/codex-conversation-projection-test.js +59 -0
  25. package/_site/_test/codex-notify-fanout-test.js +45 -0
  26. package/_site/_test/commit-boundary-freeze-test.js +147 -0
  27. package/_site/_test/commit-conversation-binding-test.js +99 -0
  28. package/_site/_test/commit-evidence-test.js +110 -0
  29. package/_site/_test/context-pack-test.js +198 -0
  30. package/_site/_test/conversation-api-test.js +119 -0
  31. package/_site/_test/conversation-store-test.js +136 -0
  32. package/_site/_test/cross-repo-isolation-e2e-test.js +187 -0
  33. package/_site/_test/data-dir-migration-test.js +77 -0
  34. package/_site/_test/desktop-browser-compat-test.js +28 -0
  35. package/_site/_test/desktop-hook-runtime-regression-test.js +204 -0
  36. package/_site/_test/embedding-config-test.js +32 -0
  37. package/_site/_test/explicit-commit-processor-test.js +37 -0
  38. package/_site/_test/fixtures/fake-browser-exit.js +10 -0
  39. package/_site/_test/fixtures/fake-browser-hang.js +10 -0
  40. package/_site/_test/fixtures/make-git-repos.js +89 -0
  41. package/_site/_test/folder-picker-output-test.js +19 -0
  42. package/_site/_test/full-integration-e2e-test.js +165 -0
  43. package/_site/_test/git-validation-test.js +81 -0
  44. package/_site/_test/hook-runtime-endpoint-test.js +88 -0
  45. package/_site/_test/hook-status-repair-api-test.js +152 -0
  46. package/_site/_test/hook-trigger-test.js +85 -0
  47. package/_site/_test/hook-worktree-test.js +42 -0
  48. package/_site/_test/import-preflight-api-test.js +233 -0
  49. package/_site/_test/import-ui-flow-test.js +171 -0
  50. package/_site/_test/index-builder-test.js +41 -0
  51. package/_site/_test/index-writer-concurrency-test.js +135 -0
  52. package/_site/_test/integration-adapters-test.js +243 -0
  53. package/_site/_test/integration-capture-setup-test.js +108 -0
  54. package/_site/_test/integration-surface-coverage-test.js +165 -0
  55. package/_site/_test/kb-framework-test.js +57 -0
  56. package/_site/_test/kbpath-follow-test.js +63 -0
  57. package/_site/_test/knowledge-db-test.js +144 -0
  58. package/_site/_test/knowledge-language-control-test.js +170 -0
  59. package/_site/_test/knowledge-maintenance-test.js +40 -0
  60. package/_site/_test/knowledge-migration-test.js +75 -0
  61. package/_site/_test/knowledge-promotion-recovery-test.js +260 -0
  62. package/_site/_test/knowledge-query-test.js +54 -0
  63. package/_site/_test/knowledge-retrieval-service-test.js +77 -0
  64. package/_site/_test/knowledge-scopes-test.js +37 -0
  65. package/_site/_test/knowledge-storage-location-test.js +28 -0
  66. package/_site/_test/knowledge-storage-startup-test.js +52 -0
  67. package/_site/_test/knowledge-store-logs-supervision-test.js +89 -0
  68. package/_site/_test/legacy-claude-session-alias-test.js +106 -0
  69. package/_site/_test/legacy-forward-compat-test.js +257 -0
  70. package/_site/_test/legacy-project-upgrade-e2e-test.js +363 -0
  71. package/_site/_test/legacy-routes-removed-test.js +47 -0
  72. package/_site/_test/legacy-workbench-exclusion-test.js +92 -0
  73. package/_site/_test/log-redaction-test.js +58 -0
  74. package/_site/_test/logging-api-test.js +89 -0
  75. package/_site/_test/logging-sse-no-gap-test.js +88 -0
  76. package/_site/_test/logging-ui-test.js +104 -0
  77. package/_site/_test/markdown-delta-overlay-test.js +85 -0
  78. package/_site/_test/markdown-maintenance-api-test.js +75 -0
  79. package/_site/_test/markdown-maintenance-test.js +11 -0
  80. package/_site/_test/mcp-server-test.js +149 -0
  81. package/_site/_test/model-context-windows-test.js +89 -0
  82. package/_site/_test/module-bridge-eventbridge-test.js +193 -0
  83. package/_site/_test/non-release-ci-test.js +34 -0
  84. package/_site/_test/offline-boundary-isolation-test.js +32 -0
  85. package/_site/_test/orphaned-claim-recovery-test.js +80 -0
  86. package/_site/_test/p0-data-migration-characterization-test.js +37 -0
  87. package/_site/_test/p0-e2e-gate-test.js +347 -0
  88. package/_site/_test/package-boundary-test.js +42 -0
  89. package/_site/_test/package-startup-test.js +56 -0
  90. package/_site/_test/packaged-ui-smoke-test.js +79 -0
  91. package/_site/_test/path-consistency-test.js +151 -0
  92. package/_site/_test/pending-sweep-test.js +9 -0
  93. package/_site/_test/post-commit-automation-test.js +87 -0
  94. package/_site/_test/pr-consumer-contract-test.js +55 -0
  95. package/_site/_test/project-control-panel-task14-test.js +63 -0
  96. package/_site/_test/project-delete-recovery-test.js +64 -0
  97. package/_site/_test/project-goal-editor-test.js +145 -0
  98. package/_site/_test/project-layout-v2-migration-test.js +127 -0
  99. package/_site/_test/project-lifecycle-transaction-test.js +106 -0
  100. package/_site/_test/project-remove-running-guard-test.js +79 -0
  101. package/_site/_test/project-store-test.js +90 -0
  102. package/_site/_test/prompt-settings-test.js +115 -0
  103. package/_site/_test/protected-architecture-gate-test.js +123 -0
  104. package/_site/_test/refactor-characterization-test.js +36 -0
  105. package/_site/_test/release-version-sync-test.js +98 -0
  106. package/_site/_test/requirement-binding-test.js +113 -0
  107. package/_site/_test/requirement-recorder-test.js +173 -0
  108. package/_site/_test/run-all-tests.js +156 -0
  109. package/_site/_test/runtime-endpoint-test.js +64 -0
  110. package/_site/_test/scanner-test.js +83 -0
  111. package/_site/_test/server-runtime-migration-safety-test.js +27 -0
  112. package/_site/_test/server-security-test.js +96 -0
  113. package/_site/_test/sessions-stream-test.js +92 -0
  114. package/_site/_test/shared-contracts-test.js +58 -0
  115. package/_site/_test/simple-import-test.js +97 -0
  116. package/_site/_test/startup-analysis-disabled-test.js +16 -0
  117. package/_site/_test/storage-foundation-test.js +78 -0
  118. package/_site/_test/structured-logger-test.js +101 -0
  119. package/_site/_test/task15-20-ui-flow-test.js +148 -0
  120. package/_site/_test/team-knowledge-store-sparse-checkout-test.js +196 -0
  121. package/_site/_test/team-knowledge-store-test.js +131 -0
  122. package/_site/_test/tracking-start-test.js +143 -0
  123. package/_site/_test/ui-i18n-toggle-test.js +114 -0
  124. package/_site/_test/ui-smoke-test.js +73 -0
  125. package/_site/_test/v4122-upgrade-data-contract-test.js +56 -0
  126. package/_site/_test/workbench-permission-test.js +118 -0
  127. package/_site/_test/workspace-ui-contract-test.js +43 -0
  128. package/_site/lib/ai-profile-resolver.js +78 -0
  129. package/_site/lib/ai-vendor-presets.js +83 -0
  130. package/_site/lib/ai-workspace.js +104 -0
  131. package/_site/lib/atomic-file.js +204 -0
  132. package/_site/lib/automation-config.js +140 -0
  133. package/_site/lib/bridge-adapter.js +256 -0
  134. package/_site/lib/bridge-consumer-service.js +402 -0
  135. package/_site/lib/claude-cli-runner.js +1599 -0
  136. package/_site/lib/commit-conversation-binder.js +246 -0
  137. package/_site/lib/commit-processing-ledger.js +27 -0
  138. package/_site/lib/commit-prompt.js +202 -0
  139. package/_site/lib/commit-reconciler.js +444 -0
  140. package/_site/lib/context-pack-builder.js +299 -0
  141. package/_site/lib/contracts.js +279 -0
  142. package/_site/lib/conversation-exclusions.js +204 -0
  143. package/_site/lib/conversation-query-service.js +161 -0
  144. package/_site/lib/conversation-store.js +385 -0
  145. package/_site/lib/data-dir.js +81 -0
  146. package/_site/lib/data-state-classifier.js +40 -0
  147. package/_site/lib/embedding-service.js +85 -0
  148. package/_site/lib/evidence-bundle.js +228 -0
  149. package/_site/lib/folder-picker-output.js +24 -0
  150. package/_site/lib/git-runner.js +62 -0
  151. package/_site/lib/github-team-store.js +1175 -0
  152. package/_site/lib/hook-manager.js +202 -0
  153. package/_site/lib/index-builder.js +157 -0
  154. package/_site/lib/index-service.js +173 -0
  155. package/_site/lib/integration-installer.js +907 -0
  156. package/_site/lib/kb-framework.js +191 -0
  157. package/_site/lib/kb-validator.js +124 -0
  158. package/_site/lib/knowledge-db.js +363 -0
  159. package/_site/lib/knowledge-promotion.js +435 -0
  160. package/_site/lib/knowledge-retrieval-service.js +401 -0
  161. package/_site/lib/knowledge-schema.js +121 -0
  162. package/_site/lib/knowledge-tool-runtime.js +359 -0
  163. package/_site/lib/legacy-data-manifest.js +26 -0
  164. package/_site/lib/llm-client.js +173 -0
  165. package/_site/lib/markdown-knowledge-indexer.js +230 -0
  166. package/_site/lib/migration-service.js +442 -0
  167. package/_site/lib/model-context-windows.js +82 -0
  168. package/_site/lib/module-bridge.js +496 -0
  169. package/_site/lib/post-commit-automation.js +141 -0
  170. package/_site/lib/project-lifecycle-service.js +511 -0
  171. package/_site/lib/project-registry-store.js +129 -0
  172. package/_site/lib/project-store.js +262 -0
  173. package/_site/lib/prompt-registry.js +148 -0
  174. package/_site/lib/requirement-adapters.js +10 -0
  175. package/_site/lib/requirement-binder.js +112 -0
  176. package/_site/lib/requirement-recorder.js +278 -0
  177. package/_site/lib/runtime-endpoint.js +160 -0
  178. package/_site/lib/scanner.js +298 -0
  179. package/_site/lib/server-app.js +1520 -0
  180. package/_site/lib/settings-store.js +113 -0
  181. package/_site/lib/storage-layout.js +162 -0
  182. package/_site/lib/structured-logger.js +574 -0
  183. package/_site/scripts/codex-notify-fanout.cjs +51 -0
  184. package/_site/scripts/folder-picker.ps1 +156 -0
  185. package/_site/scripts/hook-trigger.js +156 -0
  186. package/_site/scripts/sync-release-version.js +158 -0
  187. package/_site/server.js +12 -0
  188. package/_site/start.bat +26 -0
  189. package/_site/stop.bat +11 -0
  190. package/bin/project-knowledge-integrations.js +241 -0
  191. package/bin/project-knowledge-kb.js +65 -0
  192. package/bin/project-knowledge-mcp.js +195 -0
  193. package/bin/project-knowledge.js +439 -0
  194. package/claude-prompts.json +21 -0
  195. package/docs/README.zh-CN.md +175 -0
  196. package/docs/assets/architecture.svg +182 -0
  197. package/docs/assets/dashboard.png +0 -0
  198. package/docs/assets/draft-review.png +0 -0
  199. package/docs/assets/logo.svg +50 -0
  200. package/docs/assets/runs-drafts.png +0 -0
  201. package/docs/assets/settings.png +0 -0
  202. package/docs/project-registry-schema.md +33 -0
  203. package/docs/team-knowledge-mode-a-plan.md +73 -0
  204. package/docs/testing-strategy.md +33 -0
  205. package/package.json +100 -0
  206. package/plugins/project-knowledge/.claude-plugin/plugin.json +19 -0
  207. package/plugins/project-knowledge/.codex-plugin/plugin.json +38 -0
  208. package/plugins/project-knowledge/.mcp.json +14 -0
  209. package/plugins/project-knowledge/opencode/project-knowledge.md +3 -0
  210. package/plugins/project-knowledge/skills/project-knowledge/SKILL.md +28 -0
  211. package/plugins/project-knowledge/skills/project-knowledge/agents/openai.yaml +11 -0
  212. package/templates/change.md +30 -0
  213. package/templates/project-readme.md +52 -0
  214. package/ui/app.css +58 -0
  215. package/ui/app.js +607 -0
  216. package/ui/favicon.svg +38 -0
  217. package/ui/i18n.js +146 -0
  218. package/ui/index.html +162 -0
@@ -0,0 +1,401 @@
1
+ const fs = require('fs');
2
+ const path = require('path');
3
+ const crypto = require('crypto');
4
+ const { DomainError, validateProjectId } = require('./contracts');
5
+ const { CHUNKER_VERSION, chunkMarkdown, listMarkdownFiles, inferEntryType, parseKnowledgeMetadata } = require('./markdown-knowledge-indexer');
6
+ const { sha256: documentSha256 } = require('./knowledge-schema');
7
+
8
+ const DEFAULT_CONTEXT_BUDGET = 256 * 1024;
9
+ const DEFAULT_CANDIDATE_LIMIT = 50;
10
+
11
+ function hash(value) {
12
+ return `sha256:${crypto.createHash('sha256').update(String(value == null ? '' : value), 'utf8').digest('hex')}`;
13
+ }
14
+
15
+ function tokens(value) {
16
+ const normalized = String(value || '').toLowerCase().replace(/\\/g, '/');
17
+ const out = normalized.match(/[\p{L}\p{N}_-]+/gu) || [];
18
+ for (const segment of normalized.match(/[\p{Script=Han}]+/gu) || []) {
19
+ for (let index = 0; index < segment.length - 1; index += 1) out.push(segment.slice(index, index + 2));
20
+ }
21
+ return [...new Set(out.filter(item => item.length > 1))];
22
+ }
23
+
24
+ function normalizePath(value) {
25
+ return String(value || '').replace(/\\/g, '/').replace(/^\.\//, '').toLowerCase();
26
+ }
27
+
28
+ function titleFromMarkdown(markdown, fallback) {
29
+ const match = /^#\s+(.+)$/m.exec(String(markdown || ''));
30
+ return match ? match[1].trim() : fallback;
31
+ }
32
+
33
+ function occurrences(text, terms, cap = 12) {
34
+ const source = String(text || '').toLowerCase();
35
+ let count = 0;
36
+ for (const term of terms) {
37
+ let offset = 0;
38
+ let matches = 0;
39
+ while (term && matches < cap && (offset = source.indexOf(term, offset)) >= 0) {
40
+ matches += 1;
41
+ count += 1;
42
+ offset += term.length;
43
+ }
44
+ }
45
+ return count;
46
+ }
47
+
48
+ function safeCurrentFile(layout, knowledgePath, entryId) {
49
+ const root = path.resolve(knowledgePath);
50
+ const target = path.resolve(root, ...String(entryId).split('/'));
51
+ if (!layout.isPathInside(root, target, { realpath: true }) || !fs.existsSync(target)
52
+ || !fs.statSync(target).isFile() || fs.lstatSync(target).isSymbolicLink()) return null;
53
+ return target;
54
+ }
55
+
56
+ function manifestHash(manifest) {
57
+ const { manifestHash: ignored, ...payload } = manifest;
58
+ void ignored;
59
+ return hash(JSON.stringify(payload));
60
+ }
61
+
62
+ class KnowledgeRetrievalService {
63
+ constructor(options = {}) {
64
+ this.layout = options.layout;
65
+ this.registryStore = options.registryStore;
66
+ this.projectStore = options.projectStore;
67
+ this.databaseProvider = options.databaseProvider || (() => options.database || null);
68
+ this.embedderProvider = options.embedderProvider || (() => options.embedder || null);
69
+ this.logger = options.logger || null;
70
+ this.candidateLimit = Number(options.candidateLimit || DEFAULT_CANDIDATE_LIMIT);
71
+ }
72
+
73
+ async log(level, event, message, context) {
74
+ try { if (this.logger && typeof this.logger[level] === 'function') await this.logger[level](event, message, context); } catch {
75
+ // Retrieval truth and logging availability are independent.
76
+ }
77
+ }
78
+
79
+ project(input = {}) {
80
+ if (input.project && input.project.projectId) return input.project;
81
+ const projectId = validateProjectId(input.projectId);
82
+ const config = this.projectStore.readConfig(projectId);
83
+ const state = this.projectStore.readState(projectId);
84
+ if (config.enabled === false) throw new DomainError('PROJECT_NOT_FOUND', 'The requested project is disabled.', { status: 404 });
85
+ return {
86
+ projectId,
87
+ projectSlug: config.legacyExtensions && config.legacyExtensions.slug || projectId,
88
+ primarySpaceId: `project:${projectId}`,
89
+ knowledgePath: path.resolve(config.knowledgePath),
90
+ relatedProjectIds: Array.isArray(config.relatedProjectIds) ? config.relatedProjectIds : [],
91
+ index: state.index,
92
+ };
93
+ }
94
+
95
+ scopes(project) {
96
+ const scopes = [{ projectId: project.projectId, spaceId: project.primarySpaceId || `project:${project.projectId}`, knowledgePath: project.knowledgePath, weight: 1, reason: 'primary' }];
97
+ for (const relatedId of project.relatedProjectIds || []) {
98
+ if (!this.registryStore || !this.registryStore.readDisplaySnapshot(relatedId)) continue;
99
+ const config = this.projectStore.readConfig(relatedId);
100
+ if (config.enabled === false) continue;
101
+ scopes.push({ projectId: relatedId, spaceId: `project:${relatedId}`, knowledgePath: path.resolve(config.knowledgePath), weight: 0.88, reason: 'related' });
102
+ }
103
+ return scopes;
104
+ }
105
+
106
+ sourceManifest(projectId) {
107
+ const filePath = this.layout.getRuntimePath('index-sources', `${projectId}.json`);
108
+ try {
109
+ const value = JSON.parse(fs.readFileSync(filePath, 'utf8'));
110
+ return value && value.schema === 'knowledge-index-source-manifest/v1' ? value : null;
111
+ } catch { return null; }
112
+ }
113
+
114
+ scanCurrent(scopes) {
115
+ const candidates = new Map();
116
+ const scopeHealth = [];
117
+ for (const scope of scopes) {
118
+ const state = this.projectStore.readState(scope.projectId);
119
+ const sourceManifest = this.sourceManifest(scope.projectId);
120
+ const manifestEntries = sourceManifest && sourceManifest.entries || {};
121
+ const files = listMarkdownFiles(scope.knowledgePath);
122
+ let overlayEntries = 0;
123
+ for (const filePath of files) {
124
+ const entryId = path.relative(scope.knowledgePath, filePath).replace(/\\/g, '/');
125
+ const markdown = fs.readFileSync(filePath, 'utf8');
126
+ const documentHash = documentSha256(`chunker:${CHUNKER_VERSION}\n${markdown}`);
127
+ const metadata = parseKnowledgeMetadata(markdown);
128
+ const title = titleFromMarkdown(markdown, path.basename(entryId, '.md'));
129
+ const rawChunks = chunkMarkdown(markdown);
130
+ const fromOverlay = !sourceManifest || !manifestEntries[entryId] || manifestEntries[entryId].documentHash !== documentHash;
131
+ if (fromOverlay) overlayEntries += 1;
132
+ for (const chunk of rawChunks) {
133
+ const chunkId = `${entryId}:${chunk.chunkOrder}`;
134
+ candidates.set(`${scope.spaceId}:${chunkId}`, {
135
+ recordId: `${scope.spaceId}:${chunkId}`,
136
+ spaceId: scope.spaceId,
137
+ projectId: scope.projectId,
138
+ scopeReason: scope.reason,
139
+ scopeWeight: scope.weight,
140
+ knowledgePath: scope.knowledgePath,
141
+ entryId,
142
+ entryType: inferEntryType(entryId),
143
+ chunkId,
144
+ chunkOrder: chunk.chunkOrder,
145
+ title,
146
+ headingPath: chunk.headingPath,
147
+ chunkText: chunk.chunkText,
148
+ documentHash,
149
+ contentHash: documentSha256(chunk.chunkText),
150
+ metadata,
151
+ fromOverlay,
152
+ indexScore: 0,
153
+ indexChannels: [],
154
+ });
155
+ }
156
+ }
157
+ const deletedEntries = sourceManifest
158
+ ? Object.keys(manifestEntries).filter(entryId => !safeCurrentFile(this.layout, scope.knowledgePath, entryId)).length
159
+ : 0;
160
+ scopeHealth.push({
161
+ projectId: scope.projectId,
162
+ generation: Number(state.index && state.index.generation || 0),
163
+ dirty: state.index && state.index.dirty === true,
164
+ sourceManifest: sourceManifest ? 'available' : 'missing',
165
+ overlayEntries,
166
+ deletedEntries,
167
+ });
168
+ }
169
+ return { candidates, scopeHealth };
170
+ }
171
+
172
+ async recallIndex(scopes, query, current, health) {
173
+ if (!query || !fs.existsSync(this.layout.getIndexPath())) {
174
+ health.index = 'missing';
175
+ return;
176
+ }
177
+ const database = this.databaseProvider();
178
+ const embedder = this.embedderProvider();
179
+ if (!database || !embedder || typeof embedder.embedQuery !== 'function') {
180
+ health.index = 'unavailable';
181
+ return;
182
+ }
183
+ try {
184
+ const vector = await embedder.embedQuery(query);
185
+ const rows = await database.hybridSearch({
186
+ text: query,
187
+ vector,
188
+ spaceIds: scopes.map(scope => scope.spaceId),
189
+ limit: this.candidateLimit,
190
+ candidates: Math.min(200, this.candidateLimit * 4),
191
+ });
192
+ for (const row of rows) {
193
+ const key = `${row.space_id}:${row.chunk_id}`;
194
+ const candidate = current.get(key);
195
+ if (!candidate) continue;
196
+ candidate.indexScore = Number(row.relevance_score || 0);
197
+ candidate.indexChannels = Array.isArray(row.match_channels) ? row.match_channels : [];
198
+ }
199
+ health.index = 'ok';
200
+ } catch (error) {
201
+ health.index = 'degraded';
202
+ health.reason = 'index-query-failed';
203
+ health.errorCode = error && error.code || 'INDEX_UNAVAILABLE';
204
+ }
205
+ }
206
+
207
+ score(candidate, signals) {
208
+ const selectionSignals = [];
209
+ let score = candidate.indexScore * 1000;
210
+ if (candidate.indexScore) selectionSignals.push(...candidate.indexChannels.map(channel => `index:${channel}`));
211
+ const primaryMatches = occurrences(`${candidate.title}\n${candidate.headingPath.join(' ')}\n${candidate.chunkText}`, signals.primaryTokens);
212
+ if (primaryMatches) { score += Math.min(primaryMatches, 20) * 8; selectionSignals.push(`user-keyword:${primaryMatches}`); }
213
+ const implementationMatches = occurrences(`${candidate.entryId}\n${candidate.title}\n${candidate.chunkText}`, signals.implementationTokens);
214
+ if (implementationMatches) { score += Math.min(implementationMatches, 16) * 3; selectionSignals.push(`implementation:${implementationMatches}`); }
215
+ const assistantMatches = occurrences(candidate.chunkText, signals.assistantTokens);
216
+ if (assistantMatches) { score += Math.min(assistantMatches, 8); selectionSignals.push(`assistant-hint:${assistantMatches}`); }
217
+
218
+ const changedPaths = signals.changedPaths;
219
+ const metadataPaths = candidate.metadata.sourcePaths.map(normalizePath);
220
+ const exactPaths = metadataPaths.filter(value => changedPaths.has(value));
221
+ if (exactPaths.length) { score += 300 + exactPaths.length * 20; selectionSignals.push(`exact-source-path:${exactPaths.join(',')}`); }
222
+ const changedBasenames = signals.changedBasenames;
223
+ const basenameMatches = metadataPaths.filter(value => changedBasenames.has(path.posix.basename(value)));
224
+ if (!exactPaths.length && basenameMatches.length) { score += 70; selectionSignals.push(`source-basename:${basenameMatches.join(',')}`); }
225
+
226
+ const structured = [...candidate.metadata.symbols, ...candidate.metadata.tags, ...candidate.metadata.routes, ...candidate.metadata.affectedModules].map(value => value.toLowerCase());
227
+ const structuredMatches = structured.filter(value => signals.allTerms.has(value)
228
+ || signals.allTerms.has(normalizePath(value))
229
+ || tokens(value).some(term => signals.allTerms.has(term)));
230
+ if (structuredMatches.length) { score += 90 + structuredMatches.length * 10; selectionSignals.push(`structured:${structuredMatches.join(',')}`); }
231
+ score *= candidate.scopeWeight;
232
+ if (candidate.scopeReason === 'primary') selectionSignals.push('scope:primary');
233
+ else selectionSignals.push('scope:related');
234
+ if (candidate.fromOverlay) selectionSignals.push('markdown-delta-overlay');
235
+ return { score, selectionSignals };
236
+ }
237
+
238
+ signalSet(input = {}) {
239
+ const primaryTexts = input.primaryTexts || [];
240
+ const implementationTexts = input.implementationTexts || [];
241
+ const assistantTexts = input.assistantTexts || [];
242
+ const changedPaths = new Set((input.changedPaths || []).map(normalizePath).filter(Boolean));
243
+ const implementationTokens = tokens(implementationTexts.join(' '));
244
+ for (const changedPath of changedPaths) {
245
+ implementationTokens.push(...tokens(changedPath), ...tokens(path.posix.basename(changedPath)));
246
+ }
247
+ const primaryTokens = tokens(primaryTexts.join(' '));
248
+ const assistantTokens = tokens(assistantTexts.join(' '));
249
+ const allTerms = new Set([...primaryTokens, ...implementationTokens]);
250
+ return {
251
+ primaryTokens,
252
+ implementationTokens: [...new Set(implementationTokens)],
253
+ assistantTokens,
254
+ changedPaths,
255
+ changedBasenames: new Set([...changedPaths].map(value => path.posix.basename(value))),
256
+ allTerms,
257
+ indexQuery: [...primaryTexts, ...implementationTexts].join('\n').slice(0, 12_000),
258
+ signalHash: hash(JSON.stringify({ primaryTexts, implementationTexts, changedPaths: [...changedPaths], assistantHintHashes: assistantTexts.map(hash) })),
259
+ };
260
+ }
261
+
262
+ async retrieve(input = {}) {
263
+ const project = this.project(input);
264
+ const scopes = Array.isArray(input.scopes) && input.scopes.length ? input.scopes : this.scopes(project);
265
+ const signals = this.signalSet(input.signals);
266
+ const scanned = this.scanCurrent(scopes);
267
+ const health = { index: 'unknown', scopes: scanned.scopeHealth };
268
+ await this.recallIndex(scopes, signals.indexQuery, scanned.candidates, health);
269
+ const ranked = [];
270
+ for (const candidate of scanned.candidates.values()) {
271
+ const scored = this.score(candidate, signals);
272
+ if (scored.score <= 0) continue;
273
+ ranked.push({ ...candidate, relevanceScore: scored.score, selectionSignals: scored.selectionSignals });
274
+ }
275
+ ranked.sort((left, right) => right.relevanceScore - left.relevanceScore || left.recordId.localeCompare(right.recordId));
276
+ return { project, scopes, signals, health, candidates: ranked.slice(0, this.candidateLimit), totalCurrentChunks: scanned.candidates.size };
277
+ }
278
+
279
+ async search(input = {}) {
280
+ const query = String(input.query || '').trim();
281
+ if (!query) throw new DomainError('INVALID_ARGUMENT', 'query is required.');
282
+ const limit = Math.max(1, Math.min(Number(input.limit || 8), 30));
283
+ const retrieved = await this.retrieve({ ...input, signals: { primaryTexts: [query] } });
284
+ const results = retrieved.candidates.slice(0, limit).map(candidate => ({
285
+ record_id: candidate.recordId,
286
+ space_id: candidate.spaceId,
287
+ entry_id: candidate.entryId,
288
+ chunk_id: candidate.chunkId,
289
+ chunk_order: candidate.chunkOrder,
290
+ title: candidate.title,
291
+ heading_path: candidate.headingPath.join(' > '),
292
+ chunk_text: candidate.chunkText,
293
+ document_hash: candidate.documentHash,
294
+ content_hash: candidate.contentHash,
295
+ relevance_score: candidate.relevanceScore,
296
+ match_channels: candidate.selectionSignals,
297
+ scope_reason: candidate.scopeReason,
298
+ scope_project_id: candidate.projectId,
299
+ from_delta_overlay: candidate.fromOverlay,
300
+ }));
301
+ await this.log('info', 'retrieval.search_completed', 'Knowledge search completed against authoritative Markdown.', {
302
+ projectId: retrieved.project.projectId,
303
+ component: 'knowledge-retrieval',
304
+ context: { queryHash: hash(query), resultCount: results.length, backend: retrieved.health.index, currentChunkCount: retrieved.totalCurrentChunks },
305
+ });
306
+ return {
307
+ ok: true,
308
+ projectId: retrieved.project.projectId,
309
+ projectSlug: retrieved.project.projectSlug || retrieved.project.projectId,
310
+ query,
311
+ backend: retrieved.health.index === 'ok' ? 'hybrid+markdown-truth' : 'markdown-hybrid-fallback',
312
+ source: 'knowledge-retrieval-service',
313
+ health: retrieved.health,
314
+ results,
315
+ };
316
+ }
317
+
318
+ async retrieveForCommit(input = {}) {
319
+ const snapshot = input.conversationSnapshot || { turns: [] };
320
+ const evidence = input.commitEvidence || {};
321
+ const userTexts = (snapshot.turns || []).flatMap(turn => (turn.userEvents || []).map(event => event.content));
322
+ const assistantTexts = (snapshot.turns || []).flatMap(turn => (turn.assistantEvents || []).map(event => event.content));
323
+ const changedPaths = (evidence.files || []).flatMap(file => [file.path, file.oldPath]).filter(Boolean);
324
+ const retrieved = await this.retrieve({
325
+ projectId: input.projectId,
326
+ project: input.project,
327
+ signals: {
328
+ primaryTexts: userTexts,
329
+ assistantTexts,
330
+ implementationTexts: [evidence.subject || '', ...(evidence.files || []).map(file => `${file.status || ''} ${file.path || ''}`)],
331
+ changedPaths,
332
+ },
333
+ });
334
+ const budget = Math.max(8 * 1024, Number(input.contextBudget || DEFAULT_CONTEXT_BUDGET));
335
+ const selected = [];
336
+ let totalBytes = 0;
337
+ for (const candidate of retrieved.candidates) {
338
+ const bytes = Buffer.byteLength(candidate.chunkText, 'utf8');
339
+ if (totalBytes + bytes > budget) continue;
340
+ totalBytes += bytes;
341
+ selected.push(candidate);
342
+ }
343
+ const manifest = {
344
+ schema: 'knowledge-retrieval-manifest/v1',
345
+ projectId: retrieved.project.projectId,
346
+ commitSha: evidence.commitSha || null,
347
+ signalHash: retrieved.signals.signalHash,
348
+ backend: retrieved.health.index,
349
+ health: retrieved.health,
350
+ indexGeneration: Object.fromEntries(retrieved.health.scopes.map(scope => [scope.projectId, scope.generation])),
351
+ candidateCount: retrieved.candidates.length,
352
+ currentChunkCount: retrieved.totalCurrentChunks,
353
+ selected: selected.map(candidate => ({
354
+ projectId: candidate.projectId,
355
+ spaceId: candidate.spaceId,
356
+ entryId: candidate.entryId,
357
+ chunkId: candidate.chunkId,
358
+ chunkOrder: candidate.chunkOrder,
359
+ headingPath: candidate.headingPath,
360
+ documentHash: candidate.documentHash,
361
+ contentHash: candidate.contentHash,
362
+ selectionSignals: candidate.selectionSignals,
363
+ relevanceScore: candidate.relevanceScore,
364
+ fromDeltaOverlay: candidate.fromOverlay,
365
+ bytes: Buffer.byteLength(candidate.chunkText, 'utf8'),
366
+ })),
367
+ totalBytes,
368
+ tokenEstimate: Math.ceil(totalBytes / 4),
369
+ contextBudget: budget,
370
+ manifestHash: '',
371
+ };
372
+ manifest.manifestHash = manifestHash(manifest);
373
+ const entries = selected.map(candidate => ({
374
+ path: candidate.entryId,
375
+ projectId: candidate.projectId,
376
+ chunkId: candidate.chunkId,
377
+ headingPath: candidate.headingPath,
378
+ hash: candidate.contentHash,
379
+ documentHash: candidate.documentHash,
380
+ content: candidate.chunkText,
381
+ selectionSignals: candidate.selectionSignals,
382
+ }));
383
+ await this.log('info', 'retrieval.commit_context_frozen', 'Commit knowledge context was selected from authoritative Markdown.', {
384
+ projectId: retrieved.project.projectId,
385
+ commitSha: evidence.commitSha,
386
+ component: 'knowledge-retrieval',
387
+ context: { manifestHash: manifest.manifestHash, selectedCount: entries.length, totalBytes, candidateCount: manifest.candidateCount, backend: manifest.backend },
388
+ });
389
+ return { entries, omitted: [], totalBytes, limitBytes: budget, manifest, manifestHash: manifest.manifestHash };
390
+ }
391
+ }
392
+
393
+ module.exports = {
394
+ KnowledgeRetrievalService,
395
+ DEFAULT_CONTEXT_BUDGET,
396
+ DEFAULT_CANDIDATE_LIMIT,
397
+ hash,
398
+ tokens,
399
+ normalizePath,
400
+ manifestHash,
401
+ };
@@ -0,0 +1,121 @@
1
+ const crypto = require('crypto');
2
+ const arrow = require('apache-arrow');
3
+
4
+ const KNOWLEDGE_DB_SCHEMA_VERSION = 1;
5
+ const KNOWLEDGE_TABLE = 'knowledge_chunks';
6
+ const EMBEDDING_DIMENSIONS = 512;
7
+
8
+ function field(name, type, nullable = false) {
9
+ return new arrow.Field(name, type, nullable);
10
+ }
11
+
12
+ function knowledgeChunkSchema(dimensions = EMBEDDING_DIMENSIONS) {
13
+ return new arrow.Schema([
14
+ field('schema_version', new arrow.Int32()),
15
+ field('record_id', new arrow.Utf8()),
16
+ field('space_id', new arrow.Utf8()),
17
+ field('entry_id', new arrow.Utf8()),
18
+ field('entry_type', new arrow.Utf8()),
19
+ field('entry_version', new arrow.Int32()),
20
+ field('chunk_id', new arrow.Utf8()),
21
+ field('chunk_order', new arrow.Int32()),
22
+ field('title', new arrow.Utf8()),
23
+ field('heading_path', new arrow.Utf8()),
24
+ field('chunk_text', new arrow.Utf8()),
25
+ field('search_text', new arrow.Utf8()),
26
+ field('vector', new arrow.FixedSizeList(dimensions, field('item', new arrow.Float32()))),
27
+ field('tags_json', new arrow.Utf8()),
28
+ field('source_paths_json', new arrow.Utf8()),
29
+ field('routes_json', new arrow.Utf8()),
30
+ field('symbols_json', new arrow.Utf8()),
31
+ field('source_project_id', new arrow.Utf8()),
32
+ field('source_commit', new arrow.Utf8()),
33
+ field('document_hash', new arrow.Utf8()),
34
+ field('content_hash', new arrow.Utf8()),
35
+ field('created_at', new arrow.Utf8()),
36
+ field('updated_at', new arrow.Utf8()),
37
+ ]);
38
+ }
39
+
40
+ function sha256(value) {
41
+ return crypto.createHash('sha256').update(String(value || ''), 'utf8').digest('hex');
42
+ }
43
+
44
+ function requiredText(value, name) {
45
+ const normalized = String(value || '').trim();
46
+ if (!normalized) throw new Error(`${name} is required`);
47
+ return normalized;
48
+ }
49
+
50
+ function jsonArray(value) {
51
+ return JSON.stringify(Array.isArray(value) ? value.map(item => String(item)) : []);
52
+ }
53
+
54
+ function normalizeVector(value, dimensions = EMBEDDING_DIMENSIONS) {
55
+ const vector = Array.from(value || [], Number);
56
+ if (vector.length !== dimensions || vector.some(item => !Number.isFinite(item))) {
57
+ throw new Error(`vector must contain exactly ${dimensions} finite numbers`);
58
+ }
59
+ return vector;
60
+ }
61
+
62
+ function normalizeKnowledgeChunk(input, options = {}) {
63
+ const dimensions = options.dimensions || EMBEDDING_DIMENSIONS;
64
+ const now = options.now || new Date().toISOString();
65
+ const spaceId = requiredText(input.spaceId || input.space_id, 'spaceId');
66
+ const entryId = requiredText(input.entryId || input.entry_id, 'entryId');
67
+ const chunkOrder = Number.isInteger(input.chunkOrder) ? input.chunkOrder : Number(input.chunk_order || 0);
68
+ if (!Number.isInteger(chunkOrder) || chunkOrder < 0) throw new Error('chunkOrder must be a non-negative integer');
69
+ const chunkText = requiredText(input.chunkText || input.chunk_text, 'chunkText');
70
+ const contentHash = String(input.contentHash || input.content_hash || sha256(chunkText));
71
+ const chunkId = String(input.chunkId || input.chunk_id || `${entryId}:${chunkOrder}`);
72
+
73
+ return {
74
+ schema_version: KNOWLEDGE_DB_SCHEMA_VERSION,
75
+ record_id: `${spaceId}:${chunkId}`,
76
+ space_id: spaceId,
77
+ entry_id: entryId,
78
+ entry_type: String(input.entryType || input.entry_type || 'document'),
79
+ entry_version: Number.isInteger(input.entryVersion) ? input.entryVersion : Number(input.entry_version || 1),
80
+ chunk_id: chunkId,
81
+ chunk_order: chunkOrder,
82
+ title: String(input.title || ''),
83
+ heading_path: Array.isArray(input.headingPath) ? input.headingPath.join(' > ') : String(input.heading_path || ''),
84
+ chunk_text: chunkText,
85
+ search_text: String(input.searchText || input.search_text || `${input.title || ''}\n${chunkText}`).trim(),
86
+ vector: normalizeVector(input.vector, dimensions),
87
+ tags_json: jsonArray(input.tags),
88
+ source_paths_json: jsonArray(input.sourcePaths || input.source_paths),
89
+ routes_json: jsonArray(input.routes),
90
+ symbols_json: jsonArray(input.symbols),
91
+ source_project_id: String(input.sourceProjectId || input.source_project_id || ''),
92
+ source_commit: String(input.sourceCommit || input.source_commit || ''),
93
+ document_hash: String(input.documentHash || input.document_hash || contentHash),
94
+ content_hash: contentHash,
95
+ created_at: String(input.createdAt || input.created_at || now),
96
+ updated_at: String(input.updatedAt || input.updated_at || now),
97
+ };
98
+ }
99
+
100
+ function decodeKnowledgeChunk(row) {
101
+ const parse = (value) => {
102
+ try { return JSON.parse(value || '[]'); } catch { return []; }
103
+ };
104
+ return {
105
+ ...row,
106
+ tags: parse(row.tags_json),
107
+ sourcePaths: parse(row.source_paths_json),
108
+ routes: parse(row.routes_json),
109
+ symbols: parse(row.symbols_json),
110
+ };
111
+ }
112
+
113
+ module.exports = {
114
+ KNOWLEDGE_DB_SCHEMA_VERSION,
115
+ KNOWLEDGE_TABLE,
116
+ EMBEDDING_DIMENSIONS,
117
+ knowledgeChunkSchema,
118
+ normalizeKnowledgeChunk,
119
+ decodeKnowledgeChunk,
120
+ sha256,
121
+ };