@sanqianx/project-knowledge 4.6.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.agents/plugins/marketplace.json +20 -0
- package/.claude-plugin/marketplace.json +17 -0
- package/CHANGELOG.md +780 -0
- package/LICENSE +201 -0
- package/NOTICE +5 -0
- package/README.md +266 -0
- package/_site/README.md +30 -0
- package/_site/_test/ai-profile-resolver-test.js +137 -0
- package/_site/_test/ai-profile-test.js +75 -0
- package/_site/_test/ai-vendor-presets-test.js +91 -0
- package/_site/_test/automation-queue-test.js +14 -0
- package/_site/_test/automation-ui-test.js +40 -0
- package/_site/_test/background-task-registry-test.js +43 -0
- package/_site/_test/baseline-schema-test.js +91 -0
- package/_site/_test/bin-cli-test.js +218 -0
- package/_site/_test/bridge-adapter-test.js +95 -0
- package/_site/_test/bridge-consumer-service-test.js +310 -0
- package/_site/_test/cdp-browser-failure-test.js +100 -0
- package/_site/_test/chat-claudecodeui-match-test.js +92 -0
- package/_site/_test/claude-executable-discovery-test.js +90 -0
- package/_site/_test/claude-session-lifecycle-sweep-test.js +252 -0
- package/_site/_test/claude-workbench-test.js +170 -0
- package/_site/_test/client-capture-restart-e2e-test.js +189 -0
- package/_site/_test/codex-conversation-projection-test.js +59 -0
- package/_site/_test/codex-notify-fanout-test.js +45 -0
- package/_site/_test/commit-boundary-freeze-test.js +147 -0
- package/_site/_test/commit-conversation-binding-test.js +99 -0
- package/_site/_test/commit-evidence-test.js +110 -0
- package/_site/_test/context-pack-test.js +198 -0
- package/_site/_test/conversation-api-test.js +119 -0
- package/_site/_test/conversation-store-test.js +136 -0
- package/_site/_test/cross-repo-isolation-e2e-test.js +187 -0
- package/_site/_test/data-dir-migration-test.js +77 -0
- package/_site/_test/desktop-browser-compat-test.js +28 -0
- package/_site/_test/desktop-hook-runtime-regression-test.js +204 -0
- package/_site/_test/embedding-config-test.js +32 -0
- package/_site/_test/explicit-commit-processor-test.js +37 -0
- package/_site/_test/fixtures/fake-browser-exit.js +10 -0
- package/_site/_test/fixtures/fake-browser-hang.js +10 -0
- package/_site/_test/fixtures/make-git-repos.js +89 -0
- package/_site/_test/folder-picker-output-test.js +19 -0
- package/_site/_test/full-integration-e2e-test.js +165 -0
- package/_site/_test/git-validation-test.js +81 -0
- package/_site/_test/hook-runtime-endpoint-test.js +88 -0
- package/_site/_test/hook-status-repair-api-test.js +152 -0
- package/_site/_test/hook-trigger-test.js +85 -0
- package/_site/_test/hook-worktree-test.js +42 -0
- package/_site/_test/import-preflight-api-test.js +233 -0
- package/_site/_test/import-ui-flow-test.js +171 -0
- package/_site/_test/index-builder-test.js +41 -0
- package/_site/_test/index-writer-concurrency-test.js +135 -0
- package/_site/_test/integration-adapters-test.js +243 -0
- package/_site/_test/integration-capture-setup-test.js +108 -0
- package/_site/_test/integration-surface-coverage-test.js +165 -0
- package/_site/_test/kb-framework-test.js +57 -0
- package/_site/_test/kbpath-follow-test.js +63 -0
- package/_site/_test/knowledge-db-test.js +144 -0
- package/_site/_test/knowledge-language-control-test.js +170 -0
- package/_site/_test/knowledge-maintenance-test.js +40 -0
- package/_site/_test/knowledge-migration-test.js +75 -0
- package/_site/_test/knowledge-promotion-recovery-test.js +260 -0
- package/_site/_test/knowledge-query-test.js +54 -0
- package/_site/_test/knowledge-retrieval-service-test.js +77 -0
- package/_site/_test/knowledge-scopes-test.js +37 -0
- package/_site/_test/knowledge-storage-location-test.js +28 -0
- package/_site/_test/knowledge-storage-startup-test.js +52 -0
- package/_site/_test/knowledge-store-logs-supervision-test.js +89 -0
- package/_site/_test/legacy-claude-session-alias-test.js +106 -0
- package/_site/_test/legacy-forward-compat-test.js +257 -0
- package/_site/_test/legacy-project-upgrade-e2e-test.js +363 -0
- package/_site/_test/legacy-routes-removed-test.js +47 -0
- package/_site/_test/legacy-workbench-exclusion-test.js +92 -0
- package/_site/_test/log-redaction-test.js +58 -0
- package/_site/_test/logging-api-test.js +89 -0
- package/_site/_test/logging-sse-no-gap-test.js +88 -0
- package/_site/_test/logging-ui-test.js +104 -0
- package/_site/_test/markdown-delta-overlay-test.js +85 -0
- package/_site/_test/markdown-maintenance-api-test.js +75 -0
- package/_site/_test/markdown-maintenance-test.js +11 -0
- package/_site/_test/mcp-server-test.js +149 -0
- package/_site/_test/model-context-windows-test.js +89 -0
- package/_site/_test/module-bridge-eventbridge-test.js +193 -0
- package/_site/_test/non-release-ci-test.js +34 -0
- package/_site/_test/offline-boundary-isolation-test.js +32 -0
- package/_site/_test/orphaned-claim-recovery-test.js +80 -0
- package/_site/_test/p0-data-migration-characterization-test.js +37 -0
- package/_site/_test/p0-e2e-gate-test.js +347 -0
- package/_site/_test/package-boundary-test.js +42 -0
- package/_site/_test/package-startup-test.js +56 -0
- package/_site/_test/packaged-ui-smoke-test.js +79 -0
- package/_site/_test/path-consistency-test.js +151 -0
- package/_site/_test/pending-sweep-test.js +9 -0
- package/_site/_test/post-commit-automation-test.js +87 -0
- package/_site/_test/pr-consumer-contract-test.js +55 -0
- package/_site/_test/project-control-panel-task14-test.js +63 -0
- package/_site/_test/project-delete-recovery-test.js +64 -0
- package/_site/_test/project-goal-editor-test.js +145 -0
- package/_site/_test/project-layout-v2-migration-test.js +127 -0
- package/_site/_test/project-lifecycle-transaction-test.js +106 -0
- package/_site/_test/project-remove-running-guard-test.js +79 -0
- package/_site/_test/project-store-test.js +90 -0
- package/_site/_test/prompt-settings-test.js +115 -0
- package/_site/_test/protected-architecture-gate-test.js +123 -0
- package/_site/_test/refactor-characterization-test.js +36 -0
- package/_site/_test/release-version-sync-test.js +98 -0
- package/_site/_test/requirement-binding-test.js +113 -0
- package/_site/_test/requirement-recorder-test.js +173 -0
- package/_site/_test/run-all-tests.js +156 -0
- package/_site/_test/runtime-endpoint-test.js +64 -0
- package/_site/_test/scanner-test.js +83 -0
- package/_site/_test/server-runtime-migration-safety-test.js +27 -0
- package/_site/_test/server-security-test.js +96 -0
- package/_site/_test/sessions-stream-test.js +92 -0
- package/_site/_test/shared-contracts-test.js +58 -0
- package/_site/_test/simple-import-test.js +97 -0
- package/_site/_test/startup-analysis-disabled-test.js +16 -0
- package/_site/_test/storage-foundation-test.js +78 -0
- package/_site/_test/structured-logger-test.js +101 -0
- package/_site/_test/task15-20-ui-flow-test.js +148 -0
- package/_site/_test/team-knowledge-store-sparse-checkout-test.js +196 -0
- package/_site/_test/team-knowledge-store-test.js +131 -0
- package/_site/_test/tracking-start-test.js +143 -0
- package/_site/_test/ui-i18n-toggle-test.js +114 -0
- package/_site/_test/ui-smoke-test.js +73 -0
- package/_site/_test/v4122-upgrade-data-contract-test.js +56 -0
- package/_site/_test/workbench-permission-test.js +118 -0
- package/_site/_test/workspace-ui-contract-test.js +43 -0
- package/_site/lib/ai-profile-resolver.js +78 -0
- package/_site/lib/ai-vendor-presets.js +83 -0
- package/_site/lib/ai-workspace.js +104 -0
- package/_site/lib/atomic-file.js +204 -0
- package/_site/lib/automation-config.js +140 -0
- package/_site/lib/bridge-adapter.js +256 -0
- package/_site/lib/bridge-consumer-service.js +402 -0
- package/_site/lib/claude-cli-runner.js +1599 -0
- package/_site/lib/commit-conversation-binder.js +246 -0
- package/_site/lib/commit-processing-ledger.js +27 -0
- package/_site/lib/commit-prompt.js +202 -0
- package/_site/lib/commit-reconciler.js +444 -0
- package/_site/lib/context-pack-builder.js +299 -0
- package/_site/lib/contracts.js +279 -0
- package/_site/lib/conversation-exclusions.js +204 -0
- package/_site/lib/conversation-query-service.js +161 -0
- package/_site/lib/conversation-store.js +385 -0
- package/_site/lib/data-dir.js +81 -0
- package/_site/lib/data-state-classifier.js +40 -0
- package/_site/lib/embedding-service.js +85 -0
- package/_site/lib/evidence-bundle.js +228 -0
- package/_site/lib/folder-picker-output.js +24 -0
- package/_site/lib/git-runner.js +62 -0
- package/_site/lib/github-team-store.js +1175 -0
- package/_site/lib/hook-manager.js +202 -0
- package/_site/lib/index-builder.js +157 -0
- package/_site/lib/index-service.js +173 -0
- package/_site/lib/integration-installer.js +907 -0
- package/_site/lib/kb-framework.js +191 -0
- package/_site/lib/kb-validator.js +124 -0
- package/_site/lib/knowledge-db.js +363 -0
- package/_site/lib/knowledge-promotion.js +435 -0
- package/_site/lib/knowledge-retrieval-service.js +401 -0
- package/_site/lib/knowledge-schema.js +121 -0
- package/_site/lib/knowledge-tool-runtime.js +359 -0
- package/_site/lib/legacy-data-manifest.js +26 -0
- package/_site/lib/llm-client.js +173 -0
- package/_site/lib/markdown-knowledge-indexer.js +230 -0
- package/_site/lib/migration-service.js +442 -0
- package/_site/lib/model-context-windows.js +82 -0
- package/_site/lib/module-bridge.js +496 -0
- package/_site/lib/post-commit-automation.js +141 -0
- package/_site/lib/project-lifecycle-service.js +511 -0
- package/_site/lib/project-registry-store.js +129 -0
- package/_site/lib/project-store.js +262 -0
- package/_site/lib/prompt-registry.js +148 -0
- package/_site/lib/requirement-adapters.js +10 -0
- package/_site/lib/requirement-binder.js +112 -0
- package/_site/lib/requirement-recorder.js +278 -0
- package/_site/lib/runtime-endpoint.js +160 -0
- package/_site/lib/scanner.js +298 -0
- package/_site/lib/server-app.js +1520 -0
- package/_site/lib/settings-store.js +113 -0
- package/_site/lib/storage-layout.js +162 -0
- package/_site/lib/structured-logger.js +574 -0
- package/_site/scripts/codex-notify-fanout.cjs +51 -0
- package/_site/scripts/folder-picker.ps1 +156 -0
- package/_site/scripts/hook-trigger.js +156 -0
- package/_site/scripts/sync-release-version.js +158 -0
- package/_site/server.js +12 -0
- package/_site/start.bat +26 -0
- package/_site/stop.bat +11 -0
- package/bin/project-knowledge-integrations.js +241 -0
- package/bin/project-knowledge-kb.js +65 -0
- package/bin/project-knowledge-mcp.js +195 -0
- package/bin/project-knowledge.js +439 -0
- package/claude-prompts.json +21 -0
- package/docs/README.zh-CN.md +175 -0
- package/docs/assets/architecture.svg +182 -0
- package/docs/assets/dashboard.png +0 -0
- package/docs/assets/draft-review.png +0 -0
- package/docs/assets/logo.svg +50 -0
- package/docs/assets/runs-drafts.png +0 -0
- package/docs/assets/settings.png +0 -0
- package/docs/project-registry-schema.md +33 -0
- package/docs/team-knowledge-mode-a-plan.md +73 -0
- package/docs/testing-strategy.md +33 -0
- package/package.json +100 -0
- package/plugins/project-knowledge/.claude-plugin/plugin.json +19 -0
- package/plugins/project-knowledge/.codex-plugin/plugin.json +38 -0
- package/plugins/project-knowledge/.mcp.json +14 -0
- package/plugins/project-knowledge/opencode/project-knowledge.md +3 -0
- package/plugins/project-knowledge/skills/project-knowledge/SKILL.md +28 -0
- package/plugins/project-knowledge/skills/project-knowledge/agents/openai.yaml +11 -0
- package/templates/change.md +30 -0
- package/templates/project-readme.md +52 -0
- package/ui/app.css +58 -0
- package/ui/app.js +607 -0
- package/ui/favicon.svg +38 -0
- package/ui/i18n.js +146 -0
- package/ui/index.html +162 -0
|
@@ -0,0 +1,401 @@
|
|
|
1
|
+
const fs = require('fs');
|
|
2
|
+
const path = require('path');
|
|
3
|
+
const crypto = require('crypto');
|
|
4
|
+
const { DomainError, validateProjectId } = require('./contracts');
|
|
5
|
+
const { CHUNKER_VERSION, chunkMarkdown, listMarkdownFiles, inferEntryType, parseKnowledgeMetadata } = require('./markdown-knowledge-indexer');
|
|
6
|
+
const { sha256: documentSha256 } = require('./knowledge-schema');
|
|
7
|
+
|
|
8
|
+
const DEFAULT_CONTEXT_BUDGET = 256 * 1024;
|
|
9
|
+
const DEFAULT_CANDIDATE_LIMIT = 50;
|
|
10
|
+
|
|
11
|
+
function hash(value) {
|
|
12
|
+
return `sha256:${crypto.createHash('sha256').update(String(value == null ? '' : value), 'utf8').digest('hex')}`;
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
function tokens(value) {
|
|
16
|
+
const normalized = String(value || '').toLowerCase().replace(/\\/g, '/');
|
|
17
|
+
const out = normalized.match(/[\p{L}\p{N}_-]+/gu) || [];
|
|
18
|
+
for (const segment of normalized.match(/[\p{Script=Han}]+/gu) || []) {
|
|
19
|
+
for (let index = 0; index < segment.length - 1; index += 1) out.push(segment.slice(index, index + 2));
|
|
20
|
+
}
|
|
21
|
+
return [...new Set(out.filter(item => item.length > 1))];
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
function normalizePath(value) {
|
|
25
|
+
return String(value || '').replace(/\\/g, '/').replace(/^\.\//, '').toLowerCase();
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
function titleFromMarkdown(markdown, fallback) {
|
|
29
|
+
const match = /^#\s+(.+)$/m.exec(String(markdown || ''));
|
|
30
|
+
return match ? match[1].trim() : fallback;
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
function occurrences(text, terms, cap = 12) {
|
|
34
|
+
const source = String(text || '').toLowerCase();
|
|
35
|
+
let count = 0;
|
|
36
|
+
for (const term of terms) {
|
|
37
|
+
let offset = 0;
|
|
38
|
+
let matches = 0;
|
|
39
|
+
while (term && matches < cap && (offset = source.indexOf(term, offset)) >= 0) {
|
|
40
|
+
matches += 1;
|
|
41
|
+
count += 1;
|
|
42
|
+
offset += term.length;
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
return count;
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
function safeCurrentFile(layout, knowledgePath, entryId) {
|
|
49
|
+
const root = path.resolve(knowledgePath);
|
|
50
|
+
const target = path.resolve(root, ...String(entryId).split('/'));
|
|
51
|
+
if (!layout.isPathInside(root, target, { realpath: true }) || !fs.existsSync(target)
|
|
52
|
+
|| !fs.statSync(target).isFile() || fs.lstatSync(target).isSymbolicLink()) return null;
|
|
53
|
+
return target;
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
function manifestHash(manifest) {
|
|
57
|
+
const { manifestHash: ignored, ...payload } = manifest;
|
|
58
|
+
void ignored;
|
|
59
|
+
return hash(JSON.stringify(payload));
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
class KnowledgeRetrievalService {
|
|
63
|
+
constructor(options = {}) {
|
|
64
|
+
this.layout = options.layout;
|
|
65
|
+
this.registryStore = options.registryStore;
|
|
66
|
+
this.projectStore = options.projectStore;
|
|
67
|
+
this.databaseProvider = options.databaseProvider || (() => options.database || null);
|
|
68
|
+
this.embedderProvider = options.embedderProvider || (() => options.embedder || null);
|
|
69
|
+
this.logger = options.logger || null;
|
|
70
|
+
this.candidateLimit = Number(options.candidateLimit || DEFAULT_CANDIDATE_LIMIT);
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
async log(level, event, message, context) {
|
|
74
|
+
try { if (this.logger && typeof this.logger[level] === 'function') await this.logger[level](event, message, context); } catch {
|
|
75
|
+
// Retrieval truth and logging availability are independent.
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
project(input = {}) {
|
|
80
|
+
if (input.project && input.project.projectId) return input.project;
|
|
81
|
+
const projectId = validateProjectId(input.projectId);
|
|
82
|
+
const config = this.projectStore.readConfig(projectId);
|
|
83
|
+
const state = this.projectStore.readState(projectId);
|
|
84
|
+
if (config.enabled === false) throw new DomainError('PROJECT_NOT_FOUND', 'The requested project is disabled.', { status: 404 });
|
|
85
|
+
return {
|
|
86
|
+
projectId,
|
|
87
|
+
projectSlug: config.legacyExtensions && config.legacyExtensions.slug || projectId,
|
|
88
|
+
primarySpaceId: `project:${projectId}`,
|
|
89
|
+
knowledgePath: path.resolve(config.knowledgePath),
|
|
90
|
+
relatedProjectIds: Array.isArray(config.relatedProjectIds) ? config.relatedProjectIds : [],
|
|
91
|
+
index: state.index,
|
|
92
|
+
};
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
scopes(project) {
|
|
96
|
+
const scopes = [{ projectId: project.projectId, spaceId: project.primarySpaceId || `project:${project.projectId}`, knowledgePath: project.knowledgePath, weight: 1, reason: 'primary' }];
|
|
97
|
+
for (const relatedId of project.relatedProjectIds || []) {
|
|
98
|
+
if (!this.registryStore || !this.registryStore.readDisplaySnapshot(relatedId)) continue;
|
|
99
|
+
const config = this.projectStore.readConfig(relatedId);
|
|
100
|
+
if (config.enabled === false) continue;
|
|
101
|
+
scopes.push({ projectId: relatedId, spaceId: `project:${relatedId}`, knowledgePath: path.resolve(config.knowledgePath), weight: 0.88, reason: 'related' });
|
|
102
|
+
}
|
|
103
|
+
return scopes;
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
sourceManifest(projectId) {
|
|
107
|
+
const filePath = this.layout.getRuntimePath('index-sources', `${projectId}.json`);
|
|
108
|
+
try {
|
|
109
|
+
const value = JSON.parse(fs.readFileSync(filePath, 'utf8'));
|
|
110
|
+
return value && value.schema === 'knowledge-index-source-manifest/v1' ? value : null;
|
|
111
|
+
} catch { return null; }
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
scanCurrent(scopes) {
|
|
115
|
+
const candidates = new Map();
|
|
116
|
+
const scopeHealth = [];
|
|
117
|
+
for (const scope of scopes) {
|
|
118
|
+
const state = this.projectStore.readState(scope.projectId);
|
|
119
|
+
const sourceManifest = this.sourceManifest(scope.projectId);
|
|
120
|
+
const manifestEntries = sourceManifest && sourceManifest.entries || {};
|
|
121
|
+
const files = listMarkdownFiles(scope.knowledgePath);
|
|
122
|
+
let overlayEntries = 0;
|
|
123
|
+
for (const filePath of files) {
|
|
124
|
+
const entryId = path.relative(scope.knowledgePath, filePath).replace(/\\/g, '/');
|
|
125
|
+
const markdown = fs.readFileSync(filePath, 'utf8');
|
|
126
|
+
const documentHash = documentSha256(`chunker:${CHUNKER_VERSION}\n${markdown}`);
|
|
127
|
+
const metadata = parseKnowledgeMetadata(markdown);
|
|
128
|
+
const title = titleFromMarkdown(markdown, path.basename(entryId, '.md'));
|
|
129
|
+
const rawChunks = chunkMarkdown(markdown);
|
|
130
|
+
const fromOverlay = !sourceManifest || !manifestEntries[entryId] || manifestEntries[entryId].documentHash !== documentHash;
|
|
131
|
+
if (fromOverlay) overlayEntries += 1;
|
|
132
|
+
for (const chunk of rawChunks) {
|
|
133
|
+
const chunkId = `${entryId}:${chunk.chunkOrder}`;
|
|
134
|
+
candidates.set(`${scope.spaceId}:${chunkId}`, {
|
|
135
|
+
recordId: `${scope.spaceId}:${chunkId}`,
|
|
136
|
+
spaceId: scope.spaceId,
|
|
137
|
+
projectId: scope.projectId,
|
|
138
|
+
scopeReason: scope.reason,
|
|
139
|
+
scopeWeight: scope.weight,
|
|
140
|
+
knowledgePath: scope.knowledgePath,
|
|
141
|
+
entryId,
|
|
142
|
+
entryType: inferEntryType(entryId),
|
|
143
|
+
chunkId,
|
|
144
|
+
chunkOrder: chunk.chunkOrder,
|
|
145
|
+
title,
|
|
146
|
+
headingPath: chunk.headingPath,
|
|
147
|
+
chunkText: chunk.chunkText,
|
|
148
|
+
documentHash,
|
|
149
|
+
contentHash: documentSha256(chunk.chunkText),
|
|
150
|
+
metadata,
|
|
151
|
+
fromOverlay,
|
|
152
|
+
indexScore: 0,
|
|
153
|
+
indexChannels: [],
|
|
154
|
+
});
|
|
155
|
+
}
|
|
156
|
+
}
|
|
157
|
+
const deletedEntries = sourceManifest
|
|
158
|
+
? Object.keys(manifestEntries).filter(entryId => !safeCurrentFile(this.layout, scope.knowledgePath, entryId)).length
|
|
159
|
+
: 0;
|
|
160
|
+
scopeHealth.push({
|
|
161
|
+
projectId: scope.projectId,
|
|
162
|
+
generation: Number(state.index && state.index.generation || 0),
|
|
163
|
+
dirty: state.index && state.index.dirty === true,
|
|
164
|
+
sourceManifest: sourceManifest ? 'available' : 'missing',
|
|
165
|
+
overlayEntries,
|
|
166
|
+
deletedEntries,
|
|
167
|
+
});
|
|
168
|
+
}
|
|
169
|
+
return { candidates, scopeHealth };
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
async recallIndex(scopes, query, current, health) {
|
|
173
|
+
if (!query || !fs.existsSync(this.layout.getIndexPath())) {
|
|
174
|
+
health.index = 'missing';
|
|
175
|
+
return;
|
|
176
|
+
}
|
|
177
|
+
const database = this.databaseProvider();
|
|
178
|
+
const embedder = this.embedderProvider();
|
|
179
|
+
if (!database || !embedder || typeof embedder.embedQuery !== 'function') {
|
|
180
|
+
health.index = 'unavailable';
|
|
181
|
+
return;
|
|
182
|
+
}
|
|
183
|
+
try {
|
|
184
|
+
const vector = await embedder.embedQuery(query);
|
|
185
|
+
const rows = await database.hybridSearch({
|
|
186
|
+
text: query,
|
|
187
|
+
vector,
|
|
188
|
+
spaceIds: scopes.map(scope => scope.spaceId),
|
|
189
|
+
limit: this.candidateLimit,
|
|
190
|
+
candidates: Math.min(200, this.candidateLimit * 4),
|
|
191
|
+
});
|
|
192
|
+
for (const row of rows) {
|
|
193
|
+
const key = `${row.space_id}:${row.chunk_id}`;
|
|
194
|
+
const candidate = current.get(key);
|
|
195
|
+
if (!candidate) continue;
|
|
196
|
+
candidate.indexScore = Number(row.relevance_score || 0);
|
|
197
|
+
candidate.indexChannels = Array.isArray(row.match_channels) ? row.match_channels : [];
|
|
198
|
+
}
|
|
199
|
+
health.index = 'ok';
|
|
200
|
+
} catch (error) {
|
|
201
|
+
health.index = 'degraded';
|
|
202
|
+
health.reason = 'index-query-failed';
|
|
203
|
+
health.errorCode = error && error.code || 'INDEX_UNAVAILABLE';
|
|
204
|
+
}
|
|
205
|
+
}
|
|
206
|
+
|
|
207
|
+
score(candidate, signals) {
|
|
208
|
+
const selectionSignals = [];
|
|
209
|
+
let score = candidate.indexScore * 1000;
|
|
210
|
+
if (candidate.indexScore) selectionSignals.push(...candidate.indexChannels.map(channel => `index:${channel}`));
|
|
211
|
+
const primaryMatches = occurrences(`${candidate.title}\n${candidate.headingPath.join(' ')}\n${candidate.chunkText}`, signals.primaryTokens);
|
|
212
|
+
if (primaryMatches) { score += Math.min(primaryMatches, 20) * 8; selectionSignals.push(`user-keyword:${primaryMatches}`); }
|
|
213
|
+
const implementationMatches = occurrences(`${candidate.entryId}\n${candidate.title}\n${candidate.chunkText}`, signals.implementationTokens);
|
|
214
|
+
if (implementationMatches) { score += Math.min(implementationMatches, 16) * 3; selectionSignals.push(`implementation:${implementationMatches}`); }
|
|
215
|
+
const assistantMatches = occurrences(candidate.chunkText, signals.assistantTokens);
|
|
216
|
+
if (assistantMatches) { score += Math.min(assistantMatches, 8); selectionSignals.push(`assistant-hint:${assistantMatches}`); }
|
|
217
|
+
|
|
218
|
+
const changedPaths = signals.changedPaths;
|
|
219
|
+
const metadataPaths = candidate.metadata.sourcePaths.map(normalizePath);
|
|
220
|
+
const exactPaths = metadataPaths.filter(value => changedPaths.has(value));
|
|
221
|
+
if (exactPaths.length) { score += 300 + exactPaths.length * 20; selectionSignals.push(`exact-source-path:${exactPaths.join(',')}`); }
|
|
222
|
+
const changedBasenames = signals.changedBasenames;
|
|
223
|
+
const basenameMatches = metadataPaths.filter(value => changedBasenames.has(path.posix.basename(value)));
|
|
224
|
+
if (!exactPaths.length && basenameMatches.length) { score += 70; selectionSignals.push(`source-basename:${basenameMatches.join(',')}`); }
|
|
225
|
+
|
|
226
|
+
const structured = [...candidate.metadata.symbols, ...candidate.metadata.tags, ...candidate.metadata.routes, ...candidate.metadata.affectedModules].map(value => value.toLowerCase());
|
|
227
|
+
const structuredMatches = structured.filter(value => signals.allTerms.has(value)
|
|
228
|
+
|| signals.allTerms.has(normalizePath(value))
|
|
229
|
+
|| tokens(value).some(term => signals.allTerms.has(term)));
|
|
230
|
+
if (structuredMatches.length) { score += 90 + structuredMatches.length * 10; selectionSignals.push(`structured:${structuredMatches.join(',')}`); }
|
|
231
|
+
score *= candidate.scopeWeight;
|
|
232
|
+
if (candidate.scopeReason === 'primary') selectionSignals.push('scope:primary');
|
|
233
|
+
else selectionSignals.push('scope:related');
|
|
234
|
+
if (candidate.fromOverlay) selectionSignals.push('markdown-delta-overlay');
|
|
235
|
+
return { score, selectionSignals };
|
|
236
|
+
}
|
|
237
|
+
|
|
238
|
+
signalSet(input = {}) {
|
|
239
|
+
const primaryTexts = input.primaryTexts || [];
|
|
240
|
+
const implementationTexts = input.implementationTexts || [];
|
|
241
|
+
const assistantTexts = input.assistantTexts || [];
|
|
242
|
+
const changedPaths = new Set((input.changedPaths || []).map(normalizePath).filter(Boolean));
|
|
243
|
+
const implementationTokens = tokens(implementationTexts.join(' '));
|
|
244
|
+
for (const changedPath of changedPaths) {
|
|
245
|
+
implementationTokens.push(...tokens(changedPath), ...tokens(path.posix.basename(changedPath)));
|
|
246
|
+
}
|
|
247
|
+
const primaryTokens = tokens(primaryTexts.join(' '));
|
|
248
|
+
const assistantTokens = tokens(assistantTexts.join(' '));
|
|
249
|
+
const allTerms = new Set([...primaryTokens, ...implementationTokens]);
|
|
250
|
+
return {
|
|
251
|
+
primaryTokens,
|
|
252
|
+
implementationTokens: [...new Set(implementationTokens)],
|
|
253
|
+
assistantTokens,
|
|
254
|
+
changedPaths,
|
|
255
|
+
changedBasenames: new Set([...changedPaths].map(value => path.posix.basename(value))),
|
|
256
|
+
allTerms,
|
|
257
|
+
indexQuery: [...primaryTexts, ...implementationTexts].join('\n').slice(0, 12_000),
|
|
258
|
+
signalHash: hash(JSON.stringify({ primaryTexts, implementationTexts, changedPaths: [...changedPaths], assistantHintHashes: assistantTexts.map(hash) })),
|
|
259
|
+
};
|
|
260
|
+
}
|
|
261
|
+
|
|
262
|
+
async retrieve(input = {}) {
|
|
263
|
+
const project = this.project(input);
|
|
264
|
+
const scopes = Array.isArray(input.scopes) && input.scopes.length ? input.scopes : this.scopes(project);
|
|
265
|
+
const signals = this.signalSet(input.signals);
|
|
266
|
+
const scanned = this.scanCurrent(scopes);
|
|
267
|
+
const health = { index: 'unknown', scopes: scanned.scopeHealth };
|
|
268
|
+
await this.recallIndex(scopes, signals.indexQuery, scanned.candidates, health);
|
|
269
|
+
const ranked = [];
|
|
270
|
+
for (const candidate of scanned.candidates.values()) {
|
|
271
|
+
const scored = this.score(candidate, signals);
|
|
272
|
+
if (scored.score <= 0) continue;
|
|
273
|
+
ranked.push({ ...candidate, relevanceScore: scored.score, selectionSignals: scored.selectionSignals });
|
|
274
|
+
}
|
|
275
|
+
ranked.sort((left, right) => right.relevanceScore - left.relevanceScore || left.recordId.localeCompare(right.recordId));
|
|
276
|
+
return { project, scopes, signals, health, candidates: ranked.slice(0, this.candidateLimit), totalCurrentChunks: scanned.candidates.size };
|
|
277
|
+
}
|
|
278
|
+
|
|
279
|
+
async search(input = {}) {
|
|
280
|
+
const query = String(input.query || '').trim();
|
|
281
|
+
if (!query) throw new DomainError('INVALID_ARGUMENT', 'query is required.');
|
|
282
|
+
const limit = Math.max(1, Math.min(Number(input.limit || 8), 30));
|
|
283
|
+
const retrieved = await this.retrieve({ ...input, signals: { primaryTexts: [query] } });
|
|
284
|
+
const results = retrieved.candidates.slice(0, limit).map(candidate => ({
|
|
285
|
+
record_id: candidate.recordId,
|
|
286
|
+
space_id: candidate.spaceId,
|
|
287
|
+
entry_id: candidate.entryId,
|
|
288
|
+
chunk_id: candidate.chunkId,
|
|
289
|
+
chunk_order: candidate.chunkOrder,
|
|
290
|
+
title: candidate.title,
|
|
291
|
+
heading_path: candidate.headingPath.join(' > '),
|
|
292
|
+
chunk_text: candidate.chunkText,
|
|
293
|
+
document_hash: candidate.documentHash,
|
|
294
|
+
content_hash: candidate.contentHash,
|
|
295
|
+
relevance_score: candidate.relevanceScore,
|
|
296
|
+
match_channels: candidate.selectionSignals,
|
|
297
|
+
scope_reason: candidate.scopeReason,
|
|
298
|
+
scope_project_id: candidate.projectId,
|
|
299
|
+
from_delta_overlay: candidate.fromOverlay,
|
|
300
|
+
}));
|
|
301
|
+
await this.log('info', 'retrieval.search_completed', 'Knowledge search completed against authoritative Markdown.', {
|
|
302
|
+
projectId: retrieved.project.projectId,
|
|
303
|
+
component: 'knowledge-retrieval',
|
|
304
|
+
context: { queryHash: hash(query), resultCount: results.length, backend: retrieved.health.index, currentChunkCount: retrieved.totalCurrentChunks },
|
|
305
|
+
});
|
|
306
|
+
return {
|
|
307
|
+
ok: true,
|
|
308
|
+
projectId: retrieved.project.projectId,
|
|
309
|
+
projectSlug: retrieved.project.projectSlug || retrieved.project.projectId,
|
|
310
|
+
query,
|
|
311
|
+
backend: retrieved.health.index === 'ok' ? 'hybrid+markdown-truth' : 'markdown-hybrid-fallback',
|
|
312
|
+
source: 'knowledge-retrieval-service',
|
|
313
|
+
health: retrieved.health,
|
|
314
|
+
results,
|
|
315
|
+
};
|
|
316
|
+
}
|
|
317
|
+
|
|
318
|
+
async retrieveForCommit(input = {}) {
|
|
319
|
+
const snapshot = input.conversationSnapshot || { turns: [] };
|
|
320
|
+
const evidence = input.commitEvidence || {};
|
|
321
|
+
const userTexts = (snapshot.turns || []).flatMap(turn => (turn.userEvents || []).map(event => event.content));
|
|
322
|
+
const assistantTexts = (snapshot.turns || []).flatMap(turn => (turn.assistantEvents || []).map(event => event.content));
|
|
323
|
+
const changedPaths = (evidence.files || []).flatMap(file => [file.path, file.oldPath]).filter(Boolean);
|
|
324
|
+
const retrieved = await this.retrieve({
|
|
325
|
+
projectId: input.projectId,
|
|
326
|
+
project: input.project,
|
|
327
|
+
signals: {
|
|
328
|
+
primaryTexts: userTexts,
|
|
329
|
+
assistantTexts,
|
|
330
|
+
implementationTexts: [evidence.subject || '', ...(evidence.files || []).map(file => `${file.status || ''} ${file.path || ''}`)],
|
|
331
|
+
changedPaths,
|
|
332
|
+
},
|
|
333
|
+
});
|
|
334
|
+
const budget = Math.max(8 * 1024, Number(input.contextBudget || DEFAULT_CONTEXT_BUDGET));
|
|
335
|
+
const selected = [];
|
|
336
|
+
let totalBytes = 0;
|
|
337
|
+
for (const candidate of retrieved.candidates) {
|
|
338
|
+
const bytes = Buffer.byteLength(candidate.chunkText, 'utf8');
|
|
339
|
+
if (totalBytes + bytes > budget) continue;
|
|
340
|
+
totalBytes += bytes;
|
|
341
|
+
selected.push(candidate);
|
|
342
|
+
}
|
|
343
|
+
const manifest = {
|
|
344
|
+
schema: 'knowledge-retrieval-manifest/v1',
|
|
345
|
+
projectId: retrieved.project.projectId,
|
|
346
|
+
commitSha: evidence.commitSha || null,
|
|
347
|
+
signalHash: retrieved.signals.signalHash,
|
|
348
|
+
backend: retrieved.health.index,
|
|
349
|
+
health: retrieved.health,
|
|
350
|
+
indexGeneration: Object.fromEntries(retrieved.health.scopes.map(scope => [scope.projectId, scope.generation])),
|
|
351
|
+
candidateCount: retrieved.candidates.length,
|
|
352
|
+
currentChunkCount: retrieved.totalCurrentChunks,
|
|
353
|
+
selected: selected.map(candidate => ({
|
|
354
|
+
projectId: candidate.projectId,
|
|
355
|
+
spaceId: candidate.spaceId,
|
|
356
|
+
entryId: candidate.entryId,
|
|
357
|
+
chunkId: candidate.chunkId,
|
|
358
|
+
chunkOrder: candidate.chunkOrder,
|
|
359
|
+
headingPath: candidate.headingPath,
|
|
360
|
+
documentHash: candidate.documentHash,
|
|
361
|
+
contentHash: candidate.contentHash,
|
|
362
|
+
selectionSignals: candidate.selectionSignals,
|
|
363
|
+
relevanceScore: candidate.relevanceScore,
|
|
364
|
+
fromDeltaOverlay: candidate.fromOverlay,
|
|
365
|
+
bytes: Buffer.byteLength(candidate.chunkText, 'utf8'),
|
|
366
|
+
})),
|
|
367
|
+
totalBytes,
|
|
368
|
+
tokenEstimate: Math.ceil(totalBytes / 4),
|
|
369
|
+
contextBudget: budget,
|
|
370
|
+
manifestHash: '',
|
|
371
|
+
};
|
|
372
|
+
manifest.manifestHash = manifestHash(manifest);
|
|
373
|
+
const entries = selected.map(candidate => ({
|
|
374
|
+
path: candidate.entryId,
|
|
375
|
+
projectId: candidate.projectId,
|
|
376
|
+
chunkId: candidate.chunkId,
|
|
377
|
+
headingPath: candidate.headingPath,
|
|
378
|
+
hash: candidate.contentHash,
|
|
379
|
+
documentHash: candidate.documentHash,
|
|
380
|
+
content: candidate.chunkText,
|
|
381
|
+
selectionSignals: candidate.selectionSignals,
|
|
382
|
+
}));
|
|
383
|
+
await this.log('info', 'retrieval.commit_context_frozen', 'Commit knowledge context was selected from authoritative Markdown.', {
|
|
384
|
+
projectId: retrieved.project.projectId,
|
|
385
|
+
commitSha: evidence.commitSha,
|
|
386
|
+
component: 'knowledge-retrieval',
|
|
387
|
+
context: { manifestHash: manifest.manifestHash, selectedCount: entries.length, totalBytes, candidateCount: manifest.candidateCount, backend: manifest.backend },
|
|
388
|
+
});
|
|
389
|
+
return { entries, omitted: [], totalBytes, limitBytes: budget, manifest, manifestHash: manifest.manifestHash };
|
|
390
|
+
}
|
|
391
|
+
}
|
|
392
|
+
|
|
393
|
+
module.exports = {
|
|
394
|
+
KnowledgeRetrievalService,
|
|
395
|
+
DEFAULT_CONTEXT_BUDGET,
|
|
396
|
+
DEFAULT_CANDIDATE_LIMIT,
|
|
397
|
+
hash,
|
|
398
|
+
tokens,
|
|
399
|
+
normalizePath,
|
|
400
|
+
manifestHash,
|
|
401
|
+
};
|
|
@@ -0,0 +1,121 @@
|
|
|
1
|
+
const crypto = require('crypto');
|
|
2
|
+
const arrow = require('apache-arrow');
|
|
3
|
+
|
|
4
|
+
const KNOWLEDGE_DB_SCHEMA_VERSION = 1;
|
|
5
|
+
const KNOWLEDGE_TABLE = 'knowledge_chunks';
|
|
6
|
+
const EMBEDDING_DIMENSIONS = 512;
|
|
7
|
+
|
|
8
|
+
function field(name, type, nullable = false) {
|
|
9
|
+
return new arrow.Field(name, type, nullable);
|
|
10
|
+
}
|
|
11
|
+
|
|
12
|
+
function knowledgeChunkSchema(dimensions = EMBEDDING_DIMENSIONS) {
|
|
13
|
+
return new arrow.Schema([
|
|
14
|
+
field('schema_version', new arrow.Int32()),
|
|
15
|
+
field('record_id', new arrow.Utf8()),
|
|
16
|
+
field('space_id', new arrow.Utf8()),
|
|
17
|
+
field('entry_id', new arrow.Utf8()),
|
|
18
|
+
field('entry_type', new arrow.Utf8()),
|
|
19
|
+
field('entry_version', new arrow.Int32()),
|
|
20
|
+
field('chunk_id', new arrow.Utf8()),
|
|
21
|
+
field('chunk_order', new arrow.Int32()),
|
|
22
|
+
field('title', new arrow.Utf8()),
|
|
23
|
+
field('heading_path', new arrow.Utf8()),
|
|
24
|
+
field('chunk_text', new arrow.Utf8()),
|
|
25
|
+
field('search_text', new arrow.Utf8()),
|
|
26
|
+
field('vector', new arrow.FixedSizeList(dimensions, field('item', new arrow.Float32()))),
|
|
27
|
+
field('tags_json', new arrow.Utf8()),
|
|
28
|
+
field('source_paths_json', new arrow.Utf8()),
|
|
29
|
+
field('routes_json', new arrow.Utf8()),
|
|
30
|
+
field('symbols_json', new arrow.Utf8()),
|
|
31
|
+
field('source_project_id', new arrow.Utf8()),
|
|
32
|
+
field('source_commit', new arrow.Utf8()),
|
|
33
|
+
field('document_hash', new arrow.Utf8()),
|
|
34
|
+
field('content_hash', new arrow.Utf8()),
|
|
35
|
+
field('created_at', new arrow.Utf8()),
|
|
36
|
+
field('updated_at', new arrow.Utf8()),
|
|
37
|
+
]);
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
function sha256(value) {
|
|
41
|
+
return crypto.createHash('sha256').update(String(value || ''), 'utf8').digest('hex');
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
function requiredText(value, name) {
|
|
45
|
+
const normalized = String(value || '').trim();
|
|
46
|
+
if (!normalized) throw new Error(`${name} is required`);
|
|
47
|
+
return normalized;
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
function jsonArray(value) {
|
|
51
|
+
return JSON.stringify(Array.isArray(value) ? value.map(item => String(item)) : []);
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
function normalizeVector(value, dimensions = EMBEDDING_DIMENSIONS) {
|
|
55
|
+
const vector = Array.from(value || [], Number);
|
|
56
|
+
if (vector.length !== dimensions || vector.some(item => !Number.isFinite(item))) {
|
|
57
|
+
throw new Error(`vector must contain exactly ${dimensions} finite numbers`);
|
|
58
|
+
}
|
|
59
|
+
return vector;
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
function normalizeKnowledgeChunk(input, options = {}) {
|
|
63
|
+
const dimensions = options.dimensions || EMBEDDING_DIMENSIONS;
|
|
64
|
+
const now = options.now || new Date().toISOString();
|
|
65
|
+
const spaceId = requiredText(input.spaceId || input.space_id, 'spaceId');
|
|
66
|
+
const entryId = requiredText(input.entryId || input.entry_id, 'entryId');
|
|
67
|
+
const chunkOrder = Number.isInteger(input.chunkOrder) ? input.chunkOrder : Number(input.chunk_order || 0);
|
|
68
|
+
if (!Number.isInteger(chunkOrder) || chunkOrder < 0) throw new Error('chunkOrder must be a non-negative integer');
|
|
69
|
+
const chunkText = requiredText(input.chunkText || input.chunk_text, 'chunkText');
|
|
70
|
+
const contentHash = String(input.contentHash || input.content_hash || sha256(chunkText));
|
|
71
|
+
const chunkId = String(input.chunkId || input.chunk_id || `${entryId}:${chunkOrder}`);
|
|
72
|
+
|
|
73
|
+
return {
|
|
74
|
+
schema_version: KNOWLEDGE_DB_SCHEMA_VERSION,
|
|
75
|
+
record_id: `${spaceId}:${chunkId}`,
|
|
76
|
+
space_id: spaceId,
|
|
77
|
+
entry_id: entryId,
|
|
78
|
+
entry_type: String(input.entryType || input.entry_type || 'document'),
|
|
79
|
+
entry_version: Number.isInteger(input.entryVersion) ? input.entryVersion : Number(input.entry_version || 1),
|
|
80
|
+
chunk_id: chunkId,
|
|
81
|
+
chunk_order: chunkOrder,
|
|
82
|
+
title: String(input.title || ''),
|
|
83
|
+
heading_path: Array.isArray(input.headingPath) ? input.headingPath.join(' > ') : String(input.heading_path || ''),
|
|
84
|
+
chunk_text: chunkText,
|
|
85
|
+
search_text: String(input.searchText || input.search_text || `${input.title || ''}\n${chunkText}`).trim(),
|
|
86
|
+
vector: normalizeVector(input.vector, dimensions),
|
|
87
|
+
tags_json: jsonArray(input.tags),
|
|
88
|
+
source_paths_json: jsonArray(input.sourcePaths || input.source_paths),
|
|
89
|
+
routes_json: jsonArray(input.routes),
|
|
90
|
+
symbols_json: jsonArray(input.symbols),
|
|
91
|
+
source_project_id: String(input.sourceProjectId || input.source_project_id || ''),
|
|
92
|
+
source_commit: String(input.sourceCommit || input.source_commit || ''),
|
|
93
|
+
document_hash: String(input.documentHash || input.document_hash || contentHash),
|
|
94
|
+
content_hash: contentHash,
|
|
95
|
+
created_at: String(input.createdAt || input.created_at || now),
|
|
96
|
+
updated_at: String(input.updatedAt || input.updated_at || now),
|
|
97
|
+
};
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
function decodeKnowledgeChunk(row) {
|
|
101
|
+
const parse = (value) => {
|
|
102
|
+
try { return JSON.parse(value || '[]'); } catch { return []; }
|
|
103
|
+
};
|
|
104
|
+
return {
|
|
105
|
+
...row,
|
|
106
|
+
tags: parse(row.tags_json),
|
|
107
|
+
sourcePaths: parse(row.source_paths_json),
|
|
108
|
+
routes: parse(row.routes_json),
|
|
109
|
+
symbols: parse(row.symbols_json),
|
|
110
|
+
};
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
module.exports = {
|
|
114
|
+
KNOWLEDGE_DB_SCHEMA_VERSION,
|
|
115
|
+
KNOWLEDGE_TABLE,
|
|
116
|
+
EMBEDDING_DIMENSIONS,
|
|
117
|
+
knowledgeChunkSchema,
|
|
118
|
+
normalizeKnowledgeChunk,
|
|
119
|
+
decodeKnowledgeChunk,
|
|
120
|
+
sha256,
|
|
121
|
+
};
|