@sanqianx/project-knowledge 4.7.0 → 5.0.0

package/_site/_test/markdown-delta-overlay-test.js

@@ -1,85 +1,85 @@
-const assert = require('assert');
-const fs = require('fs');
-const os = require('os');
-const path = require('path');
-const AtomicFile = require('../lib/atomic-file');
-const { StorageLayout } = require('../lib/storage-layout');
-const { ProjectRegistryStore } = require('../lib/project-registry-store');
-const { ProjectStore } = require('../lib/project-store');
-const { KnowledgeRetrievalService } = require('../lib/knowledge-retrieval-service');
-const { CHUNKER_VERSION } = require('../lib/markdown-knowledge-indexer');
-const { sha256 } = require('../lib/knowledge-schema');
-
-(async () => {
-  const root = fs.mkdtempSync(path.join(os.tmpdir(), 'pk-delta-overlay-'));
-  const layout = new StorageLayout({ dataDir: path.join(root, 'data') });
-  const registry = new ProjectRegistryStore({ layout });
-  const projects = new ProjectStore({ layout });
-  await registry.initialize();
-  const projectId = 'project-delta';
-  const knowledgePath = path.join(root, 'knowledge');
-  fs.mkdirSync(path.join(knowledgePath, 'modules'), { recursive: true });
-  const changedPath = path.join(knowledgePath, 'modules', 'changed.md');
-  const deletedPath = path.join(knowledgePath, 'modules', 'deleted.md');
-  const oldChanged = '# Changed\n\nStale token policy from C0.\n';
-  const oldDeleted = '# Deleted\n\nStale deleted knowledge.\n';
-  fs.writeFileSync(changedPath, oldChanged, 'utf8');
-  fs.writeFileSync(deletedPath, oldDeleted, 'utf8');
-  await projects.create(projectId, {
-    displayName: 'Delta', storageName: 'delta', repoPath: root, knowledgePath,
-  }, { index: { dirty: true, generation: 2, sinceCommit: '1'.repeat(40) } });
-  await registry.add(projectId, { displayNameSnapshot: 'Delta' });
-  AtomicFile.writeJsonAtomic(layout.getRuntimePath('index-sources', `${projectId}.json`), {
-    schema: 'knowledge-index-source-manifest/v1',
-    projectId,
-    generation: 1,
-    entries: {
-      'modules/changed.md': { entryId: 'modules/changed.md', documentHash: sha256(`chunker:${CHUNKER_VERSION}\n${oldChanged}`), sourceCommit: '0'.repeat(40) },
-      'modules/deleted.md': { entryId: 'modules/deleted.md', documentHash: sha256(`chunker:${CHUNKER_VERSION}\n${oldDeleted}`), sourceCommit: '0'.repeat(40) },
-    },
-  });
-
-  const currentChanged = `---\nsourcePaths: [src/token-policy.js]\n---\n# Changed\n\nC1 current rotating token policy truth.\n`;
-  fs.writeFileSync(changedPath, currentChanged, 'utf8');
-  fs.unlinkSync(deletedPath);
-  fs.writeFileSync(path.join(knowledgePath, 'modules', 'new.md'), `---\ntags: [token-policy]\n---\n# New\n\nC1 newly documented policy.\n`, 'utf8');
-  fs.mkdirSync(layout.getIndexPath(), { recursive: true });
-
-  const staleRows = [{
-    record_id: 'project:project-delta:modules/changed.md:0',
-    space_id: 'project:project-delta', entry_id: 'modules/changed.md', chunk_id: 'modules/changed.md:0', chunk_order: 0,
-    title: 'Changed', chunk_text: oldChanged, document_hash: sha256(`chunker:${CHUNKER_VERSION}\n${oldChanged}`),
-    relevance_score: 1, match_channels: ['semantic'],
-  }, {
-    record_id: 'project:project-delta:modules/deleted.md:0',
-    space_id: 'project:project-delta', entry_id: 'modules/deleted.md', chunk_id: 'modules/deleted.md:0', chunk_order: 0,
-    title: 'Deleted', chunk_text: oldDeleted, document_hash: sha256(`chunker:${CHUNKER_VERSION}\n${oldDeleted}`),
-    relevance_score: 2, match_channels: ['semantic'],
-  }];
-  const service = new KnowledgeRetrievalService({
-    layout, registryStore: registry, projectStore: projects,
-    databaseProvider: () => ({ hybridSearch: async () => staleRows }),
-    embedderProvider: () => ({ embedQuery: async () => [1] }),
-  });
-  const result = await service.retrieveForCommit({
-    projectId,
-    conversationSnapshot: { turns: [{ userEvents: [{ content: 'Update rotating token policy.' }], assistantEvents: [] }] },
-    commitEvidence: { commitSha: '1'.repeat(40), subject: 'token policy', files: [{ status: 'M', path: 'src/token-policy.js' }] },
-  });
-  const changed = result.entries.find(entry => entry.path === 'modules/changed.md');
-  const added = result.entries.find(entry => entry.path === 'modules/new.md');
-  assert(changed && /C1 current/.test(changed.content), 'stale index content must be replaced by current Markdown truth');
-  assert(!result.entries.some(entry => /Stale token policy/.test(entry.content)), 'stale chunk text must never reach the prompt context');
-  assert(!result.entries.some(entry => entry.path === 'modules/deleted.md'), 'deleted Markdown must be removed from stale candidates');
-  assert(changed.selectionSignals.includes('markdown-delta-overlay'));
-  assert(added && added.selectionSignals.includes('markdown-delta-overlay'), 'new Markdown must enter the in-memory delta overlay immediately');
-  assert.strictEqual(result.manifest.health.scopes[0].overlayEntries, 2);
-  assert.strictEqual(result.manifest.health.scopes[0].deletedEntries, 1);
-  assert.strictEqual(result.manifest.backend, 'ok', 'a stale index may contribute recall while current Markdown remains truth');
-  assert.strictEqual(changed.documentHash, sha256(`chunker:${CHUNKER_VERSION}\n${currentChanged}`), 'retrieval manifest must freeze the current document hash');
-  assert.strictEqual(projects.readState(projectId).index.dirty, true, 'retrieval must not wait for or mutate index completion state');
-  console.log('markdown-delta-overlay-test PASS');
-})().catch(error => {
-  console.error(error.stack || error.message);
-  process.exitCode = 1;
-});
+const assert = require('assert');
+const fs = require('fs');
+const os = require('os');
+const path = require('path');
+const AtomicFile = require('../lib/atomic-file');
+const { StorageLayout } = require('../lib/storage-layout');
+const { ProjectRegistryStore } = require('../lib/project-registry-store');
+const { ProjectStore } = require('../lib/project-store');
+const { KnowledgeRetrievalService } = require('../lib/knowledge-retrieval-service');
+const { CHUNKER_VERSION } = require('../lib/markdown-knowledge-indexer');
+const { sha256 } = require('../lib/knowledge-schema');
+
+(async () => {
+  const root = fs.mkdtempSync(path.join(os.tmpdir(), 'pk-delta-overlay-'));
+  const layout = new StorageLayout({ dataDir: path.join(root, 'data') });
+  const registry = new ProjectRegistryStore({ layout });
+  const projects = new ProjectStore({ layout });
+  await registry.initialize();
+  const projectId = 'project-delta';
+  const knowledgePath = path.join(root, 'knowledge');
+  fs.mkdirSync(path.join(knowledgePath, 'modules'), { recursive: true });
+  const changedPath = path.join(knowledgePath, 'modules', 'changed.md');
+  const deletedPath = path.join(knowledgePath, 'modules', 'deleted.md');
+  const oldChanged = '# Changed\n\nStale token policy from C0.\n';
+  const oldDeleted = '# Deleted\n\nStale deleted knowledge.\n';
+  fs.writeFileSync(changedPath, oldChanged, 'utf8');
+  fs.writeFileSync(deletedPath, oldDeleted, 'utf8');
+  await projects.create(projectId, {
+    displayName: 'Delta', storageName: 'delta', repoPath: root, knowledgePath,
+  }, { index: { dirty: true, generation: 2, sinceCommit: '1'.repeat(40) } });
+  await registry.add(projectId, { displayNameSnapshot: 'Delta' });
+  AtomicFile.writeJsonAtomic(layout.getRuntimePath('index-sources', `${projectId}.json`), {
+    schema: 'knowledge-index-source-manifest/v1',
+    projectId,
+    generation: 1,
+    entries: {
+      'modules/changed.md': { entryId: 'modules/changed.md', documentHash: sha256(`chunker:${CHUNKER_VERSION}\n${oldChanged}`), sourceCommit: '0'.repeat(40) },
+      'modules/deleted.md': { entryId: 'modules/deleted.md', documentHash: sha256(`chunker:${CHUNKER_VERSION}\n${oldDeleted}`), sourceCommit: '0'.repeat(40) },
+    },
+  });
+
+  const currentChanged = `---\nsourcePaths: [src/token-policy.js]\n---\n# Changed\n\nC1 current rotating token policy truth.\n`;
+  fs.writeFileSync(changedPath, currentChanged, 'utf8');
+  fs.unlinkSync(deletedPath);
+  fs.writeFileSync(path.join(knowledgePath, 'modules', 'new.md'), `---\ntags: [token-policy]\n---\n# New\n\nC1 newly documented policy.\n`, 'utf8');
+  fs.mkdirSync(layout.getIndexPath(), { recursive: true });
+
+  const staleRows = [{
+    record_id: 'project:project-delta:modules/changed.md:0',
+    space_id: 'project:project-delta', entry_id: 'modules/changed.md', chunk_id: 'modules/changed.md:0', chunk_order: 0,
+    title: 'Changed', chunk_text: oldChanged, document_hash: sha256(`chunker:${CHUNKER_VERSION}\n${oldChanged}`),
+    relevance_score: 1, match_channels: ['semantic'],
+  }, {
+    record_id: 'project:project-delta:modules/deleted.md:0',
+    space_id: 'project:project-delta', entry_id: 'modules/deleted.md', chunk_id: 'modules/deleted.md:0', chunk_order: 0,
+    title: 'Deleted', chunk_text: oldDeleted, document_hash: sha256(`chunker:${CHUNKER_VERSION}\n${oldDeleted}`),
+    relevance_score: 2, match_channels: ['semantic'],
+  }];
+  const service = new KnowledgeRetrievalService({
+    layout, registryStore: registry, projectStore: projects,
+    databaseProvider: () => ({ hybridSearch: async () => staleRows }),
+    embedderProvider: () => ({ embedQuery: async () => [1] }),
+  });
+  const result = await service.retrieveForCommit({
+    projectId,
+    conversationSnapshot: { turns: [{ userEvents: [{ content: 'Update rotating token policy.' }], assistantEvents: [] }] },
+    commitEvidence: { commitSha: '1'.repeat(40), subject: 'token policy', files: [{ status: 'M', path: 'src/token-policy.js' }] },
+  });
+  const changed = result.entries.find(entry => entry.path === 'modules/changed.md');
+  const added = result.entries.find(entry => entry.path === 'modules/new.md');
+  assert(changed && /C1 current/.test(changed.content), 'stale index content must be replaced by current Markdown truth');
+  assert(!result.entries.some(entry => /Stale token policy/.test(entry.content)), 'stale chunk text must never reach the prompt context');
+  assert(!result.entries.some(entry => entry.path === 'modules/deleted.md'), 'deleted Markdown must be removed from stale candidates');
+  assert(changed.selectionSignals.includes('markdown-delta-overlay'));
+  assert(added && added.selectionSignals.includes('markdown-delta-overlay'), 'new Markdown must enter the in-memory delta overlay immediately');
+  assert.strictEqual(result.manifest.health.scopes[0].overlayEntries, 2);
+  assert.strictEqual(result.manifest.health.scopes[0].deletedEntries, 1);
+  assert.strictEqual(result.manifest.backend, 'ok', 'a stale index may contribute recall while current Markdown remains truth');
+  assert.strictEqual(changed.documentHash, sha256(`chunker:${CHUNKER_VERSION}\n${currentChanged}`), 'retrieval manifest must freeze the current document hash');
+  assert.strictEqual(projects.readState(projectId).index.dirty, true, 'retrieval must not wait for or mutate index completion state');
+  console.log('markdown-delta-overlay-test PASS');
+})().catch(error => {
+  console.error(error.stack || error.message);
+  process.exitCode = 1;
+});

package/_site/_test/markdown-maintenance-api-test.js

@@ -1,75 +1,75 @@
-const assert = require('assert');
-const fs = require('fs');
-const os = require('os');
-const path = require('path');
-const { execFileSync } = require('child_process');
-const { spawnServer } = require('./helpers/spawn-server');
-
-const ROOT = path.resolve(__dirname, '..', '..');
-const PORT = 7932;
-const BASE_URL = `http://127.0.0.1:${PORT}`;
-const TEMP = fs.mkdtempSync(path.join(os.tmpdir(), 'index-maintenance-api-v2-'));
-const DATA_DIR = path.join(TEMP, 'data');
-const REPO = path.join(TEMP, 'repo');
-
-function git(args) { return execFileSync('git', args, { cwd: REPO, encoding: 'utf8' }).trim(); }
-
-async function json(method, route, body) {
-  const response = await fetch(`${BASE_URL}${route}`, {
-    method,
-    headers: body ? { 'Content-Type': 'application/json' } : undefined,
-    body: body ? JSON.stringify(body) : undefined,
-  });
-  const text = await response.text();
-  const data = text ? JSON.parse(text) : {};
-  if (!response.ok) throw new Error(`${response.status}: ${JSON.stringify(data)}`);
-  return data;
-}
-
-(async () => {
-  fs.mkdirSync(REPO, { recursive: true });
-  git(['init', '--initial-branch=main']);
-  git(['config', 'user.name', 'Index Test']);
-  git(['config', 'user.email', 'index@example.test']);
-  fs.writeFileSync(path.join(REPO, 'README.md'), '# source\n', 'utf8');
-  git(['add', '.']);
-  git(['commit', '-m', 'baseline']);
-  const spawned = spawnServer({ root: ROOT, port: PORT, dataDir: DATA_DIR, tag: 'index-maintenance-v2', extraEnv: { KB_EMBEDDING_FAKE: '1' } });
-  try {
-    for (let attempt = 0; attempt < 100; attempt += 1) {
-      try { await json('GET', '/api/health'); break; } catch {
-        if (attempt === 99) throw new Error('server did not start');
-        await new Promise(resolve => setTimeout(resolve, 100));
-      }
-    }
-    await json('PATCH', '/api/settings', { knowledge: { rootPath: path.join(TEMP, 'knowledge') } });
-    const imported = await json('POST', '/api/projects/import', { projectId: 'maintenance-demo', localPath: REPO, displayName: 'Maintenance demo' });
-    const kbPath = imported.config.knowledgePath;
-    fs.mkdirSync(path.join(kbPath, 'modules'), { recursive: true });
-    fs.writeFileSync(path.join(kbPath, 'GOAL.md'), '# Goal\n\nBuild reliable derived indexes.\n', 'utf8');
-    fs.writeFileSync(path.join(kbPath, 'modules', 'core.md'), '# Core\n\nSingle writer atomic rebuild content.\n', 'utf8');
-    const markdownBefore = fs.readFileSync(path.join(kbPath, 'modules', 'core.md'), 'utf8');
-
-    const before = await json('GET', '/api/knowledge/maintenance');
-    assert.strictEqual(before.indexPath, path.join(DATA_DIR, 'index', 'knowledge.lancedb'));
-    const rebuilt = await json('POST', '/api/knowledge/maintenance/rebuild', {});
-    assert(rebuilt.ok && rebuilt.validation.ok, JSON.stringify(rebuilt));
-    assert(fs.existsSync(before.indexPath));
-    assert.strictEqual(fs.readFileSync(path.join(kbPath, 'modules', 'core.md'), 'utf8'), markdownBefore, 'index maintenance must not rewrite authoritative Markdown');
-    let search = await json('POST', '/api/knowledge/search', { projectId: 'maintenance-demo', query: 'atomic rebuild', limit: 5 });
-    assert.strictEqual(search.source, 'knowledge-retrieval-service');
-    assert.strictEqual(search.backend, 'hybrid+markdown-truth');
-    assert(search.results.some(item => item.entry_id === 'modules/core.md'));
-
-    fs.writeFileSync(path.join(kbPath, 'modules', 'core.md'), `${markdownBefore}\nSecond rebuild evidence.\n`, 'utf8');
-    const second = await json('POST', '/api/knowledge/maintenance/rebuild', {});
-    assert(second.ok && second.backup && fs.existsSync(second.backup), 'replacing a live index should retain the previous index under recovery');
-    search = await json('POST', '/api/knowledge/search', { projectId: 'maintenance-demo', query: 'Second rebuild evidence', limit: 5 });
-    assert(search.results.some(item => /Second rebuild evidence/.test(item.chunk_text)));
-    const after = await json('GET', '/api/knowledge/maintenance');
-    assert(after.projects.every(project => project.index.dirty === false));
-    console.log('markdown-maintenance-api-test: PASS');
-  } finally {
-    spawned.child.kill();
-  }
-})().catch(error => { console.error(error); process.exitCode = 1; });
+const assert = require('assert');
+const fs = require('fs');
+const os = require('os');
+const path = require('path');
+const { execFileSync } = require('child_process');
+const { spawnServer } = require('./helpers/spawn-server');
+
+const ROOT = path.resolve(__dirname, '..', '..');
+const PORT = 7932;
+const BASE_URL = `http://127.0.0.1:${PORT}`;
+const TEMP = fs.mkdtempSync(path.join(os.tmpdir(), 'index-maintenance-api-v2-'));
+const DATA_DIR = path.join(TEMP, 'data');
+const REPO = path.join(TEMP, 'repo');
+
+function git(args) { return execFileSync('git', args, { cwd: REPO, encoding: 'utf8' }).trim(); }
+
+async function json(method, route, body) {
+  const response = await fetch(`${BASE_URL}${route}`, {
+    method,
+    headers: body ? { 'Content-Type': 'application/json' } : undefined,
+    body: body ? JSON.stringify(body) : undefined,
+  });
+  const text = await response.text();
+  const data = text ? JSON.parse(text) : {};
+  if (!response.ok) throw new Error(`${response.status}: ${JSON.stringify(data)}`);
+  return data;
+}
+
+(async () => {
+  fs.mkdirSync(REPO, { recursive: true });
+  git(['init', '--initial-branch=main']);
+  git(['config', 'user.name', 'Index Test']);
+  git(['config', 'user.email', 'index@example.test']);
+  fs.writeFileSync(path.join(REPO, 'README.md'), '# source\n', 'utf8');
+  git(['add', '.']);
+  git(['commit', '-m', 'baseline']);
+  const spawned = spawnServer({ root: ROOT, port: PORT, dataDir: DATA_DIR, tag: 'index-maintenance-v2', extraEnv: { KB_EMBEDDING_FAKE: '1' } });
+  try {
+    for (let attempt = 0; attempt < 100; attempt += 1) {
+      try { await json('GET', '/api/health'); break; } catch {
+        if (attempt === 99) throw new Error('server did not start');
+        await new Promise(resolve => setTimeout(resolve, 100));
+      }
+    }
+    await json('PATCH', '/api/settings', { knowledge: { rootPath: path.join(TEMP, 'knowledge') } });
+    const imported = await json('POST', '/api/projects/import', { projectId: 'maintenance-demo', localPath: REPO, displayName: 'Maintenance demo' });
+    const kbPath = imported.config.knowledgePath;
+    fs.mkdirSync(path.join(kbPath, 'modules'), { recursive: true });
+    fs.writeFileSync(path.join(kbPath, 'GOAL.md'), '# Goal\n\nBuild reliable derived indexes.\n', 'utf8');
+    fs.writeFileSync(path.join(kbPath, 'modules', 'core.md'), '# Core\n\nSingle writer atomic rebuild content.\n', 'utf8');
+    const markdownBefore = fs.readFileSync(path.join(kbPath, 'modules', 'core.md'), 'utf8');
+
+    const before = await json('GET', '/api/knowledge/maintenance');
+    assert.strictEqual(before.indexPath, path.join(DATA_DIR, 'index', 'knowledge.lancedb'));
+    const rebuilt = await json('POST', '/api/knowledge/maintenance/rebuild', {});
+    assert(rebuilt.ok && rebuilt.validation.ok, JSON.stringify(rebuilt));
+    assert(fs.existsSync(before.indexPath));
+    assert.strictEqual(fs.readFileSync(path.join(kbPath, 'modules', 'core.md'), 'utf8'), markdownBefore, 'index maintenance must not rewrite authoritative Markdown');
+    let search = await json('POST', '/api/knowledge/search', { projectId: 'maintenance-demo', query: 'atomic rebuild', limit: 5 });
+    assert.strictEqual(search.source, 'knowledge-retrieval-service');
+    assert.strictEqual(search.backend, 'hybrid+markdown-truth');
+    assert(search.results.some(item => item.entry_id === 'modules/core.md'));
+
+    fs.writeFileSync(path.join(kbPath, 'modules', 'core.md'), `${markdownBefore}\nSecond rebuild evidence.\n`, 'utf8');
+    const second = await json('POST', '/api/knowledge/maintenance/rebuild', {});
+    assert(second.ok && second.backup && fs.existsSync(second.backup), 'replacing a live index should retain the previous index under recovery');
+    search = await json('POST', '/api/knowledge/search', { projectId: 'maintenance-demo', query: 'Second rebuild evidence', limit: 5 });
+    assert(search.results.some(item => /Second rebuild evidence/.test(item.chunk_text)));
+    const after = await json('GET', '/api/knowledge/maintenance');
+    assert(after.projects.every(project => project.index.dirty === false));
+    console.log('markdown-maintenance-api-test: PASS');
+  } finally {
+    spawned.child.kill();
+  }
+})().catch(error => { console.error(error); process.exitCode = 1; });