@kaelio/ktx 0.1.1 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (146) hide show
  1. package/assets/python/{kaelio_ktx-0.1.1-py3-none-any.whl → kaelio_ktx-0.3.0-py3-none-any.whl} +0 -0
  2. package/assets/python/manifest.json +4 -4
  3. package/dist/admin-reindex.d.ts +15 -0
  4. package/dist/admin-reindex.js +161 -0
  5. package/dist/admin-reindex.test.js +116 -0
  6. package/dist/{dev.d.ts → admin.d.ts} +1 -1
  7. package/dist/{dev.js → admin.js} +14 -12
  8. package/dist/{dev.test.js → admin.test.js} +36 -31
  9. package/dist/cli-program.js +7 -7
  10. package/dist/cli-program.test.js +1 -3
  11. package/dist/cli-project.d.ts +18 -0
  12. package/dist/cli-project.js +52 -0
  13. package/dist/cli-project.test.d.ts +1 -0
  14. package/dist/cli-project.test.js +149 -0
  15. package/dist/cli-runtime.d.ts +2 -2
  16. package/dist/cli-runtime.js +2 -8
  17. package/dist/commands/connection-commands.js +11 -10
  18. package/dist/commands/connection-selection.d.ts +11 -0
  19. package/dist/commands/connection-selection.js +9 -0
  20. package/dist/commands/ingest-commands.js +32 -26
  21. package/dist/commands/knowledge-commands.js +17 -28
  22. package/dist/commands/mcp-commands.js +17 -11
  23. package/dist/commands/runtime-commands.js +2 -2
  24. package/dist/commands/sl-commands.js +27 -32
  25. package/dist/context-build-view.js +1 -1
  26. package/dist/doctor.test.js +4 -4
  27. package/dist/example-smoke.test.js +3 -3
  28. package/dist/index.test.js +97 -85
  29. package/dist/ingest.js +9 -2
  30. package/dist/ingest.test.js +27 -3
  31. package/dist/io/print-list.test.js +4 -4
  32. package/dist/knowledge.js +1 -1
  33. package/dist/managed-local-embeddings.d.ts +0 -2
  34. package/dist/managed-local-embeddings.js +2 -5
  35. package/dist/managed-local-embeddings.test.js +5 -8
  36. package/dist/managed-python-command.js +2 -2
  37. package/dist/managed-python-command.test.js +3 -3
  38. package/dist/managed-python-daemon.js +2 -2
  39. package/dist/managed-python-daemon.test.js +1 -1
  40. package/dist/managed-python-http.js +3 -3
  41. package/dist/managed-python-http.test.js +6 -6
  42. package/dist/managed-python-runtime.d.ts +1 -1
  43. package/dist/managed-python-runtime.js +3 -3
  44. package/dist/managed-python-runtime.test.js +2 -2
  45. package/dist/memory-flow-tui.test.js +2 -2
  46. package/dist/next-steps.d.ts +6 -6
  47. package/dist/next-steps.js +4 -4
  48. package/dist/next-steps.test.js +5 -5
  49. package/dist/print-command-tree.js +0 -2
  50. package/dist/print-command-tree.test.js +1 -1
  51. package/dist/public-ingest.d.ts +4 -2
  52. package/dist/public-ingest.js +12 -8
  53. package/dist/public-ingest.test.js +7 -3
  54. package/dist/release-version.d.ts +1 -5
  55. package/dist/release-version.js +2 -39
  56. package/dist/runtime-requirements.js +1 -1
  57. package/dist/runtime.js +6 -6
  58. package/dist/runtime.test.js +8 -8
  59. package/dist/scan.js +7 -2
  60. package/dist/scan.test.js +3 -3
  61. package/dist/setup-agents.js +3 -3
  62. package/dist/setup-agents.test.js +1 -1
  63. package/dist/setup-embeddings.js +2 -2
  64. package/dist/setup-embeddings.test.js +5 -5
  65. package/dist/setup-runtime.test.js +3 -3
  66. package/dist/sl.js +1 -1
  67. package/dist/standalone-smoke.test.js +6 -2
  68. package/node_modules/@ktx/context/dist/core/git.service.d.ts +1 -0
  69. package/node_modules/@ktx/context/dist/core/git.service.js +12 -0
  70. package/node_modules/@ktx/context/dist/index-sync/index.d.ts +2 -0
  71. package/node_modules/@ktx/context/dist/index-sync/index.js +1 -0
  72. package/node_modules/@ktx/context/dist/index-sync/reindex.d.ts +20 -0
  73. package/node_modules/@ktx/context/dist/index-sync/reindex.js +141 -0
  74. package/node_modules/@ktx/context/dist/index-sync/reindex.test.d.ts +1 -0
  75. package/node_modules/@ktx/context/dist/index-sync/reindex.test.js +139 -0
  76. package/node_modules/@ktx/context/dist/index-sync/types.d.ts +29 -0
  77. package/node_modules/@ktx/context/dist/index-sync/types.js +1 -0
  78. package/node_modules/@ktx/context/dist/index.d.ts +1 -0
  79. package/node_modules/@ktx/context/dist/index.js +1 -0
  80. package/node_modules/@ktx/context/dist/ingest/adapters/historic-sql/historic-sql.adapter.d.ts +2 -1
  81. package/node_modules/@ktx/context/dist/ingest/adapters/historic-sql/historic-sql.adapter.js +18 -0
  82. package/node_modules/@ktx/context/dist/ingest/adapters/historic-sql/local-ingest-acceptance.test.js +6 -6
  83. package/node_modules/@ktx/context/dist/ingest/adapters/historic-sql/projection.d.ts +5 -0
  84. package/node_modules/@ktx/context/dist/ingest/adapters/historic-sql/projection.js +48 -0
  85. package/node_modules/@ktx/context/dist/ingest/adapters/historic-sql/projection.test.js +83 -0
  86. package/node_modules/@ktx/context/dist/ingest/adapters/live-database/daemon-introspection.js +4 -1
  87. package/node_modules/@ktx/context/dist/ingest/adapters/live-database/daemon-introspection.test.js +32 -0
  88. package/node_modules/@ktx/context/dist/ingest/finalization-scope.d.ts +22 -0
  89. package/node_modules/@ktx/context/dist/ingest/finalization-scope.js +95 -0
  90. package/node_modules/@ktx/context/dist/ingest/finalization-scope.test.d.ts +1 -0
  91. package/node_modules/@ktx/context/dist/ingest/finalization-scope.test.js +114 -0
  92. package/node_modules/@ktx/context/dist/ingest/index.d.ts +1 -2
  93. package/node_modules/@ktx/context/dist/ingest/index.js +0 -1
  94. package/node_modules/@ktx/context/dist/ingest/ingest-bundle.runner.d.ts +2 -0
  95. package/node_modules/@ktx/context/dist/ingest/ingest-bundle.runner.isolated-diff.test.js +166 -0
  96. package/node_modules/@ktx/context/dist/ingest/ingest-bundle.runner.js +235 -45
  97. package/node_modules/@ktx/context/dist/ingest/ingest-bundle.runner.test.js +193 -38
  98. package/node_modules/@ktx/context/dist/ingest/local-bundle-ingest.test.js +22 -3
  99. package/node_modules/@ktx/context/dist/ingest/local-bundle-runtime.js +3 -4
  100. package/node_modules/@ktx/context/dist/ingest/local-ingest.js +0 -7
  101. package/node_modules/@ktx/context/dist/ingest/local-stage-ingest.js +15 -5
  102. package/node_modules/@ktx/context/dist/ingest/local-stage-ingest.test.js +29 -0
  103. package/node_modules/@ktx/context/dist/ingest/memory-flow/schema.d.ts +4 -4
  104. package/node_modules/@ktx/context/dist/ingest/memory-flow/schema.js +1 -1
  105. package/node_modules/@ktx/context/dist/ingest/memory-flow/types.d.ts +1 -1
  106. package/node_modules/@ktx/context/dist/ingest/ports.d.ts +1 -20
  107. package/node_modules/@ktx/context/dist/ingest/report-snapshot.d.ts +73 -2
  108. package/node_modules/@ktx/context/dist/ingest/report-snapshot.js +27 -0
  109. package/node_modules/@ktx/context/dist/ingest/reports.d.ts +23 -5
  110. package/node_modules/@ktx/context/dist/ingest/reports.js +7 -24
  111. package/node_modules/@ktx/context/dist/ingest/types.d.ts +33 -0
  112. package/node_modules/@ktx/context/dist/llm/index.d.ts +1 -1
  113. package/node_modules/@ktx/context/dist/llm/index.js +1 -1
  114. package/node_modules/@ktx/context/dist/llm/local-config.d.ts +0 -1
  115. package/node_modules/@ktx/context/dist/llm/local-config.js +2 -12
  116. package/node_modules/@ktx/context/dist/llm/local-config.test.js +2 -23
  117. package/node_modules/@ktx/context/dist/memory/local-memory.js +9 -3
  118. package/node_modules/@ktx/context/dist/package-exports.test.js +2 -2
  119. package/node_modules/@ktx/context/dist/project/config.d.ts +16 -0
  120. package/node_modules/@ktx/context/dist/project/driver-schemas.d.ts +8 -0
  121. package/node_modules/@ktx/context/dist/project/driver-schemas.js +4 -0
  122. package/node_modules/@ktx/context/dist/scan/enabled-tables.d.ts +3 -0
  123. package/node_modules/@ktx/context/dist/scan/enabled-tables.js +15 -0
  124. package/node_modules/@ktx/context/dist/scan/local-scan.d.ts +2 -4
  125. package/node_modules/@ktx/context/dist/scan/local-scan.js +2 -15
  126. package/node_modules/@ktx/context/dist/sl/ports.d.ts +3 -3
  127. package/node_modules/@ktx/context/dist/sl/sl-search.service.d.ts +3 -2
  128. package/node_modules/@ktx/context/dist/sl/sl-search.service.js +47 -45
  129. package/node_modules/@ktx/context/dist/sl/sl-search.service.test.js +61 -0
  130. package/node_modules/@ktx/context/dist/sl/sqlite-sl-sources-index.d.ts +4 -3
  131. package/node_modules/@ktx/context/dist/sl/sqlite-sl-sources-index.js +15 -5
  132. package/node_modules/@ktx/context/dist/sl/sqlite-sl-sources-index.test.js +24 -0
  133. package/node_modules/@ktx/context/dist/wiki/knowledge-wiki.service.d.ts +3 -2
  134. package/node_modules/@ktx/context/dist/wiki/knowledge-wiki.service.js +62 -51
  135. package/node_modules/@ktx/context/dist/wiki/knowledge-wiki.service.test.js +59 -3
  136. package/node_modules/@ktx/context/dist/wiki/ports.d.ts +3 -3
  137. package/node_modules/@ktx/context/dist/wiki/sqlite-knowledge-index.d.ts +33 -0
  138. package/node_modules/@ktx/context/dist/wiki/sqlite-knowledge-index.js +155 -2
  139. package/node_modules/@ktx/context/dist/wiki/sqlite-knowledge-index.test.js +26 -0
  140. package/node_modules/@ktx/context/package.json +5 -0
  141. package/package.json +1 -1
  142. package/node_modules/@ktx/context/dist/ingest/adapters/historic-sql/post-processor.d.ts +0 -4
  143. package/node_modules/@ktx/context/dist/ingest/adapters/historic-sql/post-processor.js +0 -38
  144. package/node_modules/@ktx/context/dist/ingest/adapters/historic-sql/post-processor.test.js +0 -63
  145. /package/dist/{dev.test.d.ts → admin-reindex.test.d.ts} +0 -0
  146. /package/{node_modules/@ktx/context/dist/ingest/adapters/historic-sql/post-processor.test.d.ts → dist/admin.test.d.ts} +0 -0
package/dist/scan.js CHANGED
@@ -1,5 +1,5 @@
1
- import { loadKtxProject } from '@ktx/context/project';
2
1
  import { runLocalScan, } from '@ktx/context/scan';
2
+ import { loadKtxCliProject } from './cli-project.js';
3
3
  import { createKtxCliLocalIngestAdapters } from './local-adapters.js';
4
4
  import { createKtxCliScanConnector } from './local-scan-connectors.js';
5
5
  import { profileMark } from './startup-profile.js';
@@ -237,7 +237,12 @@ export function createCliScanProgress(io, state = { progress: 0, hasPendingTrans
237
237
  }
238
238
  export async function runKtxScan(args, io = process, deps = {}) {
239
239
  try {
240
- const project = await loadKtxProject({ projectDir: args.projectDir });
240
+ const project = await loadKtxCliProject({
241
+ projectDir: args.projectDir,
242
+ cliVersion: args.cliVersion ?? '0.0.0-private',
243
+ installPolicy: args.runtimeInstallPolicy ?? 'never',
244
+ io,
245
+ });
241
246
  const managedDaemon = managedDaemonOptionsForScanRun(args, deps.runtimeIo ?? io);
242
247
  const connector = args.mode !== 'structural' || args.detectRelationships
243
248
  ? await createKtxCliScanConnector(project, args.connectionId)
package/dist/scan.test.js CHANGED
@@ -287,14 +287,14 @@ describe('runKtxScan', () => {
287
287
  expect(io.stdout()).toContain('Report: raw-sources/warehouse/live-database/sync-1/scan-report.json');
288
288
  expect(io.stdout()).toContain('Next:\n');
289
289
  expect(io.stdout()).toContain('ktx status --project-dir ');
290
- expect(io.stdout()).not.toContain('ktx dev scan status');
291
- expect(io.stdout()).not.toContain('ktx dev scan report');
290
+ expect(io.stdout()).not.toContain('ktx admin scan status');
291
+ expect(io.stdout()).not.toContain('ktx admin scan report');
292
292
  expect(io.stdout()).not.toContain('\u001b[');
293
293
  expect(io.stdout()).not.toContain('✓');
294
294
  expect(io.stdout()).not.toContain('+1');
295
295
  expect(io.stdout()).not.toContain('/~');
296
296
  });
297
- it('passes managed daemon options to local ingest adapters when no explicit daemon URL is set', async () => {
297
+ it('passes KTX daemon options to local ingest adapters when no explicit daemon URL is set', async () => {
298
298
  await initKtxProject({ projectDir: tempDir });
299
299
  const createLocalIngestAdapters = vi.fn(() => []);
300
300
  const runLocalScan = vi.fn(async (_input) => ({
@@ -426,8 +426,8 @@ function cliInstructionContent(input) {
426
426
  'Available commands:',
427
427
  '',
428
428
  `- \`${ktxCommandLine(input.launcher, ['status', ...jsonProjectDirArgs])}\``,
429
- `- \`${ktxCommandLine(input.launcher, ['sl', 'list', ...jsonProjectDirArgs])}\``,
430
- `- \`${ktxCommandLine(input.launcher, ['sl', 'search', '<text>', ...jsonProjectDirArgs, '--connection-id', '<id>'])}\``,
429
+ `- \`${ktxCommandLine(input.launcher, ['sl', ...jsonProjectDirArgs])}\``,
430
+ `- \`${ktxCommandLine(input.launcher, ['sl', '<text>', ...jsonProjectDirArgs, '--connection-id', '<id>'])}\``,
431
431
  `- \`${ktxCommandLine(input.launcher, [
432
432
  'sl',
433
433
  'query',
@@ -442,7 +442,7 @@ function cliInstructionContent(input) {
442
442
  '--max-rows',
443
443
  '100',
444
444
  ])}\``,
445
- `- \`${ktxCommandLine(input.launcher, ['wiki', 'search', '<query>', ...jsonProjectDirArgs, '--limit', '10'])}\``,
445
+ `- \`${ktxCommandLine(input.launcher, ['wiki', '<query>', ...jsonProjectDirArgs, '--limit', '10'])}\``,
446
446
  '',
447
447
  'Use semantic-layer queries before direct database access. Do not print secrets or credential references.',
448
448
  '',
@@ -149,7 +149,7 @@ describe('setup agents', () => {
149
149
  expect(skill).toContain(`--project-dir ${tempDir}`);
150
150
  expect(skill).toContain('must not print secrets');
151
151
  expect(skill).toContain('status --json');
152
- expect(skill).toContain('sl list --json');
152
+ expect(skill).toContain('sl --json');
153
153
  expect(skill).toContain('sl query');
154
154
  expect(skill).toContain('--format json');
155
155
  expect(skill).not.toContain('sl query --json');
@@ -209,12 +209,12 @@ function localEmbeddingSetupMessage(message, stderrTail = []) {
209
209
  const lines = [
210
210
  `Local embedding health check failed: ${message}`,
211
211
  'Local embeddings use the KTX-managed Python runtime.',
212
- 'Prepare the runtime with: ktx dev runtime start --feature local-embeddings',
212
+ 'Prepare the runtime with: ktx admin runtime start --feature local-embeddings',
213
213
  'Use --yes with setup to install and start the runtime without prompting.',
214
214
  'The first run may download Python packages and the all-MiniLM-L6-v2 model.',
215
215
  ];
216
216
  if (stderrTail.length > 0) {
217
- lines.push('Recent local embeddings daemon stderr:', ...stderrTail);
217
+ lines.push('Recent KTX daemon stderr:', ...stderrTail);
218
218
  }
219
219
  return lines.join('\n');
220
220
  }
@@ -225,7 +225,7 @@ describe('setup embeddings step', () => {
225
225
  it('fails non-interactive local setup when the managed local embeddings runtime is missing', async () => {
226
226
  const io = makeIo();
227
227
  const ensureLocalEmbeddings = vi.fn(async () => {
228
- throw new Error('KTX Python runtime is required for this command. Run: ktx dev runtime install --feature local-embeddings --yes');
228
+ throw new Error('KTX Python runtime is required for this command. Run: ktx admin runtime install --feature local-embeddings --yes');
229
229
  });
230
230
  const result = await runKtxSetupEmbeddingsStep({
231
231
  projectDir: tempDir,
@@ -235,7 +235,7 @@ describe('setup embeddings step', () => {
235
235
  skipEmbeddings: false,
236
236
  }, io.io, { env: {}, ensureLocalEmbeddings });
237
237
  expect(result.status).toBe('failed');
238
- expect(io.stderr()).toContain('KTX Python runtime is required for this command. Run: ktx dev runtime install --feature local-embeddings --yes');
238
+ expect(io.stderr()).toContain('KTX Python runtime is required for this command. Run: ktx admin runtime install --feature local-embeddings --yes');
239
239
  });
240
240
  it('does not persist embedding completion when the health check fails', async () => {
241
241
  const io = makeIo();
@@ -255,7 +255,7 @@ describe('setup embeddings step', () => {
255
255
  expect(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8')).not.toContain('completed_steps:');
256
256
  expect(config.ingest.embeddings.backend).toBe('none');
257
257
  expect(io.stderr()).toContain('Local embedding health check failed: 401 invalid api key [redacted]');
258
- expect(io.stderr()).toContain('Prepare the runtime with: ktx dev runtime start --feature local-embeddings');
258
+ expect(io.stderr()).toContain('Prepare the runtime with: ktx admin runtime start --feature local-embeddings');
259
259
  expect(io.stderr()).not.toContain('skip for now');
260
260
  });
261
261
  it('prints the recent daemon stderr tail when local embedding health check fails', async () => {
@@ -275,7 +275,7 @@ describe('setup embeddings step', () => {
275
275
  healthCheck: vi.fn(async () => ({ ok: false, message: 'HTTP 500' })),
276
276
  });
277
277
  expect(result.status).toBe('failed');
278
- expect(io.stderr()).toContain('Recent local embeddings daemon stderr:');
278
+ expect(io.stderr()).toContain('Recent KTX daemon stderr:');
279
279
  expect(io.stderr()).toContain('daemon traceback line 6');
280
280
  expect(io.stderr()).toContain('daemon traceback line 45');
281
281
  expect(io.stderr()).not.toContain('daemon traceback line 5');
@@ -296,7 +296,7 @@ describe('setup embeddings step', () => {
296
296
  healthCheck: vi.fn(async () => ({ ok: false, message: 'HTTP 500' })),
297
297
  });
298
298
  expect(result.status).toBe('failed');
299
- expect(io.stderr()).not.toContain('Recent local embeddings daemon stderr:');
299
+ expect(io.stderr()).not.toContain('Recent KTX daemon stderr:');
300
300
  });
301
301
  it('uses fixed OpenAI defaults and only asks for credentials when OpenAI is selected', async () => {
302
302
  const io = makeIo();
@@ -56,7 +56,7 @@ describe('runKtxSetupRuntimeStep', () => {
56
56
  it('fails fast when required runtime features cannot be installed in no-input mode', async () => {
57
57
  const io = makeIo();
58
58
  const ensureRuntime = vi.fn(async () => {
59
- throw new Error('KTX Python runtime is required for this command. Run: ktx dev runtime install --yes');
59
+ throw new Error('KTX Python runtime is required for this command. Run: ktx admin runtime install --yes');
60
60
  });
61
61
  await expect(runKtxSetupRuntimeStep({
62
62
  projectDir: tempDir,
@@ -71,9 +71,9 @@ describe('runKtxSetupRuntimeStep', () => {
71
71
  })).resolves.toMatchObject({ status: 'failed' });
72
72
  expect(ensureRuntime).toHaveBeenCalledWith(expect.objectContaining({ installPolicy: 'never' }));
73
73
  expect((await readKtxSetupState(tempDir)).completed_steps).not.toContain('runtime');
74
- expect(io.stderr()).toContain('ktx dev runtime install --yes');
74
+ expect(io.stderr()).toContain('ktx admin runtime install --yes');
75
75
  });
76
- it('starts the managed local embeddings daemon for configured sentence-transformers embeddings', async () => {
76
+ it('starts the KTX daemon for configured sentence-transformers embeddings', async () => {
77
77
  const io = makeIo();
78
78
  const ensureLocalEmbeddings = vi.fn(async () => ({
79
79
  baseUrl: 'http://127.0.0.1:61234',
package/dist/sl.js CHANGED
@@ -99,7 +99,7 @@ export async function runKtxSl(args, io = process, deps = {}) {
99
99
  await printSlSources({
100
100
  rows: sources,
101
101
  emptyMessage: `No semantic-layer sources matched "${args.query}" in ${project.projectDir}`,
102
- emptyHint: 'Run `ktx sl list` to inspect available sources.',
102
+ emptyHint: 'Run `ktx sl` to inspect available sources.',
103
103
  command: 'sl search',
104
104
  output: args.output,
105
105
  json: args.json,
@@ -116,6 +116,10 @@ describe('standalone built ktx CLI smoke', () => {
116
116
  const init = await runSetupNewProject(projectDir);
117
117
  expectSetupStderr(init);
118
118
  expect(init.stdout).toContain(`Project: ${projectDir}`);
119
+ const reindex = await runBuiltCli(['--project-dir', projectDir, 'admin', 'reindex', '--output', 'plain']);
120
+ expect(reindex.code).toBe(0);
121
+ expect(reindex.stdout).toContain('reindex\t');
122
+ expect(reindex.stderr).toContain('wiki/global');
119
123
  const run = await runBuiltCli([
120
124
  'ingest',
121
125
  'run',
@@ -125,7 +129,7 @@ describe('standalone built ktx CLI smoke', () => {
125
129
  'fake',
126
130
  ]);
127
131
  expect(run).toMatchObject({ code: 1, stdout: '' });
128
- expect(run.stderr).toContain("unknown option '--connection-id'");
132
+ expect(run.stderr).toContain("unknown option '--adapter'");
129
133
  });
130
134
  it('rejects the removed agent command through the built binary', async () => {
131
135
  const result = await runBuiltCli(['agent']);
@@ -235,7 +239,7 @@ describe('standalone built ktx CLI smoke', () => {
235
239
  ]);
236
240
  expect(add.code).toBe(1);
237
241
  expect(add.stdout).toBe('');
238
- expect(add.stderr).toContain("unknown command 'add'");
242
+ expect(add.stderr).toMatch(/unknown (command|option)|too many arguments/);
239
243
  const yaml = await readFile(join(projectDir, 'ktx.yaml'), 'utf-8');
240
244
  expect(yaml).not.toContain('driver: notion');
241
245
  expect(yaml).not.toContain('auth_token_ref: env:NOTION_TOKEN');
@@ -101,6 +101,7 @@ export declare class GitService {
101
101
  status: 'A' | 'M' | 'D';
102
102
  path: string;
103
103
  }>>;
104
+ changedPaths(): Promise<string[]>;
104
105
  /**
105
106
  * List all paths under the working tree that match `pathSpec`, scoped to HEAD.
106
107
  * Used for the reconciler's first-ever run when there's no watermark to diff from.
@@ -424,6 +424,18 @@ export class GitService {
424
424
  }
425
425
  return out;
426
426
  }
427
+ async changedPaths() {
428
+ const raw = await this.git.raw(['status', '--porcelain=v1', '-z']);
429
+ const fields = raw.split('\0').filter(Boolean);
430
+ const paths = [];
431
+ for (const field of fields) {
432
+ const path = field.slice(3);
433
+ if (path.length > 0) {
434
+ paths.push(path);
435
+ }
436
+ }
437
+ return [...new Set(paths)].sort();
438
+ }
427
439
  /**
428
440
  * List all paths under the working tree that match `pathSpec`, scoped to HEAD.
429
441
  * Used for the reconciler's first-ever run when there's no watermark to diff from.
@@ -0,0 +1,2 @@
1
+ export type { ReindexOptions, ReindexScopeResult, ReindexSummary, ReindexWorkResult } from './types.js';
2
+ export { discoverReindexScopes, reindexLocalIndexes } from './reindex.js';
@@ -0,0 +1 @@
1
+ export { discoverReindexScopes, reindexLocalIndexes } from './reindex.js';
@@ -0,0 +1,20 @@
1
+ import { type KtxLocalProject } from '../project/index.js';
2
+ import type { ReindexOptions, ReindexSummary } from './types.js';
3
+ type DiscoveredScope = {
4
+ kind: 'wiki';
5
+ scope: 'GLOBAL';
6
+ scopeId: null;
7
+ label: 'global';
8
+ } | {
9
+ kind: 'wiki';
10
+ scope: 'USER';
11
+ scopeId: string;
12
+ label: `user/${string}`;
13
+ } | {
14
+ kind: 'sl';
15
+ connectionId: string;
16
+ label: string;
17
+ };
18
+ export declare function discoverReindexScopes(project: KtxLocalProject): Promise<DiscoveredScope[]>;
19
+ export declare function reindexLocalIndexes(project: KtxLocalProject, options: ReindexOptions): Promise<ReindexSummary>;
20
+ export {};
@@ -0,0 +1,141 @@
1
+ import { readdir, stat } from 'node:fs/promises';
2
+ import { join, relative } from 'node:path';
3
+ import { ktxLocalStateDbPath } from '../project/index.js';
4
+ import { loadLocalSlSourceRecords, SlSearchService, SqliteSlSourcesIndex } from '../sl/index.js';
5
+ import { KnowledgeWikiService, SqliteKnowledgeIndex } from '../wiki/index.js';
6
+ const ZERO = {
7
+ scanned: 0,
8
+ updated: 0,
9
+ deleted: 0,
10
+ embeddingsRecomputed: 0,
11
+ embeddingsFailed: 0,
12
+ };
13
+ async function directoryExists(path) {
14
+ try {
15
+ return (await stat(path)).isDirectory();
16
+ }
17
+ catch {
18
+ return false;
19
+ }
20
+ }
21
+ async function childDirectories(path) {
22
+ try {
23
+ const entries = await readdir(path, { withFileTypes: true });
24
+ return entries
25
+ .filter((entry) => entry.isDirectory())
26
+ .map((entry) => entry.name)
27
+ .sort((left, right) => left.localeCompare(right));
28
+ }
29
+ catch (error) {
30
+ if (error.code === 'ENOENT') {
31
+ return [];
32
+ }
33
+ throw error;
34
+ }
35
+ }
36
+ export async function discoverReindexScopes(project) {
37
+ const scopes = [];
38
+ if (await directoryExists(join(project.projectDir, 'wiki/global'))) {
39
+ scopes.push({ kind: 'wiki', scope: 'GLOBAL', scopeId: null, label: 'global' });
40
+ }
41
+ for (const userId of await childDirectories(join(project.projectDir, 'wiki/user'))) {
42
+ scopes.push({ kind: 'wiki', scope: 'USER', scopeId: userId, label: `user/${userId}` });
43
+ }
44
+ for (const connectionId of await childDirectories(join(project.projectDir, 'semantic-layer'))) {
45
+ if (connectionId !== '_schema') {
46
+ scopes.push({ kind: 'sl', connectionId, label: connectionId });
47
+ }
48
+ }
49
+ return scopes;
50
+ }
51
+ function errorMessage(error) {
52
+ if (!(error instanceof Error)) {
53
+ return String(error);
54
+ }
55
+ return error.name && error.name !== 'Error' ? `${error.name}: ${error.message}` : error.message;
56
+ }
57
+ function addTotals(left, right) {
58
+ return {
59
+ scanned: left.scanned + right.scanned,
60
+ updated: left.updated + right.updated,
61
+ deleted: left.deleted + right.deleted,
62
+ embeddingsRecomputed: left.embeddingsRecomputed + right.embeddingsRecomputed,
63
+ embeddingsFailed: left.embeddingsFailed + right.embeddingsFailed,
64
+ };
65
+ }
66
+ function durationSince(startedAt) {
67
+ return Number((process.hrtime.bigint() - startedAt) / 1000000n);
68
+ }
69
+ function embeddingFailureError(work) {
70
+ if (work.embeddingsFailed === 0) {
71
+ return undefined;
72
+ }
73
+ return `${work.embeddingsFailed} embedding recomputation${work.embeddingsFailed === 1 ? '' : 's'} failed`;
74
+ }
75
+ export async function reindexLocalIndexes(project, options) {
76
+ const startedAt = process.hrtime.bigint();
77
+ const dbPath = ktxLocalStateDbPath(project);
78
+ const scopes = await discoverReindexScopes(project);
79
+ const wikiIndex = new SqliteKnowledgeIndex({ dbPath });
80
+ const slIndex = new SqliteSlSourcesIndex({ dbPath });
81
+ const wikiService = new KnowledgeWikiService(project.fileStore, options.embeddingService, wikiIndex, project.git);
82
+ const slService = new SlSearchService(options.embeddingService, slIndex);
83
+ const results = [];
84
+ for (const scope of scopes) {
85
+ const scopeStartedAt = process.hrtime.bigint();
86
+ try {
87
+ let work;
88
+ if (scope.kind === 'wiki') {
89
+ if (options.force) {
90
+ wikiIndex.clear(scope.scope, scope.scopeId);
91
+ }
92
+ work = await wikiService.syncIndex(scope.scope, scope.scopeId);
93
+ results.push({
94
+ kind: 'wiki',
95
+ label: scope.label,
96
+ scope: scope.scope === 'GLOBAL' ? 'global' : 'user',
97
+ scopeId: scope.scopeId,
98
+ ...work,
99
+ ...(options.force ? { deleted: 0 } : {}),
100
+ ...(options.embeddingService && work.embeddingsFailed > 0 ? { error: embeddingFailureError(work) } : {}),
101
+ durationMs: durationSince(scopeStartedAt),
102
+ });
103
+ continue;
104
+ }
105
+ if (options.force) {
106
+ await slIndex.clear(scope.connectionId);
107
+ }
108
+ const records = await loadLocalSlSourceRecords(project, { connectionId: scope.connectionId });
109
+ work = await slService.indexSources(scope.connectionId, records.map((record) => record.source));
110
+ results.push({
111
+ kind: 'sl',
112
+ label: scope.label,
113
+ connectionId: scope.connectionId,
114
+ ...work,
115
+ ...(options.force ? { deleted: 0 } : {}),
116
+ ...(options.embeddingService && work.embeddingsFailed > 0 ? { error: embeddingFailureError(work) } : {}),
117
+ durationMs: durationSince(scopeStartedAt),
118
+ });
119
+ }
120
+ catch (error) {
121
+ results.push({
122
+ kind: scope.kind,
123
+ label: scope.label,
124
+ ...(scope.kind === 'wiki'
125
+ ? { scope: scope.scope === 'GLOBAL' ? 'global' : 'user', scopeId: scope.scopeId }
126
+ : { connectionId: scope.connectionId }),
127
+ ...ZERO,
128
+ durationMs: durationSince(scopeStartedAt),
129
+ error: errorMessage(error),
130
+ });
131
+ }
132
+ }
133
+ return {
134
+ scopes: results,
135
+ totals: results.reduce(addTotals, ZERO),
136
+ dbPath: relative(project.projectDir, dbPath) || dbPath,
137
+ force: options.force,
138
+ embeddingsAvailable: options.embeddingService !== null,
139
+ durationMs: durationSince(startedAt),
140
+ };
141
+ }
@@ -0,0 +1,139 @@
1
+ import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises';
2
+ import { tmpdir } from 'node:os';
3
+ import { join } from 'node:path';
4
+ import { afterEach, beforeEach, describe, expect, it } from 'vitest';
5
+ import { initKtxProject, loadKtxProject } from '../project/index.js';
6
+ import { SqliteKnowledgeIndex } from '../wiki/sqlite-knowledge-index.js';
7
+ import { reindexLocalIndexes } from './reindex.js';
8
+ class FakeEmbeddingPort {
9
+ maxBatchSize = 8;
10
+ async computeEmbedding(text) {
11
+ return [text.length, 1];
12
+ }
13
+ async computeEmbeddingsBulk(texts) {
14
+ return texts.map((text) => [text.length, 1]);
15
+ }
16
+ }
17
+ async function createProject(tempDir) {
18
+ await initKtxProject({ projectDir: tempDir, force: true });
19
+ return loadKtxProject({ projectDir: tempDir });
20
+ }
21
+ describe('reindexLocalIndexes', () => {
22
+ let tempDir;
23
+ beforeEach(async () => {
24
+ tempDir = await mkdtemp(join(tmpdir(), 'ktx-reindex-'));
25
+ });
26
+ afterEach(async () => {
27
+ await rm(tempDir, { recursive: true, force: true });
28
+ });
29
+ it('returns an empty summary when no wiki or semantic-layer directories exist', async () => {
30
+ const project = await createProject(tempDir);
31
+ await rm(join(project.projectDir, 'wiki'), { recursive: true, force: true });
32
+ await rm(join(project.projectDir, 'semantic-layer'), { recursive: true, force: true });
33
+ await expect(reindexLocalIndexes(project, { force: false, embeddingService: null })).resolves.toMatchObject({
34
+ scopes: [],
35
+ totals: { scanned: 0, updated: 0, deleted: 0, embeddingsRecomputed: 0, embeddingsFailed: 0 },
36
+ force: false,
37
+ embeddingsAvailable: false,
38
+ });
39
+ });
40
+ it('discovers empty directories as zero-row scopes', async () => {
41
+ const project = await createProject(tempDir);
42
+ await mkdir(join(project.projectDir, 'wiki/user/local'), { recursive: true });
43
+ await mkdir(join(project.projectDir, 'semantic-layer/warehouse'), { recursive: true });
44
+ const summary = await reindexLocalIndexes(project, { force: false, embeddingService: null });
45
+ expect(summary.scopes.map((scope) => scope.label)).toEqual(['global', 'user/local', 'warehouse']);
46
+ expect(summary.totals.scanned).toBe(0);
47
+ });
48
+ it('indexes mixed wiki and SL sources and reports totals', async () => {
49
+ const project = await createProject(tempDir);
50
+ await writeFile(join(project.projectDir, 'wiki/global/revenue.md'), '---\nsummary: Revenue\nusage_mode: auto\n---\n\nPaid orders.\n', 'utf-8');
51
+ await mkdir(join(project.projectDir, 'semantic-layer/warehouse'), { recursive: true });
52
+ await writeFile(join(project.projectDir, 'semantic-layer/warehouse/orders.yaml'), 'name: orders\ntable: public.orders\ngrain: [id]\ncolumns:\n - name: id\n type: number\njoins: []\nmeasures: []\n', 'utf-8');
53
+ const summary = await reindexLocalIndexes(project, {
54
+ force: false,
55
+ embeddingService: new FakeEmbeddingPort(),
56
+ });
57
+ expect(summary.scopes).toHaveLength(2);
58
+ expect(summary.totals).toMatchObject({ scanned: 2, updated: 2, deleted: 0, embeddingsRecomputed: 2 });
59
+ expect(summary.embeddingsAvailable).toBe(true);
60
+ });
61
+ it('does not report unchanged lexical-only rows as updated on repeated runs', async () => {
62
+ const project = await createProject(tempDir);
63
+ await writeFile(join(project.projectDir, 'wiki/global/revenue.md'), '---\nsummary: Revenue\nusage_mode: auto\n---\n\nPaid orders.\n', 'utf-8');
64
+ await mkdir(join(project.projectDir, 'semantic-layer/warehouse'), { recursive: true });
65
+ await writeFile(join(project.projectDir, 'semantic-layer/warehouse/orders.yaml'), 'name: orders\ntable: public.orders\ngrain: [id]\ncolumns:\n - name: id\n type: number\njoins: []\nmeasures: []\n', 'utf-8');
66
+ const first = await reindexLocalIndexes(project, { force: false, embeddingService: null });
67
+ expect(first.totals).toMatchObject({
68
+ scanned: 2,
69
+ updated: 2,
70
+ deleted: 0,
71
+ embeddingsRecomputed: 0,
72
+ embeddingsFailed: 0,
73
+ });
74
+ const second = await reindexLocalIndexes(project, { force: false, embeddingService: null });
75
+ expect(second.totals).toMatchObject({
76
+ scanned: 2,
77
+ updated: 0,
78
+ deleted: 0,
79
+ embeddingsRecomputed: 0,
80
+ embeddingsFailed: 0,
81
+ });
82
+ expect(second.scopes.map((scope) => [scope.label, scope.updated])).toEqual([
83
+ ['global', 0],
84
+ ['warehouse', 0],
85
+ ]);
86
+ });
87
+ it('force clears stale rows before rebuilding each discovered scope', async () => {
88
+ const project = await createProject(tempDir);
89
+ const wikiIndex = new SqliteKnowledgeIndex({ dbPath: join(project.projectDir, '.ktx/db.sqlite') });
90
+ wikiIndex.sync([
91
+ {
92
+ path: 'wiki/global/stale.md',
93
+ key: 'stale',
94
+ scope: 'GLOBAL',
95
+ scopeId: null,
96
+ summary: 'Stale',
97
+ content: 'Stale content',
98
+ tags: [],
99
+ embedding: [1, 0],
100
+ },
101
+ ]);
102
+ await writeFile(join(project.projectDir, 'wiki/global/revenue.md'), '---\nsummary: Revenue\nusage_mode: auto\n---\n\nPaid orders.\n', 'utf-8');
103
+ const summary = await reindexLocalIndexes(project, {
104
+ force: true,
105
+ embeddingService: new FakeEmbeddingPort(),
106
+ });
107
+ expect(summary.force).toBe(true);
108
+ expect(summary.totals).toMatchObject({ scanned: 1, updated: 1, deleted: 0 });
109
+ expect(wikiIndex.search('Stale', 10)).toEqual([]);
110
+ });
111
+ it('captures a per-scope error and continues other scopes', async () => {
112
+ const project = await createProject(tempDir);
113
+ await writeFile(join(project.projectDir, 'wiki/global/revenue.md'), '---\nsummary: Revenue\nusage_mode: auto\n---\n\nPaid orders.\n', 'utf-8');
114
+ await mkdir(join(project.projectDir, 'semantic-layer/warehouse'), { recursive: true });
115
+ await writeFile(join(project.projectDir, 'semantic-layer/warehouse/broken.yaml'), 'not: [valid', 'utf-8');
116
+ const summary = await reindexLocalIndexes(project, { force: false, embeddingService: null });
117
+ expect(summary.scopes.find((scope) => scope.label === 'global')?.error).toBeUndefined();
118
+ expect(summary.scopes.find((scope) => scope.label === 'warehouse')?.error).toContain('YAML');
119
+ });
120
+ it('marks a scope errored when configured embeddings fail', async () => {
121
+ const project = await createProject(tempDir);
122
+ await writeFile(join(project.projectDir, 'wiki/global/revenue.md'), '---\nsummary: Revenue\nusage_mode: auto\n---\n\nPaid orders.\n', 'utf-8');
123
+ const embeddingService = {
124
+ maxBatchSize: 8,
125
+ async computeEmbedding() {
126
+ throw new Error('embedding provider unavailable');
127
+ },
128
+ async computeEmbeddingsBulk() {
129
+ throw new Error('embedding provider unavailable');
130
+ },
131
+ };
132
+ const summary = await reindexLocalIndexes(project, { force: false, embeddingService });
133
+ expect(summary.scopes[0]).toMatchObject({
134
+ label: 'global',
135
+ embeddingsFailed: 1,
136
+ error: '1 embedding recomputation failed',
137
+ });
138
+ });
139
+ });
@@ -0,0 +1,29 @@
1
+ import type { KtxEmbeddingPort } from '../core/index.js';
2
+ export interface ReindexOptions {
3
+ force: boolean;
4
+ embeddingService: KtxEmbeddingPort | null;
5
+ }
6
+ export interface ReindexWorkResult {
7
+ scanned: number;
8
+ updated: number;
9
+ deleted: number;
10
+ embeddingsRecomputed: number;
11
+ embeddingsFailed: number;
12
+ }
13
+ export interface ReindexScopeResult extends ReindexWorkResult {
14
+ kind: 'wiki' | 'sl';
15
+ label: string;
16
+ scope?: 'global' | 'user';
17
+ scopeId?: string | null;
18
+ connectionId?: string;
19
+ durationMs: number;
20
+ error?: string;
21
+ }
22
+ export interface ReindexSummary {
23
+ scopes: ReindexScopeResult[];
24
+ totals: ReindexWorkResult;
25
+ dbPath: string;
26
+ force: boolean;
27
+ embeddingsAvailable: boolean;
28
+ durationMs: number;
29
+ }
@@ -0,0 +1 @@
1
+ export {};
@@ -7,6 +7,7 @@ export * from './agent/index.js';
7
7
  export * from './core/index.js';
8
8
  export * from './daemon/index.js';
9
9
  export * from './ingest/index.js';
10
+ export * from './index-sync/index.js';
10
11
  export * from './llm/index.js';
11
12
  export type { CaptureSession, CaptureSignals, MemoryAgentInput, MemoryAgentResult, MemoryAgentServiceDeps, MemoryAgentSettings, MemoryAgentSourceType, MemoryCommitMessagePort, MemoryConnectionPort, MemoryFileStorePort, MemoryKnowledgeSlRefsPort, MemoryLockPort, MemorySlSourceReconcilerPort, MemoryTelemetryPort, MemoryToolSetLike, MemoryToolsetFactoryPort, } from './memory/index.js';
12
13
  export * from './project/index.js';
@@ -6,6 +6,7 @@ export * from './agent/index.js';
6
6
  export * from './core/index.js';
7
7
  export * from './daemon/index.js';
8
8
  export * from './ingest/index.js';
9
+ export * from './index-sync/index.js';
9
10
  export * from './llm/index.js';
10
11
  export * from './project/index.js';
11
12
  export * from './prompts/index.js';
@@ -1,4 +1,4 @@
1
- import type { ChunkResult, DiffSet, FetchContext, ScopeDescriptor, SourceAdapter } from '../../types.js';
1
+ import type { ChunkResult, DeterministicFinalizationContext, DiffSet, FetchContext, FinalizationResult, ScopeDescriptor, SourceAdapter } from '../../types.js';
2
2
  import { type HistoricSqlSourceAdapterDeps } from './types.js';
3
3
  export declare class HistoricSqlSourceAdapter implements SourceAdapter {
4
4
  private readonly deps;
@@ -11,4 +11,5 @@ export declare class HistoricSqlSourceAdapter implements SourceAdapter {
11
11
  fetch(pullConfig: unknown, stagedDir: string, ctx: FetchContext): Promise<void>;
12
12
  chunk(stagedDir: string, diffSet?: DiffSet): Promise<ChunkResult>;
13
13
  describeScope(stagedDir: string): Promise<ScopeDescriptor>;
14
+ finalize(ctx: DeterministicFinalizationContext): Promise<FinalizationResult>;
14
15
  }
@@ -1,5 +1,6 @@
1
1
  import { chunkHistoricSqlUnifiedStagedDir, describeHistoricSqlUnifiedScope } from './chunk-unified.js';
2
2
  import { detectHistoricSqlStagedDir } from './detect.js';
3
+ import { projectHistoricSqlEvidence } from './projection.js';
3
4
  import { stageHistoricSqlAggregatedSnapshot } from './stage-unified.js';
4
5
  export class HistoricSqlSourceAdapter {
5
6
  deps;
@@ -30,4 +31,21 @@ export class HistoricSqlSourceAdapter {
30
31
  describeScope(stagedDir) {
31
32
  return describeHistoricSqlUnifiedScope(stagedDir);
32
33
  }
34
+ async finalize(ctx) {
35
+ const projection = await projectHistoricSqlEvidence({
36
+ workdir: ctx.workdir,
37
+ connectionId: ctx.connectionId,
38
+ syncId: ctx.syncId,
39
+ runId: ctx.runId,
40
+ overrideReplay: ctx.overrideReplay,
41
+ });
42
+ return {
43
+ result: projection,
44
+ warnings: projection.warnings,
45
+ errors: [],
46
+ touchedSources: projection.touchedSources,
47
+ changedWikiPageKeys: projection.changedWikiPageKeys,
48
+ actions: projection.actions,
49
+ };
50
+ }
33
51
  }