@kaelio/ktx 0.1.0-rc.5 → 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (180) hide show
  1. package/assets/python/kaelio_ktx-0.1.0-py3-none-any.whl +0 -0
  2. package/assets/python/manifest.json +2 -2
  3. package/dist/clack.d.ts +6 -0
  4. package/dist/clack.js +23 -0
  5. package/dist/cli-program.js +5 -2
  6. package/dist/cli-program.test.js +7 -1
  7. package/dist/cli-runtime.d.ts +4 -0
  8. package/dist/cli-runtime.js +8 -1
  9. package/dist/command-schemas.d.ts +1 -1
  10. package/dist/commands/ingest-commands.js +1 -0
  11. package/dist/commands/knowledge-commands.js +5 -0
  12. package/dist/commands/mcp-commands.js +11 -3
  13. package/dist/commands/mcp-commands.test.js +30 -1
  14. package/dist/commands/sql-commands.d.ts +3 -0
  15. package/dist/commands/sql-commands.js +43 -0
  16. package/dist/commands/sql-commands.test.d.ts +1 -0
  17. package/dist/commands/sql-commands.test.js +68 -0
  18. package/dist/context-build-view.js +5 -1
  19. package/dist/dev.test.js +27 -0
  20. package/dist/index.d.ts +1 -0
  21. package/dist/index.js +1 -0
  22. package/dist/index.test.js +56 -21
  23. package/dist/ingest.js +123 -18
  24. package/dist/ingest.test.js +206 -0
  25. package/dist/io/print-list.d.ts +2 -1
  26. package/dist/io/print-list.js +7 -0
  27. package/dist/io/print-list.test.js +13 -11
  28. package/dist/io/symbols.d.ts +2 -2
  29. package/dist/knowledge.d.ts +1 -0
  30. package/dist/knowledge.js +34 -16
  31. package/dist/knowledge.test.js +27 -0
  32. package/dist/managed-python-command.d.ts +2 -0
  33. package/dist/managed-python-command.js +17 -9
  34. package/dist/managed-python-command.test.js +59 -4
  35. package/dist/next-steps.js +1 -1
  36. package/dist/next-steps.test.js +2 -0
  37. package/dist/print-command-tree.js +7 -1
  38. package/dist/public-ingest.d.ts +9 -1
  39. package/dist/public-ingest.js +50 -7
  40. package/dist/public-ingest.test.js +69 -2
  41. package/dist/release-version.d.ts +5 -0
  42. package/dist/release-version.js +44 -0
  43. package/dist/runtime-requirements.d.ts +23 -0
  44. package/dist/runtime-requirements.js +99 -0
  45. package/dist/runtime-requirements.test.d.ts +1 -0
  46. package/dist/runtime-requirements.test.js +63 -0
  47. package/dist/setup-agents.d.ts +11 -3
  48. package/dist/setup-agents.js +397 -134
  49. package/dist/setup-agents.test.js +359 -61
  50. package/dist/setup-embeddings.js +3 -6
  51. package/dist/setup-embeddings.test.js +18 -2
  52. package/dist/setup-models.js +2 -2
  53. package/dist/setup-models.test.js +5 -3
  54. package/dist/setup-ready-menu.d.ts +1 -1
  55. package/dist/setup-ready-menu.js +2 -0
  56. package/dist/setup-ready-menu.test.js +3 -0
  57. package/dist/setup-runtime.d.ts +45 -0
  58. package/dist/setup-runtime.js +47 -0
  59. package/dist/setup-runtime.test.d.ts +1 -0
  60. package/dist/setup-runtime.test.js +110 -0
  61. package/dist/setup-sources-notion.test.d.ts +1 -0
  62. package/dist/setup-sources-notion.test.js +107 -0
  63. package/dist/setup-sources.js +5 -2
  64. package/dist/setup.d.ts +19 -1
  65. package/dist/setup.js +104 -29
  66. package/dist/setup.test.js +221 -57
  67. package/dist/sl.js +2 -2
  68. package/dist/sl.test.js +10 -0
  69. package/dist/source-mapping.js +9 -1
  70. package/dist/source-mapping.test.d.ts +1 -0
  71. package/dist/source-mapping.test.js +65 -0
  72. package/dist/sql.d.ts +22 -0
  73. package/dist/sql.js +125 -0
  74. package/dist/sql.test.d.ts +1 -0
  75. package/dist/sql.test.js +226 -0
  76. package/node_modules/@ktx/connector-clickhouse/dist/package-exports.test.js +1 -1
  77. package/node_modules/@ktx/context/dist/connections/connection-type.d.ts +4 -4
  78. package/node_modules/@ktx/context/dist/core/git.service.d.ts +3 -0
  79. package/node_modules/@ktx/context/dist/core/git.service.js +47 -1
  80. package/node_modules/@ktx/context/dist/core/git.service.patch.test.d.ts +1 -0
  81. package/node_modules/@ktx/context/dist/core/git.service.patch.test.js +40 -0
  82. package/node_modules/@ktx/context/dist/ingest/adapters/historic-sql/types.d.ts +5 -5
  83. package/node_modules/@ktx/context/dist/ingest/adapters/looker/looker.adapter.d.ts +2 -2
  84. package/node_modules/@ktx/context/dist/ingest/adapters/looker/tools/looker-query-to-sl.tool.d.ts +2 -2
  85. package/node_modules/@ktx/context/dist/ingest/adapters/looker/types.d.ts +16 -16
  86. package/node_modules/@ktx/context/dist/ingest/adapters/lookml/pull-config.d.ts +1 -1
  87. package/node_modules/@ktx/context/dist/ingest/adapters/metabase/fetch.js +16 -0
  88. package/node_modules/@ktx/context/dist/ingest/adapters/metabase/fetch.test.js +41 -0
  89. package/node_modules/@ktx/context/dist/ingest/adapters/metricflow/metricflow.adapter.d.ts +2 -1
  90. package/node_modules/@ktx/context/dist/ingest/adapters/metricflow/metricflow.adapter.js +40 -0
  91. package/node_modules/@ktx/context/dist/ingest/adapters/metricflow/metricflow.adapter.test.js +116 -1
  92. package/node_modules/@ktx/context/dist/ingest/adapters/metricflow/projection-config.d.ts +29 -0
  93. package/node_modules/@ktx/context/dist/ingest/adapters/metricflow/projection-config.js +40 -0
  94. package/node_modules/@ktx/context/dist/ingest/adapters/metricflow/pull-config.d.ts +1 -1
  95. package/node_modules/@ktx/context/dist/ingest/artifact-gates.d.ts +25 -0
  96. package/node_modules/@ktx/context/dist/ingest/artifact-gates.js +149 -0
  97. package/node_modules/@ktx/context/dist/ingest/artifact-gates.test.d.ts +1 -0
  98. package/node_modules/@ktx/context/dist/ingest/artifact-gates.test.js +167 -0
  99. package/node_modules/@ktx/context/dist/ingest/final-gate-repair.d.ts +29 -0
  100. package/node_modules/@ktx/context/dist/ingest/final-gate-repair.js +178 -0
  101. package/node_modules/@ktx/context/dist/ingest/final-gate-repair.test.d.ts +1 -0
  102. package/node_modules/@ktx/context/dist/ingest/final-gate-repair.test.js +109 -0
  103. package/node_modules/@ktx/context/dist/ingest/index.d.ts +8 -1
  104. package/node_modules/@ktx/context/dist/ingest/index.js +7 -0
  105. package/node_modules/@ktx/context/dist/ingest/ingest-bundle.runner.d.ts +18 -2
  106. package/node_modules/@ktx/context/dist/ingest/ingest-bundle.runner.isolated-diff.test.d.ts +1 -0
  107. package/node_modules/@ktx/context/dist/ingest/ingest-bundle.runner.isolated-diff.test.js +1761 -0
  108. package/node_modules/@ktx/context/dist/ingest/ingest-bundle.runner.js +1695 -901
  109. package/node_modules/@ktx/context/dist/ingest/ingest-bundle.runner.test.js +135 -118
  110. package/node_modules/@ktx/context/dist/ingest/ingest-trace.d.ts +50 -0
  111. package/node_modules/@ktx/context/dist/ingest/ingest-trace.js +88 -0
  112. package/node_modules/@ktx/context/dist/ingest/ingest-trace.test.d.ts +1 -0
  113. package/node_modules/@ktx/context/dist/ingest/ingest-trace.test.js +76 -0
  114. package/node_modules/@ktx/context/dist/ingest/isolated-diff/git-patch.d.ts +16 -0
  115. package/node_modules/@ktx/context/dist/ingest/isolated-diff/git-patch.js +78 -0
  116. package/node_modules/@ktx/context/dist/ingest/isolated-diff/git-patch.test.d.ts +1 -0
  117. package/node_modules/@ktx/context/dist/ingest/isolated-diff/git-patch.test.js +76 -0
  118. package/node_modules/@ktx/context/dist/ingest/isolated-diff/patch-integrator.d.ts +58 -0
  119. package/node_modules/@ktx/context/dist/ingest/isolated-diff/patch-integrator.js +223 -0
  120. package/node_modules/@ktx/context/dist/ingest/isolated-diff/patch-integrator.test.d.ts +1 -0
  121. package/node_modules/@ktx/context/dist/ingest/isolated-diff/patch-integrator.test.js +369 -0
  122. package/node_modules/@ktx/context/dist/ingest/isolated-diff/textual-conflict-resolver.d.ts +23 -0
  123. package/node_modules/@ktx/context/dist/ingest/isolated-diff/textual-conflict-resolver.js +190 -0
  124. package/node_modules/@ktx/context/dist/ingest/isolated-diff/textual-conflict-resolver.test.d.ts +1 -0
  125. package/node_modules/@ktx/context/dist/ingest/isolated-diff/textual-conflict-resolver.test.js +101 -0
  126. package/node_modules/@ktx/context/dist/ingest/isolated-diff/work-unit-executor.d.ts +15 -0
  127. package/node_modules/@ktx/context/dist/ingest/isolated-diff/work-unit-executor.js +61 -0
  128. package/node_modules/@ktx/context/dist/ingest/isolated-diff/work-unit-executor.test.d.ts +1 -0
  129. package/node_modules/@ktx/context/dist/ingest/isolated-diff/work-unit-executor.test.js +137 -0
  130. package/node_modules/@ktx/context/dist/ingest/local-bundle-ingest.test.js +7 -0
  131. package/node_modules/@ktx/context/dist/ingest/local-bundle-runtime.js +54 -10
  132. package/node_modules/@ktx/context/dist/ingest/local-bundle-runtime.test.js +65 -0
  133. package/node_modules/@ktx/context/dist/ingest/memory-flow/schema.d.ts +23 -5
  134. package/node_modules/@ktx/context/dist/ingest/memory-flow/schema.js +17 -0
  135. package/node_modules/@ktx/context/dist/ingest/memory-flow/schema.test.js +1 -0
  136. package/node_modules/@ktx/context/dist/ingest/memory-flow/types.d.ts +6 -0
  137. package/node_modules/@ktx/context/dist/ingest/parsed-target-table.d.ts +1 -1
  138. package/node_modules/@ktx/context/dist/ingest/ports.d.ts +3 -0
  139. package/node_modules/@ktx/context/dist/ingest/report-snapshot.d.ts +32 -7
  140. package/node_modules/@ktx/context/dist/ingest/report-snapshot.js +25 -0
  141. package/node_modules/@ktx/context/dist/ingest/report-snapshot.test.js +124 -0
  142. package/node_modules/@ktx/context/dist/ingest/reports.d.ts +23 -0
  143. package/node_modules/@ktx/context/dist/ingest/semantic-layer-target-policy.d.ts +11 -0
  144. package/node_modules/@ktx/context/dist/ingest/semantic-layer-target-policy.js +26 -0
  145. package/node_modules/@ktx/context/dist/ingest/semantic-layer-target-policy.test.d.ts +1 -0
  146. package/node_modules/@ktx/context/dist/ingest/semantic-layer-target-policy.test.js +25 -0
  147. package/node_modules/@ktx/context/dist/ingest/stages/stage-3-work-units.d.ts +4 -0
  148. package/node_modules/@ktx/context/dist/ingest/stages/stage-3-work-units.js +4 -0
  149. package/node_modules/@ktx/context/dist/ingest/stages/stage-3-work-units.test.js +29 -0
  150. package/node_modules/@ktx/context/dist/ingest/tools/emit-unmapped-fallback.tool.d.ts +1 -1
  151. package/node_modules/@ktx/context/dist/ingest/types.d.ts +24 -0
  152. package/node_modules/@ktx/context/dist/ingest/wiki-body-refs.d.ts +24 -0
  153. package/node_modules/@ktx/context/dist/ingest/wiki-body-refs.js +111 -0
  154. package/node_modules/@ktx/context/dist/ingest/wiki-body-refs.test.d.ts +1 -0
  155. package/node_modules/@ktx/context/dist/ingest/wiki-body-refs.test.js +138 -0
  156. package/node_modules/@ktx/context/dist/llm/claude-code-runtime.js +19 -2
  157. package/node_modules/@ktx/context/dist/llm/claude-code-runtime.test.js +33 -0
  158. package/node_modules/@ktx/context/dist/project/setup-config.d.ts +1 -1
  159. package/node_modules/@ktx/context/dist/project/setup-config.js +10 -1
  160. package/node_modules/@ktx/context/dist/project/setup-config.test.js +3 -2
  161. package/node_modules/@ktx/context/dist/sl/tools/sl-edit-source.tool.js +5 -1
  162. package/node_modules/@ktx/context/dist/sl/tools/sl-edit-source.tool.test.js +15 -0
  163. package/node_modules/@ktx/context/dist/sl/tools/sl-write-source.tool.js +5 -1
  164. package/node_modules/@ktx/context/dist/sl/tools/sl-write-source.tool.test.js +22 -0
  165. package/node_modules/@ktx/context/dist/tools/action-target-connection.d.ts +9 -0
  166. package/node_modules/@ktx/context/dist/tools/action-target-connection.js +14 -0
  167. package/node_modules/@ktx/context/dist/tools/context-candidate-write.tool.d.ts +4 -4
  168. package/node_modules/@ktx/context/dist/tools/index.d.ts +1 -0
  169. package/node_modules/@ktx/context/dist/tools/index.js +1 -0
  170. package/node_modules/@ktx/context/dist/wiki/local-knowledge.js +4 -1
  171. package/node_modules/@ktx/context/dist/wiki/local-knowledge.test.js +44 -0
  172. package/node_modules/@ktx/context/dist/wiki/tools/wiki-write.tool.js +3 -48
  173. package/node_modules/@ktx/context/dist/wiki/tools/wiki-write.tool.test.js +28 -0
  174. package/node_modules/@ktx/context/dist/wiki/wiki-ref-validation.d.ts +17 -0
  175. package/node_modules/@ktx/context/dist/wiki/wiki-ref-validation.js +79 -0
  176. package/node_modules/@ktx/context/dist/wiki/wiki-ref-validation.test.d.ts +1 -0
  177. package/node_modules/@ktx/context/dist/wiki/wiki-ref-validation.test.js +64 -0
  178. package/node_modules/@ktx/context/prompts/memory_agent_bundle_ingest_work_unit.md +23 -4
  179. package/node_modules/@ktx/context/skills/ingest_triage/SKILL.md +7 -3
  180. package/package.json +4 -4
@@ -1,8 +1,7 @@
1
- import { mkdir, mkdtemp, readFile, rm, stat, writeFile } from 'node:fs/promises';
1
+ import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises';
2
2
  import { tmpdir } from 'node:os';
3
3
  import { join } from 'node:path';
4
4
  import { beforeEach, describe, expect, it, vi } from 'vitest';
5
- import { GitService } from '../core/index.js';
6
5
  import { addTouchedSlSource } from '../tools/index.js';
7
6
  import { IngestBundleRunner } from './ingest-bundle.runner.js';
8
7
  import { createMemoryFlowLiveBuffer } from './memory-flow/live-buffer.js';
@@ -113,9 +112,15 @@ const makeDeps = () => {
113
112
  };
114
113
  const scopedGit = {
115
114
  revParseHead: vi.fn().mockResolvedValue('h'),
116
- commitFiles: vi.fn(),
115
+ commitFiles: vi.fn().mockResolvedValue({ created: true, commitHash: 'h' }),
116
+ commitStaged: vi.fn().mockResolvedValue({ created: false, commitHash: 'h' }),
117
117
  resetHardTo: vi.fn(),
118
118
  assertWorktreeClean: vi.fn().mockResolvedValue(undefined),
119
+ writeBinaryNoRenamePatch: vi.fn(async (_base, _head, patchPath) => {
120
+ await writeFile(patchPath, '', 'utf-8');
121
+ }),
122
+ applyPatchFile3WayIndex: vi.fn(),
123
+ diffNameStatus: vi.fn().mockResolvedValue([]),
119
124
  };
120
125
  const sessionWorktreeService = {
121
126
  create: vi.fn().mockResolvedValue({
@@ -157,10 +162,12 @@ const makeDeps = () => {
157
162
  loadPrompt: vi.fn().mockResolvedValue('base-framing'),
158
163
  };
159
164
  const wikiService = {
160
- forWorktree: vi.fn().mockReturnValue({}),
165
+ forWorktree: vi.fn(),
166
+ listPageKeys: vi.fn().mockResolvedValue([]),
161
167
  readPage: vi.fn().mockResolvedValue(null),
162
168
  syncFromCommit: vi.fn().mockResolvedValue(undefined),
163
169
  };
170
+ wikiService.forWorktree.mockReturnValue(wikiService);
164
171
  const knowledgeSlRefs = {
165
172
  syncFromWiki: vi.fn().mockResolvedValue({ inserted: 1, deleted: 0 }),
166
173
  };
@@ -168,7 +175,7 @@ const makeDeps = () => {
168
175
  listPagesForUser: vi.fn().mockResolvedValue([]),
169
176
  };
170
177
  const semanticLayerService = {
171
- forWorktree: vi.fn().mockReturnValue({}),
178
+ forWorktree: vi.fn(),
172
179
  listFilesForConnection: vi
173
180
  .fn()
174
181
  .mockImplementation((connectionId) => Promise.resolve(connectionId === 'warehouse-2' ? ['looker__orders.yaml'] : [])),
@@ -179,6 +186,7 @@ const makeDeps = () => {
179
186
  loadErrors: [],
180
187
  })),
181
188
  };
189
+ semanticLayerService.forWorktree.mockReturnValue(semanticLayerService);
182
190
  const slSearchService = {
183
191
  indexSources: vi.fn().mockResolvedValue(undefined),
184
192
  };
@@ -239,8 +247,12 @@ const buildRunner = (deps = makeDeps(), overrides = {}) => new IngestBundleRunne
239
247
  resolveUploadDir: (uploadId) => `/tmp/ktx-test/ingest-uploads/${uploadId}`,
240
248
  resolvePullDir: (jobId) => `/tmp/ktx-test/ingest-pulls/${jobId}`,
241
249
  resolveTranscriptDir: (jobId) => `/tmp/ktx-test/run/wu-transcripts/${jobId}`,
250
+ resolveTracePath: (jobId) => `/tmp/ktx-test/ingest-traces/${jobId}/trace.jsonl`,
251
+ },
252
+ settings: {
253
+ probeRowCount: 1,
254
+ memoryIngestionModel: 'test-model',
242
255
  },
243
- settings: { probeRowCount: 1, memoryIngestionModel: 'test-model' },
244
256
  skillsRegistry: deps.skillsRegistry,
245
257
  promptService: deps.promptService,
246
258
  wikiService: deps.wikiService,
@@ -377,6 +389,114 @@ describe('IngestBundleRunner — Stages 1 → 7', () => {
377
389
  // Single touched path → path-scoped diff for the LLM commit-message note.
378
390
  expect(deps.configService.enqueueCommitMessageJobForExternalCommit).toHaveBeenCalledWith({ commitHash: 'sq' }, expect.stringContaining('ingest(fake): j1'), 'raw-sources/c1/fake/s/a.yml');
379
391
  });
392
+ it('fails before squash when reconciliation leaves a touched wiki page with dangling refs', async () => {
393
+ const deps = makeDeps();
394
+ let currentToolSession = null;
395
+ const scopedWiki = {
396
+ listPageKeys: vi.fn().mockResolvedValue(['page-a']),
397
+ readPage: vi.fn().mockImplementation((_scope, _scopeId, key) => {
398
+ if (key === 'page-a') {
399
+ return Promise.resolve({
400
+ pageKey: 'page-a',
401
+ frontmatter: { summary: 'Page A', usage_mode: 'auto', refs: ['missing-page'] },
402
+ content: 'See [[missing-page]].',
403
+ });
404
+ }
405
+ return Promise.resolve(null);
406
+ }),
407
+ };
408
+ deps.wikiService.forWorktree.mockReturnValue(scopedWiki);
409
+ deps.toolsetFactory.createIngestWuToolset.mockImplementation((toolSession) => {
410
+ currentToolSession = toolSession;
411
+ return {
412
+ toRuntimeTools: vi.fn().mockReturnValue({}),
413
+ getAllTools: vi.fn().mockReturnValue([]),
414
+ getToolNames: vi.fn().mockReturnValue([]),
415
+ };
416
+ });
417
+ deps.agentRunner.runLoop.mockImplementation(async (params) => {
418
+ if (params.telemetryTags.operationName === 'ingest-bundle-wu') {
419
+ currentToolSession.actions.push({ target: 'sl', type: 'updated', key: 'orders', detail: 'Orders source' });
420
+ }
421
+ if (params.telemetryTags.operationName === 'ingest-bundle-reconcile') {
422
+ currentToolSession.actions.push({ target: 'wiki', type: 'created', key: 'page-a', detail: 'Page A' });
423
+ }
424
+ return { stopReason: 'natural' };
425
+ });
426
+ const runner = buildRunner(deps);
427
+ runner.stageRawFilesStage1 = vi.fn().mockResolvedValue({
428
+ currentHashes: new Map([['a.yml', 'h1']]),
429
+ rawDirInWorktree: 'raw-sources/c1/fake/s',
430
+ });
431
+ runner.resolveStagedDir = vi.fn().mockResolvedValue('/tmp/stage/upload-x');
432
+ await expect(runner.run({
433
+ jobId: 'j1',
434
+ connectionId: 'c1',
435
+ sourceKey: 'fake',
436
+ trigger: 'upload',
437
+ bundleRef: { kind: 'upload', uploadId: 'upload-x' },
438
+ })).rejects.toThrow(/wiki references target missing page\(s\): page-a -> missing-page/);
439
+ expect(deps.runsRepo.markFailed).toHaveBeenCalledWith('run-1');
440
+ expect(deps.gitService.squashMergeIntoMain).not.toHaveBeenCalled();
441
+ });
442
+ it('allows reconciliation to save circular wiki refs once both pages exist', async () => {
443
+ const deps = makeDeps();
444
+ let currentToolSession = null;
445
+ const scopedWiki = {
446
+ listPageKeys: vi.fn().mockResolvedValue(['page-a', 'page-b']),
447
+ readPage: vi.fn().mockImplementation((_scope, _scopeId, key) => {
448
+ if (key === 'page-a') {
449
+ return Promise.resolve({
450
+ pageKey: 'page-a',
451
+ frontmatter: { summary: 'Page A', usage_mode: 'auto', refs: ['page-b'] },
452
+ content: 'See [[page-b]].',
453
+ });
454
+ }
455
+ if (key === 'page-b') {
456
+ return Promise.resolve({
457
+ pageKey: 'page-b',
458
+ frontmatter: { summary: 'Page B', usage_mode: 'auto', refs: ['page-a'] },
459
+ content: 'See [[page-a]].',
460
+ });
461
+ }
462
+ return Promise.resolve(null);
463
+ }),
464
+ };
465
+ deps.wikiService.forWorktree.mockReturnValue(scopedWiki);
466
+ deps.toolsetFactory.createIngestWuToolset.mockImplementation((toolSession) => {
467
+ currentToolSession = toolSession;
468
+ return {
469
+ toRuntimeTools: vi.fn().mockReturnValue({}),
470
+ getAllTools: vi.fn().mockReturnValue([]),
471
+ getToolNames: vi.fn().mockReturnValue([]),
472
+ };
473
+ });
474
+ deps.agentRunner.runLoop.mockImplementation(async (params) => {
475
+ if (params.telemetryTags.operationName === 'ingest-bundle-wu') {
476
+ currentToolSession.actions.push({ target: 'sl', type: 'updated', key: 'orders', detail: 'Orders source' });
477
+ }
478
+ if (params.telemetryTags.operationName === 'ingest-bundle-reconcile') {
479
+ currentToolSession.actions.push({ target: 'wiki', type: 'created', key: 'page-a', detail: 'Page A' }, { target: 'wiki', type: 'created', key: 'page-b', detail: 'Page B' });
480
+ }
481
+ return { stopReason: 'natural' };
482
+ });
483
+ const runner = buildRunner(deps);
484
+ runner.stageRawFilesStage1 = vi.fn().mockResolvedValue({
485
+ currentHashes: new Map([['a.yml', 'h1']]),
486
+ rawDirInWorktree: 'raw-sources/c1/fake/s',
487
+ });
488
+ runner.resolveStagedDir = vi.fn().mockResolvedValue('/tmp/stage/upload-x');
489
+ const result = await runner.run({
490
+ jobId: 'j1',
491
+ connectionId: 'c1',
492
+ sourceKey: 'fake',
493
+ trigger: 'upload',
494
+ bundleRef: { kind: 'upload', uploadId: 'upload-x' },
495
+ });
496
+ expect(result.failedWorkUnits).toEqual([]);
497
+ expect(deps.gitService.squashMergeIntoMain).toHaveBeenCalled();
498
+ expect(deps.runsRepo.markFailed).not.toHaveBeenCalled();
499
+ });
380
500
  it('threads target warehouse connection names into WorkUnit and reconcile tool sessions', async () => {
381
501
  const deps = makeDeps();
382
502
  const sessions = [];
@@ -1174,7 +1294,7 @@ describe('IngestBundleRunner — Stages 1 → 7', () => {
1174
1294
  });
1175
1295
  const runner = buildRunner(deps);
1176
1296
  runner.stageRawFilesStage1 = vi.fn().mockResolvedValue({
1177
- currentHashes: new Map([['explores/b2b/sales_pipeline.json', 'h1']]),
1297
+ currentHashes: new Map([['a.yml', 'h1']]),
1178
1298
  rawDirInWorktree: 'raw-sources/looker-run/fake/s',
1179
1299
  });
1180
1300
  runner.resolveStagedDir = vi.fn().mockResolvedValue('/tmp/stage/upload-x');
@@ -1232,6 +1352,7 @@ describe('IngestBundleRunner — Stages 1 → 7', () => {
1232
1352
  workUnits: [{ unitKey: 'u1', rawFiles: ['semantic_models.yml'], peerFileIndex: [], dependencyPaths: [] }],
1233
1353
  parseArtifacts: { semanticModels: [{ name: 'orders' }] },
1234
1354
  });
1355
+ deps.adapter.listTargetConnectionIds = vi.fn().mockResolvedValue(['warehouse-2']);
1235
1356
  deps.semanticLayerService.loadAllSources.mockImplementation((connectionId) => Promise.resolve({ sources: [{ name: `${connectionId}_source` }], loadErrors: [] }));
1236
1357
  const postProcessor = {
1237
1358
  run: vi.fn().mockResolvedValue({
@@ -1567,9 +1688,15 @@ describe('IngestBundleRunner — Stages 1 → 7', () => {
1567
1688
  const assertError = new Error('Worktree has in-progress git operation (sequencer ...); refusing to proceed');
1568
1689
  const sessionGit = {
1569
1690
  revParseHead: vi.fn().mockResolvedValue('h'),
1570
- commitFiles: vi.fn(),
1691
+ commitFiles: vi.fn().mockResolvedValue({ created: true, commitHash: 'h' }),
1692
+ commitStaged: vi.fn().mockResolvedValue({ created: false, commitHash: 'h' }),
1571
1693
  resetHardTo: vi.fn(),
1572
1694
  assertWorktreeClean: vi.fn().mockRejectedValue(assertError),
1695
+ writeBinaryNoRenamePatch: vi.fn(async (_base, _head, patchPath) => {
1696
+ await writeFile(patchPath, '', 'utf-8');
1697
+ }),
1698
+ applyPatchFile3WayIndex: vi.fn(),
1699
+ diffNameStatus: vi.fn().mockResolvedValue([]),
1573
1700
  };
1574
1701
  deps.sessionWorktreeService.create.mockResolvedValue({
1575
1702
  chatId: 'j1',
@@ -1596,116 +1723,6 @@ describe('IngestBundleRunner — Stages 1 → 7', () => {
1596
1723
  expect(deps.runsRepo.markFailed).toHaveBeenCalledWith('run-1');
1597
1724
  expect(deps.gitService.squashMergeIntoMain).not.toHaveBeenCalled();
1598
1725
  });
1599
- it('squash-merges only successful WUs into main when one WU fails sl_validate', async () => {
1600
- const homeDir = await mkdtemp(join(tmpdir(), 'ingest-rollback-'));
1601
- try {
1602
- const configDir = join(homeDir, 'config');
1603
- const mainGit = new GitService({
1604
- storage: { configDir, homeDir },
1605
- git: {
1606
- userName: 'System User',
1607
- userEmail: 'system@example.com',
1608
- bootstrapMessage: 'Initialize test config repo',
1609
- bootstrapAuthor: 'test-system',
1610
- bootstrapAuthorEmail: 'system@example.com',
1611
- },
1612
- });
1613
- await mainGit.onModuleInit();
1614
- const baseSha = await mainGit.revParseHead();
1615
- if (!baseSha) {
1616
- throw new Error('no base sha');
1617
- }
1618
- const deps = makeDeps();
1619
- const sessionDir = join(homeDir, '.worktrees', 'session-j1');
1620
- const sessionBranch = 'session/j1';
1621
- let currentToolSession = null;
1622
- deps.gitService = mainGit;
1623
- deps.sessionWorktreeService.create.mockImplementation(async (_jobId, startSha) => {
1624
- await mkdir(join(homeDir, '.worktrees'), { recursive: true });
1625
- await mainGit.addWorktree(sessionDir, sessionBranch, startSha);
1626
- return {
1627
- chatId: 'j1',
1628
- workdir: sessionDir,
1629
- branch: sessionBranch,
1630
- baseSha: startSha,
1631
- createdAt: new Date(),
1632
- git: mainGit.forWorktree(sessionDir),
1633
- config: {},
1634
- };
1635
- });
1636
- deps.sessionWorktreeService.cleanup.mockResolvedValue(undefined);
1637
- deps.adapter.chunk.mockResolvedValue({
1638
- workUnits: [
1639
- { unitKey: 'wu-good', rawFiles: ['good.raw'], peerFileIndex: [], dependencyPaths: [] },
1640
- { unitKey: 'wu-bad', rawFiles: ['bad.raw'], peerFileIndex: [], dependencyPaths: [] },
1641
- ],
1642
- });
1643
- deps.toolsetFactory.createIngestWuToolset.mockImplementation((toolSession) => {
1644
- currentToolSession = toolSession;
1645
- return {
1646
- toRuntimeTools: vi.fn().mockReturnValue({}),
1647
- getAllTools: vi.fn().mockReturnValue([]),
1648
- getToolNames: vi.fn().mockReturnValue([]),
1649
- };
1650
- });
1651
- deps.slValidator.validateSingleSource.mockImplementation((_validationDeps, _connectionId, sourceName) => ({
1652
- errors: sourceName === 'bad' ? [{ message: 'bad source rejected' }] : [],
1653
- warnings: [],
1654
- }));
1655
- deps.agentRunner.runLoop.mockImplementation(async (params) => {
1656
- const unitKey = params.telemetryTags?.unitKey;
1657
- if (unitKey === 'wu-good') {
1658
- await mkdir(join(sessionDir, 'semantic-layer', 'c1'), { recursive: true });
1659
- await writeFile(join(sessionDir, 'semantic-layer', 'c1', 'good.yaml'), 'name: good\n');
1660
- addTouchedSlSource(currentToolSession.touchedSlSources, 'c1', 'good');
1661
- currentToolSession.actions.push({ target: 'sl', type: 'created', key: 'good', detail: '' });
1662
- await currentToolSession.gitService.commitFiles(['semantic-layer/c1/good.yaml'], 'test: add good source', 'KTX Test', 'system@ktx.local');
1663
- }
1664
- if (unitKey === 'wu-bad') {
1665
- await mkdir(join(sessionDir, 'semantic-layer', 'c1'), { recursive: true });
1666
- await writeFile(join(sessionDir, 'semantic-layer', 'c1', 'bad.yaml'), 'name: bad\n');
1667
- addTouchedSlSource(currentToolSession.touchedSlSources, 'c1', 'bad');
1668
- currentToolSession.actions.push({ target: 'sl', type: 'created', key: 'bad', detail: '' });
1669
- await currentToolSession.gitService.commitFiles(['semantic-layer/c1/bad.yaml'], 'test: add bad source', 'KTX Test', 'system@ktx.local');
1670
- }
1671
- return { stopReason: 'natural' };
1672
- });
1673
- const runner = buildRunner(deps);
1674
- runner.stageRawFilesStage1 = vi.fn().mockImplementation(async ({ worktreeRoot }) => {
1675
- const rawDir = join(worktreeRoot, 'raw-sources', 'c1', 'fake', 's');
1676
- await mkdir(rawDir, { recursive: true });
1677
- await writeFile(join(rawDir, 'good.raw'), 'good raw');
1678
- await writeFile(join(rawDir, 'bad.raw'), 'bad raw');
1679
- return {
1680
- currentHashes: new Map([
1681
- ['good.raw', 'good-hash'],
1682
- ['bad.raw', 'bad-hash'],
1683
- ]),
1684
- rawDirInWorktree: 'raw-sources/c1/fake/s',
1685
- };
1686
- });
1687
- runner.resolveStagedDir = vi.fn().mockResolvedValue('/tmp/stage/upload-x');
1688
- const result = await runner.run({
1689
- jobId: 'j1',
1690
- connectionId: 'c1',
1691
- sourceKey: 'fake',
1692
- trigger: 'upload',
1693
- bundleRef: { kind: 'upload', uploadId: 'upload-x' },
1694
- });
1695
- expect(result.failedWorkUnits).toEqual(['wu-bad']);
1696
- expect(await readFile(join(configDir, 'semantic-layer', 'c1', 'good.yaml'), 'utf-8')).toContain('good');
1697
- expect(await readFile(join(configDir, 'semantic-layer', 'c1', 'bad.yaml'), 'utf-8').catch(() => null)).toBeNull();
1698
- expect(deps.reportsRepo.create).toHaveBeenCalledWith(expect.objectContaining({
1699
- body: expect.objectContaining({
1700
- failedWorkUnits: ['wu-bad'],
1701
- }),
1702
- }));
1703
- await expect(stat(join(configDir, '.git', 'sequencer'))).rejects.toThrow();
1704
- }
1705
- finally {
1706
- await rm(homeDir, { recursive: true, force: true });
1707
- }
1708
- });
1709
1726
  it('fails the run and rethrows when the adapter cannot detect the bundle', async () => {
1710
1727
  const deps = makeDeps();
1711
1728
  deps.adapter.detect.mockResolvedValue(false);
@@ -0,0 +1,50 @@
1
+ export type IngestTraceLevel = 'info' | 'debug' | 'trace' | 'error';
2
+ export interface IngestTraceContext {
3
+ tracePath: string;
4
+ jobId: string;
5
+ connectionId: string;
6
+ sourceKey: string;
7
+ runId?: string;
8
+ syncId?: string;
9
+ level?: IngestTraceLevel;
10
+ }
11
+ export interface IngestTraceEvent {
12
+ schemaVersion: 1;
13
+ at: string;
14
+ level: IngestTraceLevel;
15
+ jobId: string;
16
+ connectionId: string;
17
+ sourceKey: string;
18
+ runId?: string;
19
+ syncId?: string;
20
+ phase: string;
21
+ event: string;
22
+ durationMs?: number;
23
+ data?: Record<string, unknown>;
24
+ error?: {
25
+ name: string;
26
+ message: string;
27
+ stack?: string;
28
+ };
29
+ }
30
+ export interface IngestTraceWriter {
31
+ readonly tracePath: string;
32
+ readonly context: IngestTraceContext;
33
+ withContext(context: Partial<Pick<IngestTraceContext, 'runId' | 'syncId'>>): IngestTraceWriter;
34
+ event(level: IngestTraceLevel, phase: string, event: string, data?: Record<string, unknown>, error?: unknown, durationMs?: number): Promise<void>;
35
+ }
36
+ export declare function ingestTracePathForJob(homeDir: string, jobId: string): string;
37
+ export declare class FileIngestTraceWriter implements IngestTraceWriter {
38
+ readonly tracePath: string;
39
+ readonly context: IngestTraceContext;
40
+ constructor(context: IngestTraceContext);
41
+ withContext(context: Partial<Pick<IngestTraceContext, 'runId' | 'syncId'>>): IngestTraceWriter;
42
+ event(level: IngestTraceLevel, phase: string, event: string, data?: Record<string, unknown>, error?: unknown, durationMs?: number): Promise<void>;
43
+ }
44
+ export declare class NoopIngestTraceWriter implements IngestTraceWriter {
45
+ readonly tracePath = "";
46
+ readonly context: IngestTraceContext;
47
+ withContext(): IngestTraceWriter;
48
+ event(): Promise<void>;
49
+ }
50
+ export declare function traceTimed<T>(trace: IngestTraceWriter, phase: string, event: string, data: Record<string, unknown>, fn: () => Promise<T>): Promise<T>;
@@ -0,0 +1,88 @@
1
+ import { appendFile, mkdir } from 'node:fs/promises';
2
+ import { dirname, join } from 'node:path';
3
+ const TRACE_LEVEL_RANK = {
4
+ error: 0,
5
+ info: 1,
6
+ debug: 2,
7
+ trace: 3,
8
+ };
9
+ export function ingestTracePathForJob(homeDir, jobId) {
10
+ return join(homeDir, 'ingest-traces', jobId, 'trace.jsonl');
11
+ }
12
+ function serializeError(error) {
13
+ if (error === undefined || error === null) {
14
+ return undefined;
15
+ }
16
+ if (error instanceof Error) {
17
+ return {
18
+ name: error.name,
19
+ message: error.message,
20
+ ...(error.stack ? { stack: error.stack } : {}),
21
+ };
22
+ }
23
+ return { name: 'Error', message: String(error) };
24
+ }
25
+ function shouldWrite(configured, incoming) {
26
+ return TRACE_LEVEL_RANK[incoming] <= TRACE_LEVEL_RANK[configured];
27
+ }
28
+ export class FileIngestTraceWriter {
29
+ tracePath;
30
+ context;
31
+ constructor(context) {
32
+ this.context = { ...context, level: context.level ?? 'debug' };
33
+ this.tracePath = context.tracePath;
34
+ }
35
+ withContext(context) {
36
+ return new FileIngestTraceWriter({ ...this.context, ...context, tracePath: this.tracePath });
37
+ }
38
+ async event(level, phase, event, data, error, durationMs) {
39
+ if (!shouldWrite(this.context.level ?? 'debug', level)) {
40
+ return;
41
+ }
42
+ const serializedError = serializeError(error);
43
+ const payload = {
44
+ schemaVersion: 1,
45
+ at: new Date().toISOString(),
46
+ level,
47
+ jobId: this.context.jobId,
48
+ connectionId: this.context.connectionId,
49
+ sourceKey: this.context.sourceKey,
50
+ ...(this.context.runId ? { runId: this.context.runId } : {}),
51
+ ...(this.context.syncId ? { syncId: this.context.syncId } : {}),
52
+ phase,
53
+ event,
54
+ ...(durationMs !== undefined ? { durationMs } : {}),
55
+ ...(data ? { data } : {}),
56
+ ...(serializedError ? { error: serializedError } : {}),
57
+ };
58
+ await mkdir(dirname(this.tracePath), { recursive: true });
59
+ await appendFile(this.tracePath, `${JSON.stringify(payload)}\n`, 'utf-8');
60
+ }
61
+ }
62
+ export class NoopIngestTraceWriter {
63
+ tracePath = '';
64
+ context = {
65
+ tracePath: '',
66
+ jobId: '',
67
+ connectionId: '',
68
+ sourceKey: '',
69
+ level: 'error',
70
+ };
71
+ withContext() {
72
+ return this;
73
+ }
74
+ async event() { }
75
+ }
76
+ export async function traceTimed(trace, phase, event, data, fn) {
77
+ await trace.event('debug', phase, `${event}_started`, data);
78
+ const started = Date.now();
79
+ try {
80
+ const result = await fn();
81
+ await trace.event('debug', phase, `${event}_finished`, data, undefined, Date.now() - started);
82
+ return result;
83
+ }
84
+ catch (error) {
85
+ await trace.event('error', phase, `${event}_failed`, data, error, Date.now() - started);
86
+ throw error;
87
+ }
88
+ }
@@ -0,0 +1,76 @@
1
+ import { mkdtemp, readFile } from 'node:fs/promises';
2
+ import { tmpdir } from 'node:os';
3
+ import { join } from 'node:path';
4
+ import { describe, expect, it, vi } from 'vitest';
5
+ import { FileIngestTraceWriter, ingestTracePathForJob, traceTimed } from './ingest-trace.js';
6
+ describe('FileIngestTraceWriter', () => {
7
+ it('persists structured trace events as JSONL', async () => {
8
+ const root = await mkdtemp(join(tmpdir(), 'ktx-trace-'));
9
+ const tracePath = ingestTracePathForJob(root, 'job-1');
10
+ const trace = new FileIngestTraceWriter({
11
+ tracePath,
12
+ jobId: 'job-1',
13
+ connectionId: 'metabase-main',
14
+ sourceKey: 'metabase',
15
+ level: 'debug',
16
+ });
17
+ await trace.event('debug', 'snapshot', 'input_snapshot', {
18
+ baseSha: 'abc123',
19
+ rawFileCount: 2,
20
+ diffSummary: { added: 1, modified: 1, deleted: 0, unchanged: 3 },
21
+ });
22
+ const lines = (await readFile(tracePath, 'utf-8'))
23
+ .trim()
24
+ .split('\n')
25
+ .map((line) => JSON.parse(line));
26
+ expect(lines).toHaveLength(1);
27
+ expect(lines[0]).toMatchObject({
28
+ schemaVersion: 1,
29
+ jobId: 'job-1',
30
+ connectionId: 'metabase-main',
31
+ sourceKey: 'metabase',
32
+ level: 'debug',
33
+ phase: 'snapshot',
34
+ event: 'input_snapshot',
35
+ data: {
36
+ baseSha: 'abc123',
37
+ rawFileCount: 2,
38
+ diffSummary: { added: 1, modified: 1, deleted: 0, unchanged: 3 },
39
+ },
40
+ });
41
+ expect(typeof lines[0].at).toBe('string');
42
+ });
43
+ it('records timing and error context for postmortem inspection', async () => {
44
+ vi.useFakeTimers();
45
+ vi.setSystemTime(new Date('2026-05-17T12:00:00.000Z'));
46
+ const root = await mkdtemp(join(tmpdir(), 'ktx-trace-'));
47
+ const tracePath = ingestTracePathForJob(root, 'job-2');
48
+ const trace = new FileIngestTraceWriter({
49
+ tracePath,
50
+ jobId: 'job-2',
51
+ connectionId: 'c1',
52
+ sourceKey: 'fake',
53
+ level: 'trace',
54
+ });
55
+ await expect(traceTimed(trace, 'integration', 'apply_patch', { unitKey: 'wu-1' }, async () => {
56
+ vi.advanceTimersByTime(17);
57
+ throw new Error('patch conflict');
58
+ })).rejects.toThrow('patch conflict');
59
+ const lines = (await readFile(tracePath, 'utf-8'))
60
+ .trim()
61
+ .split('\n')
62
+ .map((line) => JSON.parse(line));
63
+ expect(lines.map((line) => line.event)).toEqual(['apply_patch_started', 'apply_patch_failed']);
64
+ expect(lines[1]).toMatchObject({
65
+ level: 'error',
66
+ phase: 'integration',
67
+ data: { unitKey: 'wu-1' },
68
+ error: { name: 'Error', message: 'patch conflict' },
69
+ });
70
+ expect(lines[1].durationMs).toBe(17);
71
+ vi.useRealTimers();
72
+ });
73
+ it('uses the documented trace path layout', () => {
74
+ expect(ingestTracePathForJob('/project/.ktx', 'job-3')).toBe('/project/.ktx/ingest-traces/job-3/trace.jsonl');
75
+ });
76
+ });
@@ -0,0 +1,16 @@
1
+ export declare const textArtifactRoots: readonly ["wiki/", "semantic-layer/"];
2
+ export interface PatchTouchedPath {
3
+ path: string;
4
+ oldPath: string;
5
+ newPath: string;
6
+ mode: string | null;
7
+ binary: boolean;
8
+ }
9
+ export interface PatchPolicyInput {
10
+ unitKey: string;
11
+ patch: string;
12
+ slDisallowed: boolean;
13
+ allowedTargetConnectionIds?: ReadonlySet<string>;
14
+ }
15
+ export declare function parsePatchTouchedPaths(patch: string): PatchTouchedPath[];
16
+ export declare function assertPatchAllowedForWorkUnit(input: PatchPolicyInput): PatchTouchedPath[];
@@ -0,0 +1,78 @@
1
+ import { assertSemanticLayerTargetPathsAllowed } from '../semantic-layer-target-policy.js';
2
+ export const textArtifactRoots = ['wiki/', 'semantic-layer/'];
3
+ function stripPrefix(path) {
4
+ return path.replace(/^[ab]\//, '');
5
+ }
6
+ function isTextArtifactPath(path) {
7
+ return textArtifactRoots.some((root) => path.startsWith(root));
8
+ }
9
+ export function parsePatchTouchedPaths(patch) {
10
+ const lines = patch.split('\n');
11
+ const entries = [];
12
+ let current = null;
13
+ const pushCurrent = () => {
14
+ if (current) {
15
+ entries.push(current);
16
+ }
17
+ };
18
+ for (const line of lines) {
19
+ const diffMatch = /^diff --git (.+) (.+)$/.exec(line);
20
+ if (diffMatch) {
21
+ pushCurrent();
22
+ const oldPath = stripPrefix(diffMatch[1] ?? '');
23
+ const newPath = stripPrefix(diffMatch[2] ?? '');
24
+ current = {
25
+ path: newPath === '/dev/null' ? oldPath : newPath,
26
+ oldPath,
27
+ newPath,
28
+ mode: null,
29
+ binary: false,
30
+ };
31
+ continue;
32
+ }
33
+ if (!current) {
34
+ continue;
35
+ }
36
+ const indexMode = /^index [0-9a-f]+\.\.[0-9a-f]+(?: ([0-7]{6}))?$/.exec(line);
37
+ if (indexMode?.[1]) {
38
+ current.mode = indexMode[1];
39
+ }
40
+ const newMode = /^new mode ([0-7]{6})$/.exec(line);
41
+ if (newMode) {
42
+ current.mode = newMode[1] ?? current.mode;
43
+ }
44
+ const newFileMode = /^new file mode ([0-7]{6})$/.exec(line);
45
+ if (newFileMode) {
46
+ current.mode = newFileMode[1] ?? current.mode;
47
+ }
48
+ if (line === 'GIT binary patch' || line.startsWith('Binary files ')) {
49
+ current.binary = true;
50
+ }
51
+ }
52
+ pushCurrent();
53
+ return entries;
54
+ }
55
+ export function assertPatchAllowedForWorkUnit(input) {
56
+ const touched = parsePatchTouchedPaths(input.patch);
57
+ if (input.allowedTargetConnectionIds) {
58
+ assertSemanticLayerTargetPathsAllowed({
59
+ paths: touched.map((entry) => entry.path),
60
+ allowedConnectionIds: input.allowedTargetConnectionIds,
61
+ });
62
+ }
63
+ for (const entry of touched) {
64
+ if (input.slDisallowed && entry.path.startsWith('semantic-layer/')) {
65
+ throw new Error(`slDisallowed WorkUnit ${input.unitKey} touched ${entry.path}`);
66
+ }
67
+ if (!isTextArtifactPath(entry.path)) {
68
+ continue;
69
+ }
70
+ if (entry.binary) {
71
+ throw new Error(`unexpected binary patch under ${entry.path}`);
72
+ }
73
+ if (entry.mode && entry.mode !== '100644') {
74
+ throw new Error(`unexpected executable mode under ${entry.path}: ${entry.mode}`);
75
+ }
76
+ }
77
+ return touched;
78
+ }