@kaelio/ktx 0.1.0-rc.6 → 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (55) hide show
  1. package/assets/python/{kaelio_ktx-0.1.0rc6-py3-none-any.whl → kaelio_ktx-0.1.0-py3-none-any.whl} +0 -0
  2. package/assets/python/manifest.json +4 -4
  3. package/dist/commands/mcp-commands.js +11 -3
  4. package/dist/commands/mcp-commands.test.js +30 -1
  5. package/dist/ingest.test.js +2 -26
  6. package/dist/next-steps.js +1 -1
  7. package/dist/next-steps.test.js +2 -0
  8. package/dist/runtime-requirements.d.ts +1 -2
  9. package/dist/runtime-requirements.js +0 -7
  10. package/dist/runtime-requirements.test.js +2 -2
  11. package/dist/setup-agents.d.ts +11 -3
  12. package/dist/setup-agents.js +397 -134
  13. package/dist/setup-agents.test.js +359 -61
  14. package/dist/setup-runtime.d.ts +0 -1
  15. package/dist/setup-runtime.js +0 -1
  16. package/dist/setup-runtime.test.js +7 -13
  17. package/dist/setup.d.ts +3 -0
  18. package/dist/setup.js +51 -25
  19. package/dist/setup.test.js +112 -16
  20. package/node_modules/@ktx/connector-clickhouse/dist/package-exports.test.js +1 -1
  21. package/node_modules/@ktx/context/dist/core/git.service.d.ts +0 -1
  22. package/node_modules/@ktx/context/dist/core/git.service.js +0 -12
  23. package/node_modules/@ktx/context/dist/ingest/adapters/historic-sql/historic-sql.adapter.d.ts +1 -2
  24. package/node_modules/@ktx/context/dist/ingest/adapters/historic-sql/historic-sql.adapter.js +0 -18
  25. package/node_modules/@ktx/context/dist/ingest/adapters/historic-sql/local-ingest-acceptance.test.js +6 -6
  26. package/node_modules/@ktx/context/dist/ingest/adapters/historic-sql/post-processor.d.ts +4 -0
  27. package/node_modules/@ktx/context/dist/ingest/adapters/historic-sql/post-processor.js +38 -0
  28. package/node_modules/@ktx/context/dist/ingest/adapters/historic-sql/post-processor.test.js +63 -0
  29. package/node_modules/@ktx/context/dist/ingest/adapters/historic-sql/projection.d.ts +0 -5
  30. package/node_modules/@ktx/context/dist/ingest/adapters/historic-sql/projection.js +0 -48
  31. package/node_modules/@ktx/context/dist/ingest/adapters/historic-sql/projection.test.js +0 -83
  32. package/node_modules/@ktx/context/dist/ingest/index.d.ts +2 -1
  33. package/node_modules/@ktx/context/dist/ingest/index.js +1 -0
  34. package/node_modules/@ktx/context/dist/ingest/ingest-bundle.runner.d.ts +0 -2
  35. package/node_modules/@ktx/context/dist/ingest/ingest-bundle.runner.isolated-diff.test.js +0 -166
  36. package/node_modules/@ktx/context/dist/ingest/ingest-bundle.runner.js +45 -235
  37. package/node_modules/@ktx/context/dist/ingest/ingest-bundle.runner.test.js +38 -193
  38. package/node_modules/@ktx/context/dist/ingest/local-bundle-ingest.test.js +3 -22
  39. package/node_modules/@ktx/context/dist/ingest/local-bundle-runtime.js +4 -0
  40. package/node_modules/@ktx/context/dist/ingest/local-ingest.js +7 -0
  41. package/node_modules/@ktx/context/dist/ingest/memory-flow/schema.d.ts +4 -4
  42. package/node_modules/@ktx/context/dist/ingest/memory-flow/schema.js +1 -1
  43. package/node_modules/@ktx/context/dist/ingest/memory-flow/types.d.ts +1 -1
  44. package/node_modules/@ktx/context/dist/ingest/ports.d.ts +20 -1
  45. package/node_modules/@ktx/context/dist/ingest/report-snapshot.d.ts +2 -73
  46. package/node_modules/@ktx/context/dist/ingest/report-snapshot.js +0 -27
  47. package/node_modules/@ktx/context/dist/ingest/reports.d.ts +5 -23
  48. package/node_modules/@ktx/context/dist/ingest/reports.js +24 -7
  49. package/node_modules/@ktx/context/dist/ingest/types.d.ts +0 -33
  50. package/node_modules/@ktx/context/dist/package-exports.test.js +1 -2
  51. package/package.json +4 -4
  52. package/node_modules/@ktx/context/dist/ingest/finalization-scope.d.ts +0 -22
  53. package/node_modules/@ktx/context/dist/ingest/finalization-scope.js +0 -95
  54. package/node_modules/@ktx/context/dist/ingest/finalization-scope.test.js +0 -114
  55. /package/node_modules/@ktx/context/dist/ingest/{finalization-scope.test.d.ts → adapters/historic-sql/post-processor.test.d.ts} +0 -0
@@ -85,7 +85,6 @@ const makeDeps = () => {
85
85
  triageSupported: undefined,
86
86
  detect: vi.fn().mockResolvedValue(true),
87
87
  listTargetConnectionIds: undefined,
88
- finalize: undefined,
89
88
  chunk: vi.fn().mockResolvedValue({
90
89
  workUnits: [{ unitKey: 'u1', rawFiles: ['a.yml'], peerFileIndex: [], dependencyPaths: [] }],
91
90
  }),
@@ -122,7 +121,6 @@ const makeDeps = () => {
122
121
  }),
123
122
  applyPatchFile3WayIndex: vi.fn(),
124
123
  diffNameStatus: vi.fn().mockResolvedValue([]),
125
- changedPaths: vi.fn().mockResolvedValue([]),
126
124
  };
127
125
  const sessionWorktreeService = {
128
126
  create: vi.fn().mockResolvedValue({
@@ -1346,65 +1344,25 @@ describe('IngestBundleRunner — Stages 1 → 7', () => {
1346
1344
  }),
1347
1345
  }));
1348
1346
  });
1349
- it('runs adapter finalization before squash, records the outcome, and reindexes touched sources', async () => {
1347
+ it('runs a registered post-processor before squash, records the outcome, and reindexes touched sources after squash', async () => {
1350
1348
  const deps = makeDeps();
1351
1349
  deps.adapter.source = 'metricflow';
1352
1350
  deps.registry.get.mockReturnValue(deps.adapter);
1353
1351
  deps.adapter.chunk.mockResolvedValue({
1354
- workUnits: [],
1352
+ workUnits: [{ unitKey: 'u1', rawFiles: ['semantic_models.yml'], peerFileIndex: [], dependencyPaths: [] }],
1355
1353
  parseArtifacts: { semanticModels: [{ name: 'orders' }] },
1356
1354
  });
1357
1355
  deps.adapter.listTargetConnectionIds = vi.fn().mockResolvedValue(['warehouse-2']);
1358
- deps.adapter.finalize = vi.fn().mockResolvedValue({
1359
- result: { sourcesTouched: 1 },
1360
- warnings: ['kept going'],
1361
- errors: [],
1362
- touchedSources: [{ connectionId: 'warehouse-2', sourceName: 'orders' }],
1363
- changedWikiPageKeys: [],
1364
- actions: [
1365
- {
1366
- target: 'sl',
1367
- type: 'updated',
1368
- key: 'orders',
1369
- targetConnectionId: 'warehouse-2',
1370
- detail: 'Finalized orders usage',
1371
- rawPaths: ['semantic_models.yml'],
1372
- },
1373
- ],
1374
- });
1375
1356
  deps.semanticLayerService.loadAllSources.mockImplementation((connectionId) => Promise.resolve({ sources: [{ name: `${connectionId}_source` }], loadErrors: [] }));
1376
- let head = 'pre-finalization';
1377
- const git = {
1378
- revParseHead: vi.fn(async () => head),
1379
- commitFiles: vi.fn().mockImplementation(async (paths) => {
1380
- if (paths.includes('semantic-layer/warehouse-2/orders.yaml')) {
1381
- head = 'post-finalization';
1382
- return { created: true, commitHash: 'finalization-sha' };
1383
- }
1384
- return { created: true, commitHash: head };
1385
- }),
1386
- commitStaged: vi.fn().mockResolvedValue({ created: false, commitHash: 'post-finalization' }),
1387
- resetHardTo: vi.fn(),
1388
- assertWorktreeClean: vi.fn().mockResolvedValue(undefined),
1389
- writeBinaryNoRenamePatch: vi.fn(async (_base, _head, patchPath) => {
1390
- await writeFile(patchPath, '', 'utf-8');
1357
+ const postProcessor = {
1358
+ run: vi.fn().mockResolvedValue({
1359
+ result: { sourcesCreated: 1 },
1360
+ warnings: ['kept going'],
1361
+ errors: [],
1362
+ touchedSources: [{ connectionId: 'warehouse-2', sourceName: 'orders' }],
1391
1363
  }),
1392
- applyPatchFile3WayIndex: vi.fn(),
1393
- diffNameStatus: vi.fn().mockImplementation(async (from, to) => from === 'pre-finalization' && to === 'post-finalization'
1394
- ? [{ status: 'M', path: 'semantic-layer/warehouse-2/orders.yaml' }]
1395
- : []),
1396
- changedPaths: vi.fn().mockResolvedValue(['semantic-layer/warehouse-2/orders.yaml']),
1397
1364
  };
1398
- deps.sessionWorktreeService.create.mockResolvedValue({
1399
- chatId: 'j1',
1400
- workdir: '/tmp/wt',
1401
- branch: 'session/j1',
1402
- baseSha: 'b',
1403
- createdAt: new Date(),
1404
- git,
1405
- config: {},
1406
- });
1407
- const runner = buildRunner(deps);
1365
+ const runner = buildRunner(deps, { postProcessors: { metricflow: postProcessor } });
1408
1366
  runner.stageRawFilesStage1 = vi.fn().mockResolvedValue({
1409
1367
  currentHashes: new Map([['semantic_models.yml', 'h1']]),
1410
1368
  rawDirInWorktree: 'raw-sources/c1/metricflow/s',
@@ -1417,7 +1375,7 @@ describe('IngestBundleRunner — Stages 1 → 7', () => {
1417
1375
  trigger: 'upload',
1418
1376
  bundleRef: { kind: 'upload', uploadId: 'upload-x' },
1419
1377
  });
1420
- expect(deps.adapter.finalize).toHaveBeenCalledWith(expect.objectContaining({
1378
+ expect(postProcessor.run).toHaveBeenCalledWith({
1421
1379
  connectionId: 'c1',
1422
1380
  sourceKey: 'metricflow',
1423
1381
  syncId: expect.any(String),
@@ -1425,25 +1383,24 @@ describe('IngestBundleRunner — Stages 1 → 7', () => {
1425
1383
  runId: 'run-1',
1426
1384
  workdir: '/tmp/wt',
1427
1385
  parseArtifacts: { semanticModels: [{ name: 'orders' }] },
1428
- }));
1386
+ });
1429
1387
  expect(deps.reportsRepo.create).toHaveBeenCalledWith(expect.objectContaining({
1430
1388
  body: expect.objectContaining({
1431
- finalization: expect.objectContaining({
1389
+ postProcessor: {
1432
1390
  sourceKey: 'metricflow',
1433
1391
  status: 'success',
1434
- commitSha: 'finalization-sha',
1435
- touchedPaths: ['semantic-layer/warehouse-2/orders.yaml'],
1436
- derivedTouchedSources: [{ connectionId: 'warehouse-2', sourceName: 'orders' }],
1437
- declaredTouchedSources: [{ connectionId: 'warehouse-2', sourceName: 'orders' }],
1438
- actions: [expect.objectContaining({ key: 'orders' })],
1439
- }),
1392
+ result: { sourcesCreated: 1 },
1393
+ warnings: ['kept going'],
1394
+ errors: [],
1395
+ touchedSources: [{ connectionId: 'warehouse-2', sourceName: 'orders' }],
1396
+ },
1440
1397
  }),
1441
1398
  }));
1442
1399
  expect(deps.semanticLayerService.loadAllSources).toHaveBeenCalledWith('warehouse-2');
1443
1400
  expect(deps.slSearchService.indexSources).toHaveBeenCalledWith('warehouse-2', [{ name: 'warehouse-2_source' }]);
1444
1401
  expect(deps.sessionWorktreeService.cleanup).toHaveBeenCalledWith(expect.any(Object), 'success');
1445
1402
  });
1446
- it('includes finalization actions in memory-flow saved counts', async () => {
1403
+ it('includes historic-sql post-processor output in memory-flow saved counts', async () => {
1447
1404
  const deps = makeDeps();
1448
1405
  deps.adapter.source = 'historic-sql';
1449
1406
  deps.registry.get.mockReturnValue(deps.adapter);
@@ -1457,19 +1414,21 @@ describe('IngestBundleRunner — Stages 1 → 7', () => {
1457
1414
  },
1458
1415
  ],
1459
1416
  });
1460
- deps.adapter.finalize = vi.fn().mockResolvedValue({
1461
- warnings: [],
1462
- errors: [],
1463
- touchedSources: [],
1464
- changedWikiPageKeys: [],
1465
- actions: [
1466
- { target: 'sl', type: 'updated', key: 'orders', detail: 'Merged usage' },
1467
- { target: 'sl', type: 'updated', key: 'customers', detail: 'Merged usage' },
1468
- { target: 'wiki', type: 'created', key: 'historic-sql-orders', detail: 'Projected pattern' },
1469
- { target: 'wiki', type: 'updated', key: 'historic-sql-customers', detail: 'Projected pattern' },
1470
- ],
1471
- });
1472
- const runner = buildRunner(deps);
1417
+ const postProcessor = {
1418
+ run: vi.fn().mockResolvedValue({
1419
+ result: {
1420
+ tableUsageMerged: 2,
1421
+ staleTablesMarked: 1,
1422
+ patternPagesWritten: 3,
1423
+ stalePatternPagesMarked: 1,
1424
+ archivedPatternPages: 1,
1425
+ },
1426
+ warnings: [],
1427
+ errors: [],
1428
+ touchedSources: [{ connectionId: 'c1', sourceName: 'orders' }],
1429
+ }),
1430
+ };
1431
+ const runner = buildRunner(deps, { postProcessors: { 'historic-sql': postProcessor } });
1473
1432
  runner.stageRawFilesStage1 = vi.fn().mockResolvedValue({
1474
1433
  currentHashes: new Map([['tables/public/orders.json', 'h1']]),
1475
1434
  rawDirInWorktree: 'raw-sources/c1/historic-sql/s',
@@ -1489,11 +1448,11 @@ describe('IngestBundleRunner — Stages 1 → 7', () => {
1489
1448
  });
1490
1449
  expect(memoryFlow.snapshot().events).toContainEqual(expect.objectContaining({
1491
1450
  type: 'saved',
1492
- wikiCount: 2,
1493
- slCount: 2,
1451
+ wikiCount: 5,
1452
+ slCount: 3,
1494
1453
  }));
1495
1454
  });
1496
- it('marks finalization infrastructure failure as failed and preserves worktree cleanup state', async () => {
1455
+ it('marks post-processor infrastructure failure as failed and preserves worktree cleanup state', async () => {
1497
1456
  const deps = makeDeps();
1498
1457
  deps.adapter.source = 'metricflow';
1499
1458
  deps.registry.get.mockReturnValue(deps.adapter);
@@ -1501,8 +1460,8 @@ describe('IngestBundleRunner — Stages 1 → 7', () => {
1501
1460
  workUnits: [{ unitKey: 'u1', rawFiles: ['semantic_models.yml'], peerFileIndex: [], dependencyPaths: [] }],
1502
1461
  parseArtifacts: { semanticModels: [{ name: 'orders' }] },
1503
1462
  });
1504
- deps.adapter.finalize = vi.fn().mockRejectedValue(new Error('worktree write failed'));
1505
- const runner = buildRunner(deps);
1463
+ const postProcessor = { run: vi.fn().mockRejectedValue(new Error('worktree write failed')) };
1464
+ const runner = buildRunner(deps, { postProcessors: { metricflow: postProcessor } });
1506
1465
  runner.stageRawFilesStage1 = vi.fn().mockResolvedValue({
1507
1466
  currentHashes: new Map([['semantic_models.yml', 'h1']]),
1508
1467
  rawDirInWorktree: 'raw-sources/c1/metricflow/s',
@@ -1519,120 +1478,6 @@ describe('IngestBundleRunner — Stages 1 → 7', () => {
1519
1478
  expect(deps.gitService.squashMergeIntoMain).not.toHaveBeenCalled();
1520
1479
  expect(deps.sessionWorktreeService.cleanup).toHaveBeenCalledWith(expect.any(Object), 'crash');
1521
1480
  });
1522
- it('reports finalization actions excluded from provenance when raw paths are not defensible', async () => {
1523
- const deps = makeDeps();
1524
- deps.adapter.finalize = vi.fn().mockResolvedValue({
1525
- warnings: [],
1526
- errors: [],
1527
- touchedSources: [],
1528
- changedWikiPageKeys: [],
1529
- actions: [
1530
- { target: 'wiki', type: 'updated', key: 'historic-sql-pattern', detail: 'No raw path' },
1531
- { target: 'sl', type: 'updated', key: 'orders', detail: 'Invalid raw path', rawPaths: ['missing.json'] },
1532
- ],
1533
- });
1534
- const runner = buildRunner(deps);
1535
- runner.stageRawFilesStage1 = vi.fn().mockResolvedValue({
1536
- currentHashes: new Map([['current.json', 'h1']]),
1537
- rawDirInWorktree: 'raw-sources/c1/fake/s',
1538
- });
1539
- runner.resolveStagedDir = vi.fn().mockResolvedValue('/tmp/stage/upload-x');
1540
- await runner.run({
1541
- jobId: 'j1',
1542
- connectionId: 'c1',
1543
- sourceKey: 'fake',
1544
- trigger: 'upload',
1545
- bundleRef: { kind: 'upload', uploadId: 'upload-x' },
1546
- });
1547
- expect(deps.reportsRepo.create).toHaveBeenCalledWith(expect.objectContaining({
1548
- body: expect.objectContaining({
1549
- finalization: expect.objectContaining({
1550
- provenanceExclusions: [
1551
- expect.objectContaining({ reason: 'missing_raw_paths' }),
1552
- expect.objectContaining({ reason: 'raw_path_not_defensible', invalidRawPaths: ['missing.json'] }),
1553
- ],
1554
- }),
1555
- }),
1556
- }));
1557
- expect(deps.provenanceRepo.insertMany).not.toHaveBeenCalledWith(expect.arrayContaining([expect.objectContaining({ rawPath: 'missing.json' })]));
1558
- });
1559
- it('passes explicit override replay metadata and no current work unit outcomes', async () => {
1560
- const deps = makeDeps();
1561
- deps.reportsRepo.findByJobId.mockResolvedValue({
1562
- id: 'prior-report',
1563
- runId: 'prior-run',
1564
- jobId: 'prior-job',
1565
- connectionId: 'c1',
1566
- sourceKey: 'fake',
1567
- createdAt: '2026-05-18T00:00:00.000Z',
1568
- body: {
1569
- status: 'completed',
1570
- syncId: 'prior-sync',
1571
- diffSummary: { added: 0, modified: 0, deleted: 0, unchanged: 0 },
1572
- commitSha: 'prior-sha',
1573
- workUnits: [
1574
- {
1575
- unitKey: 'prior-unit',
1576
- rawFiles: ['prior.json'],
1577
- status: 'success',
1578
- actions: [{ target: 'wiki', type: 'created', key: 'prior', detail: 'prior' }],
1579
- touchedSlSources: [],
1580
- },
1581
- ],
1582
- failedWorkUnits: [],
1583
- reconciliationSkipped: false,
1584
- conflictsResolved: [],
1585
- evictionsApplied: [
1586
- {
1587
- rawPath: 'do-not-replay.json',
1588
- artifactKind: 'wiki',
1589
- artifactKey: 'old',
1590
- action: 'removed',
1591
- reason: 'prior',
1592
- },
1593
- ],
1594
- unmappedFallbacks: [],
1595
- artifactResolutions: [],
1596
- evictionInputs: ['evicted-from-prior-report.json'],
1597
- unresolvedCards: [],
1598
- supersededBy: null,
1599
- overrideOf: null,
1600
- provenanceRows: [],
1601
- toolTranscripts: [],
1602
- },
1603
- });
1604
- deps.adapter.finalize = vi.fn().mockResolvedValue({
1605
- warnings: [],
1606
- errors: [],
1607
- touchedSources: [],
1608
- changedWikiPageKeys: [],
1609
- actions: [],
1610
- });
1611
- deps.gitService.listFilesAtHead.mockResolvedValue(['raw-sources/c1/fake/prior-sync/prior.json']);
1612
- deps.gitService.getFileAtCommit.mockResolvedValue('{"id":1}\n');
1613
- const runner = buildRunner(deps);
1614
- runner.stageRawFilesStage1 = vi.fn().mockResolvedValue({
1615
- currentHashes: new Map([['prior.json', 'h1']]),
1616
- rawDirInWorktree: 'raw-sources/c1/fake/prior-sync',
1617
- });
1618
- runner.resolveStagedDir = vi.fn().mockResolvedValue('/tmp/stage/prior');
1619
- await runner.run({
1620
- jobId: 'override-job',
1621
- connectionId: 'c1',
1622
- sourceKey: 'fake',
1623
- trigger: 'manual_override',
1624
- bundleRef: { kind: 'override', priorJobId: 'prior-job' },
1625
- });
1626
- expect(deps.adapter.finalize).toHaveBeenCalledWith(expect.objectContaining({
1627
- workUnitOutcomes: [],
1628
- overrideReplay: {
1629
- priorJobId: 'prior-job',
1630
- priorRunId: 'prior-run',
1631
- priorSyncId: 'prior-sync',
1632
- evictionRawPaths: ['evicted-from-prior-report.json'],
1633
- },
1634
- }));
1635
- });
1636
1481
  it('includes existing global wiki pages in WorkUnit prompts', async () => {
1637
1482
  const deps = makeDeps();
1638
1483
  deps.knowledgeIndex.listPagesForUser.mockResolvedValue([
@@ -7,7 +7,6 @@ import { initKtxProject, loadKtxProject } from '../project/index.js';
7
7
  import { makeLocalGitRepo } from '../test/make-local-git-repo.js';
8
8
  import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
9
9
  import { FakeSourceAdapter } from './adapters/fake/fake.adapter.js';
10
- import { projectHistoricSqlEvidence } from './adapters/historic-sql/projection.js';
11
10
  import { LocalLookerRuntimeStore } from './adapters/looker/local-runtime-store.js';
12
11
  import { createDefaultLocalIngestAdapters, localPullConfigForAdapter } from './local-adapters.js';
13
12
  import { getLocalIngestStatus, runLocalIngest } from './local-ingest.js';
@@ -160,23 +159,6 @@ class HistoricSqlEvidenceTestAdapter {
160
159
  ],
161
160
  });
162
161
  }
163
- async finalize(ctx) {
164
- const projection = await projectHistoricSqlEvidence({
165
- workdir: ctx.workdir,
166
- connectionId: ctx.connectionId,
167
- syncId: ctx.syncId,
168
- runId: ctx.runId,
169
- overrideReplay: ctx.overrideReplay,
170
- });
171
- return {
172
- result: projection,
173
- warnings: projection.warnings,
174
- errors: [],
175
- touchedSources: projection.touchedSources,
176
- changedWikiPageKeys: projection.changedWikiPageKeys,
177
- actions: projection.actions,
178
- };
179
- }
180
162
  }
181
163
  function makeLookerRuntimeClient() {
182
164
  const lookerModels = {
@@ -444,7 +426,7 @@ describe('canonical local ingest', () => {
444
426
  },
445
427
  ]);
446
428
  });
447
- it('runs historic-SQL evidence projection through local bundle finalization', async () => {
429
+ it('runs historic-SQL evidence projection through the local bundle post-processor', async () => {
448
430
  const projectDir = join(tempDir, 'historic-sql-project');
449
431
  await initKtxProject({ projectDir });
450
432
  await writeFile(join(projectDir, 'ktx.yaml'), [
@@ -495,12 +477,11 @@ describe('canonical local ingest', () => {
495
477
  agentRunner,
496
478
  });
497
479
  expect(result.result.failedWorkUnits).toEqual([]);
498
- expect(result.report.body.finalization).toMatchObject({
480
+ expect(result.report.body.postProcessor).toMatchObject({
499
481
  sourceKey: 'historic-sql',
500
482
  status: 'success',
501
483
  result: { tableUsageMerged: 1 },
502
- declaredTouchedSources: [{ connectionId: 'warehouse', sourceName: 'orders' }],
503
- derivedTouchedSources: [{ connectionId: 'warehouse', sourceName: 'orders' }],
484
+ touchedSources: [{ connectionId: 'warehouse', sourceName: 'orders' }],
504
485
  });
505
486
  await expect(readFile(join(projectDir, 'semantic-layer/warehouse/_schema/public.yaml'), 'utf-8')).resolves.toContain('Orders are repeatedly queried by lifecycle status.');
506
487
  });
@@ -13,6 +13,7 @@ import { ContextCandidateMarkTool, ContextCandidateWriteTool, ContextEvidenceNei
13
13
  import { buildKnowledgeSearchText, KnowledgeWikiService, searchLocalKnowledgePages, SqliteKnowledgeIndex, WikiListTagsTool, WikiReadTool, WikiRemoveTool, WikiSearchTool, WikiWriteTool, } from '../wiki/index.js';
14
14
  import { CandidateDedupService, ContextCandidateCarryforwardService, CuratorPaginationService, } from './context-candidates/index.js';
15
15
  import { createEmitHistoricSqlEvidenceTool } from './adapters/historic-sql/evidence-tool.js';
16
+ import { HistoricSqlProjectionPostProcessor } from './adapters/historic-sql/post-processor.js';
16
17
  import { ContextEvidenceIndexService, SqliteContextEvidenceStore } from './context-evidence/index.js';
17
18
  import { DiffSetService } from './diff-set.service.js';
18
19
  import { ingestTracePathForJob } from './ingest-trace.js';
@@ -585,6 +586,9 @@ export function createLocalBundleIngestRuntime(options) {
585
586
  settings: { batchSize: 8, maxPasses: 8, stepBudgetPerPass: 60 },
586
587
  logger,
587
588
  }),
589
+ postProcessors: {
590
+ 'historic-sql': new HistoricSqlProjectionPostProcessor(),
591
+ },
588
592
  logger,
589
593
  };
590
594
  return {
@@ -178,6 +178,13 @@ async function recordLocalMetabaseChildFailure(options) {
178
178
  overrideOf: null,
179
179
  provenanceRows: [],
180
180
  toolTranscripts: [],
181
+ postProcessor: {
182
+ sourceKey: 'metabase',
183
+ status: 'failed',
184
+ errors: [reason],
185
+ warnings: [],
186
+ touchedSources: [],
187
+ },
181
188
  };
182
189
  const report = await store.create({
183
190
  runId: run.id,
@@ -43,8 +43,8 @@ export declare const memoryFlowEventSchema: z.ZodDiscriminatedUnion<[z.ZodObject
43
43
  stage: z.ZodEnum<{
44
44
  source: "source";
45
45
  workUnits: "workUnits";
46
- chunks: "chunks";
47
46
  actions: "actions";
47
+ chunks: "chunks";
48
48
  gates: "gates";
49
49
  saved: "saved";
50
50
  }>;
@@ -58,7 +58,7 @@ export declare const memoryFlowEventSchema: z.ZodDiscriminatedUnion<[z.ZodObject
58
58
  report: "report";
59
59
  integration: "integration";
60
60
  reconciliation: "reconciliation";
61
- finalization: "finalization";
61
+ post_processor: "post_processor";
62
62
  wiki_sl_ref_repair: "wiki_sl_ref_repair";
63
63
  final_gates: "final_gates";
64
64
  save: "save";
@@ -143,8 +143,8 @@ export declare const memoryFlowActionDetailSchema: z.ZodObject<{
143
143
  }>;
144
144
  action: z.ZodEnum<{
145
145
  created: "created";
146
- updated: "updated";
147
146
  removed: "removed";
147
+ updated: "updated";
148
148
  }>;
149
149
  key: z.ZodString;
150
150
  summary: z.ZodString;
@@ -163,8 +163,8 @@ export declare const memoryFlowDetailSectionsSchema: z.ZodObject<{
163
163
  }>;
164
164
  action: z.ZodEnum<{
165
165
  created: "created";
166
- updated: "updated";
167
166
  removed: "removed";
167
+ updated: "updated";
168
168
  }>;
169
169
  key: z.ZodString;
170
170
  summary: z.ZodString;
@@ -53,7 +53,7 @@ export const memoryFlowEventSchema = z.discriminatedUnion('type', [
53
53
  'source',
54
54
  'integration',
55
55
  'reconciliation',
56
- 'finalization',
56
+ 'post_processor',
57
57
  'wiki_sl_ref_repair',
58
58
  'final_gates',
59
59
  'save',
@@ -40,7 +40,7 @@ type MemoryFlowEventPayload = {
40
40
  reason: string;
41
41
  } | {
42
42
  type: 'stage_progress';
43
- stage: 'source' | 'integration' | 'reconciliation' | 'finalization' | 'wiki_sl_ref_repair' | 'final_gates' | 'save' | 'provenance' | 'report';
43
+ stage: 'source' | 'integration' | 'reconciliation' | 'post_processor' | 'wiki_sl_ref_repair' | 'final_gates' | 'save' | 'provenance' | 'report';
44
44
  percent: number;
45
45
  message: string;
46
46
  transient?: boolean;
@@ -6,7 +6,7 @@ import type { CaptureSession, MemoryAction, MemoryKnowledgeSlRefsPort } from '..
6
6
  import type { PromptService } from '../prompts/index.js';
7
7
  import type { SkillsRegistryService } from '../skills/index.js';
8
8
  import type { SemanticLayerService, SlConnectionCatalogPort, SlSearchService, SlSourcesIndexPort, SlValidationDeps, SlValidatorPort } from '../sl/index.js';
9
- import type { ToolContext, ToolSession } from '../tools/index.js';
9
+ import type { ToolContext, ToolSession, TouchedSlSource } from '../tools/index.js';
10
10
  import type { KnowledgeIndexPort, KnowledgeWikiService } from '../wiki/index.js';
11
11
  import type { CanonicalPin } from './canonical-pins.js';
12
12
  import type { IngestTraceLevel } from './ingest-trace.js';
@@ -281,6 +281,24 @@ export interface CuratorPaginationPort {
281
281
  warnings: string[];
282
282
  }>;
283
283
  }
284
+ export interface IngestBundlePostProcessorInput {
285
+ connectionId: string;
286
+ sourceKey: string;
287
+ syncId: string;
288
+ jobId: string;
289
+ runId: string;
290
+ workdir: string;
291
+ parseArtifacts: unknown;
292
+ }
293
+ export interface IngestBundlePostProcessorResult {
294
+ result?: unknown;
295
+ warnings: string[];
296
+ errors: string[];
297
+ touchedSources: TouchedSlSource[];
298
+ }
299
+ export interface IngestBundlePostProcessorPort {
300
+ run(input: IngestBundlePostProcessorInput): Promise<IngestBundlePostProcessorResult>;
301
+ }
284
302
  export interface IngestBundleRunnerDeps {
285
303
  runs: IngestRunsPort;
286
304
  provenance: IngestProvenancePort;
@@ -314,6 +332,7 @@ export interface IngestBundleRunnerDeps {
314
332
  candidateDedup?: CandidateDedupPort;
315
333
  contextCandidateCarryforward?: ContextCandidateCarryforwardPort;
316
334
  curatorPagination?: CuratorPaginationPort;
335
+ postProcessors?: Record<string, IngestBundlePostProcessorPort>;
317
336
  logger?: KtxLogger;
318
337
  }
319
338
  export interface IngestCaptureState {
@@ -103,8 +103,8 @@ export declare const ingestReportSnapshotSchema: z.ZodObject<{
103
103
  }>;
104
104
  type: z.ZodEnum<{
105
105
  created: "created";
106
- updated: "updated";
107
106
  removed: "removed";
107
+ updated: "updated";
108
108
  }>;
109
109
  key: z.ZodString;
110
110
  detail: z.ZodString;
@@ -129,8 +129,8 @@ export declare const ingestReportSnapshotSchema: z.ZodObject<{
129
129
  }>;
130
130
  type: z.ZodEnum<{
131
131
  created: "created";
132
- updated: "updated";
133
132
  removed: "removed";
133
+ updated: "updated";
134
134
  }>;
135
135
  key: z.ZodString;
136
136
  detail: z.ZodString;
@@ -221,77 +221,6 @@ export declare const ingestReportSnapshotSchema: z.ZodObject<{
221
221
  errorCount: z.ZodNumber;
222
222
  toolNames: z.ZodArray<z.ZodString>;
223
223
  }, z.core.$strip>>>;
224
- finalization: z.ZodOptional<z.ZodObject<{
225
- sourceKey: z.ZodString;
226
- status: z.ZodEnum<{
227
- success: "success";
228
- skipped: "skipped";
229
- failed: "failed";
230
- }>;
231
- commitSha: z.ZodNullable<z.ZodString>;
232
- touchedPaths: z.ZodArray<z.ZodString>;
233
- declaredTouchedSources: z.ZodArray<z.ZodObject<{
234
- connectionId: z.ZodString;
235
- sourceName: z.ZodString;
236
- }, z.core.$strip>>;
237
- derivedTouchedSources: z.ZodArray<z.ZodObject<{
238
- connectionId: z.ZodString;
239
- sourceName: z.ZodString;
240
- }, z.core.$strip>>;
241
- declaredChangedWikiPageKeys: z.ZodArray<z.ZodString>;
242
- derivedChangedWikiPageKeys: z.ZodArray<z.ZodString>;
243
- mismatches: z.ZodDefault<z.ZodArray<z.ZodObject<{
244
- artifactKind: z.ZodEnum<{
245
- wiki: "wiki";
246
- sl: "sl";
247
- }>;
248
- key: z.ZodString;
249
- direction: z.ZodEnum<{
250
- missing_from_adapter_declaration: "missing_from_adapter_declaration";
251
- extra_in_adapter_declaration: "extra_in_adapter_declaration";
252
- }>;
253
- }, z.core.$strip>>>;
254
- result: z.ZodOptional<z.ZodUnknown>;
255
- errors: z.ZodArray<z.ZodString>;
256
- warnings: z.ZodArray<z.ZodString>;
257
- actions: z.ZodDefault<z.ZodArray<z.ZodObject<{
258
- target: z.ZodEnum<{
259
- wiki: "wiki";
260
- sl: "sl";
261
- }>;
262
- type: z.ZodEnum<{
263
- created: "created";
264
- updated: "updated";
265
- removed: "removed";
266
- }>;
267
- key: z.ZodString;
268
- detail: z.ZodString;
269
- targetConnectionId: z.ZodDefault<z.ZodNullable<z.ZodString>>;
270
- rawPaths: z.ZodOptional<z.ZodArray<z.ZodString>>;
271
- }, z.core.$strip>>>;
272
- provenanceExclusions: z.ZodDefault<z.ZodArray<z.ZodObject<{
273
- action: z.ZodObject<{
274
- target: z.ZodEnum<{
275
- wiki: "wiki";
276
- sl: "sl";
277
- }>;
278
- type: z.ZodEnum<{
279
- created: "created";
280
- updated: "updated";
281
- removed: "removed";
282
- }>;
283
- key: z.ZodString;
284
- detail: z.ZodString;
285
- targetConnectionId: z.ZodDefault<z.ZodNullable<z.ZodString>>;
286
- rawPaths: z.ZodOptional<z.ZodArray<z.ZodString>>;
287
- }, z.core.$strip>;
288
- reason: z.ZodEnum<{
289
- missing_raw_paths: "missing_raw_paths";
290
- raw_path_not_defensible: "raw_path_not_defensible";
291
- }>;
292
- invalidRawPaths: z.ZodOptional<z.ZodArray<z.ZodString>>;
293
- }, z.core.$strip>>>;
294
- }, z.core.$strip>>;
295
224
  memoryFlow: z.ZodOptional<z.ZodType<import("./index.js").MemoryFlowReplayInput, unknown, z.core.$ZodTypeInternals<import("./index.js").MemoryFlowReplayInput, unknown>>>;
296
225
  }, z.core.$loose>;
297
226
  }, z.core.$loose>;
@@ -114,32 +114,6 @@ const ingestReportFailureSchema = z.object({
114
114
  message: z.string().min(1),
115
115
  details: z.record(z.string(), z.unknown()).optional(),
116
116
  });
117
- const finalizationMismatchSchema = z.object({
118
- artifactKind: z.enum(['sl', 'wiki']),
119
- key: z.string().min(1),
120
- direction: z.enum(['missing_from_adapter_declaration', 'extra_in_adapter_declaration']),
121
- });
122
- const finalizationProvenanceExclusionSchema = z.object({
123
- action: ingestActionSchema,
124
- reason: z.enum(['missing_raw_paths', 'raw_path_not_defensible']),
125
- invalidRawPaths: z.array(z.string()).optional(),
126
- });
127
- const finalizationOutcomeSchema = z.object({
128
- sourceKey: z.string().min(1),
129
- status: z.enum(['success', 'failed', 'skipped']),
130
- commitSha: z.string().nullable(),
131
- touchedPaths: z.array(z.string()),
132
- declaredTouchedSources: z.array(touchedSlSourceSchema),
133
- derivedTouchedSources: z.array(touchedSlSourceSchema),
134
- declaredChangedWikiPageKeys: z.array(z.string()),
135
- derivedChangedWikiPageKeys: z.array(z.string()),
136
- mismatches: z.array(finalizationMismatchSchema).default([]),
137
- result: z.unknown().optional(),
138
- errors: z.array(z.string()),
139
- warnings: z.array(z.string()),
140
- actions: z.array(ingestActionSchema).default([]),
141
- provenanceExclusions: z.array(finalizationProvenanceExclusionSchema).default([]),
142
- });
143
117
  export const ingestReportSnapshotSchema = z
144
118
  .object({
145
119
  id: z.string().min(1),
@@ -197,7 +171,6 @@ export const ingestReportSnapshotSchema = z
197
171
  overrideOf: z.string().nullable().default(null),
198
172
  provenanceRows: z.array(provenanceDetailSchema).default([]),
199
173
  toolTranscripts: z.array(toolTranscriptSummarySchema).default([]),
200
- finalization: finalizationOutcomeSchema.optional(),
201
174
  memoryFlow: memoryFlowReplayInputSchema.optional(),
202
175
  })
203
176
  .passthrough(),