@kaelio/ktx 0.1.0-rc.6 → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/assets/python/{kaelio_ktx-0.1.0rc6-py3-none-any.whl → kaelio_ktx-0.1.0-py3-none-any.whl} +0 -0
- package/assets/python/manifest.json +4 -4
- package/dist/commands/mcp-commands.js +11 -3
- package/dist/commands/mcp-commands.test.js +30 -1
- package/dist/ingest.test.js +2 -26
- package/dist/next-steps.js +1 -1
- package/dist/next-steps.test.js +2 -0
- package/dist/runtime-requirements.d.ts +1 -2
- package/dist/runtime-requirements.js +0 -7
- package/dist/runtime-requirements.test.js +2 -2
- package/dist/setup-agents.d.ts +11 -3
- package/dist/setup-agents.js +397 -134
- package/dist/setup-agents.test.js +359 -61
- package/dist/setup-runtime.d.ts +0 -1
- package/dist/setup-runtime.js +0 -1
- package/dist/setup-runtime.test.js +7 -13
- package/dist/setup.d.ts +3 -0
- package/dist/setup.js +51 -25
- package/dist/setup.test.js +112 -16
- package/node_modules/@ktx/connector-clickhouse/dist/package-exports.test.js +1 -1
- package/node_modules/@ktx/context/dist/core/git.service.d.ts +0 -1
- package/node_modules/@ktx/context/dist/core/git.service.js +0 -12
- package/node_modules/@ktx/context/dist/ingest/adapters/historic-sql/historic-sql.adapter.d.ts +1 -2
- package/node_modules/@ktx/context/dist/ingest/adapters/historic-sql/historic-sql.adapter.js +0 -18
- package/node_modules/@ktx/context/dist/ingest/adapters/historic-sql/local-ingest-acceptance.test.js +6 -6
- package/node_modules/@ktx/context/dist/ingest/adapters/historic-sql/post-processor.d.ts +4 -0
- package/node_modules/@ktx/context/dist/ingest/adapters/historic-sql/post-processor.js +38 -0
- package/node_modules/@ktx/context/dist/ingest/adapters/historic-sql/post-processor.test.js +63 -0
- package/node_modules/@ktx/context/dist/ingest/adapters/historic-sql/projection.d.ts +0 -5
- package/node_modules/@ktx/context/dist/ingest/adapters/historic-sql/projection.js +0 -48
- package/node_modules/@ktx/context/dist/ingest/adapters/historic-sql/projection.test.js +0 -83
- package/node_modules/@ktx/context/dist/ingest/index.d.ts +2 -1
- package/node_modules/@ktx/context/dist/ingest/index.js +1 -0
- package/node_modules/@ktx/context/dist/ingest/ingest-bundle.runner.d.ts +0 -2
- package/node_modules/@ktx/context/dist/ingest/ingest-bundle.runner.isolated-diff.test.js +0 -166
- package/node_modules/@ktx/context/dist/ingest/ingest-bundle.runner.js +45 -235
- package/node_modules/@ktx/context/dist/ingest/ingest-bundle.runner.test.js +38 -193
- package/node_modules/@ktx/context/dist/ingest/local-bundle-ingest.test.js +3 -22
- package/node_modules/@ktx/context/dist/ingest/local-bundle-runtime.js +4 -0
- package/node_modules/@ktx/context/dist/ingest/local-ingest.js +7 -0
- package/node_modules/@ktx/context/dist/ingest/memory-flow/schema.d.ts +4 -4
- package/node_modules/@ktx/context/dist/ingest/memory-flow/schema.js +1 -1
- package/node_modules/@ktx/context/dist/ingest/memory-flow/types.d.ts +1 -1
- package/node_modules/@ktx/context/dist/ingest/ports.d.ts +20 -1
- package/node_modules/@ktx/context/dist/ingest/report-snapshot.d.ts +2 -73
- package/node_modules/@ktx/context/dist/ingest/report-snapshot.js +0 -27
- package/node_modules/@ktx/context/dist/ingest/reports.d.ts +5 -23
- package/node_modules/@ktx/context/dist/ingest/reports.js +24 -7
- package/node_modules/@ktx/context/dist/ingest/types.d.ts +0 -33
- package/node_modules/@ktx/context/dist/package-exports.test.js +1 -2
- package/package.json +4 -4
- package/node_modules/@ktx/context/dist/ingest/finalization-scope.d.ts +0 -22
- package/node_modules/@ktx/context/dist/ingest/finalization-scope.js +0 -95
- package/node_modules/@ktx/context/dist/ingest/finalization-scope.test.js +0 -114
- /package/node_modules/@ktx/context/dist/ingest/{finalization-scope.test.d.ts → adapters/historic-sql/post-processor.test.d.ts} +0 -0
|
@@ -85,7 +85,6 @@ const makeDeps = () => {
|
|
|
85
85
|
triageSupported: undefined,
|
|
86
86
|
detect: vi.fn().mockResolvedValue(true),
|
|
87
87
|
listTargetConnectionIds: undefined,
|
|
88
|
-
finalize: undefined,
|
|
89
88
|
chunk: vi.fn().mockResolvedValue({
|
|
90
89
|
workUnits: [{ unitKey: 'u1', rawFiles: ['a.yml'], peerFileIndex: [], dependencyPaths: [] }],
|
|
91
90
|
}),
|
|
@@ -122,7 +121,6 @@ const makeDeps = () => {
|
|
|
122
121
|
}),
|
|
123
122
|
applyPatchFile3WayIndex: vi.fn(),
|
|
124
123
|
diffNameStatus: vi.fn().mockResolvedValue([]),
|
|
125
|
-
changedPaths: vi.fn().mockResolvedValue([]),
|
|
126
124
|
};
|
|
127
125
|
const sessionWorktreeService = {
|
|
128
126
|
create: vi.fn().mockResolvedValue({
|
|
@@ -1346,65 +1344,25 @@ describe('IngestBundleRunner — Stages 1 → 7', () => {
|
|
|
1346
1344
|
}),
|
|
1347
1345
|
}));
|
|
1348
1346
|
});
|
|
1349
|
-
it('runs
|
|
1347
|
+
it('runs a registered post-processor before squash, records the outcome, and reindexes touched sources after squash', async () => {
|
|
1350
1348
|
const deps = makeDeps();
|
|
1351
1349
|
deps.adapter.source = 'metricflow';
|
|
1352
1350
|
deps.registry.get.mockReturnValue(deps.adapter);
|
|
1353
1351
|
deps.adapter.chunk.mockResolvedValue({
|
|
1354
|
-
workUnits: [],
|
|
1352
|
+
workUnits: [{ unitKey: 'u1', rawFiles: ['semantic_models.yml'], peerFileIndex: [], dependencyPaths: [] }],
|
|
1355
1353
|
parseArtifacts: { semanticModels: [{ name: 'orders' }] },
|
|
1356
1354
|
});
|
|
1357
1355
|
deps.adapter.listTargetConnectionIds = vi.fn().mockResolvedValue(['warehouse-2']);
|
|
1358
|
-
deps.adapter.finalize = vi.fn().mockResolvedValue({
|
|
1359
|
-
result: { sourcesTouched: 1 },
|
|
1360
|
-
warnings: ['kept going'],
|
|
1361
|
-
errors: [],
|
|
1362
|
-
touchedSources: [{ connectionId: 'warehouse-2', sourceName: 'orders' }],
|
|
1363
|
-
changedWikiPageKeys: [],
|
|
1364
|
-
actions: [
|
|
1365
|
-
{
|
|
1366
|
-
target: 'sl',
|
|
1367
|
-
type: 'updated',
|
|
1368
|
-
key: 'orders',
|
|
1369
|
-
targetConnectionId: 'warehouse-2',
|
|
1370
|
-
detail: 'Finalized orders usage',
|
|
1371
|
-
rawPaths: ['semantic_models.yml'],
|
|
1372
|
-
},
|
|
1373
|
-
],
|
|
1374
|
-
});
|
|
1375
1356
|
deps.semanticLayerService.loadAllSources.mockImplementation((connectionId) => Promise.resolve({ sources: [{ name: `${connectionId}_source` }], loadErrors: [] }));
|
|
1376
|
-
|
|
1377
|
-
|
|
1378
|
-
|
|
1379
|
-
|
|
1380
|
-
|
|
1381
|
-
|
|
1382
|
-
return { created: true, commitHash: 'finalization-sha' };
|
|
1383
|
-
}
|
|
1384
|
-
return { created: true, commitHash: head };
|
|
1385
|
-
}),
|
|
1386
|
-
commitStaged: vi.fn().mockResolvedValue({ created: false, commitHash: 'post-finalization' }),
|
|
1387
|
-
resetHardTo: vi.fn(),
|
|
1388
|
-
assertWorktreeClean: vi.fn().mockResolvedValue(undefined),
|
|
1389
|
-
writeBinaryNoRenamePatch: vi.fn(async (_base, _head, patchPath) => {
|
|
1390
|
-
await writeFile(patchPath, '', 'utf-8');
|
|
1357
|
+
const postProcessor = {
|
|
1358
|
+
run: vi.fn().mockResolvedValue({
|
|
1359
|
+
result: { sourcesCreated: 1 },
|
|
1360
|
+
warnings: ['kept going'],
|
|
1361
|
+
errors: [],
|
|
1362
|
+
touchedSources: [{ connectionId: 'warehouse-2', sourceName: 'orders' }],
|
|
1391
1363
|
}),
|
|
1392
|
-
applyPatchFile3WayIndex: vi.fn(),
|
|
1393
|
-
diffNameStatus: vi.fn().mockImplementation(async (from, to) => from === 'pre-finalization' && to === 'post-finalization'
|
|
1394
|
-
? [{ status: 'M', path: 'semantic-layer/warehouse-2/orders.yaml' }]
|
|
1395
|
-
: []),
|
|
1396
|
-
changedPaths: vi.fn().mockResolvedValue(['semantic-layer/warehouse-2/orders.yaml']),
|
|
1397
1364
|
};
|
|
1398
|
-
deps
|
|
1399
|
-
chatId: 'j1',
|
|
1400
|
-
workdir: '/tmp/wt',
|
|
1401
|
-
branch: 'session/j1',
|
|
1402
|
-
baseSha: 'b',
|
|
1403
|
-
createdAt: new Date(),
|
|
1404
|
-
git,
|
|
1405
|
-
config: {},
|
|
1406
|
-
});
|
|
1407
|
-
const runner = buildRunner(deps);
|
|
1365
|
+
const runner = buildRunner(deps, { postProcessors: { metricflow: postProcessor } });
|
|
1408
1366
|
runner.stageRawFilesStage1 = vi.fn().mockResolvedValue({
|
|
1409
1367
|
currentHashes: new Map([['semantic_models.yml', 'h1']]),
|
|
1410
1368
|
rawDirInWorktree: 'raw-sources/c1/metricflow/s',
|
|
@@ -1417,7 +1375,7 @@ describe('IngestBundleRunner — Stages 1 → 7', () => {
|
|
|
1417
1375
|
trigger: 'upload',
|
|
1418
1376
|
bundleRef: { kind: 'upload', uploadId: 'upload-x' },
|
|
1419
1377
|
});
|
|
1420
|
-
expect(
|
|
1378
|
+
expect(postProcessor.run).toHaveBeenCalledWith({
|
|
1421
1379
|
connectionId: 'c1',
|
|
1422
1380
|
sourceKey: 'metricflow',
|
|
1423
1381
|
syncId: expect.any(String),
|
|
@@ -1425,25 +1383,24 @@ describe('IngestBundleRunner — Stages 1 → 7', () => {
|
|
|
1425
1383
|
runId: 'run-1',
|
|
1426
1384
|
workdir: '/tmp/wt',
|
|
1427
1385
|
parseArtifacts: { semanticModels: [{ name: 'orders' }] },
|
|
1428
|
-
})
|
|
1386
|
+
});
|
|
1429
1387
|
expect(deps.reportsRepo.create).toHaveBeenCalledWith(expect.objectContaining({
|
|
1430
1388
|
body: expect.objectContaining({
|
|
1431
|
-
|
|
1389
|
+
postProcessor: {
|
|
1432
1390
|
sourceKey: 'metricflow',
|
|
1433
1391
|
status: 'success',
|
|
1434
|
-
|
|
1435
|
-
|
|
1436
|
-
|
|
1437
|
-
|
|
1438
|
-
|
|
1439
|
-
}),
|
|
1392
|
+
result: { sourcesCreated: 1 },
|
|
1393
|
+
warnings: ['kept going'],
|
|
1394
|
+
errors: [],
|
|
1395
|
+
touchedSources: [{ connectionId: 'warehouse-2', sourceName: 'orders' }],
|
|
1396
|
+
},
|
|
1440
1397
|
}),
|
|
1441
1398
|
}));
|
|
1442
1399
|
expect(deps.semanticLayerService.loadAllSources).toHaveBeenCalledWith('warehouse-2');
|
|
1443
1400
|
expect(deps.slSearchService.indexSources).toHaveBeenCalledWith('warehouse-2', [{ name: 'warehouse-2_source' }]);
|
|
1444
1401
|
expect(deps.sessionWorktreeService.cleanup).toHaveBeenCalledWith(expect.any(Object), 'success');
|
|
1445
1402
|
});
|
|
1446
|
-
it('includes
|
|
1403
|
+
it('includes historic-sql post-processor output in memory-flow saved counts', async () => {
|
|
1447
1404
|
const deps = makeDeps();
|
|
1448
1405
|
deps.adapter.source = 'historic-sql';
|
|
1449
1406
|
deps.registry.get.mockReturnValue(deps.adapter);
|
|
@@ -1457,19 +1414,21 @@ describe('IngestBundleRunner — Stages 1 → 7', () => {
|
|
|
1457
1414
|
},
|
|
1458
1415
|
],
|
|
1459
1416
|
});
|
|
1460
|
-
|
|
1461
|
-
|
|
1462
|
-
|
|
1463
|
-
|
|
1464
|
-
|
|
1465
|
-
|
|
1466
|
-
|
|
1467
|
-
|
|
1468
|
-
|
|
1469
|
-
|
|
1470
|
-
|
|
1471
|
-
|
|
1472
|
-
|
|
1417
|
+
const postProcessor = {
|
|
1418
|
+
run: vi.fn().mockResolvedValue({
|
|
1419
|
+
result: {
|
|
1420
|
+
tableUsageMerged: 2,
|
|
1421
|
+
staleTablesMarked: 1,
|
|
1422
|
+
patternPagesWritten: 3,
|
|
1423
|
+
stalePatternPagesMarked: 1,
|
|
1424
|
+
archivedPatternPages: 1,
|
|
1425
|
+
},
|
|
1426
|
+
warnings: [],
|
|
1427
|
+
errors: [],
|
|
1428
|
+
touchedSources: [{ connectionId: 'c1', sourceName: 'orders' }],
|
|
1429
|
+
}),
|
|
1430
|
+
};
|
|
1431
|
+
const runner = buildRunner(deps, { postProcessors: { 'historic-sql': postProcessor } });
|
|
1473
1432
|
runner.stageRawFilesStage1 = vi.fn().mockResolvedValue({
|
|
1474
1433
|
currentHashes: new Map([['tables/public/orders.json', 'h1']]),
|
|
1475
1434
|
rawDirInWorktree: 'raw-sources/c1/historic-sql/s',
|
|
@@ -1489,11 +1448,11 @@ describe('IngestBundleRunner — Stages 1 → 7', () => {
|
|
|
1489
1448
|
});
|
|
1490
1449
|
expect(memoryFlow.snapshot().events).toContainEqual(expect.objectContaining({
|
|
1491
1450
|
type: 'saved',
|
|
1492
|
-
wikiCount:
|
|
1493
|
-
slCount:
|
|
1451
|
+
wikiCount: 5,
|
|
1452
|
+
slCount: 3,
|
|
1494
1453
|
}));
|
|
1495
1454
|
});
|
|
1496
|
-
it('marks
|
|
1455
|
+
it('marks post-processor infrastructure failure as failed and preserves worktree cleanup state', async () => {
|
|
1497
1456
|
const deps = makeDeps();
|
|
1498
1457
|
deps.adapter.source = 'metricflow';
|
|
1499
1458
|
deps.registry.get.mockReturnValue(deps.adapter);
|
|
@@ -1501,8 +1460,8 @@ describe('IngestBundleRunner — Stages 1 → 7', () => {
|
|
|
1501
1460
|
workUnits: [{ unitKey: 'u1', rawFiles: ['semantic_models.yml'], peerFileIndex: [], dependencyPaths: [] }],
|
|
1502
1461
|
parseArtifacts: { semanticModels: [{ name: 'orders' }] },
|
|
1503
1462
|
});
|
|
1504
|
-
|
|
1505
|
-
const runner = buildRunner(deps);
|
|
1463
|
+
const postProcessor = { run: vi.fn().mockRejectedValue(new Error('worktree write failed')) };
|
|
1464
|
+
const runner = buildRunner(deps, { postProcessors: { metricflow: postProcessor } });
|
|
1506
1465
|
runner.stageRawFilesStage1 = vi.fn().mockResolvedValue({
|
|
1507
1466
|
currentHashes: new Map([['semantic_models.yml', 'h1']]),
|
|
1508
1467
|
rawDirInWorktree: 'raw-sources/c1/metricflow/s',
|
|
@@ -1519,120 +1478,6 @@ describe('IngestBundleRunner — Stages 1 → 7', () => {
|
|
|
1519
1478
|
expect(deps.gitService.squashMergeIntoMain).not.toHaveBeenCalled();
|
|
1520
1479
|
expect(deps.sessionWorktreeService.cleanup).toHaveBeenCalledWith(expect.any(Object), 'crash');
|
|
1521
1480
|
});
|
|
1522
|
-
it('reports finalization actions excluded from provenance when raw paths are not defensible', async () => {
|
|
1523
|
-
const deps = makeDeps();
|
|
1524
|
-
deps.adapter.finalize = vi.fn().mockResolvedValue({
|
|
1525
|
-
warnings: [],
|
|
1526
|
-
errors: [],
|
|
1527
|
-
touchedSources: [],
|
|
1528
|
-
changedWikiPageKeys: [],
|
|
1529
|
-
actions: [
|
|
1530
|
-
{ target: 'wiki', type: 'updated', key: 'historic-sql-pattern', detail: 'No raw path' },
|
|
1531
|
-
{ target: 'sl', type: 'updated', key: 'orders', detail: 'Invalid raw path', rawPaths: ['missing.json'] },
|
|
1532
|
-
],
|
|
1533
|
-
});
|
|
1534
|
-
const runner = buildRunner(deps);
|
|
1535
|
-
runner.stageRawFilesStage1 = vi.fn().mockResolvedValue({
|
|
1536
|
-
currentHashes: new Map([['current.json', 'h1']]),
|
|
1537
|
-
rawDirInWorktree: 'raw-sources/c1/fake/s',
|
|
1538
|
-
});
|
|
1539
|
-
runner.resolveStagedDir = vi.fn().mockResolvedValue('/tmp/stage/upload-x');
|
|
1540
|
-
await runner.run({
|
|
1541
|
-
jobId: 'j1',
|
|
1542
|
-
connectionId: 'c1',
|
|
1543
|
-
sourceKey: 'fake',
|
|
1544
|
-
trigger: 'upload',
|
|
1545
|
-
bundleRef: { kind: 'upload', uploadId: 'upload-x' },
|
|
1546
|
-
});
|
|
1547
|
-
expect(deps.reportsRepo.create).toHaveBeenCalledWith(expect.objectContaining({
|
|
1548
|
-
body: expect.objectContaining({
|
|
1549
|
-
finalization: expect.objectContaining({
|
|
1550
|
-
provenanceExclusions: [
|
|
1551
|
-
expect.objectContaining({ reason: 'missing_raw_paths' }),
|
|
1552
|
-
expect.objectContaining({ reason: 'raw_path_not_defensible', invalidRawPaths: ['missing.json'] }),
|
|
1553
|
-
],
|
|
1554
|
-
}),
|
|
1555
|
-
}),
|
|
1556
|
-
}));
|
|
1557
|
-
expect(deps.provenanceRepo.insertMany).not.toHaveBeenCalledWith(expect.arrayContaining([expect.objectContaining({ rawPath: 'missing.json' })]));
|
|
1558
|
-
});
|
|
1559
|
-
it('passes explicit override replay metadata and no current work unit outcomes', async () => {
|
|
1560
|
-
const deps = makeDeps();
|
|
1561
|
-
deps.reportsRepo.findByJobId.mockResolvedValue({
|
|
1562
|
-
id: 'prior-report',
|
|
1563
|
-
runId: 'prior-run',
|
|
1564
|
-
jobId: 'prior-job',
|
|
1565
|
-
connectionId: 'c1',
|
|
1566
|
-
sourceKey: 'fake',
|
|
1567
|
-
createdAt: '2026-05-18T00:00:00.000Z',
|
|
1568
|
-
body: {
|
|
1569
|
-
status: 'completed',
|
|
1570
|
-
syncId: 'prior-sync',
|
|
1571
|
-
diffSummary: { added: 0, modified: 0, deleted: 0, unchanged: 0 },
|
|
1572
|
-
commitSha: 'prior-sha',
|
|
1573
|
-
workUnits: [
|
|
1574
|
-
{
|
|
1575
|
-
unitKey: 'prior-unit',
|
|
1576
|
-
rawFiles: ['prior.json'],
|
|
1577
|
-
status: 'success',
|
|
1578
|
-
actions: [{ target: 'wiki', type: 'created', key: 'prior', detail: 'prior' }],
|
|
1579
|
-
touchedSlSources: [],
|
|
1580
|
-
},
|
|
1581
|
-
],
|
|
1582
|
-
failedWorkUnits: [],
|
|
1583
|
-
reconciliationSkipped: false,
|
|
1584
|
-
conflictsResolved: [],
|
|
1585
|
-
evictionsApplied: [
|
|
1586
|
-
{
|
|
1587
|
-
rawPath: 'do-not-replay.json',
|
|
1588
|
-
artifactKind: 'wiki',
|
|
1589
|
-
artifactKey: 'old',
|
|
1590
|
-
action: 'removed',
|
|
1591
|
-
reason: 'prior',
|
|
1592
|
-
},
|
|
1593
|
-
],
|
|
1594
|
-
unmappedFallbacks: [],
|
|
1595
|
-
artifactResolutions: [],
|
|
1596
|
-
evictionInputs: ['evicted-from-prior-report.json'],
|
|
1597
|
-
unresolvedCards: [],
|
|
1598
|
-
supersededBy: null,
|
|
1599
|
-
overrideOf: null,
|
|
1600
|
-
provenanceRows: [],
|
|
1601
|
-
toolTranscripts: [],
|
|
1602
|
-
},
|
|
1603
|
-
});
|
|
1604
|
-
deps.adapter.finalize = vi.fn().mockResolvedValue({
|
|
1605
|
-
warnings: [],
|
|
1606
|
-
errors: [],
|
|
1607
|
-
touchedSources: [],
|
|
1608
|
-
changedWikiPageKeys: [],
|
|
1609
|
-
actions: [],
|
|
1610
|
-
});
|
|
1611
|
-
deps.gitService.listFilesAtHead.mockResolvedValue(['raw-sources/c1/fake/prior-sync/prior.json']);
|
|
1612
|
-
deps.gitService.getFileAtCommit.mockResolvedValue('{"id":1}\n');
|
|
1613
|
-
const runner = buildRunner(deps);
|
|
1614
|
-
runner.stageRawFilesStage1 = vi.fn().mockResolvedValue({
|
|
1615
|
-
currentHashes: new Map([['prior.json', 'h1']]),
|
|
1616
|
-
rawDirInWorktree: 'raw-sources/c1/fake/prior-sync',
|
|
1617
|
-
});
|
|
1618
|
-
runner.resolveStagedDir = vi.fn().mockResolvedValue('/tmp/stage/prior');
|
|
1619
|
-
await runner.run({
|
|
1620
|
-
jobId: 'override-job',
|
|
1621
|
-
connectionId: 'c1',
|
|
1622
|
-
sourceKey: 'fake',
|
|
1623
|
-
trigger: 'manual_override',
|
|
1624
|
-
bundleRef: { kind: 'override', priorJobId: 'prior-job' },
|
|
1625
|
-
});
|
|
1626
|
-
expect(deps.adapter.finalize).toHaveBeenCalledWith(expect.objectContaining({
|
|
1627
|
-
workUnitOutcomes: [],
|
|
1628
|
-
overrideReplay: {
|
|
1629
|
-
priorJobId: 'prior-job',
|
|
1630
|
-
priorRunId: 'prior-run',
|
|
1631
|
-
priorSyncId: 'prior-sync',
|
|
1632
|
-
evictionRawPaths: ['evicted-from-prior-report.json'],
|
|
1633
|
-
},
|
|
1634
|
-
}));
|
|
1635
|
-
});
|
|
1636
1481
|
it('includes existing global wiki pages in WorkUnit prompts', async () => {
|
|
1637
1482
|
const deps = makeDeps();
|
|
1638
1483
|
deps.knowledgeIndex.listPagesForUser.mockResolvedValue([
|
|
@@ -7,7 +7,6 @@ import { initKtxProject, loadKtxProject } from '../project/index.js';
|
|
|
7
7
|
import { makeLocalGitRepo } from '../test/make-local-git-repo.js';
|
|
8
8
|
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
|
9
9
|
import { FakeSourceAdapter } from './adapters/fake/fake.adapter.js';
|
|
10
|
-
import { projectHistoricSqlEvidence } from './adapters/historic-sql/projection.js';
|
|
11
10
|
import { LocalLookerRuntimeStore } from './adapters/looker/local-runtime-store.js';
|
|
12
11
|
import { createDefaultLocalIngestAdapters, localPullConfigForAdapter } from './local-adapters.js';
|
|
13
12
|
import { getLocalIngestStatus, runLocalIngest } from './local-ingest.js';
|
|
@@ -160,23 +159,6 @@ class HistoricSqlEvidenceTestAdapter {
|
|
|
160
159
|
],
|
|
161
160
|
});
|
|
162
161
|
}
|
|
163
|
-
async finalize(ctx) {
|
|
164
|
-
const projection = await projectHistoricSqlEvidence({
|
|
165
|
-
workdir: ctx.workdir,
|
|
166
|
-
connectionId: ctx.connectionId,
|
|
167
|
-
syncId: ctx.syncId,
|
|
168
|
-
runId: ctx.runId,
|
|
169
|
-
overrideReplay: ctx.overrideReplay,
|
|
170
|
-
});
|
|
171
|
-
return {
|
|
172
|
-
result: projection,
|
|
173
|
-
warnings: projection.warnings,
|
|
174
|
-
errors: [],
|
|
175
|
-
touchedSources: projection.touchedSources,
|
|
176
|
-
changedWikiPageKeys: projection.changedWikiPageKeys,
|
|
177
|
-
actions: projection.actions,
|
|
178
|
-
};
|
|
179
|
-
}
|
|
180
162
|
}
|
|
181
163
|
function makeLookerRuntimeClient() {
|
|
182
164
|
const lookerModels = {
|
|
@@ -444,7 +426,7 @@ describe('canonical local ingest', () => {
|
|
|
444
426
|
},
|
|
445
427
|
]);
|
|
446
428
|
});
|
|
447
|
-
it('runs historic-SQL evidence projection through local bundle
|
|
429
|
+
it('runs historic-SQL evidence projection through the local bundle post-processor', async () => {
|
|
448
430
|
const projectDir = join(tempDir, 'historic-sql-project');
|
|
449
431
|
await initKtxProject({ projectDir });
|
|
450
432
|
await writeFile(join(projectDir, 'ktx.yaml'), [
|
|
@@ -495,12 +477,11 @@ describe('canonical local ingest', () => {
|
|
|
495
477
|
agentRunner,
|
|
496
478
|
});
|
|
497
479
|
expect(result.result.failedWorkUnits).toEqual([]);
|
|
498
|
-
expect(result.report.body.
|
|
480
|
+
expect(result.report.body.postProcessor).toMatchObject({
|
|
499
481
|
sourceKey: 'historic-sql',
|
|
500
482
|
status: 'success',
|
|
501
483
|
result: { tableUsageMerged: 1 },
|
|
502
|
-
|
|
503
|
-
derivedTouchedSources: [{ connectionId: 'warehouse', sourceName: 'orders' }],
|
|
484
|
+
touchedSources: [{ connectionId: 'warehouse', sourceName: 'orders' }],
|
|
504
485
|
});
|
|
505
486
|
await expect(readFile(join(projectDir, 'semantic-layer/warehouse/_schema/public.yaml'), 'utf-8')).resolves.toContain('Orders are repeatedly queried by lifecycle status.');
|
|
506
487
|
});
|
|
@@ -13,6 +13,7 @@ import { ContextCandidateMarkTool, ContextCandidateWriteTool, ContextEvidenceNei
|
|
|
13
13
|
import { buildKnowledgeSearchText, KnowledgeWikiService, searchLocalKnowledgePages, SqliteKnowledgeIndex, WikiListTagsTool, WikiReadTool, WikiRemoveTool, WikiSearchTool, WikiWriteTool, } from '../wiki/index.js';
|
|
14
14
|
import { CandidateDedupService, ContextCandidateCarryforwardService, CuratorPaginationService, } from './context-candidates/index.js';
|
|
15
15
|
import { createEmitHistoricSqlEvidenceTool } from './adapters/historic-sql/evidence-tool.js';
|
|
16
|
+
import { HistoricSqlProjectionPostProcessor } from './adapters/historic-sql/post-processor.js';
|
|
16
17
|
import { ContextEvidenceIndexService, SqliteContextEvidenceStore } from './context-evidence/index.js';
|
|
17
18
|
import { DiffSetService } from './diff-set.service.js';
|
|
18
19
|
import { ingestTracePathForJob } from './ingest-trace.js';
|
|
@@ -585,6 +586,9 @@ export function createLocalBundleIngestRuntime(options) {
|
|
|
585
586
|
settings: { batchSize: 8, maxPasses: 8, stepBudgetPerPass: 60 },
|
|
586
587
|
logger,
|
|
587
588
|
}),
|
|
589
|
+
postProcessors: {
|
|
590
|
+
'historic-sql': new HistoricSqlProjectionPostProcessor(),
|
|
591
|
+
},
|
|
588
592
|
logger,
|
|
589
593
|
};
|
|
590
594
|
return {
|
|
@@ -178,6 +178,13 @@ async function recordLocalMetabaseChildFailure(options) {
|
|
|
178
178
|
overrideOf: null,
|
|
179
179
|
provenanceRows: [],
|
|
180
180
|
toolTranscripts: [],
|
|
181
|
+
postProcessor: {
|
|
182
|
+
sourceKey: 'metabase',
|
|
183
|
+
status: 'failed',
|
|
184
|
+
errors: [reason],
|
|
185
|
+
warnings: [],
|
|
186
|
+
touchedSources: [],
|
|
187
|
+
},
|
|
181
188
|
};
|
|
182
189
|
const report = await store.create({
|
|
183
190
|
runId: run.id,
|
|
@@ -43,8 +43,8 @@ export declare const memoryFlowEventSchema: z.ZodDiscriminatedUnion<[z.ZodObject
|
|
|
43
43
|
stage: z.ZodEnum<{
|
|
44
44
|
source: "source";
|
|
45
45
|
workUnits: "workUnits";
|
|
46
|
-
chunks: "chunks";
|
|
47
46
|
actions: "actions";
|
|
47
|
+
chunks: "chunks";
|
|
48
48
|
gates: "gates";
|
|
49
49
|
saved: "saved";
|
|
50
50
|
}>;
|
|
@@ -58,7 +58,7 @@ export declare const memoryFlowEventSchema: z.ZodDiscriminatedUnion<[z.ZodObject
|
|
|
58
58
|
report: "report";
|
|
59
59
|
integration: "integration";
|
|
60
60
|
reconciliation: "reconciliation";
|
|
61
|
-
|
|
61
|
+
post_processor: "post_processor";
|
|
62
62
|
wiki_sl_ref_repair: "wiki_sl_ref_repair";
|
|
63
63
|
final_gates: "final_gates";
|
|
64
64
|
save: "save";
|
|
@@ -143,8 +143,8 @@ export declare const memoryFlowActionDetailSchema: z.ZodObject<{
|
|
|
143
143
|
}>;
|
|
144
144
|
action: z.ZodEnum<{
|
|
145
145
|
created: "created";
|
|
146
|
-
updated: "updated";
|
|
147
146
|
removed: "removed";
|
|
147
|
+
updated: "updated";
|
|
148
148
|
}>;
|
|
149
149
|
key: z.ZodString;
|
|
150
150
|
summary: z.ZodString;
|
|
@@ -163,8 +163,8 @@ export declare const memoryFlowDetailSectionsSchema: z.ZodObject<{
|
|
|
163
163
|
}>;
|
|
164
164
|
action: z.ZodEnum<{
|
|
165
165
|
created: "created";
|
|
166
|
-
updated: "updated";
|
|
167
166
|
removed: "removed";
|
|
167
|
+
updated: "updated";
|
|
168
168
|
}>;
|
|
169
169
|
key: z.ZodString;
|
|
170
170
|
summary: z.ZodString;
|
|
@@ -40,7 +40,7 @@ type MemoryFlowEventPayload = {
|
|
|
40
40
|
reason: string;
|
|
41
41
|
} | {
|
|
42
42
|
type: 'stage_progress';
|
|
43
|
-
stage: 'source' | 'integration' | 'reconciliation' | '
|
|
43
|
+
stage: 'source' | 'integration' | 'reconciliation' | 'post_processor' | 'wiki_sl_ref_repair' | 'final_gates' | 'save' | 'provenance' | 'report';
|
|
44
44
|
percent: number;
|
|
45
45
|
message: string;
|
|
46
46
|
transient?: boolean;
|
|
@@ -6,7 +6,7 @@ import type { CaptureSession, MemoryAction, MemoryKnowledgeSlRefsPort } from '..
|
|
|
6
6
|
import type { PromptService } from '../prompts/index.js';
|
|
7
7
|
import type { SkillsRegistryService } from '../skills/index.js';
|
|
8
8
|
import type { SemanticLayerService, SlConnectionCatalogPort, SlSearchService, SlSourcesIndexPort, SlValidationDeps, SlValidatorPort } from '../sl/index.js';
|
|
9
|
-
import type { ToolContext, ToolSession } from '../tools/index.js';
|
|
9
|
+
import type { ToolContext, ToolSession, TouchedSlSource } from '../tools/index.js';
|
|
10
10
|
import type { KnowledgeIndexPort, KnowledgeWikiService } from '../wiki/index.js';
|
|
11
11
|
import type { CanonicalPin } from './canonical-pins.js';
|
|
12
12
|
import type { IngestTraceLevel } from './ingest-trace.js';
|
|
@@ -281,6 +281,24 @@ export interface CuratorPaginationPort {
|
|
|
281
281
|
warnings: string[];
|
|
282
282
|
}>;
|
|
283
283
|
}
|
|
284
|
+
export interface IngestBundlePostProcessorInput {
|
|
285
|
+
connectionId: string;
|
|
286
|
+
sourceKey: string;
|
|
287
|
+
syncId: string;
|
|
288
|
+
jobId: string;
|
|
289
|
+
runId: string;
|
|
290
|
+
workdir: string;
|
|
291
|
+
parseArtifacts: unknown;
|
|
292
|
+
}
|
|
293
|
+
export interface IngestBundlePostProcessorResult {
|
|
294
|
+
result?: unknown;
|
|
295
|
+
warnings: string[];
|
|
296
|
+
errors: string[];
|
|
297
|
+
touchedSources: TouchedSlSource[];
|
|
298
|
+
}
|
|
299
|
+
export interface IngestBundlePostProcessorPort {
|
|
300
|
+
run(input: IngestBundlePostProcessorInput): Promise<IngestBundlePostProcessorResult>;
|
|
301
|
+
}
|
|
284
302
|
export interface IngestBundleRunnerDeps {
|
|
285
303
|
runs: IngestRunsPort;
|
|
286
304
|
provenance: IngestProvenancePort;
|
|
@@ -314,6 +332,7 @@ export interface IngestBundleRunnerDeps {
|
|
|
314
332
|
candidateDedup?: CandidateDedupPort;
|
|
315
333
|
contextCandidateCarryforward?: ContextCandidateCarryforwardPort;
|
|
316
334
|
curatorPagination?: CuratorPaginationPort;
|
|
335
|
+
postProcessors?: Record<string, IngestBundlePostProcessorPort>;
|
|
317
336
|
logger?: KtxLogger;
|
|
318
337
|
}
|
|
319
338
|
export interface IngestCaptureState {
|
|
@@ -103,8 +103,8 @@ export declare const ingestReportSnapshotSchema: z.ZodObject<{
|
|
|
103
103
|
}>;
|
|
104
104
|
type: z.ZodEnum<{
|
|
105
105
|
created: "created";
|
|
106
|
-
updated: "updated";
|
|
107
106
|
removed: "removed";
|
|
107
|
+
updated: "updated";
|
|
108
108
|
}>;
|
|
109
109
|
key: z.ZodString;
|
|
110
110
|
detail: z.ZodString;
|
|
@@ -129,8 +129,8 @@ export declare const ingestReportSnapshotSchema: z.ZodObject<{
|
|
|
129
129
|
}>;
|
|
130
130
|
type: z.ZodEnum<{
|
|
131
131
|
created: "created";
|
|
132
|
-
updated: "updated";
|
|
133
132
|
removed: "removed";
|
|
133
|
+
updated: "updated";
|
|
134
134
|
}>;
|
|
135
135
|
key: z.ZodString;
|
|
136
136
|
detail: z.ZodString;
|
|
@@ -221,77 +221,6 @@ export declare const ingestReportSnapshotSchema: z.ZodObject<{
|
|
|
221
221
|
errorCount: z.ZodNumber;
|
|
222
222
|
toolNames: z.ZodArray<z.ZodString>;
|
|
223
223
|
}, z.core.$strip>>>;
|
|
224
|
-
finalization: z.ZodOptional<z.ZodObject<{
|
|
225
|
-
sourceKey: z.ZodString;
|
|
226
|
-
status: z.ZodEnum<{
|
|
227
|
-
success: "success";
|
|
228
|
-
skipped: "skipped";
|
|
229
|
-
failed: "failed";
|
|
230
|
-
}>;
|
|
231
|
-
commitSha: z.ZodNullable<z.ZodString>;
|
|
232
|
-
touchedPaths: z.ZodArray<z.ZodString>;
|
|
233
|
-
declaredTouchedSources: z.ZodArray<z.ZodObject<{
|
|
234
|
-
connectionId: z.ZodString;
|
|
235
|
-
sourceName: z.ZodString;
|
|
236
|
-
}, z.core.$strip>>;
|
|
237
|
-
derivedTouchedSources: z.ZodArray<z.ZodObject<{
|
|
238
|
-
connectionId: z.ZodString;
|
|
239
|
-
sourceName: z.ZodString;
|
|
240
|
-
}, z.core.$strip>>;
|
|
241
|
-
declaredChangedWikiPageKeys: z.ZodArray<z.ZodString>;
|
|
242
|
-
derivedChangedWikiPageKeys: z.ZodArray<z.ZodString>;
|
|
243
|
-
mismatches: z.ZodDefault<z.ZodArray<z.ZodObject<{
|
|
244
|
-
artifactKind: z.ZodEnum<{
|
|
245
|
-
wiki: "wiki";
|
|
246
|
-
sl: "sl";
|
|
247
|
-
}>;
|
|
248
|
-
key: z.ZodString;
|
|
249
|
-
direction: z.ZodEnum<{
|
|
250
|
-
missing_from_adapter_declaration: "missing_from_adapter_declaration";
|
|
251
|
-
extra_in_adapter_declaration: "extra_in_adapter_declaration";
|
|
252
|
-
}>;
|
|
253
|
-
}, z.core.$strip>>>;
|
|
254
|
-
result: z.ZodOptional<z.ZodUnknown>;
|
|
255
|
-
errors: z.ZodArray<z.ZodString>;
|
|
256
|
-
warnings: z.ZodArray<z.ZodString>;
|
|
257
|
-
actions: z.ZodDefault<z.ZodArray<z.ZodObject<{
|
|
258
|
-
target: z.ZodEnum<{
|
|
259
|
-
wiki: "wiki";
|
|
260
|
-
sl: "sl";
|
|
261
|
-
}>;
|
|
262
|
-
type: z.ZodEnum<{
|
|
263
|
-
created: "created";
|
|
264
|
-
updated: "updated";
|
|
265
|
-
removed: "removed";
|
|
266
|
-
}>;
|
|
267
|
-
key: z.ZodString;
|
|
268
|
-
detail: z.ZodString;
|
|
269
|
-
targetConnectionId: z.ZodDefault<z.ZodNullable<z.ZodString>>;
|
|
270
|
-
rawPaths: z.ZodOptional<z.ZodArray<z.ZodString>>;
|
|
271
|
-
}, z.core.$strip>>>;
|
|
272
|
-
provenanceExclusions: z.ZodDefault<z.ZodArray<z.ZodObject<{
|
|
273
|
-
action: z.ZodObject<{
|
|
274
|
-
target: z.ZodEnum<{
|
|
275
|
-
wiki: "wiki";
|
|
276
|
-
sl: "sl";
|
|
277
|
-
}>;
|
|
278
|
-
type: z.ZodEnum<{
|
|
279
|
-
created: "created";
|
|
280
|
-
updated: "updated";
|
|
281
|
-
removed: "removed";
|
|
282
|
-
}>;
|
|
283
|
-
key: z.ZodString;
|
|
284
|
-
detail: z.ZodString;
|
|
285
|
-
targetConnectionId: z.ZodDefault<z.ZodNullable<z.ZodString>>;
|
|
286
|
-
rawPaths: z.ZodOptional<z.ZodArray<z.ZodString>>;
|
|
287
|
-
}, z.core.$strip>;
|
|
288
|
-
reason: z.ZodEnum<{
|
|
289
|
-
missing_raw_paths: "missing_raw_paths";
|
|
290
|
-
raw_path_not_defensible: "raw_path_not_defensible";
|
|
291
|
-
}>;
|
|
292
|
-
invalidRawPaths: z.ZodOptional<z.ZodArray<z.ZodString>>;
|
|
293
|
-
}, z.core.$strip>>>;
|
|
294
|
-
}, z.core.$strip>>;
|
|
295
224
|
memoryFlow: z.ZodOptional<z.ZodType<import("./index.js").MemoryFlowReplayInput, unknown, z.core.$ZodTypeInternals<import("./index.js").MemoryFlowReplayInput, unknown>>>;
|
|
296
225
|
}, z.core.$loose>;
|
|
297
226
|
}, z.core.$loose>;
|
|
@@ -114,32 +114,6 @@ const ingestReportFailureSchema = z.object({
|
|
|
114
114
|
message: z.string().min(1),
|
|
115
115
|
details: z.record(z.string(), z.unknown()).optional(),
|
|
116
116
|
});
|
|
117
|
-
const finalizationMismatchSchema = z.object({
|
|
118
|
-
artifactKind: z.enum(['sl', 'wiki']),
|
|
119
|
-
key: z.string().min(1),
|
|
120
|
-
direction: z.enum(['missing_from_adapter_declaration', 'extra_in_adapter_declaration']),
|
|
121
|
-
});
|
|
122
|
-
const finalizationProvenanceExclusionSchema = z.object({
|
|
123
|
-
action: ingestActionSchema,
|
|
124
|
-
reason: z.enum(['missing_raw_paths', 'raw_path_not_defensible']),
|
|
125
|
-
invalidRawPaths: z.array(z.string()).optional(),
|
|
126
|
-
});
|
|
127
|
-
const finalizationOutcomeSchema = z.object({
|
|
128
|
-
sourceKey: z.string().min(1),
|
|
129
|
-
status: z.enum(['success', 'failed', 'skipped']),
|
|
130
|
-
commitSha: z.string().nullable(),
|
|
131
|
-
touchedPaths: z.array(z.string()),
|
|
132
|
-
declaredTouchedSources: z.array(touchedSlSourceSchema),
|
|
133
|
-
derivedTouchedSources: z.array(touchedSlSourceSchema),
|
|
134
|
-
declaredChangedWikiPageKeys: z.array(z.string()),
|
|
135
|
-
derivedChangedWikiPageKeys: z.array(z.string()),
|
|
136
|
-
mismatches: z.array(finalizationMismatchSchema).default([]),
|
|
137
|
-
result: z.unknown().optional(),
|
|
138
|
-
errors: z.array(z.string()),
|
|
139
|
-
warnings: z.array(z.string()),
|
|
140
|
-
actions: z.array(ingestActionSchema).default([]),
|
|
141
|
-
provenanceExclusions: z.array(finalizationProvenanceExclusionSchema).default([]),
|
|
142
|
-
});
|
|
143
117
|
export const ingestReportSnapshotSchema = z
|
|
144
118
|
.object({
|
|
145
119
|
id: z.string().min(1),
|
|
@@ -197,7 +171,6 @@ export const ingestReportSnapshotSchema = z
|
|
|
197
171
|
overrideOf: z.string().nullable().default(null),
|
|
198
172
|
provenanceRows: z.array(provenanceDetailSchema).default([]),
|
|
199
173
|
toolTranscripts: z.array(toolTranscriptSummarySchema).default([]),
|
|
200
|
-
finalization: finalizationOutcomeSchema.optional(),
|
|
201
174
|
memoryFlow: memoryFlowReplayInputSchema.optional(),
|
|
202
175
|
})
|
|
203
176
|
.passthrough(),
|