@kaelio/ktx 0.2.0 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (87) hide show
  1. package/assets/python/{kaelio_ktx-0.2.0-py3-none-any.whl → kaelio_ktx-0.3.0-py3-none-any.whl} +0 -0
  2. package/assets/python/manifest.json +4 -4
  3. package/dist/admin-reindex.js +10 -17
  4. package/dist/admin-reindex.test.js +1 -1
  5. package/dist/cli-program.test.js +0 -2
  6. package/dist/cli-project.d.ts +18 -0
  7. package/dist/cli-project.js +52 -0
  8. package/dist/cli-project.test.js +149 -0
  9. package/dist/cli-runtime.d.ts +0 -2
  10. package/dist/cli-runtime.js +2 -8
  11. package/dist/commands/runtime-commands.js +2 -2
  12. package/dist/context-build-view.js +1 -1
  13. package/dist/index.test.js +21 -25
  14. package/dist/ingest.js +9 -2
  15. package/dist/ingest.test.js +27 -3
  16. package/dist/managed-local-embeddings.d.ts +0 -2
  17. package/dist/managed-local-embeddings.js +2 -5
  18. package/dist/managed-local-embeddings.test.js +5 -8
  19. package/dist/managed-python-daemon.js +2 -2
  20. package/dist/managed-python-daemon.test.js +1 -1
  21. package/dist/managed-python-http.js +3 -3
  22. package/dist/managed-python-http.test.js +6 -6
  23. package/dist/print-command-tree.js +0 -2
  24. package/dist/public-ingest.d.ts +4 -2
  25. package/dist/public-ingest.js +9 -3
  26. package/dist/release-version.d.ts +1 -5
  27. package/dist/release-version.js +2 -39
  28. package/dist/runtime-requirements.js +1 -1
  29. package/dist/runtime.js +6 -6
  30. package/dist/runtime.test.js +7 -7
  31. package/dist/scan.js +7 -2
  32. package/dist/scan.test.js +1 -1
  33. package/dist/setup-embeddings.js +1 -1
  34. package/dist/setup-embeddings.test.js +2 -2
  35. package/dist/setup-runtime.test.js +1 -1
  36. package/node_modules/@ktx/context/dist/core/git.service.d.ts +1 -0
  37. package/node_modules/@ktx/context/dist/core/git.service.js +12 -0
  38. package/node_modules/@ktx/context/dist/ingest/adapters/historic-sql/historic-sql.adapter.d.ts +2 -1
  39. package/node_modules/@ktx/context/dist/ingest/adapters/historic-sql/historic-sql.adapter.js +18 -0
  40. package/node_modules/@ktx/context/dist/ingest/adapters/historic-sql/local-ingest-acceptance.test.js +6 -6
  41. package/node_modules/@ktx/context/dist/ingest/adapters/historic-sql/projection.d.ts +5 -0
  42. package/node_modules/@ktx/context/dist/ingest/adapters/historic-sql/projection.js +48 -0
  43. package/node_modules/@ktx/context/dist/ingest/adapters/historic-sql/projection.test.js +83 -0
  44. package/node_modules/@ktx/context/dist/ingest/adapters/live-database/daemon-introspection.js +4 -1
  45. package/node_modules/@ktx/context/dist/ingest/adapters/live-database/daemon-introspection.test.js +32 -0
  46. package/node_modules/@ktx/context/dist/ingest/finalization-scope.d.ts +22 -0
  47. package/node_modules/@ktx/context/dist/ingest/finalization-scope.js +95 -0
  48. package/node_modules/@ktx/context/dist/ingest/finalization-scope.test.d.ts +1 -0
  49. package/node_modules/@ktx/context/dist/ingest/finalization-scope.test.js +114 -0
  50. package/node_modules/@ktx/context/dist/ingest/index.d.ts +1 -2
  51. package/node_modules/@ktx/context/dist/ingest/index.js +0 -1
  52. package/node_modules/@ktx/context/dist/ingest/ingest-bundle.runner.d.ts +2 -0
  53. package/node_modules/@ktx/context/dist/ingest/ingest-bundle.runner.isolated-diff.test.js +166 -0
  54. package/node_modules/@ktx/context/dist/ingest/ingest-bundle.runner.js +235 -45
  55. package/node_modules/@ktx/context/dist/ingest/ingest-bundle.runner.test.js +193 -38
  56. package/node_modules/@ktx/context/dist/ingest/local-bundle-ingest.test.js +22 -3
  57. package/node_modules/@ktx/context/dist/ingest/local-bundle-runtime.js +0 -4
  58. package/node_modules/@ktx/context/dist/ingest/local-ingest.js +0 -7
  59. package/node_modules/@ktx/context/dist/ingest/local-stage-ingest.js +15 -5
  60. package/node_modules/@ktx/context/dist/ingest/local-stage-ingest.test.js +29 -0
  61. package/node_modules/@ktx/context/dist/ingest/memory-flow/schema.d.ts +2 -2
  62. package/node_modules/@ktx/context/dist/ingest/memory-flow/schema.js +1 -1
  63. package/node_modules/@ktx/context/dist/ingest/memory-flow/types.d.ts +1 -1
  64. package/node_modules/@ktx/context/dist/ingest/ports.d.ts +1 -20
  65. package/node_modules/@ktx/context/dist/ingest/report-snapshot.d.ts +71 -0
  66. package/node_modules/@ktx/context/dist/ingest/report-snapshot.js +27 -0
  67. package/node_modules/@ktx/context/dist/ingest/reports.d.ts +23 -5
  68. package/node_modules/@ktx/context/dist/ingest/reports.js +7 -24
  69. package/node_modules/@ktx/context/dist/ingest/types.d.ts +33 -0
  70. package/node_modules/@ktx/context/dist/llm/index.d.ts +1 -1
  71. package/node_modules/@ktx/context/dist/llm/index.js +1 -1
  72. package/node_modules/@ktx/context/dist/llm/local-config.d.ts +0 -1
  73. package/node_modules/@ktx/context/dist/llm/local-config.js +2 -12
  74. package/node_modules/@ktx/context/dist/llm/local-config.test.js +2 -23
  75. package/node_modules/@ktx/context/dist/package-exports.test.js +2 -2
  76. package/node_modules/@ktx/context/dist/project/config.d.ts +16 -0
  77. package/node_modules/@ktx/context/dist/project/driver-schemas.d.ts +8 -0
  78. package/node_modules/@ktx/context/dist/project/driver-schemas.js +4 -0
  79. package/node_modules/@ktx/context/dist/scan/enabled-tables.d.ts +3 -0
  80. package/node_modules/@ktx/context/dist/scan/enabled-tables.js +15 -0
  81. package/node_modules/@ktx/context/dist/scan/local-scan.d.ts +2 -4
  82. package/node_modules/@ktx/context/dist/scan/local-scan.js +2 -15
  83. package/package.json +1 -1
  84. package/node_modules/@ktx/context/dist/ingest/adapters/historic-sql/post-processor.d.ts +0 -4
  85. package/node_modules/@ktx/context/dist/ingest/adapters/historic-sql/post-processor.js +0 -38
  86. package/node_modules/@ktx/context/dist/ingest/adapters/historic-sql/post-processor.test.js +0 -63
  87. /package/{node_modules/@ktx/context/dist/ingest/adapters/historic-sql/post-processor.test.d.ts → dist/cli-project.test.d.ts} +0 -0
@@ -177,6 +177,8 @@ export async function projectHistoricSqlEvidence(input) {
177
177
  stalePatternPagesMarked: 0,
178
178
  archivedPatternPages: 0,
179
179
  touchedSources: [],
180
+ changedWikiPageKeys: [],
181
+ actions: [],
180
182
  warnings: [],
181
183
  };
182
184
  const touchedKeys = new Set();
@@ -184,6 +186,14 @@ export async function projectHistoricSqlEvidence(input) {
184
186
  const manifest = stagedManifestSchema.parse(await readJson(join(rawDir, 'manifest.json')));
185
187
  const currentTables = await currentStagedTables(rawDir);
186
188
  const evidence = await loadEvidence(input.workdir, input.runId);
189
+ if (input.overrideReplay && evidence.length === 0) {
190
+ result.warnings.push('historic-sql finalization skipped stale/archive cleanup during override replay without current-run evidence');
191
+ return result;
192
+ }
193
+ if (evidence.length === 0) {
194
+ result.warnings.push('historic-sql finalization skipped because no current-run evidence was emitted');
195
+ return result;
196
+ }
187
197
  const tableEvidence = evidence.filter((entry) => entry.kind === 'table_usage');
188
198
  const patternEvidence = evidence.filter((entry) => entry.kind === 'pattern');
189
199
  const schemaRoot = join(input.workdir, 'semantic-layer', input.connectionId, '_schema');
@@ -207,6 +217,14 @@ export async function projectHistoricSqlEvidence(input) {
207
217
  touchedKeys.add(key);
208
218
  result.touchedSources.push({ connectionId: input.connectionId, sourceName });
209
219
  }
220
+ result.actions.push({
221
+ target: 'sl',
222
+ type: 'updated',
223
+ key: sourceName,
224
+ targetConnectionId: input.connectionId,
225
+ detail: `Merged historic-SQL usage for ${matchingEvidence.table}`,
226
+ rawPaths: [matchingEvidence.rawPath],
227
+ });
210
228
  }
211
229
  }
212
230
  else if (entry.usage && !currentTables.has(tableRef)) {
@@ -220,6 +238,13 @@ export async function projectHistoricSqlEvidence(input) {
220
238
  touchedKeys.add(key);
221
239
  result.touchedSources.push({ connectionId: input.connectionId, sourceName });
222
240
  }
241
+ result.actions.push({
242
+ target: 'sl',
243
+ type: 'updated',
244
+ key: sourceName,
245
+ targetConnectionId: input.connectionId,
246
+ detail: `Marked historic-SQL usage stale for ${tableRef}`,
247
+ });
223
248
  }
224
249
  }
225
250
  }
@@ -254,6 +279,14 @@ export async function projectHistoricSqlEvidence(input) {
254
279
  await writeFile(pagePath, renderMarkdownPage(frontmatter, renderPatternMarkdown(pattern)), 'utf-8');
255
280
  writtenKeys.add(key);
256
281
  result.patternPagesWritten += 1;
282
+ result.changedWikiPageKeys.push(key);
283
+ result.actions.push({
284
+ target: 'wiki',
285
+ type: reusable ? 'updated' : 'created',
286
+ key,
287
+ detail: `Projected historic-SQL pattern ${pattern.pattern.title}`,
288
+ rawPaths: [pattern.rawPath],
289
+ });
257
290
  }
258
291
  for (const page of patternPages) {
259
292
  if (writtenKeys.has(page.key))
@@ -262,11 +295,26 @@ export async function projectHistoricSqlEvidence(input) {
262
295
  const tags = [...new Set([...stringArray(page.frontmatter.tags), 'archived'])];
263
296
  await writeFile(page.path, renderMarkdownPage({ ...page.frontmatter, tags, archived_since: manifest.fetchedAt }, page.content), 'utf-8');
264
297
  result.archivedPatternPages += 1;
298
+ result.changedWikiPageKeys.push(page.key);
299
+ result.actions.push({
300
+ target: 'wiki',
301
+ type: 'updated',
302
+ key: page.key,
303
+ detail: `Archived stale historic-SQL pattern page ${page.key}`,
304
+ });
265
305
  continue;
266
306
  }
267
307
  const tags = [...new Set([...stringArray(page.frontmatter.tags), 'stale'])];
268
308
  await writeFile(page.path, renderMarkdownPage({ ...page.frontmatter, tags, stale_since: manifest.fetchedAt }, page.content), 'utf-8');
269
309
  result.stalePatternPagesMarked += 1;
310
+ result.changedWikiPageKeys.push(page.key);
311
+ result.actions.push({
312
+ target: 'wiki',
313
+ type: 'updated',
314
+ key: page.key,
315
+ detail: `Marked historic-SQL pattern page ${page.key} stale`,
316
+ });
270
317
  }
318
+ result.changedWikiPageKeys = [...new Set(result.changedWikiPageKeys)].sort();
271
319
  return result;
272
320
  }
@@ -64,6 +64,13 @@ describe('projectHistoricSqlEvidence', () => {
64
64
  });
65
65
  const result = await projectHistoricSqlEvidence({ workdir, connectionId: 'warehouse', syncId: 'sync-1', runId: 'run-1' });
66
66
  expect(result.touchedSources).toEqual([{ connectionId: 'warehouse', sourceName: 'orders' }]);
67
+ expect(result.actions).toEqual(expect.arrayContaining([
68
+ expect.objectContaining({
69
+ target: 'sl',
70
+ key: 'orders',
71
+ rawPaths: ['tables/public.orders.json'],
72
+ }),
73
+ ]));
67
74
  const shard = YAML.parse(await readFile(join(workdir, 'semantic-layer/warehouse/_schema/public.yaml'), 'utf-8'));
68
75
  expect(shard.tables.orders.usage).toEqual({
69
76
  ownerNote: 'keep me',
@@ -143,6 +150,14 @@ describe('projectHistoricSqlEvidence', () => {
143
150
  });
144
151
  const result = await projectHistoricSqlEvidence({ workdir, connectionId: 'warehouse', syncId: 'sync-1', runId: 'run-1' });
145
152
  expect(result.patternPagesWritten).toBe(1);
153
+ expect(result.changedWikiPageKeys).toContain('historic-sql-old-order-lifecycle');
154
+ expect(result.actions).toEqual(expect.arrayContaining([
155
+ expect.objectContaining({
156
+ target: 'wiki',
157
+ key: 'historic-sql-old-order-lifecycle',
158
+ rawPaths: ['patterns-input.json'],
159
+ }),
160
+ ]));
146
161
  await expect(readFile(join(workdir, 'wiki/global/historic-sql-old-order-lifecycle.md'), 'utf-8')).resolves.toContain('Order Lifecycle Analysis');
147
162
  await expect(readFile(join(workdir, 'wiki/global/historic-sql-retired-pattern.md'), 'utf-8')).resolves.toContain('stale_since: "2026-05-11T00:00:00.000Z"');
148
163
  });
@@ -274,6 +289,19 @@ describe('projectHistoricSqlEvidence', () => {
274
289
  probeWarnings: [],
275
290
  staleArchiveAfterDays: 90,
276
291
  });
292
+ await writeJson(workdir, '.ktx/ingest-evidence/historic-sql/run-1/customers.json', {
293
+ kind: 'table_usage',
294
+ connectionId: 'warehouse',
295
+ table: 'public.customers',
296
+ rawPath: 'tables/public.customers.json',
297
+ usage: {
298
+ narrative: 'Customers were queried.',
299
+ frequencyTier: 'low',
300
+ commonFilters: [],
301
+ commonJoins: [],
302
+ staleSince: null,
303
+ },
304
+ });
277
305
  await writeText(workdir, 'wiki/global/historic-sql-old-template.md', [
278
306
  '---',
279
307
  YAML.stringify({
@@ -294,6 +322,9 @@ describe('projectHistoricSqlEvidence', () => {
294
322
  const result = await projectHistoricSqlEvidence({ workdir, connectionId: 'warehouse', syncId: 'sync-1', runId: 'run-1' });
295
323
  expect(result.staleTablesMarked).toBe(1);
296
324
  expect(result.touchedSources).toEqual([{ connectionId: 'warehouse', sourceName: 'orders' }]);
325
+ const staleAction = result.actions.find((action) => action.target === 'sl' && action.key === 'orders');
326
+ expect(staleAction).toEqual(expect.objectContaining({ target: 'sl', key: 'orders' }));
327
+ expect(staleAction?.rawPaths).toBeUndefined();
297
328
  const shard = YAML.parse(await readFile(join(workdir, 'semantic-layer/warehouse/_schema/public.yaml'), 'utf-8'));
298
329
  expect(shard.tables.orders.usage).toEqual({
299
330
  ownerNote: 'keep analyst annotation',
@@ -306,4 +337,56 @@ describe('projectHistoricSqlEvidence', () => {
306
337
  });
307
338
  await expect(readFile(join(workdir, 'wiki/global/historic-sql-old-template.md'), 'utf-8')).resolves.toContain('Old body');
308
339
  });
340
+ it('does not mark stale or archive pages when override replay has no current-run evidence', async () => {
341
+ const workdir = await tempWorkdir();
342
+ await writeText(workdir, 'semantic-layer/warehouse/_schema/public.yaml', YAML.stringify({
343
+ tables: {
344
+ orders: {
345
+ table: 'public.orders',
346
+ usage: {
347
+ narrative: 'Orders were active before.',
348
+ frequencyTier: 'high',
349
+ commonFilters: ['status'],
350
+ commonGroupBys: ['status'],
351
+ commonJoins: [],
352
+ },
353
+ columns: [{ name: 'id', type: 'string' }],
354
+ },
355
+ },
356
+ }));
357
+ await writeJson(workdir, 'raw-sources/warehouse/historic-sql/override-sync/manifest.json', {
358
+ source: 'historic-sql',
359
+ connectionId: 'warehouse',
360
+ dialect: 'postgres',
361
+ fetchedAt: '2026-05-11T00:00:00.000Z',
362
+ windowStart: '2026-02-10T00:00:00.000Z',
363
+ windowEnd: '2026-05-11T00:00:00.000Z',
364
+ snapshotRowCount: 0,
365
+ touchedTableCount: 0,
366
+ parseFailures: 0,
367
+ warnings: [],
368
+ probeWarnings: [],
369
+ staleArchiveAfterDays: 90,
370
+ });
371
+ const result = await projectHistoricSqlEvidence({
372
+ workdir,
373
+ connectionId: 'warehouse',
374
+ syncId: 'override-sync',
375
+ runId: 'override-run',
376
+ overrideReplay: {
377
+ priorJobId: 'prior-job',
378
+ priorRunId: 'prior-run',
379
+ priorSyncId: 'prior-sync',
380
+ evictionRawPaths: ['tables/public/orders.json'],
381
+ },
382
+ });
383
+ expect(result.tableUsageMerged).toBe(0);
384
+ expect(result.staleTablesMarked).toBe(0);
385
+ expect(result.patternPagesWritten).toBe(0);
386
+ expect(result.stalePatternPagesMarked).toBe(0);
387
+ expect(result.archivedPatternPages).toBe(0);
388
+ expect(result.touchedSources).toEqual([]);
389
+ expect(result.changedWikiPageKeys).toEqual([]);
390
+ expect(result.actions).toEqual([]);
391
+ });
309
392
  });
@@ -2,6 +2,7 @@ import { spawn } from 'node:child_process';
2
2
  import { request as httpRequest } from 'node:http';
3
3
  import { request as httpsRequest } from 'node:https';
4
4
  import { URL } from 'node:url';
5
+ import { filterSnapshotTables, resolveEnabledTables } from '../../../scan/enabled-tables.js';
5
6
  import { inferKtxDimensionType, normalizeKtxNativeType } from '../../../scan/type-normalization.js';
6
7
  const DEFAULT_SCHEMAS = ['public'];
7
8
  function parseJsonObject(raw, subcommand) {
@@ -177,11 +178,13 @@ export function createDaemonLiveDatabaseIntrospection(options) {
177
178
  const raw = requestJson
178
179
  ? await requestJson('/database/introspect', payload)
179
180
  : await runJson('database-introspect', payload);
180
- return mapDaemonSnapshot(raw, {
181
+ const snapshot = mapDaemonSnapshot(raw, {
181
182
  connectionId,
182
183
  extractedAt: now().toISOString(),
183
184
  schemas,
184
185
  });
186
+ const enabledTables = resolveEnabledTables(connection);
187
+ return enabledTables ? filterSnapshotTables(snapshot, enabledTables) : snapshot;
185
188
  },
186
189
  };
187
190
  }
@@ -201,4 +201,36 @@ describe('createDaemonLiveDatabaseIntrospection', () => {
201
201
  await expect(introspection.extractSchema('warehouse')).rejects.toThrow('Local live-database ingest cannot run driver "snowflake".');
202
202
  expect(runJson).not.toHaveBeenCalled();
203
203
  });
204
+ it('filters out tables not on the enabled_tables allowlist', async () => {
205
+ const runJson = vi.fn(async () => daemonResponse);
206
+ const introspection = createDaemonLiveDatabaseIntrospection({
207
+ connections: {
208
+ warehouse: {
209
+ driver: 'postgres',
210
+ url: 'postgres://localhost:5432/warehouse',
211
+ enabled_tables: ['public.orders'],
212
+ },
213
+ },
214
+ schemas: ['public'],
215
+ runJson,
216
+ });
217
+ const snapshot = await introspection.extractSchema('warehouse');
218
+ expect(snapshot.tables.map((table) => `${table.db}.${table.name}`)).toEqual(['public.orders']);
219
+ });
220
+ it('passes through every table when enabled_tables is omitted or empty', async () => {
221
+ const runJson = vi.fn(async () => daemonResponse);
222
+ const introspection = createDaemonLiveDatabaseIntrospection({
223
+ connections: {
224
+ warehouse: {
225
+ driver: 'postgres',
226
+ url: 'postgres://localhost:5432/warehouse',
227
+ enabled_tables: [],
228
+ },
229
+ },
230
+ schemas: ['public'],
231
+ runJson,
232
+ });
233
+ const snapshot = await introspection.extractSchema('warehouse');
234
+ expect(snapshot.tables.map((table) => table.name)).toEqual(['customers', 'orders']);
235
+ });
204
236
  });
@@ -0,0 +1,22 @@
1
+ import type { SemanticLayerSource } from '../sl/index.js';
2
+ import type { TouchedSlSource } from '../tools/index.js';
3
+ import type { IngestReportFinalizationMismatch } from './reports.js';
4
+ interface DeriveTouchedSourcesInput {
5
+ changedPaths: string[];
6
+ beforeSourcesByConnection: Map<string, SemanticLayerSource[]>;
7
+ afterSourcesByConnection: Map<string, SemanticLayerSource[]>;
8
+ }
9
+ interface DeriveTouchedSourcesResult {
10
+ touchedSources: TouchedSlSource[];
11
+ unresolvedPaths: string[];
12
+ }
13
+ interface CompareFinalizationDeclarationsInput {
14
+ declaredTouchedSources: TouchedSlSource[];
15
+ derivedTouchedSources: TouchedSlSource[];
16
+ declaredChangedWikiPageKeys: string[];
17
+ derivedChangedWikiPageKeys: string[];
18
+ }
19
+ export declare function deriveFinalizationWikiPageKeys(paths: string[]): string[];
20
+ export declare function deriveFinalizationTouchedSources(input: DeriveTouchedSourcesInput): Promise<DeriveTouchedSourcesResult>;
21
+ export declare function compareFinalizationDeclarations(input: CompareFinalizationDeclarationsInput): IngestReportFinalizationMismatch[];
22
+ export {};
@@ -0,0 +1,95 @@
1
+ function uniqueSorted(values) {
2
+ return [...new Set(values.filter((value) => value.length > 0))].sort();
3
+ }
4
+ function touchedKey(source) {
5
+ return `${source.connectionId}:${source.sourceName}`;
6
+ }
7
+ function stableJson(value) {
8
+ if (Array.isArray(value)) {
9
+ return `[${value.map((entry) => stableJson(entry)).join(',')}]`;
10
+ }
11
+ if (value && typeof value === 'object') {
12
+ const record = value;
13
+ return `{${Object.keys(record)
14
+ .sort()
15
+ .map((key) => `${JSON.stringify(key)}:${stableJson(record[key])}`)
16
+ .join(',')}}`;
17
+ }
18
+ return JSON.stringify(value);
19
+ }
20
+ function changedSourceNames(beforeSources, afterSources) {
21
+ const before = new Map(beforeSources.map((source) => [source.name, stableJson(source)]));
22
+ const after = new Map(afterSources.map((source) => [source.name, stableJson(source)]));
23
+ return uniqueSorted(uniqueSorted([...before.keys(), ...after.keys()]).filter((sourceName) => before.get(sourceName) !== after.get(sourceName)));
24
+ }
25
+ export function deriveFinalizationWikiPageKeys(paths) {
26
+ return uniqueSorted(paths
27
+ .filter((path) => path.startsWith('wiki/global/') && path.endsWith('.md'))
28
+ .filter((path) => !path.slice('wiki/global/'.length, -'.md'.length).includes('/'))
29
+ .map((path) => path.slice('wiki/global/'.length, -'.md'.length)));
30
+ }
31
+ export async function deriveFinalizationTouchedSources(input) {
32
+ const touched = new Map();
33
+ const unresolvedPaths = [];
34
+ for (const path of input.changedPaths) {
35
+ if (!path.startsWith('semantic-layer/') || !(path.endsWith('.yaml') || path.endsWith('.yml'))) {
36
+ continue;
37
+ }
38
+ const parts = path.split('/');
39
+ const connectionId = parts[1] ?? '';
40
+ if (!connectionId) {
41
+ unresolvedPaths.push(path);
42
+ continue;
43
+ }
44
+ if (parts[2] !== '_schema') {
45
+ const fileName = parts.at(-1) ?? '';
46
+ const sourceName = fileName.replace(/\.ya?ml$/, '');
47
+ if (!sourceName) {
48
+ unresolvedPaths.push(path);
49
+ continue;
50
+ }
51
+ touched.set(`${connectionId}:${sourceName}`, { connectionId, sourceName });
52
+ continue;
53
+ }
54
+ const changedNames = changedSourceNames(input.beforeSourcesByConnection.get(connectionId) ?? [], input.afterSourcesByConnection.get(connectionId) ?? []);
55
+ if (changedNames.length === 0) {
56
+ unresolvedPaths.push(path);
57
+ continue;
58
+ }
59
+ for (const sourceName of changedNames) {
60
+ touched.set(`${connectionId}:${sourceName}`, { connectionId, sourceName });
61
+ }
62
+ }
63
+ return {
64
+ touchedSources: [...touched.values()].sort((left, right) => touchedKey(left).localeCompare(touchedKey(right))),
65
+ unresolvedPaths: uniqueSorted(unresolvedPaths),
66
+ };
67
+ }
68
+ export function compareFinalizationDeclarations(input) {
69
+ const mismatches = [];
70
+ const declaredSl = new Set(input.declaredTouchedSources.map(touchedKey));
71
+ const derivedSl = new Set(input.derivedTouchedSources.map(touchedKey));
72
+ const declaredWiki = new Set(input.declaredChangedWikiPageKeys);
73
+ const derivedWiki = new Set(input.derivedChangedWikiPageKeys);
74
+ for (const key of [...derivedSl].sort()) {
75
+ if (!declaredSl.has(key)) {
76
+ mismatches.push({ artifactKind: 'sl', key, direction: 'missing_from_adapter_declaration' });
77
+ }
78
+ }
79
+ for (const key of [...declaredSl].sort()) {
80
+ if (!derivedSl.has(key)) {
81
+ mismatches.push({ artifactKind: 'sl', key, direction: 'extra_in_adapter_declaration' });
82
+ }
83
+ }
84
+ for (const key of [...derivedWiki].sort()) {
85
+ if (!declaredWiki.has(key)) {
86
+ mismatches.push({ artifactKind: 'wiki', key, direction: 'missing_from_adapter_declaration' });
87
+ }
88
+ }
89
+ for (const key of [...declaredWiki].sort()) {
90
+ if (!derivedWiki.has(key)) {
91
+ mismatches.push({ artifactKind: 'wiki', key, direction: 'extra_in_adapter_declaration' });
92
+ }
93
+ }
94
+ return mismatches;
95
+ }
@@ -0,0 +1,114 @@
1
+ import { describe, expect, it } from 'vitest';
2
+ import { compareFinalizationDeclarations, deriveFinalizationTouchedSources, deriveFinalizationWikiPageKeys, } from './finalization-scope.js';
3
+ describe('deriveFinalizationWikiPageKeys', () => {
4
+ it('maps changed global wiki markdown paths to page keys', () => {
5
+ expect(deriveFinalizationWikiPageKeys([
6
+ 'wiki/global/historic-sql-orders.md',
7
+ 'wiki/global/nested/page.md',
8
+ 'README.md',
9
+ ])).toEqual(['historic-sql-orders']);
10
+ });
11
+ });
12
+ describe('deriveFinalizationTouchedSources', () => {
13
+ it('maps standalone semantic-layer files directly', async () => {
14
+ const result = await deriveFinalizationTouchedSources({
15
+ changedPaths: ['semantic-layer/warehouse/orders.yaml'],
16
+ beforeSourcesByConnection: new Map(),
17
+ afterSourcesByConnection: new Map(),
18
+ });
19
+ expect(result).toEqual({
20
+ touchedSources: [{ connectionId: 'warehouse', sourceName: 'orders' }],
21
+ unresolvedPaths: [],
22
+ });
23
+ });
24
+ it('resolves aggregate _schema changes by comparing loaded source snapshots', async () => {
25
+ const beforeSourcesByConnection = new Map([
26
+ [
27
+ 'warehouse',
28
+ [
29
+ {
30
+ name: 'orders',
31
+ grain: ['order_id'],
32
+ columns: [{ name: 'order_id', type: 'string' }],
33
+ joins: [],
34
+ measures: [],
35
+ usage: {
36
+ narrative: 'old',
37
+ frequencyTier: 'low',
38
+ commonFilters: [],
39
+ commonJoins: [],
40
+ },
41
+ },
42
+ ],
43
+ ],
44
+ ]);
45
+ const afterSourcesByConnection = new Map([
46
+ [
47
+ 'warehouse',
48
+ [
49
+ {
50
+ name: 'orders',
51
+ grain: ['order_id'],
52
+ columns: [{ name: 'order_id', type: 'string' }],
53
+ joins: [],
54
+ measures: [],
55
+ usage: {
56
+ narrative: 'new',
57
+ frequencyTier: 'high',
58
+ commonFilters: [],
59
+ commonJoins: [],
60
+ },
61
+ },
62
+ ],
63
+ ],
64
+ ]);
65
+ const result = await deriveFinalizationTouchedSources({
66
+ changedPaths: ['semantic-layer/warehouse/_schema/public.yaml'],
67
+ beforeSourcesByConnection,
68
+ afterSourcesByConnection,
69
+ });
70
+ expect(result).toEqual({
71
+ touchedSources: [{ connectionId: 'warehouse', sourceName: 'orders' }],
72
+ unresolvedPaths: [],
73
+ });
74
+ });
75
+ it('flags aggregate _schema changes that cannot be resolved to logical sources', async () => {
76
+ const beforeSourcesByConnection = new Map([['warehouse', []]]);
77
+ const afterSourcesByConnection = new Map([['warehouse', []]]);
78
+ const result = await deriveFinalizationTouchedSources({
79
+ changedPaths: ['semantic-layer/warehouse/_schema/public.yaml'],
80
+ beforeSourcesByConnection,
81
+ afterSourcesByConnection,
82
+ });
83
+ expect(result).toEqual({
84
+ touchedSources: [],
85
+ unresolvedPaths: ['semantic-layer/warehouse/_schema/public.yaml'],
86
+ });
87
+ });
88
+ });
89
+ describe('compareFinalizationDeclarations', () => {
90
+ it('reports missing and extra adapter declarations', () => {
91
+ expect(compareFinalizationDeclarations({
92
+ declaredTouchedSources: [{ connectionId: 'warehouse', sourceName: 'orders' }],
93
+ derivedTouchedSources: [{ connectionId: 'warehouse', sourceName: 'customers' }],
94
+ declaredChangedWikiPageKeys: ['orders'],
95
+ derivedChangedWikiPageKeys: ['orders', 'patterns'],
96
+ })).toEqual([
97
+ {
98
+ artifactKind: 'sl',
99
+ key: 'warehouse:customers',
100
+ direction: 'missing_from_adapter_declaration',
101
+ },
102
+ {
103
+ artifactKind: 'sl',
104
+ key: 'warehouse:orders',
105
+ direction: 'extra_in_adapter_declaration',
106
+ },
107
+ {
108
+ artifactKind: 'wiki',
109
+ key: 'patterns',
110
+ direction: 'missing_from_adapter_declaration',
111
+ },
112
+ ]);
113
+ });
114
+ });
@@ -77,7 +77,6 @@ export { stageHistoricSqlAggregatedSnapshot } from './adapters/historic-sql/stag
77
77
  export { historicSqlEvidenceEnvelopeSchema, historicSqlEvidencePath, historicSqlPatternEvidenceSchema, historicSqlTableUsageEvidenceSchema, serializeHistoricSqlEvidence, } from './adapters/historic-sql/evidence.js';
78
78
  export type { HistoricSqlEvidenceEnvelope, HistoricSqlPatternEvidence, HistoricSqlTableUsageEvidence, } from './adapters/historic-sql/evidence.js';
79
79
  export { createEmitHistoricSqlEvidenceTool } from './adapters/historic-sql/evidence-tool.js';
80
- export { HistoricSqlProjectionPostProcessor } from './adapters/historic-sql/post-processor.js';
81
80
  export { projectHistoricSqlEvidence } from './adapters/historic-sql/projection.js';
82
81
  export type { HistoricSqlProjectionInput, HistoricSqlProjectionResult } from './adapters/historic-sql/projection.js';
83
82
  export { patternOutputSchema, patternsArraySchema, tableUsageOutputSchema, } from './adapters/historic-sql/skill-schemas.js';
@@ -151,5 +150,5 @@ export { buildReconcileSystemPrompt, buildReconcileToolSet, buildReconcileUserPr
151
150
  export type { ReconciliationOutcome } from './stages/stage-4-reconciliation.js';
152
151
  export { runReconciliationStage4 } from './stages/stage-4-reconciliation.js';
153
152
  export type { StageIndex } from './stages/stage-index.types.js';
154
- export type { ChunkResult, DiffSet, EvictionUnit, FetchContext, IngestBundleJob, IngestBundleRef, IngestBundleResult, IngestDiffSummary, IngestJobContext, IngestJobPhase, IngestTrigger, ScopeDescriptor, SourceAdapter, SourceFetchIssue, SourceFetchReport, TriageLane, TriageSignals, UnresolvedCardInfo, WorkUnit, DeterministicProjectionContext, ProjectionResult, } from './types.js';
153
+ export type { ChunkResult, DiffSet, EvictionUnit, FetchContext, IngestBundleJob, IngestBundleRef, IngestBundleResult, IngestDiffSummary, IngestJobContext, IngestJobPhase, IngestTrigger, ScopeDescriptor, SourceAdapter, SourceFetchIssue, SourceFetchReport, TriageLane, TriageSignals, UnresolvedCardInfo, WorkUnit, DeterministicProjectionContext, ProjectionResult, DeterministicFinalizationContext, FinalizationOverrideReplay, FinalizationResult, } from './types.js';
155
154
  export * from './wiki-body-refs.js';
@@ -51,7 +51,6 @@ export { SnowflakeHistoricSqlQueryHistoryReader } from './adapters/historic-sql/
51
51
  export { stageHistoricSqlAggregatedSnapshot } from './adapters/historic-sql/stage-unified.js';
52
52
  export { historicSqlEvidenceEnvelopeSchema, historicSqlEvidencePath, historicSqlPatternEvidenceSchema, historicSqlTableUsageEvidenceSchema, serializeHistoricSqlEvidence, } from './adapters/historic-sql/evidence.js';
53
53
  export { createEmitHistoricSqlEvidenceTool } from './adapters/historic-sql/evidence-tool.js';
54
- export { HistoricSqlProjectionPostProcessor } from './adapters/historic-sql/post-processor.js';
55
54
  export { projectHistoricSqlEvidence } from './adapters/historic-sql/projection.js';
56
55
  export { patternOutputSchema, patternsArraySchema, tableUsageOutputSchema, } from './adapters/historic-sql/skill-schemas.js';
57
56
  export { HISTORIC_SQL_SOURCE_KEY, aggregatedTemplateSchema, historicSqlUnifiedPullConfigSchema, stagedManifestSchema, stagedPatternsInputSchema, stagedTableInputSchema, } from './adapters/historic-sql/types.js';
@@ -32,11 +32,13 @@ export declare class IngestBundleRunner {
32
32
  private buildWikiIndex;
33
33
  private buildSlIndex;
34
34
  private tableRefExistsInSemanticLayer;
35
+ private loadSourcesByConnection;
35
36
  private resolveContextCuratorBudget;
36
37
  private filterWorkUnitsForTriage;
37
38
  private createTrace;
38
39
  private errorMessage;
39
40
  private buildProvenancePlan;
41
+ private partitionFinalizationActionsForProvenance;
40
42
  private toReportProvenanceRows;
41
43
  private toReportWorkUnits;
42
44
  private provenanceValidationTraceData;