@kaelio/ktx 0.2.0 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/assets/python/{kaelio_ktx-0.2.0-py3-none-any.whl → kaelio_ktx-0.4.0-py3-none-any.whl} +0 -0
- package/assets/python/manifest.json +4 -4
- package/dist/admin-reindex.js +10 -17
- package/dist/admin-reindex.test.js +1 -1
- package/dist/cli-program.test.js +0 -2
- package/dist/cli-project.d.ts +18 -0
- package/dist/cli-project.js +52 -0
- package/dist/cli-project.test.js +149 -0
- package/dist/cli-runtime.d.ts +0 -2
- package/dist/cli-runtime.js +2 -8
- package/dist/commands/runtime-commands.js +2 -2
- package/dist/context-build-view.js +1 -1
- package/dist/index.test.js +21 -25
- package/dist/ingest.js +9 -2
- package/dist/ingest.test.js +27 -3
- package/dist/managed-local-embeddings.d.ts +0 -2
- package/dist/managed-local-embeddings.js +2 -5
- package/dist/managed-local-embeddings.test.js +5 -8
- package/dist/managed-python-daemon.js +2 -2
- package/dist/managed-python-daemon.test.js +1 -1
- package/dist/managed-python-http.js +3 -3
- package/dist/managed-python-http.test.js +6 -6
- package/dist/print-command-tree.js +0 -2
- package/dist/public-ingest.d.ts +4 -2
- package/dist/public-ingest.js +9 -3
- package/dist/release-version.d.ts +1 -5
- package/dist/release-version.js +2 -39
- package/dist/runtime-requirements.js +1 -1
- package/dist/runtime.js +6 -6
- package/dist/runtime.test.js +7 -7
- package/dist/scan.js +7 -2
- package/dist/scan.test.js +1 -1
- package/dist/setup-embeddings.js +1 -1
- package/dist/setup-embeddings.test.js +2 -2
- package/dist/setup-runtime.test.js +1 -1
- package/node_modules/@ktx/context/dist/core/git.service.d.ts +1 -0
- package/node_modules/@ktx/context/dist/core/git.service.js +12 -0
- package/node_modules/@ktx/context/dist/ingest/adapters/historic-sql/historic-sql.adapter.d.ts +2 -1
- package/node_modules/@ktx/context/dist/ingest/adapters/historic-sql/historic-sql.adapter.js +18 -0
- package/node_modules/@ktx/context/dist/ingest/adapters/historic-sql/local-ingest-acceptance.test.js +6 -6
- package/node_modules/@ktx/context/dist/ingest/adapters/historic-sql/projection.d.ts +5 -0
- package/node_modules/@ktx/context/dist/ingest/adapters/historic-sql/projection.js +48 -0
- package/node_modules/@ktx/context/dist/ingest/adapters/historic-sql/projection.test.js +83 -0
- package/node_modules/@ktx/context/dist/ingest/adapters/live-database/daemon-introspection.js +4 -1
- package/node_modules/@ktx/context/dist/ingest/adapters/live-database/daemon-introspection.test.js +32 -0
- package/node_modules/@ktx/context/dist/ingest/finalization-scope.d.ts +22 -0
- package/node_modules/@ktx/context/dist/ingest/finalization-scope.js +95 -0
- package/node_modules/@ktx/context/dist/ingest/finalization-scope.test.d.ts +1 -0
- package/node_modules/@ktx/context/dist/ingest/finalization-scope.test.js +114 -0
- package/node_modules/@ktx/context/dist/ingest/index.d.ts +1 -2
- package/node_modules/@ktx/context/dist/ingest/index.js +0 -1
- package/node_modules/@ktx/context/dist/ingest/ingest-bundle.runner.d.ts +2 -0
- package/node_modules/@ktx/context/dist/ingest/ingest-bundle.runner.isolated-diff.test.js +166 -0
- package/node_modules/@ktx/context/dist/ingest/ingest-bundle.runner.js +235 -45
- package/node_modules/@ktx/context/dist/ingest/ingest-bundle.runner.test.js +193 -38
- package/node_modules/@ktx/context/dist/ingest/local-bundle-ingest.test.js +22 -3
- package/node_modules/@ktx/context/dist/ingest/local-bundle-runtime.js +0 -4
- package/node_modules/@ktx/context/dist/ingest/local-ingest.js +0 -7
- package/node_modules/@ktx/context/dist/ingest/local-stage-ingest.js +15 -5
- package/node_modules/@ktx/context/dist/ingest/local-stage-ingest.test.js +29 -0
- package/node_modules/@ktx/context/dist/ingest/memory-flow/schema.d.ts +2 -2
- package/node_modules/@ktx/context/dist/ingest/memory-flow/schema.js +1 -1
- package/node_modules/@ktx/context/dist/ingest/memory-flow/types.d.ts +1 -1
- package/node_modules/@ktx/context/dist/ingest/ports.d.ts +1 -20
- package/node_modules/@ktx/context/dist/ingest/report-snapshot.d.ts +71 -0
- package/node_modules/@ktx/context/dist/ingest/report-snapshot.js +27 -0
- package/node_modules/@ktx/context/dist/ingest/reports.d.ts +23 -5
- package/node_modules/@ktx/context/dist/ingest/reports.js +7 -24
- package/node_modules/@ktx/context/dist/ingest/types.d.ts +33 -0
- package/node_modules/@ktx/context/dist/llm/index.d.ts +1 -1
- package/node_modules/@ktx/context/dist/llm/index.js +1 -1
- package/node_modules/@ktx/context/dist/llm/local-config.d.ts +0 -1
- package/node_modules/@ktx/context/dist/llm/local-config.js +2 -12
- package/node_modules/@ktx/context/dist/llm/local-config.test.js +2 -23
- package/node_modules/@ktx/context/dist/package-exports.test.js +2 -2
- package/node_modules/@ktx/context/dist/project/config.d.ts +16 -0
- package/node_modules/@ktx/context/dist/project/driver-schemas.d.ts +8 -0
- package/node_modules/@ktx/context/dist/project/driver-schemas.js +4 -0
- package/node_modules/@ktx/context/dist/scan/enabled-tables.d.ts +3 -0
- package/node_modules/@ktx/context/dist/scan/enabled-tables.js +15 -0
- package/node_modules/@ktx/context/dist/scan/local-scan.d.ts +2 -4
- package/node_modules/@ktx/context/dist/scan/local-scan.js +2 -15
- package/package.json +1 -1
- package/node_modules/@ktx/context/dist/ingest/adapters/historic-sql/post-processor.d.ts +0 -4
- package/node_modules/@ktx/context/dist/ingest/adapters/historic-sql/post-processor.js +0 -38
- package/node_modules/@ktx/context/dist/ingest/adapters/historic-sql/post-processor.test.js +0 -63
- /package/{node_modules/@ktx/context/dist/ingest/adapters/historic-sql/post-processor.test.d.ts → dist/cli-project.test.d.ts} +0 -0
|
@@ -177,6 +177,8 @@ export async function projectHistoricSqlEvidence(input) {
|
|
|
177
177
|
stalePatternPagesMarked: 0,
|
|
178
178
|
archivedPatternPages: 0,
|
|
179
179
|
touchedSources: [],
|
|
180
|
+
changedWikiPageKeys: [],
|
|
181
|
+
actions: [],
|
|
180
182
|
warnings: [],
|
|
181
183
|
};
|
|
182
184
|
const touchedKeys = new Set();
|
|
@@ -184,6 +186,14 @@ export async function projectHistoricSqlEvidence(input) {
|
|
|
184
186
|
const manifest = stagedManifestSchema.parse(await readJson(join(rawDir, 'manifest.json')));
|
|
185
187
|
const currentTables = await currentStagedTables(rawDir);
|
|
186
188
|
const evidence = await loadEvidence(input.workdir, input.runId);
|
|
189
|
+
if (input.overrideReplay && evidence.length === 0) {
|
|
190
|
+
result.warnings.push('historic-sql finalization skipped stale/archive cleanup during override replay without current-run evidence');
|
|
191
|
+
return result;
|
|
192
|
+
}
|
|
193
|
+
if (evidence.length === 0) {
|
|
194
|
+
result.warnings.push('historic-sql finalization skipped because no current-run evidence was emitted');
|
|
195
|
+
return result;
|
|
196
|
+
}
|
|
187
197
|
const tableEvidence = evidence.filter((entry) => entry.kind === 'table_usage');
|
|
188
198
|
const patternEvidence = evidence.filter((entry) => entry.kind === 'pattern');
|
|
189
199
|
const schemaRoot = join(input.workdir, 'semantic-layer', input.connectionId, '_schema');
|
|
@@ -207,6 +217,14 @@ export async function projectHistoricSqlEvidence(input) {
|
|
|
207
217
|
touchedKeys.add(key);
|
|
208
218
|
result.touchedSources.push({ connectionId: input.connectionId, sourceName });
|
|
209
219
|
}
|
|
220
|
+
result.actions.push({
|
|
221
|
+
target: 'sl',
|
|
222
|
+
type: 'updated',
|
|
223
|
+
key: sourceName,
|
|
224
|
+
targetConnectionId: input.connectionId,
|
|
225
|
+
detail: `Merged historic-SQL usage for ${matchingEvidence.table}`,
|
|
226
|
+
rawPaths: [matchingEvidence.rawPath],
|
|
227
|
+
});
|
|
210
228
|
}
|
|
211
229
|
}
|
|
212
230
|
else if (entry.usage && !currentTables.has(tableRef)) {
|
|
@@ -220,6 +238,13 @@ export async function projectHistoricSqlEvidence(input) {
|
|
|
220
238
|
touchedKeys.add(key);
|
|
221
239
|
result.touchedSources.push({ connectionId: input.connectionId, sourceName });
|
|
222
240
|
}
|
|
241
|
+
result.actions.push({
|
|
242
|
+
target: 'sl',
|
|
243
|
+
type: 'updated',
|
|
244
|
+
key: sourceName,
|
|
245
|
+
targetConnectionId: input.connectionId,
|
|
246
|
+
detail: `Marked historic-SQL usage stale for ${tableRef}`,
|
|
247
|
+
});
|
|
223
248
|
}
|
|
224
249
|
}
|
|
225
250
|
}
|
|
@@ -254,6 +279,14 @@ export async function projectHistoricSqlEvidence(input) {
|
|
|
254
279
|
await writeFile(pagePath, renderMarkdownPage(frontmatter, renderPatternMarkdown(pattern)), 'utf-8');
|
|
255
280
|
writtenKeys.add(key);
|
|
256
281
|
result.patternPagesWritten += 1;
|
|
282
|
+
result.changedWikiPageKeys.push(key);
|
|
283
|
+
result.actions.push({
|
|
284
|
+
target: 'wiki',
|
|
285
|
+
type: reusable ? 'updated' : 'created',
|
|
286
|
+
key,
|
|
287
|
+
detail: `Projected historic-SQL pattern ${pattern.pattern.title}`,
|
|
288
|
+
rawPaths: [pattern.rawPath],
|
|
289
|
+
});
|
|
257
290
|
}
|
|
258
291
|
for (const page of patternPages) {
|
|
259
292
|
if (writtenKeys.has(page.key))
|
|
@@ -262,11 +295,26 @@ export async function projectHistoricSqlEvidence(input) {
|
|
|
262
295
|
const tags = [...new Set([...stringArray(page.frontmatter.tags), 'archived'])];
|
|
263
296
|
await writeFile(page.path, renderMarkdownPage({ ...page.frontmatter, tags, archived_since: manifest.fetchedAt }, page.content), 'utf-8');
|
|
264
297
|
result.archivedPatternPages += 1;
|
|
298
|
+
result.changedWikiPageKeys.push(page.key);
|
|
299
|
+
result.actions.push({
|
|
300
|
+
target: 'wiki',
|
|
301
|
+
type: 'updated',
|
|
302
|
+
key: page.key,
|
|
303
|
+
detail: `Archived stale historic-SQL pattern page ${page.key}`,
|
|
304
|
+
});
|
|
265
305
|
continue;
|
|
266
306
|
}
|
|
267
307
|
const tags = [...new Set([...stringArray(page.frontmatter.tags), 'stale'])];
|
|
268
308
|
await writeFile(page.path, renderMarkdownPage({ ...page.frontmatter, tags, stale_since: manifest.fetchedAt }, page.content), 'utf-8');
|
|
269
309
|
result.stalePatternPagesMarked += 1;
|
|
310
|
+
result.changedWikiPageKeys.push(page.key);
|
|
311
|
+
result.actions.push({
|
|
312
|
+
target: 'wiki',
|
|
313
|
+
type: 'updated',
|
|
314
|
+
key: page.key,
|
|
315
|
+
detail: `Marked historic-SQL pattern page ${page.key} stale`,
|
|
316
|
+
});
|
|
270
317
|
}
|
|
318
|
+
result.changedWikiPageKeys = [...new Set(result.changedWikiPageKeys)].sort();
|
|
271
319
|
return result;
|
|
272
320
|
}
|
|
@@ -64,6 +64,13 @@ describe('projectHistoricSqlEvidence', () => {
|
|
|
64
64
|
});
|
|
65
65
|
const result = await projectHistoricSqlEvidence({ workdir, connectionId: 'warehouse', syncId: 'sync-1', runId: 'run-1' });
|
|
66
66
|
expect(result.touchedSources).toEqual([{ connectionId: 'warehouse', sourceName: 'orders' }]);
|
|
67
|
+
expect(result.actions).toEqual(expect.arrayContaining([
|
|
68
|
+
expect.objectContaining({
|
|
69
|
+
target: 'sl',
|
|
70
|
+
key: 'orders',
|
|
71
|
+
rawPaths: ['tables/public.orders.json'],
|
|
72
|
+
}),
|
|
73
|
+
]));
|
|
67
74
|
const shard = YAML.parse(await readFile(join(workdir, 'semantic-layer/warehouse/_schema/public.yaml'), 'utf-8'));
|
|
68
75
|
expect(shard.tables.orders.usage).toEqual({
|
|
69
76
|
ownerNote: 'keep me',
|
|
@@ -143,6 +150,14 @@ describe('projectHistoricSqlEvidence', () => {
|
|
|
143
150
|
});
|
|
144
151
|
const result = await projectHistoricSqlEvidence({ workdir, connectionId: 'warehouse', syncId: 'sync-1', runId: 'run-1' });
|
|
145
152
|
expect(result.patternPagesWritten).toBe(1);
|
|
153
|
+
expect(result.changedWikiPageKeys).toContain('historic-sql-old-order-lifecycle');
|
|
154
|
+
expect(result.actions).toEqual(expect.arrayContaining([
|
|
155
|
+
expect.objectContaining({
|
|
156
|
+
target: 'wiki',
|
|
157
|
+
key: 'historic-sql-old-order-lifecycle',
|
|
158
|
+
rawPaths: ['patterns-input.json'],
|
|
159
|
+
}),
|
|
160
|
+
]));
|
|
146
161
|
await expect(readFile(join(workdir, 'wiki/global/historic-sql-old-order-lifecycle.md'), 'utf-8')).resolves.toContain('Order Lifecycle Analysis');
|
|
147
162
|
await expect(readFile(join(workdir, 'wiki/global/historic-sql-retired-pattern.md'), 'utf-8')).resolves.toContain('stale_since: "2026-05-11T00:00:00.000Z"');
|
|
148
163
|
});
|
|
@@ -274,6 +289,19 @@ describe('projectHistoricSqlEvidence', () => {
|
|
|
274
289
|
probeWarnings: [],
|
|
275
290
|
staleArchiveAfterDays: 90,
|
|
276
291
|
});
|
|
292
|
+
await writeJson(workdir, '.ktx/ingest-evidence/historic-sql/run-1/customers.json', {
|
|
293
|
+
kind: 'table_usage',
|
|
294
|
+
connectionId: 'warehouse',
|
|
295
|
+
table: 'public.customers',
|
|
296
|
+
rawPath: 'tables/public.customers.json',
|
|
297
|
+
usage: {
|
|
298
|
+
narrative: 'Customers were queried.',
|
|
299
|
+
frequencyTier: 'low',
|
|
300
|
+
commonFilters: [],
|
|
301
|
+
commonJoins: [],
|
|
302
|
+
staleSince: null,
|
|
303
|
+
},
|
|
304
|
+
});
|
|
277
305
|
await writeText(workdir, 'wiki/global/historic-sql-old-template.md', [
|
|
278
306
|
'---',
|
|
279
307
|
YAML.stringify({
|
|
@@ -294,6 +322,9 @@ describe('projectHistoricSqlEvidence', () => {
|
|
|
294
322
|
const result = await projectHistoricSqlEvidence({ workdir, connectionId: 'warehouse', syncId: 'sync-1', runId: 'run-1' });
|
|
295
323
|
expect(result.staleTablesMarked).toBe(1);
|
|
296
324
|
expect(result.touchedSources).toEqual([{ connectionId: 'warehouse', sourceName: 'orders' }]);
|
|
325
|
+
const staleAction = result.actions.find((action) => action.target === 'sl' && action.key === 'orders');
|
|
326
|
+
expect(staleAction).toEqual(expect.objectContaining({ target: 'sl', key: 'orders' }));
|
|
327
|
+
expect(staleAction?.rawPaths).toBeUndefined();
|
|
297
328
|
const shard = YAML.parse(await readFile(join(workdir, 'semantic-layer/warehouse/_schema/public.yaml'), 'utf-8'));
|
|
298
329
|
expect(shard.tables.orders.usage).toEqual({
|
|
299
330
|
ownerNote: 'keep analyst annotation',
|
|
@@ -306,4 +337,56 @@ describe('projectHistoricSqlEvidence', () => {
|
|
|
306
337
|
});
|
|
307
338
|
await expect(readFile(join(workdir, 'wiki/global/historic-sql-old-template.md'), 'utf-8')).resolves.toContain('Old body');
|
|
308
339
|
});
|
|
340
|
+
it('does not mark stale or archive pages when override replay has no current-run evidence', async () => {
|
|
341
|
+
const workdir = await tempWorkdir();
|
|
342
|
+
await writeText(workdir, 'semantic-layer/warehouse/_schema/public.yaml', YAML.stringify({
|
|
343
|
+
tables: {
|
|
344
|
+
orders: {
|
|
345
|
+
table: 'public.orders',
|
|
346
|
+
usage: {
|
|
347
|
+
narrative: 'Orders were active before.',
|
|
348
|
+
frequencyTier: 'high',
|
|
349
|
+
commonFilters: ['status'],
|
|
350
|
+
commonGroupBys: ['status'],
|
|
351
|
+
commonJoins: [],
|
|
352
|
+
},
|
|
353
|
+
columns: [{ name: 'id', type: 'string' }],
|
|
354
|
+
},
|
|
355
|
+
},
|
|
356
|
+
}));
|
|
357
|
+
await writeJson(workdir, 'raw-sources/warehouse/historic-sql/override-sync/manifest.json', {
|
|
358
|
+
source: 'historic-sql',
|
|
359
|
+
connectionId: 'warehouse',
|
|
360
|
+
dialect: 'postgres',
|
|
361
|
+
fetchedAt: '2026-05-11T00:00:00.000Z',
|
|
362
|
+
windowStart: '2026-02-10T00:00:00.000Z',
|
|
363
|
+
windowEnd: '2026-05-11T00:00:00.000Z',
|
|
364
|
+
snapshotRowCount: 0,
|
|
365
|
+
touchedTableCount: 0,
|
|
366
|
+
parseFailures: 0,
|
|
367
|
+
warnings: [],
|
|
368
|
+
probeWarnings: [],
|
|
369
|
+
staleArchiveAfterDays: 90,
|
|
370
|
+
});
|
|
371
|
+
const result = await projectHistoricSqlEvidence({
|
|
372
|
+
workdir,
|
|
373
|
+
connectionId: 'warehouse',
|
|
374
|
+
syncId: 'override-sync',
|
|
375
|
+
runId: 'override-run',
|
|
376
|
+
overrideReplay: {
|
|
377
|
+
priorJobId: 'prior-job',
|
|
378
|
+
priorRunId: 'prior-run',
|
|
379
|
+
priorSyncId: 'prior-sync',
|
|
380
|
+
evictionRawPaths: ['tables/public/orders.json'],
|
|
381
|
+
},
|
|
382
|
+
});
|
|
383
|
+
expect(result.tableUsageMerged).toBe(0);
|
|
384
|
+
expect(result.staleTablesMarked).toBe(0);
|
|
385
|
+
expect(result.patternPagesWritten).toBe(0);
|
|
386
|
+
expect(result.stalePatternPagesMarked).toBe(0);
|
|
387
|
+
expect(result.archivedPatternPages).toBe(0);
|
|
388
|
+
expect(result.touchedSources).toEqual([]);
|
|
389
|
+
expect(result.changedWikiPageKeys).toEqual([]);
|
|
390
|
+
expect(result.actions).toEqual([]);
|
|
391
|
+
});
|
|
309
392
|
});
|
package/node_modules/@ktx/context/dist/ingest/adapters/live-database/daemon-introspection.js
CHANGED
|
@@ -2,6 +2,7 @@ import { spawn } from 'node:child_process';
|
|
|
2
2
|
import { request as httpRequest } from 'node:http';
|
|
3
3
|
import { request as httpsRequest } from 'node:https';
|
|
4
4
|
import { URL } from 'node:url';
|
|
5
|
+
import { filterSnapshotTables, resolveEnabledTables } from '../../../scan/enabled-tables.js';
|
|
5
6
|
import { inferKtxDimensionType, normalizeKtxNativeType } from '../../../scan/type-normalization.js';
|
|
6
7
|
const DEFAULT_SCHEMAS = ['public'];
|
|
7
8
|
function parseJsonObject(raw, subcommand) {
|
|
@@ -177,11 +178,13 @@ export function createDaemonLiveDatabaseIntrospection(options) {
|
|
|
177
178
|
const raw = requestJson
|
|
178
179
|
? await requestJson('/database/introspect', payload)
|
|
179
180
|
: await runJson('database-introspect', payload);
|
|
180
|
-
|
|
181
|
+
const snapshot = mapDaemonSnapshot(raw, {
|
|
181
182
|
connectionId,
|
|
182
183
|
extractedAt: now().toISOString(),
|
|
183
184
|
schemas,
|
|
184
185
|
});
|
|
186
|
+
const enabledTables = resolveEnabledTables(connection);
|
|
187
|
+
return enabledTables ? filterSnapshotTables(snapshot, enabledTables) : snapshot;
|
|
185
188
|
},
|
|
186
189
|
};
|
|
187
190
|
}
|
package/node_modules/@ktx/context/dist/ingest/adapters/live-database/daemon-introspection.test.js
CHANGED
|
@@ -201,4 +201,36 @@ describe('createDaemonLiveDatabaseIntrospection', () => {
|
|
|
201
201
|
await expect(introspection.extractSchema('warehouse')).rejects.toThrow('Local live-database ingest cannot run driver "snowflake".');
|
|
202
202
|
expect(runJson).not.toHaveBeenCalled();
|
|
203
203
|
});
|
|
204
|
+
it('filters out tables not on the enabled_tables allowlist', async () => {
|
|
205
|
+
const runJson = vi.fn(async () => daemonResponse);
|
|
206
|
+
const introspection = createDaemonLiveDatabaseIntrospection({
|
|
207
|
+
connections: {
|
|
208
|
+
warehouse: {
|
|
209
|
+
driver: 'postgres',
|
|
210
|
+
url: 'postgres://localhost:5432/warehouse',
|
|
211
|
+
enabled_tables: ['public.orders'],
|
|
212
|
+
},
|
|
213
|
+
},
|
|
214
|
+
schemas: ['public'],
|
|
215
|
+
runJson,
|
|
216
|
+
});
|
|
217
|
+
const snapshot = await introspection.extractSchema('warehouse');
|
|
218
|
+
expect(snapshot.tables.map((table) => `${table.db}.${table.name}`)).toEqual(['public.orders']);
|
|
219
|
+
});
|
|
220
|
+
it('passes through every table when enabled_tables is omitted or empty', async () => {
|
|
221
|
+
const runJson = vi.fn(async () => daemonResponse);
|
|
222
|
+
const introspection = createDaemonLiveDatabaseIntrospection({
|
|
223
|
+
connections: {
|
|
224
|
+
warehouse: {
|
|
225
|
+
driver: 'postgres',
|
|
226
|
+
url: 'postgres://localhost:5432/warehouse',
|
|
227
|
+
enabled_tables: [],
|
|
228
|
+
},
|
|
229
|
+
},
|
|
230
|
+
schemas: ['public'],
|
|
231
|
+
runJson,
|
|
232
|
+
});
|
|
233
|
+
const snapshot = await introspection.extractSchema('warehouse');
|
|
234
|
+
expect(snapshot.tables.map((table) => table.name)).toEqual(['customers', 'orders']);
|
|
235
|
+
});
|
|
204
236
|
});
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
import type { SemanticLayerSource } from '../sl/index.js';
|
|
2
|
+
import type { TouchedSlSource } from '../tools/index.js';
|
|
3
|
+
import type { IngestReportFinalizationMismatch } from './reports.js';
|
|
4
|
+
interface DeriveTouchedSourcesInput {
|
|
5
|
+
changedPaths: string[];
|
|
6
|
+
beforeSourcesByConnection: Map<string, SemanticLayerSource[]>;
|
|
7
|
+
afterSourcesByConnection: Map<string, SemanticLayerSource[]>;
|
|
8
|
+
}
|
|
9
|
+
interface DeriveTouchedSourcesResult {
|
|
10
|
+
touchedSources: TouchedSlSource[];
|
|
11
|
+
unresolvedPaths: string[];
|
|
12
|
+
}
|
|
13
|
+
interface CompareFinalizationDeclarationsInput {
|
|
14
|
+
declaredTouchedSources: TouchedSlSource[];
|
|
15
|
+
derivedTouchedSources: TouchedSlSource[];
|
|
16
|
+
declaredChangedWikiPageKeys: string[];
|
|
17
|
+
derivedChangedWikiPageKeys: string[];
|
|
18
|
+
}
|
|
19
|
+
export declare function deriveFinalizationWikiPageKeys(paths: string[]): string[];
|
|
20
|
+
export declare function deriveFinalizationTouchedSources(input: DeriveTouchedSourcesInput): Promise<DeriveTouchedSourcesResult>;
|
|
21
|
+
export declare function compareFinalizationDeclarations(input: CompareFinalizationDeclarationsInput): IngestReportFinalizationMismatch[];
|
|
22
|
+
export {};
|
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
function uniqueSorted(values) {
|
|
2
|
+
return [...new Set(values.filter((value) => value.length > 0))].sort();
|
|
3
|
+
}
|
|
4
|
+
function touchedKey(source) {
|
|
5
|
+
return `${source.connectionId}:${source.sourceName}`;
|
|
6
|
+
}
|
|
7
|
+
function stableJson(value) {
|
|
8
|
+
if (Array.isArray(value)) {
|
|
9
|
+
return `[${value.map((entry) => stableJson(entry)).join(',')}]`;
|
|
10
|
+
}
|
|
11
|
+
if (value && typeof value === 'object') {
|
|
12
|
+
const record = value;
|
|
13
|
+
return `{${Object.keys(record)
|
|
14
|
+
.sort()
|
|
15
|
+
.map((key) => `${JSON.stringify(key)}:${stableJson(record[key])}`)
|
|
16
|
+
.join(',')}}`;
|
|
17
|
+
}
|
|
18
|
+
return JSON.stringify(value);
|
|
19
|
+
}
|
|
20
|
+
function changedSourceNames(beforeSources, afterSources) {
|
|
21
|
+
const before = new Map(beforeSources.map((source) => [source.name, stableJson(source)]));
|
|
22
|
+
const after = new Map(afterSources.map((source) => [source.name, stableJson(source)]));
|
|
23
|
+
return uniqueSorted(uniqueSorted([...before.keys(), ...after.keys()]).filter((sourceName) => before.get(sourceName) !== after.get(sourceName)));
|
|
24
|
+
}
|
|
25
|
+
export function deriveFinalizationWikiPageKeys(paths) {
|
|
26
|
+
return uniqueSorted(paths
|
|
27
|
+
.filter((path) => path.startsWith('wiki/global/') && path.endsWith('.md'))
|
|
28
|
+
.filter((path) => !path.slice('wiki/global/'.length, -'.md'.length).includes('/'))
|
|
29
|
+
.map((path) => path.slice('wiki/global/'.length, -'.md'.length)));
|
|
30
|
+
}
|
|
31
|
+
export async function deriveFinalizationTouchedSources(input) {
|
|
32
|
+
const touched = new Map();
|
|
33
|
+
const unresolvedPaths = [];
|
|
34
|
+
for (const path of input.changedPaths) {
|
|
35
|
+
if (!path.startsWith('semantic-layer/') || !(path.endsWith('.yaml') || path.endsWith('.yml'))) {
|
|
36
|
+
continue;
|
|
37
|
+
}
|
|
38
|
+
const parts = path.split('/');
|
|
39
|
+
const connectionId = parts[1] ?? '';
|
|
40
|
+
if (!connectionId) {
|
|
41
|
+
unresolvedPaths.push(path);
|
|
42
|
+
continue;
|
|
43
|
+
}
|
|
44
|
+
if (parts[2] !== '_schema') {
|
|
45
|
+
const fileName = parts.at(-1) ?? '';
|
|
46
|
+
const sourceName = fileName.replace(/\.ya?ml$/, '');
|
|
47
|
+
if (!sourceName) {
|
|
48
|
+
unresolvedPaths.push(path);
|
|
49
|
+
continue;
|
|
50
|
+
}
|
|
51
|
+
touched.set(`${connectionId}:${sourceName}`, { connectionId, sourceName });
|
|
52
|
+
continue;
|
|
53
|
+
}
|
|
54
|
+
const changedNames = changedSourceNames(input.beforeSourcesByConnection.get(connectionId) ?? [], input.afterSourcesByConnection.get(connectionId) ?? []);
|
|
55
|
+
if (changedNames.length === 0) {
|
|
56
|
+
unresolvedPaths.push(path);
|
|
57
|
+
continue;
|
|
58
|
+
}
|
|
59
|
+
for (const sourceName of changedNames) {
|
|
60
|
+
touched.set(`${connectionId}:${sourceName}`, { connectionId, sourceName });
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
return {
|
|
64
|
+
touchedSources: [...touched.values()].sort((left, right) => touchedKey(left).localeCompare(touchedKey(right))),
|
|
65
|
+
unresolvedPaths: uniqueSorted(unresolvedPaths),
|
|
66
|
+
};
|
|
67
|
+
}
|
|
68
|
+
export function compareFinalizationDeclarations(input) {
|
|
69
|
+
const mismatches = [];
|
|
70
|
+
const declaredSl = new Set(input.declaredTouchedSources.map(touchedKey));
|
|
71
|
+
const derivedSl = new Set(input.derivedTouchedSources.map(touchedKey));
|
|
72
|
+
const declaredWiki = new Set(input.declaredChangedWikiPageKeys);
|
|
73
|
+
const derivedWiki = new Set(input.derivedChangedWikiPageKeys);
|
|
74
|
+
for (const key of [...derivedSl].sort()) {
|
|
75
|
+
if (!declaredSl.has(key)) {
|
|
76
|
+
mismatches.push({ artifactKind: 'sl', key, direction: 'missing_from_adapter_declaration' });
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
for (const key of [...declaredSl].sort()) {
|
|
80
|
+
if (!derivedSl.has(key)) {
|
|
81
|
+
mismatches.push({ artifactKind: 'sl', key, direction: 'extra_in_adapter_declaration' });
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
for (const key of [...derivedWiki].sort()) {
|
|
85
|
+
if (!declaredWiki.has(key)) {
|
|
86
|
+
mismatches.push({ artifactKind: 'wiki', key, direction: 'missing_from_adapter_declaration' });
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
for (const key of [...declaredWiki].sort()) {
|
|
90
|
+
if (!derivedWiki.has(key)) {
|
|
91
|
+
mismatches.push({ artifactKind: 'wiki', key, direction: 'extra_in_adapter_declaration' });
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
return mismatches;
|
|
95
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,114 @@
|
|
|
1
|
+
import { describe, expect, it } from 'vitest';
|
|
2
|
+
import { compareFinalizationDeclarations, deriveFinalizationTouchedSources, deriveFinalizationWikiPageKeys, } from './finalization-scope.js';
|
|
3
|
+
describe('deriveFinalizationWikiPageKeys', () => {
|
|
4
|
+
it('maps changed global wiki markdown paths to page keys', () => {
|
|
5
|
+
expect(deriveFinalizationWikiPageKeys([
|
|
6
|
+
'wiki/global/historic-sql-orders.md',
|
|
7
|
+
'wiki/global/nested/page.md',
|
|
8
|
+
'README.md',
|
|
9
|
+
])).toEqual(['historic-sql-orders']);
|
|
10
|
+
});
|
|
11
|
+
});
|
|
12
|
+
describe('deriveFinalizationTouchedSources', () => {
|
|
13
|
+
it('maps standalone semantic-layer files directly', async () => {
|
|
14
|
+
const result = await deriveFinalizationTouchedSources({
|
|
15
|
+
changedPaths: ['semantic-layer/warehouse/orders.yaml'],
|
|
16
|
+
beforeSourcesByConnection: new Map(),
|
|
17
|
+
afterSourcesByConnection: new Map(),
|
|
18
|
+
});
|
|
19
|
+
expect(result).toEqual({
|
|
20
|
+
touchedSources: [{ connectionId: 'warehouse', sourceName: 'orders' }],
|
|
21
|
+
unresolvedPaths: [],
|
|
22
|
+
});
|
|
23
|
+
});
|
|
24
|
+
it('resolves aggregate _schema changes by comparing loaded source snapshots', async () => {
|
|
25
|
+
const beforeSourcesByConnection = new Map([
|
|
26
|
+
[
|
|
27
|
+
'warehouse',
|
|
28
|
+
[
|
|
29
|
+
{
|
|
30
|
+
name: 'orders',
|
|
31
|
+
grain: ['order_id'],
|
|
32
|
+
columns: [{ name: 'order_id', type: 'string' }],
|
|
33
|
+
joins: [],
|
|
34
|
+
measures: [],
|
|
35
|
+
usage: {
|
|
36
|
+
narrative: 'old',
|
|
37
|
+
frequencyTier: 'low',
|
|
38
|
+
commonFilters: [],
|
|
39
|
+
commonJoins: [],
|
|
40
|
+
},
|
|
41
|
+
},
|
|
42
|
+
],
|
|
43
|
+
],
|
|
44
|
+
]);
|
|
45
|
+
const afterSourcesByConnection = new Map([
|
|
46
|
+
[
|
|
47
|
+
'warehouse',
|
|
48
|
+
[
|
|
49
|
+
{
|
|
50
|
+
name: 'orders',
|
|
51
|
+
grain: ['order_id'],
|
|
52
|
+
columns: [{ name: 'order_id', type: 'string' }],
|
|
53
|
+
joins: [],
|
|
54
|
+
measures: [],
|
|
55
|
+
usage: {
|
|
56
|
+
narrative: 'new',
|
|
57
|
+
frequencyTier: 'high',
|
|
58
|
+
commonFilters: [],
|
|
59
|
+
commonJoins: [],
|
|
60
|
+
},
|
|
61
|
+
},
|
|
62
|
+
],
|
|
63
|
+
],
|
|
64
|
+
]);
|
|
65
|
+
const result = await deriveFinalizationTouchedSources({
|
|
66
|
+
changedPaths: ['semantic-layer/warehouse/_schema/public.yaml'],
|
|
67
|
+
beforeSourcesByConnection,
|
|
68
|
+
afterSourcesByConnection,
|
|
69
|
+
});
|
|
70
|
+
expect(result).toEqual({
|
|
71
|
+
touchedSources: [{ connectionId: 'warehouse', sourceName: 'orders' }],
|
|
72
|
+
unresolvedPaths: [],
|
|
73
|
+
});
|
|
74
|
+
});
|
|
75
|
+
it('flags aggregate _schema changes that cannot be resolved to logical sources', async () => {
|
|
76
|
+
const beforeSourcesByConnection = new Map([['warehouse', []]]);
|
|
77
|
+
const afterSourcesByConnection = new Map([['warehouse', []]]);
|
|
78
|
+
const result = await deriveFinalizationTouchedSources({
|
|
79
|
+
changedPaths: ['semantic-layer/warehouse/_schema/public.yaml'],
|
|
80
|
+
beforeSourcesByConnection,
|
|
81
|
+
afterSourcesByConnection,
|
|
82
|
+
});
|
|
83
|
+
expect(result).toEqual({
|
|
84
|
+
touchedSources: [],
|
|
85
|
+
unresolvedPaths: ['semantic-layer/warehouse/_schema/public.yaml'],
|
|
86
|
+
});
|
|
87
|
+
});
|
|
88
|
+
});
|
|
89
|
+
describe('compareFinalizationDeclarations', () => {
|
|
90
|
+
it('reports missing and extra adapter declarations', () => {
|
|
91
|
+
expect(compareFinalizationDeclarations({
|
|
92
|
+
declaredTouchedSources: [{ connectionId: 'warehouse', sourceName: 'orders' }],
|
|
93
|
+
derivedTouchedSources: [{ connectionId: 'warehouse', sourceName: 'customers' }],
|
|
94
|
+
declaredChangedWikiPageKeys: ['orders'],
|
|
95
|
+
derivedChangedWikiPageKeys: ['orders', 'patterns'],
|
|
96
|
+
})).toEqual([
|
|
97
|
+
{
|
|
98
|
+
artifactKind: 'sl',
|
|
99
|
+
key: 'warehouse:customers',
|
|
100
|
+
direction: 'missing_from_adapter_declaration',
|
|
101
|
+
},
|
|
102
|
+
{
|
|
103
|
+
artifactKind: 'sl',
|
|
104
|
+
key: 'warehouse:orders',
|
|
105
|
+
direction: 'extra_in_adapter_declaration',
|
|
106
|
+
},
|
|
107
|
+
{
|
|
108
|
+
artifactKind: 'wiki',
|
|
109
|
+
key: 'patterns',
|
|
110
|
+
direction: 'missing_from_adapter_declaration',
|
|
111
|
+
},
|
|
112
|
+
]);
|
|
113
|
+
});
|
|
114
|
+
});
|
|
@@ -77,7 +77,6 @@ export { stageHistoricSqlAggregatedSnapshot } from './adapters/historic-sql/stag
|
|
|
77
77
|
export { historicSqlEvidenceEnvelopeSchema, historicSqlEvidencePath, historicSqlPatternEvidenceSchema, historicSqlTableUsageEvidenceSchema, serializeHistoricSqlEvidence, } from './adapters/historic-sql/evidence.js';
|
|
78
78
|
export type { HistoricSqlEvidenceEnvelope, HistoricSqlPatternEvidence, HistoricSqlTableUsageEvidence, } from './adapters/historic-sql/evidence.js';
|
|
79
79
|
export { createEmitHistoricSqlEvidenceTool } from './adapters/historic-sql/evidence-tool.js';
|
|
80
|
-
export { HistoricSqlProjectionPostProcessor } from './adapters/historic-sql/post-processor.js';
|
|
81
80
|
export { projectHistoricSqlEvidence } from './adapters/historic-sql/projection.js';
|
|
82
81
|
export type { HistoricSqlProjectionInput, HistoricSqlProjectionResult } from './adapters/historic-sql/projection.js';
|
|
83
82
|
export { patternOutputSchema, patternsArraySchema, tableUsageOutputSchema, } from './adapters/historic-sql/skill-schemas.js';
|
|
@@ -151,5 +150,5 @@ export { buildReconcileSystemPrompt, buildReconcileToolSet, buildReconcileUserPr
|
|
|
151
150
|
export type { ReconciliationOutcome } from './stages/stage-4-reconciliation.js';
|
|
152
151
|
export { runReconciliationStage4 } from './stages/stage-4-reconciliation.js';
|
|
153
152
|
export type { StageIndex } from './stages/stage-index.types.js';
|
|
154
|
-
export type { ChunkResult, DiffSet, EvictionUnit, FetchContext, IngestBundleJob, IngestBundleRef, IngestBundleResult, IngestDiffSummary, IngestJobContext, IngestJobPhase, IngestTrigger, ScopeDescriptor, SourceAdapter, SourceFetchIssue, SourceFetchReport, TriageLane, TriageSignals, UnresolvedCardInfo, WorkUnit, DeterministicProjectionContext, ProjectionResult, } from './types.js';
|
|
153
|
+
export type { ChunkResult, DiffSet, EvictionUnit, FetchContext, IngestBundleJob, IngestBundleRef, IngestBundleResult, IngestDiffSummary, IngestJobContext, IngestJobPhase, IngestTrigger, ScopeDescriptor, SourceAdapter, SourceFetchIssue, SourceFetchReport, TriageLane, TriageSignals, UnresolvedCardInfo, WorkUnit, DeterministicProjectionContext, ProjectionResult, DeterministicFinalizationContext, FinalizationOverrideReplay, FinalizationResult, } from './types.js';
|
|
155
154
|
export * from './wiki-body-refs.js';
|
|
@@ -51,7 +51,6 @@ export { SnowflakeHistoricSqlQueryHistoryReader } from './adapters/historic-sql/
|
|
|
51
51
|
export { stageHistoricSqlAggregatedSnapshot } from './adapters/historic-sql/stage-unified.js';
|
|
52
52
|
export { historicSqlEvidenceEnvelopeSchema, historicSqlEvidencePath, historicSqlPatternEvidenceSchema, historicSqlTableUsageEvidenceSchema, serializeHistoricSqlEvidence, } from './adapters/historic-sql/evidence.js';
|
|
53
53
|
export { createEmitHistoricSqlEvidenceTool } from './adapters/historic-sql/evidence-tool.js';
|
|
54
|
-
export { HistoricSqlProjectionPostProcessor } from './adapters/historic-sql/post-processor.js';
|
|
55
54
|
export { projectHistoricSqlEvidence } from './adapters/historic-sql/projection.js';
|
|
56
55
|
export { patternOutputSchema, patternsArraySchema, tableUsageOutputSchema, } from './adapters/historic-sql/skill-schemas.js';
|
|
57
56
|
export { HISTORIC_SQL_SOURCE_KEY, aggregatedTemplateSchema, historicSqlUnifiedPullConfigSchema, stagedManifestSchema, stagedPatternsInputSchema, stagedTableInputSchema, } from './adapters/historic-sql/types.js';
|
|
@@ -32,11 +32,13 @@ export declare class IngestBundleRunner {
|
|
|
32
32
|
private buildWikiIndex;
|
|
33
33
|
private buildSlIndex;
|
|
34
34
|
private tableRefExistsInSemanticLayer;
|
|
35
|
+
private loadSourcesByConnection;
|
|
35
36
|
private resolveContextCuratorBudget;
|
|
36
37
|
private filterWorkUnitsForTriage;
|
|
37
38
|
private createTrace;
|
|
38
39
|
private errorMessage;
|
|
39
40
|
private buildProvenancePlan;
|
|
41
|
+
private partitionFinalizationActionsForProvenance;
|
|
40
42
|
private toReportProvenanceRows;
|
|
41
43
|
private toReportWorkUnits;
|
|
42
44
|
private provenanceValidationTraceData;
|