@kaelio/ktx 0.1.0-rc.6 → 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/assets/python/{kaelio_ktx-0.1.0rc6-py3-none-any.whl → kaelio_ktx-0.1.1-py3-none-any.whl} +0 -0
- package/assets/python/manifest.json +4 -4
- package/dist/commands/mcp-commands.js +11 -3
- package/dist/commands/mcp-commands.test.js +30 -1
- package/dist/commands/setup-commands.js +14 -26
- package/dist/doctor.test.js +3 -4
- package/dist/index.test.js +26 -10
- package/dist/ingest-depth.js +0 -1
- package/dist/ingest.test-utils.js +2 -2
- package/dist/ingest.test.js +6 -30
- package/dist/managed-local-embeddings.d.ts +2 -0
- package/dist/managed-local-embeddings.js +2 -0
- package/dist/managed-local-embeddings.test.js +2 -0
- package/dist/managed-mcp-daemon.js +3 -2
- package/dist/managed-mcp-daemon.test.js +25 -0
- package/dist/managed-python-command.test.js +1 -0
- package/dist/managed-python-daemon.js +3 -2
- package/dist/managed-python-daemon.test.js +20 -0
- package/dist/managed-python-runtime.d.ts +4 -0
- package/dist/managed-python-runtime.js +47 -3
- package/dist/managed-python-runtime.test.js +51 -21
- package/dist/next-steps.js +1 -1
- package/dist/next-steps.test.js +2 -0
- package/dist/proxy-env.d.ts +1 -0
- package/dist/proxy-env.js +23 -0
- package/dist/proxy-env.test.js +17 -0
- package/dist/runtime-requirements.d.ts +1 -2
- package/dist/runtime-requirements.js +0 -7
- package/dist/runtime-requirements.test.js +2 -2
- package/dist/runtime.test.js +1 -0
- package/dist/setup-agents.d.ts +11 -3
- package/dist/setup-agents.js +400 -135
- package/dist/setup-agents.test.js +394 -62
- package/dist/setup-embeddings.d.ts +1 -0
- package/dist/setup-embeddings.js +28 -6
- package/dist/setup-embeddings.test.js +46 -4
- package/dist/setup-models.d.ts +0 -1
- package/dist/setup-models.js +2 -3
- package/dist/setup-models.test.js +8 -10
- package/dist/setup-project.d.ts +9 -1
- package/dist/setup-project.js +52 -25
- package/dist/setup-project.test.js +8 -8
- package/dist/setup-runtime.d.ts +0 -1
- package/dist/setup-runtime.js +0 -1
- package/dist/setup-runtime.test.js +9 -13
- package/dist/setup.d.ts +4 -2
- package/dist/setup.js +72 -30
- package/dist/setup.test.js +271 -58
- package/dist/sl.test.js +2 -1
- package/dist/standalone-smoke.test.js +2 -3
- package/dist/status-project.js +1 -10
- package/node_modules/@ktx/connector-clickhouse/dist/package-exports.test.js +1 -1
- package/node_modules/@ktx/context/dist/core/git.service.d.ts +0 -1
- package/node_modules/@ktx/context/dist/core/git.service.js +0 -12
- package/node_modules/@ktx/context/dist/ingest/adapters/historic-sql/historic-sql.adapter.d.ts +1 -2
- package/node_modules/@ktx/context/dist/ingest/adapters/historic-sql/historic-sql.adapter.js +0 -18
- package/node_modules/@ktx/context/dist/ingest/adapters/historic-sql/local-ingest-acceptance.test.js +7 -7
- package/node_modules/@ktx/context/dist/ingest/adapters/historic-sql/post-processor.d.ts +4 -0
- package/node_modules/@ktx/context/dist/ingest/adapters/historic-sql/post-processor.js +38 -0
- package/node_modules/@ktx/context/dist/ingest/adapters/historic-sql/post-processor.test.d.ts +1 -0
- package/node_modules/@ktx/context/dist/ingest/adapters/historic-sql/post-processor.test.js +63 -0
- package/node_modules/@ktx/context/dist/ingest/adapters/historic-sql/projection.d.ts +0 -5
- package/node_modules/@ktx/context/dist/ingest/adapters/historic-sql/projection.js +0 -48
- package/node_modules/@ktx/context/dist/ingest/adapters/historic-sql/projection.test.js +0 -83
- package/node_modules/@ktx/context/dist/ingest/index.d.ts +2 -1
- package/node_modules/@ktx/context/dist/ingest/index.js +1 -0
- package/node_modules/@ktx/context/dist/ingest/ingest-bundle.runner.d.ts +0 -2
- package/node_modules/@ktx/context/dist/ingest/ingest-bundle.runner.isolated-diff.test.js +0 -166
- package/node_modules/@ktx/context/dist/ingest/ingest-bundle.runner.js +45 -235
- package/node_modules/@ktx/context/dist/ingest/ingest-bundle.runner.test.js +38 -193
- package/node_modules/@ktx/context/dist/ingest/local-bundle-ingest.test.js +11 -30
- package/node_modules/@ktx/context/dist/ingest/local-bundle-runtime.js +5 -1
- package/node_modules/@ktx/context/dist/ingest/local-bundle-runtime.test.js +3 -3
- package/node_modules/@ktx/context/dist/ingest/local-embedding-provider.integration.test.js +9 -10
- package/node_modules/@ktx/context/dist/ingest/local-ingest.js +7 -0
- package/node_modules/@ktx/context/dist/ingest/memory-flow/schema.d.ts +4 -4
- package/node_modules/@ktx/context/dist/ingest/memory-flow/schema.js +1 -1
- package/node_modules/@ktx/context/dist/ingest/memory-flow/types.d.ts +1 -1
- package/node_modules/@ktx/context/dist/ingest/ports.d.ts +20 -1
- package/node_modules/@ktx/context/dist/ingest/report-snapshot.d.ts +2 -73
- package/node_modules/@ktx/context/dist/ingest/report-snapshot.js +0 -27
- package/node_modules/@ktx/context/dist/ingest/reports.d.ts +5 -23
- package/node_modules/@ktx/context/dist/ingest/reports.js +24 -7
- package/node_modules/@ktx/context/dist/ingest/types.d.ts +0 -33
- package/node_modules/@ktx/context/dist/llm/local-config.js +2 -15
- package/node_modules/@ktx/context/dist/llm/local-config.test.js +3 -7
- package/node_modules/@ktx/context/dist/package-exports.test.js +1 -2
- package/node_modules/@ktx/context/dist/project/config.d.ts +0 -5
- package/node_modules/@ktx/context/dist/project/config.js +5 -5
- package/node_modules/@ktx/context/dist/project/config.test.js +4 -7
- package/node_modules/@ktx/context/dist/scan/enrichment-state.test.js +4 -4
- package/node_modules/@ktx/context/dist/scan/index.d.ts +1 -1
- package/node_modules/@ktx/context/dist/scan/local-enrichment.d.ts +2 -6
- package/node_modules/@ktx/context/dist/scan/local-enrichment.js +31 -47
- package/node_modules/@ktx/context/dist/scan/local-enrichment.test.js +35 -18
- package/node_modules/@ktx/context/dist/scan/local-scan.test.js +2 -3
- package/node_modules/@ktx/llm/dist/embedding-provider.d.ts +0 -7
- package/node_modules/@ktx/llm/dist/embedding-provider.js +12 -138
- package/node_modules/@ktx/llm/dist/embedding-provider.test.js +10 -25
- package/node_modules/@ktx/llm/dist/types.d.ts +1 -1
- package/package.json +4 -4
- package/node_modules/@ktx/context/dist/ingest/finalization-scope.d.ts +0 -22
- package/node_modules/@ktx/context/dist/ingest/finalization-scope.js +0 -95
- package/node_modules/@ktx/context/dist/ingest/finalization-scope.test.js +0 -114
- /package/{node_modules/@ktx/context/dist/ingest/finalization-scope.test.d.ts → dist/proxy-env.test.d.ts} +0 -0
|
@@ -76,7 +76,7 @@ async function writeSqliteScanConfig(projectDir, dbPath, enrich = false) {
|
|
|
76
76
|
' enrichment:',
|
|
77
77
|
' mode: deterministic',
|
|
78
78
|
' embeddings:',
|
|
79
|
-
' backend:
|
|
79
|
+
' backend: none',
|
|
80
80
|
' dimensions: 6',
|
|
81
81
|
]
|
|
82
82
|
: []),
|
|
@@ -94,7 +94,6 @@ async function runSetupNewProject(projectDir) {
|
|
|
94
94
|
'setup',
|
|
95
95
|
'--project-dir',
|
|
96
96
|
projectDir,
|
|
97
|
-
'--new',
|
|
98
97
|
'--no-input',
|
|
99
98
|
'--yes',
|
|
100
99
|
'--skip-llm',
|
|
@@ -135,7 +134,7 @@ describe('standalone built ktx CLI smoke', () => {
|
|
|
135
134
|
expect(result.stderr).toContain("unknown command 'agent'");
|
|
136
135
|
});
|
|
137
136
|
it('runs status setup checks through the built binary', async () => {
|
|
138
|
-
const result = await runBuiltCli(['status', '--verbose', '--no-input']);
|
|
137
|
+
const result = await runBuiltCli(['status', '--verbose', '--no-input'], { cwd: tempDir });
|
|
139
138
|
expect(result.stdout).toMatch(/KTX status/);
|
|
140
139
|
if (result.stdout.includes('No project here yet.')) {
|
|
141
140
|
expect(result.stdout).toContain('ktx setup');
|
package/dist/status-project.js
CHANGED
|
@@ -120,15 +120,6 @@ function buildEmbeddingsStatus(config, env) {
|
|
|
120
120
|
detail: 'disabled — semantic search will be skipped',
|
|
121
121
|
};
|
|
122
122
|
}
|
|
123
|
-
if (backend === 'deterministic') {
|
|
124
|
-
return {
|
|
125
|
-
backend,
|
|
126
|
-
model,
|
|
127
|
-
dimensions,
|
|
128
|
-
status: 'warn',
|
|
129
|
-
detail: 'deterministic — semantic search degraded (lexical/dictionary lanes still work)',
|
|
130
|
-
};
|
|
131
|
-
}
|
|
132
123
|
if (backend === 'openai') {
|
|
133
124
|
const ref = config.openai?.api_key;
|
|
134
125
|
const resolved = resolveRef(ref, env);
|
|
@@ -469,7 +460,7 @@ function buildVerdict(llm, embeddings, connections, queryHistory, warnings) {
|
|
|
469
460
|
if (llm.status === 'warn')
|
|
470
461
|
reasons.push('LLM credentials missing');
|
|
471
462
|
if (embeddings.status === 'warn') {
|
|
472
|
-
if (embeddings.backend === '
|
|
463
|
+
if (embeddings.backend === 'none') {
|
|
473
464
|
reasons.push('semantic search disabled');
|
|
474
465
|
}
|
|
475
466
|
else {
|
|
@@ -6,5 +6,5 @@ describe('@ktx/connector-clickhouse package exports', () => {
|
|
|
6
6
|
expect(connector.KtxClickHouseScanConnector).toBeTypeOf('function');
|
|
7
7
|
expect(connector.clickHouseClientConfigFromConfig).toBeTypeOf('function');
|
|
8
8
|
expect(connector.createClickHouseLiveDatabaseIntrospection).toBeTypeOf('function');
|
|
9
|
-
});
|
|
9
|
+
}, 20_000);
|
|
10
10
|
});
|
|
@@ -101,7 +101,6 @@ export declare class GitService {
|
|
|
101
101
|
status: 'A' | 'M' | 'D';
|
|
102
102
|
path: string;
|
|
103
103
|
}>>;
|
|
104
|
-
changedPaths(): Promise<string[]>;
|
|
105
104
|
/**
|
|
106
105
|
* List all paths under the working tree that match `pathSpec`, scoped to HEAD.
|
|
107
106
|
* Used for the reconciler's first-ever run when there's no watermark to diff from.
|
|
@@ -424,18 +424,6 @@ export class GitService {
|
|
|
424
424
|
}
|
|
425
425
|
return out;
|
|
426
426
|
}
|
|
427
|
-
async changedPaths() {
|
|
428
|
-
const raw = await this.git.raw(['status', '--porcelain=v1', '-z']);
|
|
429
|
-
const fields = raw.split('\0').filter(Boolean);
|
|
430
|
-
const paths = [];
|
|
431
|
-
for (const field of fields) {
|
|
432
|
-
const path = field.slice(3);
|
|
433
|
-
if (path.length > 0) {
|
|
434
|
-
paths.push(path);
|
|
435
|
-
}
|
|
436
|
-
}
|
|
437
|
-
return [...new Set(paths)].sort();
|
|
438
|
-
}
|
|
439
427
|
/**
|
|
440
428
|
* List all paths under the working tree that match `pathSpec`, scoped to HEAD.
|
|
441
429
|
* Used for the reconciler's first-ever run when there's no watermark to diff from.
|
package/node_modules/@ktx/context/dist/ingest/adapters/historic-sql/historic-sql.adapter.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type { ChunkResult,
|
|
1
|
+
import type { ChunkResult, DiffSet, FetchContext, ScopeDescriptor, SourceAdapter } from '../../types.js';
|
|
2
2
|
import { type HistoricSqlSourceAdapterDeps } from './types.js';
|
|
3
3
|
export declare class HistoricSqlSourceAdapter implements SourceAdapter {
|
|
4
4
|
private readonly deps;
|
|
@@ -11,5 +11,4 @@ export declare class HistoricSqlSourceAdapter implements SourceAdapter {
|
|
|
11
11
|
fetch(pullConfig: unknown, stagedDir: string, ctx: FetchContext): Promise<void>;
|
|
12
12
|
chunk(stagedDir: string, diffSet?: DiffSet): Promise<ChunkResult>;
|
|
13
13
|
describeScope(stagedDir: string): Promise<ScopeDescriptor>;
|
|
14
|
-
finalize(ctx: DeterministicFinalizationContext): Promise<FinalizationResult>;
|
|
15
14
|
}
|
|
@@ -1,6 +1,5 @@
|
|
|
1
1
|
import { chunkHistoricSqlUnifiedStagedDir, describeHistoricSqlUnifiedScope } from './chunk-unified.js';
|
|
2
2
|
import { detectHistoricSqlStagedDir } from './detect.js';
|
|
3
|
-
import { projectHistoricSqlEvidence } from './projection.js';
|
|
4
3
|
import { stageHistoricSqlAggregatedSnapshot } from './stage-unified.js';
|
|
5
4
|
export class HistoricSqlSourceAdapter {
|
|
6
5
|
deps;
|
|
@@ -31,21 +30,4 @@ export class HistoricSqlSourceAdapter {
|
|
|
31
30
|
describeScope(stagedDir) {
|
|
32
31
|
return describeHistoricSqlUnifiedScope(stagedDir);
|
|
33
32
|
}
|
|
34
|
-
async finalize(ctx) {
|
|
35
|
-
const projection = await projectHistoricSqlEvidence({
|
|
36
|
-
workdir: ctx.workdir,
|
|
37
|
-
connectionId: ctx.connectionId,
|
|
38
|
-
syncId: ctx.syncId,
|
|
39
|
-
runId: ctx.runId,
|
|
40
|
-
overrideReplay: ctx.overrideReplay,
|
|
41
|
-
});
|
|
42
|
-
return {
|
|
43
|
-
result: projection,
|
|
44
|
-
warnings: projection.warnings,
|
|
45
|
-
errors: [],
|
|
46
|
-
touchedSources: projection.touchedSources,
|
|
47
|
-
changedWikiPageKeys: projection.changedWikiPageKeys,
|
|
48
|
-
actions: projection.actions,
|
|
49
|
-
};
|
|
50
|
-
}
|
|
51
33
|
}
|
package/node_modules/@ktx/context/dist/ingest/adapters/historic-sql/local-ingest-acceptance.test.js
CHANGED
|
@@ -132,7 +132,7 @@ async function writeHistoricSqlProject(project) {
|
|
|
132
132
|
' adapters:',
|
|
133
133
|
' - historic-sql',
|
|
134
134
|
' embeddings:',
|
|
135
|
-
' backend:
|
|
135
|
+
' backend: none',
|
|
136
136
|
'storage:',
|
|
137
137
|
' state: sqlite',
|
|
138
138
|
' search: sqlite-fts5',
|
|
@@ -194,12 +194,12 @@ describe('historic-SQL local ingest retrieval acceptance', () => {
|
|
|
194
194
|
expect(result.result.failedWorkUnits).toEqual([]);
|
|
195
195
|
expect(result.result.workUnitCount).toBe(3);
|
|
196
196
|
expect(agentRunner.runLoop).toHaveBeenCalledTimes(3);
|
|
197
|
-
const
|
|
198
|
-
expect(
|
|
199
|
-
if (!
|
|
200
|
-
throw new Error('Expected historic-SQL
|
|
197
|
+
const postProcessor = result.report.body.postProcessor;
|
|
198
|
+
expect(postProcessor).toBeDefined();
|
|
199
|
+
if (!postProcessor) {
|
|
200
|
+
throw new Error('Expected historic-SQL post-processor result');
|
|
201
201
|
}
|
|
202
|
-
expect(
|
|
202
|
+
expect(postProcessor).toMatchObject({
|
|
203
203
|
sourceKey: 'historic-sql',
|
|
204
204
|
status: 'success',
|
|
205
205
|
result: {
|
|
@@ -207,7 +207,7 @@ describe('historic-SQL local ingest retrieval acceptance', () => {
|
|
|
207
207
|
patternPagesWritten: 1,
|
|
208
208
|
},
|
|
209
209
|
});
|
|
210
|
-
expect(
|
|
210
|
+
expect(postProcessor.touchedSources).toEqual(expect.arrayContaining([
|
|
211
211
|
{ connectionId: 'warehouse', sourceName: 'customers' },
|
|
212
212
|
{ connectionId: 'warehouse', sourceName: 'orders' },
|
|
213
213
|
]));
|
|
@@ -0,0 +1,4 @@
|
|
|
1
|
+
import type { IngestBundlePostProcessorInput, IngestBundlePostProcessorPort, IngestBundlePostProcessorResult } from '../../ports.js';
|
|
2
|
+
export declare class HistoricSqlProjectionPostProcessor implements IngestBundlePostProcessorPort {
|
|
3
|
+
run(input: IngestBundlePostProcessorInput): Promise<IngestBundlePostProcessorResult>;
|
|
4
|
+
}
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
import { createSimpleGit } from '../../../core/git-env.js';
|
|
2
|
+
import { projectHistoricSqlEvidence } from './projection.js';
|
|
3
|
+
async function commitProjectionChanges(workdir) {
|
|
4
|
+
const git = createSimpleGit(workdir);
|
|
5
|
+
if (!(await git.checkIsRepo().catch(() => false))) {
|
|
6
|
+
return;
|
|
7
|
+
}
|
|
8
|
+
const status = await git.status();
|
|
9
|
+
const paths = status.files
|
|
10
|
+
.map((file) => file.path)
|
|
11
|
+
.filter((path) => path.startsWith('semantic-layer/') || path.startsWith('wiki/global/historic-sql'));
|
|
12
|
+
if (paths.length === 0) {
|
|
13
|
+
return;
|
|
14
|
+
}
|
|
15
|
+
await git.add(paths);
|
|
16
|
+
const staged = await git.diff(['--cached', '--name-only']);
|
|
17
|
+
if (!staged.trim()) {
|
|
18
|
+
return;
|
|
19
|
+
}
|
|
20
|
+
await git.commit('Project historic SQL evidence', { '--author': 'System User <system@example.com>' });
|
|
21
|
+
}
|
|
22
|
+
export class HistoricSqlProjectionPostProcessor {
|
|
23
|
+
async run(input) {
|
|
24
|
+
const projection = await projectHistoricSqlEvidence({
|
|
25
|
+
workdir: input.workdir,
|
|
26
|
+
connectionId: input.connectionId,
|
|
27
|
+
syncId: input.syncId,
|
|
28
|
+
runId: input.runId,
|
|
29
|
+
});
|
|
30
|
+
await commitProjectionChanges(input.workdir);
|
|
31
|
+
return {
|
|
32
|
+
result: projection,
|
|
33
|
+
warnings: projection.warnings,
|
|
34
|
+
errors: [],
|
|
35
|
+
touchedSources: projection.touchedSources,
|
|
36
|
+
};
|
|
37
|
+
}
|
|
38
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
import { mkdir, mkdtemp, readFile, writeFile } from 'node:fs/promises';
|
|
2
|
+
import { tmpdir } from 'node:os';
|
|
3
|
+
import { join } from 'node:path';
|
|
4
|
+
import YAML from 'yaml';
|
|
5
|
+
import { describe, expect, it } from 'vitest';
|
|
6
|
+
import { HistoricSqlProjectionPostProcessor } from './post-processor.js';
|
|
7
|
+
async function tempWorkdir() {
|
|
8
|
+
return mkdtemp(join(tmpdir(), 'historic-sql-post-processor-'));
|
|
9
|
+
}
|
|
10
|
+
async function writeJson(root, relPath, value) {
|
|
11
|
+
const target = join(root, relPath);
|
|
12
|
+
await mkdir(join(target, '..'), { recursive: true });
|
|
13
|
+
await writeFile(target, `${JSON.stringify(value, null, 2)}\n`, 'utf-8');
|
|
14
|
+
}
|
|
15
|
+
describe('HistoricSqlProjectionPostProcessor', () => {
|
|
16
|
+
it('projects current run evidence before the ingest squash commit', async () => {
|
|
17
|
+
const workdir = await tempWorkdir();
|
|
18
|
+
await mkdir(join(workdir, 'semantic-layer/warehouse/_schema'), { recursive: true });
|
|
19
|
+
await writeFile(join(workdir, 'semantic-layer/warehouse/_schema/public.yaml'), YAML.stringify({ tables: { orders: { table: 'public.orders', columns: [{ name: 'id', type: 'string' }] } } }), 'utf-8');
|
|
20
|
+
await writeJson(workdir, 'raw-sources/warehouse/historic-sql/sync-1/manifest.json', {
|
|
21
|
+
source: 'historic-sql',
|
|
22
|
+
connectionId: 'warehouse',
|
|
23
|
+
dialect: 'postgres',
|
|
24
|
+
fetchedAt: '2026-05-11T00:00:00.000Z',
|
|
25
|
+
windowStart: '2026-02-10T00:00:00.000Z',
|
|
26
|
+
windowEnd: '2026-05-11T00:00:00.000Z',
|
|
27
|
+
snapshotRowCount: 1,
|
|
28
|
+
touchedTableCount: 1,
|
|
29
|
+
parseFailures: 0,
|
|
30
|
+
warnings: [],
|
|
31
|
+
probeWarnings: [],
|
|
32
|
+
staleArchiveAfterDays: 90,
|
|
33
|
+
});
|
|
34
|
+
await writeJson(workdir, 'raw-sources/warehouse/historic-sql/sync-1/tables/public.orders.json', { table: 'public.orders' });
|
|
35
|
+
await writeJson(workdir, '.ktx/ingest-evidence/historic-sql/run-1/orders.json', {
|
|
36
|
+
kind: 'table_usage',
|
|
37
|
+
connectionId: 'warehouse',
|
|
38
|
+
table: 'public.orders',
|
|
39
|
+
rawPath: 'tables/public.orders.json',
|
|
40
|
+
usage: {
|
|
41
|
+
narrative: 'Orders are repeatedly queried by lifecycle status.',
|
|
42
|
+
frequencyTier: 'high',
|
|
43
|
+
commonFilters: ['status'],
|
|
44
|
+
commonJoins: [],
|
|
45
|
+
staleSince: null,
|
|
46
|
+
},
|
|
47
|
+
});
|
|
48
|
+
const result = await new HistoricSqlProjectionPostProcessor().run({
|
|
49
|
+
connectionId: 'warehouse',
|
|
50
|
+
sourceKey: 'historic-sql',
|
|
51
|
+
syncId: 'sync-1',
|
|
52
|
+
jobId: 'job-1',
|
|
53
|
+
runId: 'run-1',
|
|
54
|
+
workdir,
|
|
55
|
+
parseArtifacts: null,
|
|
56
|
+
});
|
|
57
|
+
expect(result.errors).toEqual([]);
|
|
58
|
+
expect(result.warnings).toEqual([]);
|
|
59
|
+
expect(result.touchedSources).toEqual([{ connectionId: 'warehouse', sourceName: 'orders' }]);
|
|
60
|
+
expect(result.result).toMatchObject({ tableUsageMerged: 1 });
|
|
61
|
+
await expect(readFile(join(workdir, 'semantic-layer/warehouse/_schema/public.yaml'), 'utf-8')).resolves.toContain('Orders are repeatedly queried by lifecycle status.');
|
|
62
|
+
});
|
|
63
|
+
});
|
|
@@ -1,11 +1,8 @@
|
|
|
1
|
-
import type { MemoryAction } from '../../../memory/index.js';
|
|
2
|
-
import type { FinalizationOverrideReplay } from '../../types.js';
|
|
3
1
|
export interface HistoricSqlProjectionInput {
|
|
4
2
|
workdir: string;
|
|
5
3
|
connectionId: string;
|
|
6
4
|
syncId: string;
|
|
7
5
|
runId: string;
|
|
8
|
-
overrideReplay?: FinalizationOverrideReplay;
|
|
9
6
|
}
|
|
10
7
|
export interface HistoricSqlProjectionResult {
|
|
11
8
|
tableUsageMerged: number;
|
|
@@ -17,8 +14,6 @@ export interface HistoricSqlProjectionResult {
|
|
|
17
14
|
connectionId: string;
|
|
18
15
|
sourceName: string;
|
|
19
16
|
}>;
|
|
20
|
-
changedWikiPageKeys: string[];
|
|
21
|
-
actions: MemoryAction[];
|
|
22
17
|
warnings: string[];
|
|
23
18
|
}
|
|
24
19
|
export declare function projectHistoricSqlEvidence(input: HistoricSqlProjectionInput): Promise<HistoricSqlProjectionResult>;
|
|
@@ -177,8 +177,6 @@ export async function projectHistoricSqlEvidence(input) {
|
|
|
177
177
|
stalePatternPagesMarked: 0,
|
|
178
178
|
archivedPatternPages: 0,
|
|
179
179
|
touchedSources: [],
|
|
180
|
-
changedWikiPageKeys: [],
|
|
181
|
-
actions: [],
|
|
182
180
|
warnings: [],
|
|
183
181
|
};
|
|
184
182
|
const touchedKeys = new Set();
|
|
@@ -186,14 +184,6 @@ export async function projectHistoricSqlEvidence(input) {
|
|
|
186
184
|
const manifest = stagedManifestSchema.parse(await readJson(join(rawDir, 'manifest.json')));
|
|
187
185
|
const currentTables = await currentStagedTables(rawDir);
|
|
188
186
|
const evidence = await loadEvidence(input.workdir, input.runId);
|
|
189
|
-
if (input.overrideReplay && evidence.length === 0) {
|
|
190
|
-
result.warnings.push('historic-sql finalization skipped stale/archive cleanup during override replay without current-run evidence');
|
|
191
|
-
return result;
|
|
192
|
-
}
|
|
193
|
-
if (evidence.length === 0) {
|
|
194
|
-
result.warnings.push('historic-sql finalization skipped because no current-run evidence was emitted');
|
|
195
|
-
return result;
|
|
196
|
-
}
|
|
197
187
|
const tableEvidence = evidence.filter((entry) => entry.kind === 'table_usage');
|
|
198
188
|
const patternEvidence = evidence.filter((entry) => entry.kind === 'pattern');
|
|
199
189
|
const schemaRoot = join(input.workdir, 'semantic-layer', input.connectionId, '_schema');
|
|
@@ -217,14 +207,6 @@ export async function projectHistoricSqlEvidence(input) {
|
|
|
217
207
|
touchedKeys.add(key);
|
|
218
208
|
result.touchedSources.push({ connectionId: input.connectionId, sourceName });
|
|
219
209
|
}
|
|
220
|
-
result.actions.push({
|
|
221
|
-
target: 'sl',
|
|
222
|
-
type: 'updated',
|
|
223
|
-
key: sourceName,
|
|
224
|
-
targetConnectionId: input.connectionId,
|
|
225
|
-
detail: `Merged historic-SQL usage for ${matchingEvidence.table}`,
|
|
226
|
-
rawPaths: [matchingEvidence.rawPath],
|
|
227
|
-
});
|
|
228
210
|
}
|
|
229
211
|
}
|
|
230
212
|
else if (entry.usage && !currentTables.has(tableRef)) {
|
|
@@ -238,13 +220,6 @@ export async function projectHistoricSqlEvidence(input) {
|
|
|
238
220
|
touchedKeys.add(key);
|
|
239
221
|
result.touchedSources.push({ connectionId: input.connectionId, sourceName });
|
|
240
222
|
}
|
|
241
|
-
result.actions.push({
|
|
242
|
-
target: 'sl',
|
|
243
|
-
type: 'updated',
|
|
244
|
-
key: sourceName,
|
|
245
|
-
targetConnectionId: input.connectionId,
|
|
246
|
-
detail: `Marked historic-SQL usage stale for ${tableRef}`,
|
|
247
|
-
});
|
|
248
223
|
}
|
|
249
224
|
}
|
|
250
225
|
}
|
|
@@ -279,14 +254,6 @@ export async function projectHistoricSqlEvidence(input) {
|
|
|
279
254
|
await writeFile(pagePath, renderMarkdownPage(frontmatter, renderPatternMarkdown(pattern)), 'utf-8');
|
|
280
255
|
writtenKeys.add(key);
|
|
281
256
|
result.patternPagesWritten += 1;
|
|
282
|
-
result.changedWikiPageKeys.push(key);
|
|
283
|
-
result.actions.push({
|
|
284
|
-
target: 'wiki',
|
|
285
|
-
type: reusable ? 'updated' : 'created',
|
|
286
|
-
key,
|
|
287
|
-
detail: `Projected historic-SQL pattern ${pattern.pattern.title}`,
|
|
288
|
-
rawPaths: [pattern.rawPath],
|
|
289
|
-
});
|
|
290
257
|
}
|
|
291
258
|
for (const page of patternPages) {
|
|
292
259
|
if (writtenKeys.has(page.key))
|
|
@@ -295,26 +262,11 @@ export async function projectHistoricSqlEvidence(input) {
|
|
|
295
262
|
const tags = [...new Set([...stringArray(page.frontmatter.tags), 'archived'])];
|
|
296
263
|
await writeFile(page.path, renderMarkdownPage({ ...page.frontmatter, tags, archived_since: manifest.fetchedAt }, page.content), 'utf-8');
|
|
297
264
|
result.archivedPatternPages += 1;
|
|
298
|
-
result.changedWikiPageKeys.push(page.key);
|
|
299
|
-
result.actions.push({
|
|
300
|
-
target: 'wiki',
|
|
301
|
-
type: 'updated',
|
|
302
|
-
key: page.key,
|
|
303
|
-
detail: `Archived stale historic-SQL pattern page ${page.key}`,
|
|
304
|
-
});
|
|
305
265
|
continue;
|
|
306
266
|
}
|
|
307
267
|
const tags = [...new Set([...stringArray(page.frontmatter.tags), 'stale'])];
|
|
308
268
|
await writeFile(page.path, renderMarkdownPage({ ...page.frontmatter, tags, stale_since: manifest.fetchedAt }, page.content), 'utf-8');
|
|
309
269
|
result.stalePatternPagesMarked += 1;
|
|
310
|
-
result.changedWikiPageKeys.push(page.key);
|
|
311
|
-
result.actions.push({
|
|
312
|
-
target: 'wiki',
|
|
313
|
-
type: 'updated',
|
|
314
|
-
key: page.key,
|
|
315
|
-
detail: `Marked historic-SQL pattern page ${page.key} stale`,
|
|
316
|
-
});
|
|
317
270
|
}
|
|
318
|
-
result.changedWikiPageKeys = [...new Set(result.changedWikiPageKeys)].sort();
|
|
319
271
|
return result;
|
|
320
272
|
}
|
|
@@ -64,13 +64,6 @@ describe('projectHistoricSqlEvidence', () => {
|
|
|
64
64
|
});
|
|
65
65
|
const result = await projectHistoricSqlEvidence({ workdir, connectionId: 'warehouse', syncId: 'sync-1', runId: 'run-1' });
|
|
66
66
|
expect(result.touchedSources).toEqual([{ connectionId: 'warehouse', sourceName: 'orders' }]);
|
|
67
|
-
expect(result.actions).toEqual(expect.arrayContaining([
|
|
68
|
-
expect.objectContaining({
|
|
69
|
-
target: 'sl',
|
|
70
|
-
key: 'orders',
|
|
71
|
-
rawPaths: ['tables/public.orders.json'],
|
|
72
|
-
}),
|
|
73
|
-
]));
|
|
74
67
|
const shard = YAML.parse(await readFile(join(workdir, 'semantic-layer/warehouse/_schema/public.yaml'), 'utf-8'));
|
|
75
68
|
expect(shard.tables.orders.usage).toEqual({
|
|
76
69
|
ownerNote: 'keep me',
|
|
@@ -150,14 +143,6 @@ describe('projectHistoricSqlEvidence', () => {
|
|
|
150
143
|
});
|
|
151
144
|
const result = await projectHistoricSqlEvidence({ workdir, connectionId: 'warehouse', syncId: 'sync-1', runId: 'run-1' });
|
|
152
145
|
expect(result.patternPagesWritten).toBe(1);
|
|
153
|
-
expect(result.changedWikiPageKeys).toContain('historic-sql-old-order-lifecycle');
|
|
154
|
-
expect(result.actions).toEqual(expect.arrayContaining([
|
|
155
|
-
expect.objectContaining({
|
|
156
|
-
target: 'wiki',
|
|
157
|
-
key: 'historic-sql-old-order-lifecycle',
|
|
158
|
-
rawPaths: ['patterns-input.json'],
|
|
159
|
-
}),
|
|
160
|
-
]));
|
|
161
146
|
await expect(readFile(join(workdir, 'wiki/global/historic-sql-old-order-lifecycle.md'), 'utf-8')).resolves.toContain('Order Lifecycle Analysis');
|
|
162
147
|
await expect(readFile(join(workdir, 'wiki/global/historic-sql-retired-pattern.md'), 'utf-8')).resolves.toContain('stale_since: "2026-05-11T00:00:00.000Z"');
|
|
163
148
|
});
|
|
@@ -289,19 +274,6 @@ describe('projectHistoricSqlEvidence', () => {
|
|
|
289
274
|
probeWarnings: [],
|
|
290
275
|
staleArchiveAfterDays: 90,
|
|
291
276
|
});
|
|
292
|
-
await writeJson(workdir, '.ktx/ingest-evidence/historic-sql/run-1/customers.json', {
|
|
293
|
-
kind: 'table_usage',
|
|
294
|
-
connectionId: 'warehouse',
|
|
295
|
-
table: 'public.customers',
|
|
296
|
-
rawPath: 'tables/public.customers.json',
|
|
297
|
-
usage: {
|
|
298
|
-
narrative: 'Customers were queried.',
|
|
299
|
-
frequencyTier: 'low',
|
|
300
|
-
commonFilters: [],
|
|
301
|
-
commonJoins: [],
|
|
302
|
-
staleSince: null,
|
|
303
|
-
},
|
|
304
|
-
});
|
|
305
277
|
await writeText(workdir, 'wiki/global/historic-sql-old-template.md', [
|
|
306
278
|
'---',
|
|
307
279
|
YAML.stringify({
|
|
@@ -322,9 +294,6 @@ describe('projectHistoricSqlEvidence', () => {
|
|
|
322
294
|
const result = await projectHistoricSqlEvidence({ workdir, connectionId: 'warehouse', syncId: 'sync-1', runId: 'run-1' });
|
|
323
295
|
expect(result.staleTablesMarked).toBe(1);
|
|
324
296
|
expect(result.touchedSources).toEqual([{ connectionId: 'warehouse', sourceName: 'orders' }]);
|
|
325
|
-
const staleAction = result.actions.find((action) => action.target === 'sl' && action.key === 'orders');
|
|
326
|
-
expect(staleAction).toEqual(expect.objectContaining({ target: 'sl', key: 'orders' }));
|
|
327
|
-
expect(staleAction?.rawPaths).toBeUndefined();
|
|
328
297
|
const shard = YAML.parse(await readFile(join(workdir, 'semantic-layer/warehouse/_schema/public.yaml'), 'utf-8'));
|
|
329
298
|
expect(shard.tables.orders.usage).toEqual({
|
|
330
299
|
ownerNote: 'keep analyst annotation',
|
|
@@ -337,56 +306,4 @@ describe('projectHistoricSqlEvidence', () => {
|
|
|
337
306
|
});
|
|
338
307
|
await expect(readFile(join(workdir, 'wiki/global/historic-sql-old-template.md'), 'utf-8')).resolves.toContain('Old body');
|
|
339
308
|
});
|
|
340
|
-
it('does not mark stale or archive pages when override replay has no current-run evidence', async () => {
|
|
341
|
-
const workdir = await tempWorkdir();
|
|
342
|
-
await writeText(workdir, 'semantic-layer/warehouse/_schema/public.yaml', YAML.stringify({
|
|
343
|
-
tables: {
|
|
344
|
-
orders: {
|
|
345
|
-
table: 'public.orders',
|
|
346
|
-
usage: {
|
|
347
|
-
narrative: 'Orders were active before.',
|
|
348
|
-
frequencyTier: 'high',
|
|
349
|
-
commonFilters: ['status'],
|
|
350
|
-
commonGroupBys: ['status'],
|
|
351
|
-
commonJoins: [],
|
|
352
|
-
},
|
|
353
|
-
columns: [{ name: 'id', type: 'string' }],
|
|
354
|
-
},
|
|
355
|
-
},
|
|
356
|
-
}));
|
|
357
|
-
await writeJson(workdir, 'raw-sources/warehouse/historic-sql/override-sync/manifest.json', {
|
|
358
|
-
source: 'historic-sql',
|
|
359
|
-
connectionId: 'warehouse',
|
|
360
|
-
dialect: 'postgres',
|
|
361
|
-
fetchedAt: '2026-05-11T00:00:00.000Z',
|
|
362
|
-
windowStart: '2026-02-10T00:00:00.000Z',
|
|
363
|
-
windowEnd: '2026-05-11T00:00:00.000Z',
|
|
364
|
-
snapshotRowCount: 0,
|
|
365
|
-
touchedTableCount: 0,
|
|
366
|
-
parseFailures: 0,
|
|
367
|
-
warnings: [],
|
|
368
|
-
probeWarnings: [],
|
|
369
|
-
staleArchiveAfterDays: 90,
|
|
370
|
-
});
|
|
371
|
-
const result = await projectHistoricSqlEvidence({
|
|
372
|
-
workdir,
|
|
373
|
-
connectionId: 'warehouse',
|
|
374
|
-
syncId: 'override-sync',
|
|
375
|
-
runId: 'override-run',
|
|
376
|
-
overrideReplay: {
|
|
377
|
-
priorJobId: 'prior-job',
|
|
378
|
-
priorRunId: 'prior-run',
|
|
379
|
-
priorSyncId: 'prior-sync',
|
|
380
|
-
evictionRawPaths: ['tables/public/orders.json'],
|
|
381
|
-
},
|
|
382
|
-
});
|
|
383
|
-
expect(result.tableUsageMerged).toBe(0);
|
|
384
|
-
expect(result.staleTablesMarked).toBe(0);
|
|
385
|
-
expect(result.patternPagesWritten).toBe(0);
|
|
386
|
-
expect(result.stalePatternPagesMarked).toBe(0);
|
|
387
|
-
expect(result.archivedPatternPages).toBe(0);
|
|
388
|
-
expect(result.touchedSources).toEqual([]);
|
|
389
|
-
expect(result.changedWikiPageKeys).toEqual([]);
|
|
390
|
-
expect(result.actions).toEqual([]);
|
|
391
|
-
});
|
|
392
309
|
});
|
|
@@ -77,6 +77,7 @@ export { stageHistoricSqlAggregatedSnapshot } from './adapters/historic-sql/stag
|
|
|
77
77
|
export { historicSqlEvidenceEnvelopeSchema, historicSqlEvidencePath, historicSqlPatternEvidenceSchema, historicSqlTableUsageEvidenceSchema, serializeHistoricSqlEvidence, } from './adapters/historic-sql/evidence.js';
|
|
78
78
|
export type { HistoricSqlEvidenceEnvelope, HistoricSqlPatternEvidence, HistoricSqlTableUsageEvidence, } from './adapters/historic-sql/evidence.js';
|
|
79
79
|
export { createEmitHistoricSqlEvidenceTool } from './adapters/historic-sql/evidence-tool.js';
|
|
80
|
+
export { HistoricSqlProjectionPostProcessor } from './adapters/historic-sql/post-processor.js';
|
|
80
81
|
export { projectHistoricSqlEvidence } from './adapters/historic-sql/projection.js';
|
|
81
82
|
export type { HistoricSqlProjectionInput, HistoricSqlProjectionResult } from './adapters/historic-sql/projection.js';
|
|
82
83
|
export { patternOutputSchema, patternsArraySchema, tableUsageOutputSchema, } from './adapters/historic-sql/skill-schemas.js';
|
|
@@ -150,5 +151,5 @@ export { buildReconcileSystemPrompt, buildReconcileToolSet, buildReconcileUserPr
|
|
|
150
151
|
export type { ReconciliationOutcome } from './stages/stage-4-reconciliation.js';
|
|
151
152
|
export { runReconciliationStage4 } from './stages/stage-4-reconciliation.js';
|
|
152
153
|
export type { StageIndex } from './stages/stage-index.types.js';
|
|
153
|
-
export type { ChunkResult, DiffSet, EvictionUnit, FetchContext, IngestBundleJob, IngestBundleRef, IngestBundleResult, IngestDiffSummary, IngestJobContext, IngestJobPhase, IngestTrigger, ScopeDescriptor, SourceAdapter, SourceFetchIssue, SourceFetchReport, TriageLane, TriageSignals, UnresolvedCardInfo, WorkUnit, DeterministicProjectionContext, ProjectionResult,
|
|
154
|
+
export type { ChunkResult, DiffSet, EvictionUnit, FetchContext, IngestBundleJob, IngestBundleRef, IngestBundleResult, IngestDiffSummary, IngestJobContext, IngestJobPhase, IngestTrigger, ScopeDescriptor, SourceAdapter, SourceFetchIssue, SourceFetchReport, TriageLane, TriageSignals, UnresolvedCardInfo, WorkUnit, DeterministicProjectionContext, ProjectionResult, } from './types.js';
|
|
154
155
|
export * from './wiki-body-refs.js';
|
|
@@ -51,6 +51,7 @@ export { SnowflakeHistoricSqlQueryHistoryReader } from './adapters/historic-sql/
|
|
|
51
51
|
export { stageHistoricSqlAggregatedSnapshot } from './adapters/historic-sql/stage-unified.js';
|
|
52
52
|
export { historicSqlEvidenceEnvelopeSchema, historicSqlEvidencePath, historicSqlPatternEvidenceSchema, historicSqlTableUsageEvidenceSchema, serializeHistoricSqlEvidence, } from './adapters/historic-sql/evidence.js';
|
|
53
53
|
export { createEmitHistoricSqlEvidenceTool } from './adapters/historic-sql/evidence-tool.js';
|
|
54
|
+
export { HistoricSqlProjectionPostProcessor } from './adapters/historic-sql/post-processor.js';
|
|
54
55
|
export { projectHistoricSqlEvidence } from './adapters/historic-sql/projection.js';
|
|
55
56
|
export { patternOutputSchema, patternsArraySchema, tableUsageOutputSchema, } from './adapters/historic-sql/skill-schemas.js';
|
|
56
57
|
export { HISTORIC_SQL_SOURCE_KEY, aggregatedTemplateSchema, historicSqlUnifiedPullConfigSchema, stagedManifestSchema, stagedPatternsInputSchema, stagedTableInputSchema, } from './adapters/historic-sql/types.js';
|
|
@@ -32,13 +32,11 @@ export declare class IngestBundleRunner {
|
|
|
32
32
|
private buildWikiIndex;
|
|
33
33
|
private buildSlIndex;
|
|
34
34
|
private tableRefExistsInSemanticLayer;
|
|
35
|
-
private loadSourcesByConnection;
|
|
36
35
|
private resolveContextCuratorBudget;
|
|
37
36
|
private filterWorkUnitsForTriage;
|
|
38
37
|
private createTrace;
|
|
39
38
|
private errorMessage;
|
|
40
39
|
private buildProvenancePlan;
|
|
41
|
-
private partitionFinalizationActionsForProvenance;
|
|
42
40
|
private toReportProvenanceRows;
|
|
43
41
|
private toReportWorkUnits;
|
|
44
42
|
private provenanceValidationTraceData;
|