@kaelio/ktx 0.1.0-rc.5 → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/assets/python/kaelio_ktx-0.1.0-py3-none-any.whl +0 -0
- package/assets/python/manifest.json +2 -2
- package/dist/clack.d.ts +6 -0
- package/dist/clack.js +23 -0
- package/dist/cli-program.js +5 -2
- package/dist/cli-program.test.js +7 -1
- package/dist/cli-runtime.d.ts +4 -0
- package/dist/cli-runtime.js +8 -1
- package/dist/command-schemas.d.ts +1 -1
- package/dist/commands/ingest-commands.js +1 -0
- package/dist/commands/knowledge-commands.js +5 -0
- package/dist/commands/mcp-commands.js +11 -3
- package/dist/commands/mcp-commands.test.js +30 -1
- package/dist/commands/sql-commands.d.ts +3 -0
- package/dist/commands/sql-commands.js +43 -0
- package/dist/commands/sql-commands.test.d.ts +1 -0
- package/dist/commands/sql-commands.test.js +68 -0
- package/dist/context-build-view.js +5 -1
- package/dist/dev.test.js +27 -0
- package/dist/index.d.ts +1 -0
- package/dist/index.js +1 -0
- package/dist/index.test.js +56 -21
- package/dist/ingest.js +123 -18
- package/dist/ingest.test.js +206 -0
- package/dist/io/print-list.d.ts +2 -1
- package/dist/io/print-list.js +7 -0
- package/dist/io/print-list.test.js +13 -11
- package/dist/io/symbols.d.ts +2 -2
- package/dist/knowledge.d.ts +1 -0
- package/dist/knowledge.js +34 -16
- package/dist/knowledge.test.js +27 -0
- package/dist/managed-python-command.d.ts +2 -0
- package/dist/managed-python-command.js +17 -9
- package/dist/managed-python-command.test.js +59 -4
- package/dist/next-steps.js +1 -1
- package/dist/next-steps.test.js +2 -0
- package/dist/print-command-tree.js +7 -1
- package/dist/public-ingest.d.ts +9 -1
- package/dist/public-ingest.js +50 -7
- package/dist/public-ingest.test.js +69 -2
- package/dist/release-version.d.ts +5 -0
- package/dist/release-version.js +44 -0
- package/dist/runtime-requirements.d.ts +23 -0
- package/dist/runtime-requirements.js +99 -0
- package/dist/runtime-requirements.test.d.ts +1 -0
- package/dist/runtime-requirements.test.js +63 -0
- package/dist/setup-agents.d.ts +11 -3
- package/dist/setup-agents.js +397 -134
- package/dist/setup-agents.test.js +359 -61
- package/dist/setup-embeddings.js +3 -6
- package/dist/setup-embeddings.test.js +18 -2
- package/dist/setup-models.js +2 -2
- package/dist/setup-models.test.js +5 -3
- package/dist/setup-ready-menu.d.ts +1 -1
- package/dist/setup-ready-menu.js +2 -0
- package/dist/setup-ready-menu.test.js +3 -0
- package/dist/setup-runtime.d.ts +45 -0
- package/dist/setup-runtime.js +47 -0
- package/dist/setup-runtime.test.d.ts +1 -0
- package/dist/setup-runtime.test.js +110 -0
- package/dist/setup-sources-notion.test.d.ts +1 -0
- package/dist/setup-sources-notion.test.js +107 -0
- package/dist/setup-sources.js +5 -2
- package/dist/setup.d.ts +19 -1
- package/dist/setup.js +104 -29
- package/dist/setup.test.js +221 -57
- package/dist/sl.js +2 -2
- package/dist/sl.test.js +10 -0
- package/dist/source-mapping.js +9 -1
- package/dist/source-mapping.test.d.ts +1 -0
- package/dist/source-mapping.test.js +65 -0
- package/dist/sql.d.ts +22 -0
- package/dist/sql.js +125 -0
- package/dist/sql.test.d.ts +1 -0
- package/dist/sql.test.js +226 -0
- package/node_modules/@ktx/connector-clickhouse/dist/package-exports.test.js +1 -1
- package/node_modules/@ktx/context/dist/connections/connection-type.d.ts +4 -4
- package/node_modules/@ktx/context/dist/core/git.service.d.ts +3 -0
- package/node_modules/@ktx/context/dist/core/git.service.js +47 -1
- package/node_modules/@ktx/context/dist/core/git.service.patch.test.d.ts +1 -0
- package/node_modules/@ktx/context/dist/core/git.service.patch.test.js +40 -0
- package/node_modules/@ktx/context/dist/ingest/adapters/historic-sql/types.d.ts +5 -5
- package/node_modules/@ktx/context/dist/ingest/adapters/looker/looker.adapter.d.ts +2 -2
- package/node_modules/@ktx/context/dist/ingest/adapters/looker/tools/looker-query-to-sl.tool.d.ts +2 -2
- package/node_modules/@ktx/context/dist/ingest/adapters/looker/types.d.ts +16 -16
- package/node_modules/@ktx/context/dist/ingest/adapters/lookml/pull-config.d.ts +1 -1
- package/node_modules/@ktx/context/dist/ingest/adapters/metabase/fetch.js +16 -0
- package/node_modules/@ktx/context/dist/ingest/adapters/metabase/fetch.test.js +41 -0
- package/node_modules/@ktx/context/dist/ingest/adapters/metricflow/metricflow.adapter.d.ts +2 -1
- package/node_modules/@ktx/context/dist/ingest/adapters/metricflow/metricflow.adapter.js +40 -0
- package/node_modules/@ktx/context/dist/ingest/adapters/metricflow/metricflow.adapter.test.js +116 -1
- package/node_modules/@ktx/context/dist/ingest/adapters/metricflow/projection-config.d.ts +29 -0
- package/node_modules/@ktx/context/dist/ingest/adapters/metricflow/projection-config.js +40 -0
- package/node_modules/@ktx/context/dist/ingest/adapters/metricflow/pull-config.d.ts +1 -1
- package/node_modules/@ktx/context/dist/ingest/artifact-gates.d.ts +25 -0
- package/node_modules/@ktx/context/dist/ingest/artifact-gates.js +149 -0
- package/node_modules/@ktx/context/dist/ingest/artifact-gates.test.d.ts +1 -0
- package/node_modules/@ktx/context/dist/ingest/artifact-gates.test.js +167 -0
- package/node_modules/@ktx/context/dist/ingest/final-gate-repair.d.ts +29 -0
- package/node_modules/@ktx/context/dist/ingest/final-gate-repair.js +178 -0
- package/node_modules/@ktx/context/dist/ingest/final-gate-repair.test.d.ts +1 -0
- package/node_modules/@ktx/context/dist/ingest/final-gate-repair.test.js +109 -0
- package/node_modules/@ktx/context/dist/ingest/index.d.ts +8 -1
- package/node_modules/@ktx/context/dist/ingest/index.js +7 -0
- package/node_modules/@ktx/context/dist/ingest/ingest-bundle.runner.d.ts +18 -2
- package/node_modules/@ktx/context/dist/ingest/ingest-bundle.runner.isolated-diff.test.d.ts +1 -0
- package/node_modules/@ktx/context/dist/ingest/ingest-bundle.runner.isolated-diff.test.js +1761 -0
- package/node_modules/@ktx/context/dist/ingest/ingest-bundle.runner.js +1695 -901
- package/node_modules/@ktx/context/dist/ingest/ingest-bundle.runner.test.js +135 -118
- package/node_modules/@ktx/context/dist/ingest/ingest-trace.d.ts +50 -0
- package/node_modules/@ktx/context/dist/ingest/ingest-trace.js +88 -0
- package/node_modules/@ktx/context/dist/ingest/ingest-trace.test.d.ts +1 -0
- package/node_modules/@ktx/context/dist/ingest/ingest-trace.test.js +76 -0
- package/node_modules/@ktx/context/dist/ingest/isolated-diff/git-patch.d.ts +16 -0
- package/node_modules/@ktx/context/dist/ingest/isolated-diff/git-patch.js +78 -0
- package/node_modules/@ktx/context/dist/ingest/isolated-diff/git-patch.test.d.ts +1 -0
- package/node_modules/@ktx/context/dist/ingest/isolated-diff/git-patch.test.js +76 -0
- package/node_modules/@ktx/context/dist/ingest/isolated-diff/patch-integrator.d.ts +58 -0
- package/node_modules/@ktx/context/dist/ingest/isolated-diff/patch-integrator.js +223 -0
- package/node_modules/@ktx/context/dist/ingest/isolated-diff/patch-integrator.test.d.ts +1 -0
- package/node_modules/@ktx/context/dist/ingest/isolated-diff/patch-integrator.test.js +369 -0
- package/node_modules/@ktx/context/dist/ingest/isolated-diff/textual-conflict-resolver.d.ts +23 -0
- package/node_modules/@ktx/context/dist/ingest/isolated-diff/textual-conflict-resolver.js +190 -0
- package/node_modules/@ktx/context/dist/ingest/isolated-diff/textual-conflict-resolver.test.d.ts +1 -0
- package/node_modules/@ktx/context/dist/ingest/isolated-diff/textual-conflict-resolver.test.js +101 -0
- package/node_modules/@ktx/context/dist/ingest/isolated-diff/work-unit-executor.d.ts +15 -0
- package/node_modules/@ktx/context/dist/ingest/isolated-diff/work-unit-executor.js +61 -0
- package/node_modules/@ktx/context/dist/ingest/isolated-diff/work-unit-executor.test.d.ts +1 -0
- package/node_modules/@ktx/context/dist/ingest/isolated-diff/work-unit-executor.test.js +137 -0
- package/node_modules/@ktx/context/dist/ingest/local-bundle-ingest.test.js +7 -0
- package/node_modules/@ktx/context/dist/ingest/local-bundle-runtime.js +54 -10
- package/node_modules/@ktx/context/dist/ingest/local-bundle-runtime.test.js +65 -0
- package/node_modules/@ktx/context/dist/ingest/memory-flow/schema.d.ts +23 -5
- package/node_modules/@ktx/context/dist/ingest/memory-flow/schema.js +17 -0
- package/node_modules/@ktx/context/dist/ingest/memory-flow/schema.test.js +1 -0
- package/node_modules/@ktx/context/dist/ingest/memory-flow/types.d.ts +6 -0
- package/node_modules/@ktx/context/dist/ingest/parsed-target-table.d.ts +1 -1
- package/node_modules/@ktx/context/dist/ingest/ports.d.ts +3 -0
- package/node_modules/@ktx/context/dist/ingest/report-snapshot.d.ts +32 -7
- package/node_modules/@ktx/context/dist/ingest/report-snapshot.js +25 -0
- package/node_modules/@ktx/context/dist/ingest/report-snapshot.test.js +124 -0
- package/node_modules/@ktx/context/dist/ingest/reports.d.ts +23 -0
- package/node_modules/@ktx/context/dist/ingest/semantic-layer-target-policy.d.ts +11 -0
- package/node_modules/@ktx/context/dist/ingest/semantic-layer-target-policy.js +26 -0
- package/node_modules/@ktx/context/dist/ingest/semantic-layer-target-policy.test.d.ts +1 -0
- package/node_modules/@ktx/context/dist/ingest/semantic-layer-target-policy.test.js +25 -0
- package/node_modules/@ktx/context/dist/ingest/stages/stage-3-work-units.d.ts +4 -0
- package/node_modules/@ktx/context/dist/ingest/stages/stage-3-work-units.js +4 -0
- package/node_modules/@ktx/context/dist/ingest/stages/stage-3-work-units.test.js +29 -0
- package/node_modules/@ktx/context/dist/ingest/tools/emit-unmapped-fallback.tool.d.ts +1 -1
- package/node_modules/@ktx/context/dist/ingest/types.d.ts +24 -0
- package/node_modules/@ktx/context/dist/ingest/wiki-body-refs.d.ts +24 -0
- package/node_modules/@ktx/context/dist/ingest/wiki-body-refs.js +111 -0
- package/node_modules/@ktx/context/dist/ingest/wiki-body-refs.test.d.ts +1 -0
- package/node_modules/@ktx/context/dist/ingest/wiki-body-refs.test.js +138 -0
- package/node_modules/@ktx/context/dist/llm/claude-code-runtime.js +19 -2
- package/node_modules/@ktx/context/dist/llm/claude-code-runtime.test.js +33 -0
- package/node_modules/@ktx/context/dist/project/setup-config.d.ts +1 -1
- package/node_modules/@ktx/context/dist/project/setup-config.js +10 -1
- package/node_modules/@ktx/context/dist/project/setup-config.test.js +3 -2
- package/node_modules/@ktx/context/dist/sl/tools/sl-edit-source.tool.js +5 -1
- package/node_modules/@ktx/context/dist/sl/tools/sl-edit-source.tool.test.js +15 -0
- package/node_modules/@ktx/context/dist/sl/tools/sl-write-source.tool.js +5 -1
- package/node_modules/@ktx/context/dist/sl/tools/sl-write-source.tool.test.js +22 -0
- package/node_modules/@ktx/context/dist/tools/action-target-connection.d.ts +9 -0
- package/node_modules/@ktx/context/dist/tools/action-target-connection.js +14 -0
- package/node_modules/@ktx/context/dist/tools/context-candidate-write.tool.d.ts +4 -4
- package/node_modules/@ktx/context/dist/tools/index.d.ts +1 -0
- package/node_modules/@ktx/context/dist/tools/index.js +1 -0
- package/node_modules/@ktx/context/dist/wiki/local-knowledge.js +4 -1
- package/node_modules/@ktx/context/dist/wiki/local-knowledge.test.js +44 -0
- package/node_modules/@ktx/context/dist/wiki/tools/wiki-write.tool.js +3 -48
- package/node_modules/@ktx/context/dist/wiki/tools/wiki-write.tool.test.js +28 -0
- package/node_modules/@ktx/context/dist/wiki/wiki-ref-validation.d.ts +17 -0
- package/node_modules/@ktx/context/dist/wiki/wiki-ref-validation.js +79 -0
- package/node_modules/@ktx/context/dist/wiki/wiki-ref-validation.test.d.ts +1 -0
- package/node_modules/@ktx/context/dist/wiki/wiki-ref-validation.test.js +64 -0
- package/node_modules/@ktx/context/prompts/memory_agent_bundle_ingest_work_unit.md +23 -4
- package/node_modules/@ktx/context/skills/ingest_triage/SKILL.md +7 -3
- package/package.json +4 -4
package/dist/index.test.js
CHANGED
|
@@ -31,7 +31,9 @@ describe('getKtxCliPackageInfo', () => {
|
|
|
31
31
|
it('identifies the CLI package and its context dependency', () => {
|
|
32
32
|
expect(getKtxCliPackageInfo()).toEqual({
|
|
33
33
|
name: '@ktx/cli',
|
|
34
|
-
version: '0.
|
|
34
|
+
version: '0.1.0-rc.1',
|
|
35
|
+
packageVersion: '0.0.0-private',
|
|
36
|
+
runtimeVersion: '0.1.0-rc.1',
|
|
35
37
|
contextPackageName: '@ktx/context',
|
|
36
38
|
});
|
|
37
39
|
});
|
|
@@ -49,6 +51,8 @@ describe('getKtxCliPackageInfo', () => {
|
|
|
49
51
|
})).toEqual({
|
|
50
52
|
name: '@kaelio/ktx',
|
|
51
53
|
version: '0.1.0',
|
|
54
|
+
packageVersion: '0.1.0',
|
|
55
|
+
runtimeVersion: '0.1.0',
|
|
52
56
|
contextPackageName: '@ktx/context',
|
|
53
57
|
});
|
|
54
58
|
});
|
|
@@ -86,7 +90,7 @@ describe('runKtxCli', () => {
|
|
|
86
90
|
it('prints version information', async () => {
|
|
87
91
|
const testIo = makeIo();
|
|
88
92
|
await expect(runKtxCli(['--version'], testIo.io)).resolves.toBe(0);
|
|
89
|
-
expect(testIo.stdout()).toBe('@ktx/cli 0.
|
|
93
|
+
expect(testIo.stdout()).toBe('@ktx/cli 0.1.0-rc.1\n');
|
|
90
94
|
expect(testIo.stderr()).toBe('');
|
|
91
95
|
});
|
|
92
96
|
it('prints the public command surface in root help', async () => {
|
|
@@ -129,6 +133,16 @@ describe('runKtxCli', () => {
|
|
|
129
133
|
json: false,
|
|
130
134
|
limit: 5,
|
|
131
135
|
}, searchIo.io);
|
|
136
|
+
const debugSearchIo = makeIo();
|
|
137
|
+
await expect(runKtxCli(['--project-dir', tempDir, '--debug', 'wiki', 'search', 'revenue'], debugSearchIo.io, { knowledge })).resolves.toBe(0);
|
|
138
|
+
expect(knowledge).toHaveBeenLastCalledWith({
|
|
139
|
+
command: 'search',
|
|
140
|
+
projectDir: tempDir,
|
|
141
|
+
query: 'revenue',
|
|
142
|
+
userId: 'local',
|
|
143
|
+
json: false,
|
|
144
|
+
debug: true,
|
|
145
|
+
}, debugSearchIo.io);
|
|
132
146
|
});
|
|
133
147
|
it('rejects removed public wiki read and write commands', async () => {
|
|
134
148
|
const knowledge = vi.fn(async () => 0);
|
|
@@ -171,7 +185,7 @@ describe('runKtxCli', () => {
|
|
|
171
185
|
await expect(runKtxCli(['--project-dir', tempDir, 'sl', 'list', '--query', 'revenue'], listIo.io, { sl })).resolves.toBe(1);
|
|
172
186
|
expect(listIo.stderr()).toContain("unknown option '--query'");
|
|
173
187
|
});
|
|
174
|
-
it('routes runtime management commands with the
|
|
188
|
+
it('routes runtime management commands with the release runtime version', async () => {
|
|
175
189
|
const runtime = vi.fn(async () => 0);
|
|
176
190
|
const installIo = makeIo();
|
|
177
191
|
const startIo = makeIo();
|
|
@@ -189,32 +203,32 @@ describe('runKtxCli', () => {
|
|
|
189
203
|
await expect(runKtxCli(['dev', 'runtime', 'prune', '--dry-run'], pruneIo.io, { runtime })).resolves.toBe(1);
|
|
190
204
|
expect(runtime).toHaveBeenNthCalledWith(1, {
|
|
191
205
|
command: 'install',
|
|
192
|
-
cliVersion: '0.
|
|
206
|
+
cliVersion: '0.1.0-rc.1',
|
|
193
207
|
feature: 'local-embeddings',
|
|
194
208
|
force: true,
|
|
195
209
|
}, installIo.io);
|
|
196
210
|
expect(runtime).toHaveBeenNthCalledWith(2, {
|
|
197
211
|
command: 'start',
|
|
198
|
-
cliVersion: '0.
|
|
212
|
+
cliVersion: '0.1.0-rc.1',
|
|
199
213
|
projectDir: expect.any(String),
|
|
200
214
|
feature: 'local-embeddings',
|
|
201
215
|
force: true,
|
|
202
216
|
}, startIo.io);
|
|
203
217
|
expect(runtime).toHaveBeenNthCalledWith(3, {
|
|
204
218
|
command: 'stop',
|
|
205
|
-
cliVersion: '0.
|
|
219
|
+
cliVersion: '0.1.0-rc.1',
|
|
206
220
|
projectDir: expect.any(String),
|
|
207
221
|
all: false,
|
|
208
222
|
}, stopIo.io);
|
|
209
223
|
expect(runtime).toHaveBeenNthCalledWith(4, {
|
|
210
224
|
command: 'stop',
|
|
211
|
-
cliVersion: '0.
|
|
225
|
+
cliVersion: '0.1.0-rc.1',
|
|
212
226
|
projectDir: expect.any(String),
|
|
213
227
|
all: true,
|
|
214
228
|
}, stopAllIo.io);
|
|
215
229
|
expect(runtime).toHaveBeenNthCalledWith(5, {
|
|
216
230
|
command: 'status',
|
|
217
|
-
cliVersion: '0.
|
|
231
|
+
cliVersion: '0.1.0-rc.1',
|
|
218
232
|
json: true,
|
|
219
233
|
}, statusIo.io);
|
|
220
234
|
expect(runtime).toHaveBeenCalledTimes(5);
|
|
@@ -261,7 +275,7 @@ describe('runKtxCli', () => {
|
|
|
261
275
|
expect(sl).toHaveBeenLastCalledWith(expect.objectContaining({
|
|
262
276
|
command: 'query',
|
|
263
277
|
projectDir: tempDir,
|
|
264
|
-
cliVersion: '0.
|
|
278
|
+
cliVersion: '0.1.0-rc.1',
|
|
265
279
|
runtimeInstallPolicy: 'prompt',
|
|
266
280
|
query: expect.objectContaining({ measures: ['orders.order_count'], dimensions: [] }),
|
|
267
281
|
}), promptIo.io);
|
|
@@ -270,13 +284,13 @@ describe('runKtxCli', () => {
|
|
|
270
284
|
sl,
|
|
271
285
|
})).resolves.toBe(0);
|
|
272
286
|
expect(sl).toHaveBeenLastCalledWith(expect.objectContaining({
|
|
273
|
-
cliVersion: '0.
|
|
287
|
+
cliVersion: '0.1.0-rc.1',
|
|
274
288
|
runtimeInstallPolicy: 'auto',
|
|
275
289
|
}), autoIo.io);
|
|
276
290
|
const noInputIo = makeIo();
|
|
277
291
|
await expect(runKtxCli(['--project-dir', tempDir, 'sl', 'query', '--measure', 'orders.order_count', '--no-input'], noInputIo.io, { sl })).resolves.toBe(0);
|
|
278
292
|
expect(sl).toHaveBeenLastCalledWith(expect.objectContaining({
|
|
279
|
-
cliVersion: '0.
|
|
293
|
+
cliVersion: '0.1.0-rc.1',
|
|
280
294
|
runtimeInstallPolicy: 'never',
|
|
281
295
|
}), noInputIo.io);
|
|
282
296
|
});
|
|
@@ -389,7 +403,7 @@ describe('runKtxCli', () => {
|
|
|
389
403
|
skipAgents: false,
|
|
390
404
|
inputMode: 'auto',
|
|
391
405
|
yes: false,
|
|
392
|
-
cliVersion: '0.
|
|
406
|
+
cliVersion: '0.1.0-rc.1',
|
|
393
407
|
skipLlm: false,
|
|
394
408
|
skipEmbeddings: false,
|
|
395
409
|
databaseSchemas: [],
|
|
@@ -498,7 +512,7 @@ describe('runKtxCli', () => {
|
|
|
498
512
|
inputMode: 'disabled',
|
|
499
513
|
depth: 'fast',
|
|
500
514
|
queryHistory: 'default',
|
|
501
|
-
cliVersion: '0.
|
|
515
|
+
cliVersion: '0.1.0-rc.1',
|
|
502
516
|
runtimeInstallPolicy: 'never',
|
|
503
517
|
}, testIo.io);
|
|
504
518
|
expect(testIo.stderr()).toBe(`Project: ${tempDir}\n`);
|
|
@@ -517,11 +531,32 @@ describe('runKtxCli', () => {
|
|
|
517
531
|
inputMode: 'auto',
|
|
518
532
|
depth: 'deep',
|
|
519
533
|
queryHistory: 'default',
|
|
520
|
-
cliVersion: '0.
|
|
534
|
+
cliVersion: '0.1.0-rc.1',
|
|
521
535
|
runtimeInstallPolicy: 'prompt',
|
|
522
536
|
}, testIo.io);
|
|
523
537
|
expect(testIo.stderr()).toBe('');
|
|
524
538
|
});
|
|
539
|
+
it('routes public ingest --yes as automatic runtime installation', async () => {
|
|
540
|
+
const testIo = makeIo();
|
|
541
|
+
const publicIngest = vi.fn().mockResolvedValue(0);
|
|
542
|
+
await expect(runKtxCli(['--project-dir', tempDir, 'ingest', 'warehouse', '--yes'], testIo.io, {
|
|
543
|
+
publicIngest,
|
|
544
|
+
})).resolves.toBe(0);
|
|
545
|
+
expect(publicIngest).toHaveBeenCalledWith(expect.objectContaining({
|
|
546
|
+
projectDir: tempDir,
|
|
547
|
+
targetConnectionId: 'warehouse',
|
|
548
|
+
runtimeInstallPolicy: 'auto',
|
|
549
|
+
}), testIo.io);
|
|
550
|
+
});
|
|
551
|
+
it('rejects conflicting public ingest runtime install modes', async () => {
|
|
552
|
+
const testIo = makeIo();
|
|
553
|
+
const publicIngest = vi.fn().mockResolvedValue(0);
|
|
554
|
+
await expect(runKtxCli(['--project-dir', tempDir, 'ingest', 'warehouse', '--yes', '--no-input'], testIo.io, {
|
|
555
|
+
publicIngest,
|
|
556
|
+
})).resolves.toBe(1);
|
|
557
|
+
expect(publicIngest).not.toHaveBeenCalled();
|
|
558
|
+
expect(testIo.stderr()).toContain('Choose only one runtime install mode: --yes or --no-input');
|
|
559
|
+
});
|
|
525
560
|
it('rejects mutually exclusive public ingest depth flags before dispatch', async () => {
|
|
526
561
|
const testIo = makeIo();
|
|
527
562
|
const publicIngest = vi.fn().mockResolvedValue(0);
|
|
@@ -545,7 +580,7 @@ describe('runKtxCli', () => {
|
|
|
545
580
|
json: false,
|
|
546
581
|
inputMode: 'disabled',
|
|
547
582
|
queryHistory: 'default',
|
|
548
|
-
cliVersion: '0.
|
|
583
|
+
cliVersion: '0.1.0-rc.1',
|
|
549
584
|
runtimeInstallPolicy: 'never',
|
|
550
585
|
}, testIo.io);
|
|
551
586
|
});
|
|
@@ -750,7 +785,7 @@ describe('runKtxCli', () => {
|
|
|
750
785
|
command: 'run',
|
|
751
786
|
projectDir: tempDir,
|
|
752
787
|
inputMode: 'disabled',
|
|
753
|
-
cliVersion: '0.
|
|
788
|
+
cliVersion: '0.1.0-rc.1',
|
|
754
789
|
anthropicApiKeyEnv: 'ANTHROPIC_API_KEY', // pragma: allowlist secret
|
|
755
790
|
anthropicModel: 'claude-sonnet-4-6',
|
|
756
791
|
skipLlm: false,
|
|
@@ -777,7 +812,7 @@ describe('runKtxCli', () => {
|
|
|
777
812
|
command: 'run',
|
|
778
813
|
projectDir: tempDir,
|
|
779
814
|
inputMode: 'disabled',
|
|
780
|
-
cliVersion: '0.
|
|
815
|
+
cliVersion: '0.1.0-rc.1',
|
|
781
816
|
llmBackend: 'vertex',
|
|
782
817
|
vertexProject: 'local-gcp-project',
|
|
783
818
|
vertexLocation: 'us-east5',
|
|
@@ -802,7 +837,7 @@ describe('runKtxCli', () => {
|
|
|
802
837
|
command: 'run',
|
|
803
838
|
projectDir: tempDir,
|
|
804
839
|
inputMode: 'disabled',
|
|
805
|
-
cliVersion: '0.
|
|
840
|
+
cliVersion: '0.1.0-rc.1',
|
|
806
841
|
llmBackend: 'claude-code',
|
|
807
842
|
llmModel: 'opus',
|
|
808
843
|
skipLlm: false,
|
|
@@ -877,7 +912,7 @@ describe('runKtxCli', () => {
|
|
|
877
912
|
projectDir: '/tmp/project',
|
|
878
913
|
inputMode: 'disabled',
|
|
879
914
|
yes: true,
|
|
880
|
-
cliVersion: '0.
|
|
915
|
+
cliVersion: '0.1.0-rc.1',
|
|
881
916
|
skipLlm: true,
|
|
882
917
|
skipEmbeddings: true,
|
|
883
918
|
databaseDrivers: ['postgres'],
|
|
@@ -1080,7 +1115,7 @@ describe('runKtxCli', () => {
|
|
|
1080
1115
|
queryFile: '/tmp/query.json',
|
|
1081
1116
|
execute: false,
|
|
1082
1117
|
format: 'json',
|
|
1083
|
-
cliVersion: '0.
|
|
1118
|
+
cliVersion: '0.1.0-rc.1',
|
|
1084
1119
|
runtimeInstallPolicy: 'auto',
|
|
1085
1120
|
}, autoIo.io);
|
|
1086
1121
|
expect(sl).toHaveBeenNthCalledWith(2, {
|
|
@@ -1090,7 +1125,7 @@ describe('runKtxCli', () => {
|
|
|
1090
1125
|
queryFile: '/tmp/query.json',
|
|
1091
1126
|
execute: false,
|
|
1092
1127
|
format: 'json',
|
|
1093
|
-
cliVersion: '0.
|
|
1128
|
+
cliVersion: '0.1.0-rc.1',
|
|
1094
1129
|
runtimeInstallPolicy: 'never',
|
|
1095
1130
|
}, neverIo.io);
|
|
1096
1131
|
expect(conflictIo.stderr()).toContain('Choose only one runtime install mode: --yes or --no-input');
|
package/dist/ingest.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { buildMemoryFlowViewModel, createMemoryFlowLiveBuffer, formatMemoryFlowFinalSummary, getLatestLocalIngestStatus, getLocalIngestStatus, ingestReportToMemoryFlowReplay, renderMemoryFlowReplay, runLocalIngest, runLocalMetabaseIngest, savedMemoryCountsForReport, } from '@ktx/context/ingest';
|
|
1
|
+
import { buildMemoryFlowViewModel, createMemoryFlowLiveBuffer, formatMemoryFlowFinalSummary, getLatestLocalIngestStatus, getLocalIngestStatus, ingestReportToMemoryFlowReplay, renderMemoryFlowReplay, runLocalIngest, runLocalMetabaseIngest, savedMemoryCountsForReport, sanitizeMemoryFlowError, } from '@ktx/context/ingest';
|
|
2
2
|
import { loadKtxProject } from '@ktx/context/project';
|
|
3
3
|
import { createKtxCliIngestQueryExecutor } from './ingest-query-executor.js';
|
|
4
4
|
import { readIngestReportSnapshotFile } from './ingest-report-file.js';
|
|
@@ -10,7 +10,7 @@ import { resolveVizFallback, warnVizFallbackOnce } from './viz-fallback.js';
|
|
|
10
10
|
import { profileMark } from './startup-profile.js';
|
|
11
11
|
profileMark('module:ingest');
|
|
12
12
|
function reportStatus(report) {
|
|
13
|
-
return report.body.failedWorkUnits.length > 0 ? 'error' : 'done';
|
|
13
|
+
return report.body.status === 'failed' || report.body.failedWorkUnits.length > 0 ? 'error' : 'done';
|
|
14
14
|
}
|
|
15
15
|
const REPORT_SOURCE_LABELS = new Map([
|
|
16
16
|
['live-database', 'Database schema'],
|
|
@@ -33,17 +33,83 @@ function reportSourceLabel(sourceKey) {
|
|
|
33
33
|
.map((part) => `${part[0]?.toUpperCase() ?? ''}${part.slice(1)}`)
|
|
34
34
|
.join(' ');
|
|
35
35
|
}
|
|
36
|
+
function jsonObjectFromFailureReason(reason) {
|
|
37
|
+
const trimmed = reason.trim();
|
|
38
|
+
const start = trimmed.indexOf('{');
|
|
39
|
+
const end = trimmed.lastIndexOf('}');
|
|
40
|
+
if (start < 0 || end < start) {
|
|
41
|
+
return null;
|
|
42
|
+
}
|
|
43
|
+
try {
|
|
44
|
+
const parsed = JSON.parse(trimmed.slice(start, end + 1));
|
|
45
|
+
return parsed && typeof parsed === 'object' && !Array.isArray(parsed) ? parsed : null;
|
|
46
|
+
}
|
|
47
|
+
catch {
|
|
48
|
+
return null;
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
function stringField(record, key) {
|
|
52
|
+
const value = record[key];
|
|
53
|
+
return typeof value === 'string' && value.trim().length > 0 ? value.trim() : null;
|
|
54
|
+
}
|
|
55
|
+
function isGoogleReauthFailure(record) {
|
|
56
|
+
const error = stringField(record, 'error')?.toLowerCase() ?? '';
|
|
57
|
+
const description = stringField(record, 'error_description')?.toLowerCase() ?? '';
|
|
58
|
+
const subtype = stringField(record, 'error_subtype')?.toLowerCase() ?? '';
|
|
59
|
+
return error === 'invalid_grant' && (description.includes('reauth') || subtype === 'invalid_rapt');
|
|
60
|
+
}
|
|
61
|
+
function formatFailureReason(sourceKey, reason) {
|
|
62
|
+
const parsed = jsonObjectFromFailureReason(reason);
|
|
63
|
+
if (!parsed) {
|
|
64
|
+
return sanitizeMemoryFlowError(reason);
|
|
65
|
+
}
|
|
66
|
+
if (sourceKey === 'historic-sql' && isGoogleReauthFailure(parsed)) {
|
|
67
|
+
return 'Google Cloud authentication failed while analyzing query history: application-default credentials expired or require reauthentication (invalid_grant / invalid_rapt). Run `gcloud auth application-default login`, then retry.';
|
|
68
|
+
}
|
|
69
|
+
const error = stringField(parsed, 'error');
|
|
70
|
+
const description = stringField(parsed, 'error_description');
|
|
71
|
+
const subtype = stringField(parsed, 'error_subtype');
|
|
72
|
+
const parts = [error, description].filter((part) => Boolean(part));
|
|
73
|
+
const message = parts.length > 0 ? parts.join(': ') : reason;
|
|
74
|
+
return subtype ? `${message} (${subtype})` : message;
|
|
75
|
+
}
|
|
76
|
+
function failedReportMessage(report) {
|
|
77
|
+
if (report.body.status === 'failed' && report.body.failure?.message) {
|
|
78
|
+
return sanitizeMemoryFlowError(report.body.failure.message);
|
|
79
|
+
}
|
|
80
|
+
const failedCount = report.body.failedWorkUnits.length;
|
|
81
|
+
if (failedCount === 0) {
|
|
82
|
+
return null;
|
|
83
|
+
}
|
|
84
|
+
const firstFailure = report.body.workUnits.find((workUnit) => workUnit.status === 'failed' && typeof workUnit.reason === 'string' && workUnit.reason.trim());
|
|
85
|
+
const sourceLabel = reportSourceLabel(report.sourceKey);
|
|
86
|
+
const prefix = `${sourceLabel} failed for ${pluralize(failedCount, 'task')}.`;
|
|
87
|
+
if (!firstFailure?.reason) {
|
|
88
|
+
return prefix;
|
|
89
|
+
}
|
|
90
|
+
return `${prefix} First failure: ${formatFailureReason(report.sourceKey, firstFailure.reason)}`;
|
|
91
|
+
}
|
|
36
92
|
function writeReportStatus(report, io) {
|
|
37
93
|
const counts = savedMemoryCountsForReport(report);
|
|
94
|
+
const failedMessage = failedReportMessage(report);
|
|
38
95
|
io.stdout.write(`Report: ${report.id}\n`);
|
|
39
96
|
io.stdout.write(`Run: ${report.runId}\n`);
|
|
40
97
|
io.stdout.write(`Job: ${report.jobId}\n`);
|
|
98
|
+
if (report.body.tracePath) {
|
|
99
|
+
io.stdout.write(`Trace: ${report.body.tracePath}\n`);
|
|
100
|
+
}
|
|
41
101
|
io.stdout.write(`Status: ${reportStatus(report)}\n`);
|
|
42
102
|
io.stdout.write(`Source: ${reportSourceLabel(report.sourceKey)}\n`);
|
|
43
103
|
io.stdout.write(`Connection: ${report.connectionId}\n`);
|
|
44
104
|
io.stdout.write(`Sync: ${report.body.syncId}\n`);
|
|
45
105
|
io.stdout.write(`Diff: +${report.body.diffSummary.added}/~${report.body.diffSummary.modified}/-${report.body.diffSummary.deleted}/=${report.body.diffSummary.unchanged}\n`);
|
|
46
106
|
io.stdout.write(`Tasks: ${report.body.workUnits.length}\n`);
|
|
107
|
+
if (report.body.failedWorkUnits.length > 0) {
|
|
108
|
+
io.stdout.write(`Failed tasks: ${report.body.failedWorkUnits.length}\n`);
|
|
109
|
+
}
|
|
110
|
+
if (failedMessage) {
|
|
111
|
+
io.stdout.write(`Error: ${failedMessage}\n`);
|
|
112
|
+
}
|
|
47
113
|
io.stdout.write(`Saved memory: ${counts.wikiCount} wiki, ${counts.slCount} SL\n`);
|
|
48
114
|
io.stdout.write(`Provenance rows: ${report.body.provenanceRows.length}\n`);
|
|
49
115
|
}
|
|
@@ -106,7 +172,11 @@ function formatDiffProgress(event) {
|
|
|
106
172
|
return `+${event.added}/~${event.modified}/-${event.deleted}/=${event.unchanged}`;
|
|
107
173
|
}
|
|
108
174
|
function workUnitEventsThrough(snapshot, eventIndex) {
|
|
109
|
-
|
|
175
|
+
const latestPlanIndex = snapshot.events
|
|
176
|
+
.slice(0, eventIndex + 1)
|
|
177
|
+
.findLastIndex((event) => event.type === 'chunks_planned');
|
|
178
|
+
const startIndex = latestPlanIndex >= 0 ? latestPlanIndex + 1 : 0;
|
|
179
|
+
return snapshot.events.slice(startIndex, eventIndex + 1);
|
|
110
180
|
}
|
|
111
181
|
function completedWorkUnitCountThrough(snapshot, eventIndex) {
|
|
112
182
|
return workUnitEventsThrough(snapshot, eventIndex).filter((event) => event.type === 'work_unit_finished').length;
|
|
@@ -127,7 +197,8 @@ function plannedWorkUnitCountThrough(snapshot, eventIndex) {
|
|
|
127
197
|
if (snapshot.plannedWorkUnits.length > 0) {
|
|
128
198
|
return snapshot.plannedWorkUnits.length;
|
|
129
199
|
}
|
|
130
|
-
const planEvent =
|
|
200
|
+
const planEvent = snapshot.events
|
|
201
|
+
.slice(0, eventIndex + 1)
|
|
131
202
|
.filter((event) => event.type === 'chunks_planned')
|
|
132
203
|
.at(-1);
|
|
133
204
|
return planEvent?.workUnitCount ?? completedWorkUnitCountThrough(snapshot, eventIndex);
|
|
@@ -167,6 +238,12 @@ function plainIngestEventProgress(event, snapshot, eventIndex) {
|
|
|
167
238
|
};
|
|
168
239
|
case 'stage_skipped':
|
|
169
240
|
return { percent: 45, message: `Skipped ${event.stage}: ${event.reason}` };
|
|
241
|
+
case 'stage_progress':
|
|
242
|
+
return {
|
|
243
|
+
percent: event.percent,
|
|
244
|
+
message: event.message,
|
|
245
|
+
...(event.transient !== undefined ? { transient: event.transient } : {}),
|
|
246
|
+
};
|
|
170
247
|
case 'work_unit_started': {
|
|
171
248
|
const total = plannedWorkUnitCountThrough(snapshot, eventIndex);
|
|
172
249
|
const ordinal = workUnitOrdinalThrough(snapshot, eventIndex, event.unitKey);
|
|
@@ -431,26 +508,54 @@ export async function runKtxIngest(args, io = process, deps = {}) {
|
|
|
431
508
|
}
|
|
432
509
|
if (args.adapter === 'metabase') {
|
|
433
510
|
const executeMetabaseFanout = deps.runLocalMetabaseIngest ?? runLocalMetabaseIngest;
|
|
511
|
+
const runOutputMode = effectiveIngestOutputMode(args.outputMode, io, env, {
|
|
512
|
+
requireInput: (args.inputMode ?? 'auto') === 'auto',
|
|
513
|
+
});
|
|
514
|
+
const plainProgress = shouldWritePlainIngestProgress(runOutputMode, io, env)
|
|
515
|
+
? createPlainIngestProgressRenderer(args, io)
|
|
516
|
+
: null;
|
|
517
|
+
const structuredProgress = deps.progress
|
|
518
|
+
? createPlainIngestProgressObserver(args, deps.progress)
|
|
519
|
+
: null;
|
|
520
|
+
const initialMemoryFlow = plainProgress || structuredProgress ? initialRunMemoryFlowInput(args, 'pending') : undefined;
|
|
521
|
+
const memoryFlow = initialMemoryFlow
|
|
522
|
+
? createMemoryFlowLiveBuffer(initialMemoryFlow, {
|
|
523
|
+
onChange: (snapshot) => {
|
|
524
|
+
plainProgress?.update(snapshot);
|
|
525
|
+
structuredProgress?.update(snapshot);
|
|
526
|
+
},
|
|
527
|
+
})
|
|
528
|
+
: undefined;
|
|
434
529
|
const progress = args.outputMode === 'json' && !deps.progress
|
|
435
530
|
? undefined
|
|
436
531
|
: createMetabaseFanoutProgress(args.connectionId, args.outputMode === 'json'
|
|
437
532
|
? { ...io, stderr: { write: () => undefined } }
|
|
438
533
|
: io, deps.progress);
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
534
|
+
plainProgress?.start();
|
|
535
|
+
structuredProgress?.start();
|
|
536
|
+
let result;
|
|
537
|
+
try {
|
|
538
|
+
result = await executeMetabaseFanout({
|
|
539
|
+
project: ingestProject,
|
|
540
|
+
adapters: createAdapters(ingestProject, adapterOptions),
|
|
541
|
+
metabaseConnectionId: args.connectionId,
|
|
542
|
+
...localIngestOptions,
|
|
543
|
+
queryExecutor,
|
|
544
|
+
trigger: 'manual_resync',
|
|
545
|
+
jobIdFactory: deps.jobIdFactory,
|
|
546
|
+
...(memoryFlow ? { memoryFlow } : {}),
|
|
547
|
+
...(progress ? { progress } : {}),
|
|
548
|
+
});
|
|
549
|
+
plainProgress?.flush();
|
|
550
|
+
if (args.outputMode === 'json') {
|
|
551
|
+
io.stdout.write(`${JSON.stringify(result, null, 2)}\n`);
|
|
552
|
+
}
|
|
553
|
+
else {
|
|
554
|
+
writeMetabaseFanoutStatus(result, io);
|
|
555
|
+
}
|
|
451
556
|
}
|
|
452
|
-
|
|
453
|
-
|
|
557
|
+
finally {
|
|
558
|
+
plainProgress?.flush();
|
|
454
559
|
}
|
|
455
560
|
return result.status === 'all_succeeded' ? 0 : 1;
|
|
456
561
|
}
|
package/dist/ingest.test.js
CHANGED
|
@@ -12,6 +12,11 @@ describe('runKtxIngest', () => {
|
|
|
12
12
|
let tempDir;
|
|
13
13
|
let originalTerm;
|
|
14
14
|
const interactiveEnv = () => ({ ...process.env, CI: 'false' });
|
|
15
|
+
const runtimeReady = (projectDir) => ({
|
|
16
|
+
status: 'ready',
|
|
17
|
+
projectDir,
|
|
18
|
+
requirements: { features: ['core'], requirements: [] },
|
|
19
|
+
});
|
|
15
20
|
beforeEach(async () => {
|
|
16
21
|
resetVizFallbackWarningsForTest();
|
|
17
22
|
originalTerm = process.env.TERM;
|
|
@@ -203,6 +208,7 @@ describe('runKtxIngest', () => {
|
|
|
203
208
|
historicSqlProbe: async () => ({ ok: true, lines: ['PASS Historic SQL probe skipped in test'] }),
|
|
204
209
|
},
|
|
205
210
|
context: async () => ({ status: 'skipped', projectDir }),
|
|
211
|
+
runtime: async () => runtimeReady(projectDir),
|
|
206
212
|
})).resolves.toBe(0);
|
|
207
213
|
const sourceDir = join(tempDir, 'source');
|
|
208
214
|
await mkdir(join(sourceDir, 'orders'), { recursive: true });
|
|
@@ -485,6 +491,106 @@ describe('runKtxIngest', () => {
|
|
|
485
491
|
expect(io.stdout()).toContain('"status": "all_succeeded"');
|
|
486
492
|
expect(io.stderr()).not.toContain('Metabase ingest: prod-metabase');
|
|
487
493
|
});
|
|
494
|
+
it('emits structured child ingest progress during Metabase fan-out', async () => {
|
|
495
|
+
const projectDir = join(tempDir, 'project');
|
|
496
|
+
await writeMetabaseConfig(projectDir);
|
|
497
|
+
const io = makeIo();
|
|
498
|
+
const progressEvents = [];
|
|
499
|
+
await expect(runKtxIngest({
|
|
500
|
+
command: 'run',
|
|
501
|
+
projectDir,
|
|
502
|
+
connectionId: 'prod-metabase',
|
|
503
|
+
adapter: 'metabase',
|
|
504
|
+
outputMode: 'json',
|
|
505
|
+
}, io.io, {
|
|
506
|
+
progress: (event) => progressEvents.push(event),
|
|
507
|
+
runLocalMetabaseIngest: async (input) => {
|
|
508
|
+
input.progress?.onMetabaseFanoutPlanned?.({
|
|
509
|
+
metabaseConnectionId: 'prod-metabase',
|
|
510
|
+
children: [{ metabaseDatabaseId: 1, targetConnectionId: 'warehouse_a' }],
|
|
511
|
+
});
|
|
512
|
+
input.progress?.onMetabaseChildStarted?.({
|
|
513
|
+
metabaseConnectionId: 'prod-metabase',
|
|
514
|
+
metabaseDatabaseId: 1,
|
|
515
|
+
targetConnectionId: 'warehouse_a',
|
|
516
|
+
jobId: 'metabase-child-1',
|
|
517
|
+
});
|
|
518
|
+
input.memoryFlow?.update({
|
|
519
|
+
plannedWorkUnits: [
|
|
520
|
+
{
|
|
521
|
+
unitKey: 'metabase-col-6',
|
|
522
|
+
rawFiles: ['cards/40.json'],
|
|
523
|
+
peerFileCount: 0,
|
|
524
|
+
dependencyCount: 0,
|
|
525
|
+
},
|
|
526
|
+
],
|
|
527
|
+
});
|
|
528
|
+
input.memoryFlow?.emit({ type: 'chunks_planned', chunkCount: 1, workUnitCount: 1, evictionCount: 0 });
|
|
529
|
+
input.memoryFlow?.emit({
|
|
530
|
+
type: 'work_unit_started',
|
|
531
|
+
unitKey: 'metabase-col-6',
|
|
532
|
+
skills: ['sl_capture'],
|
|
533
|
+
stepBudget: 40,
|
|
534
|
+
});
|
|
535
|
+
input.memoryFlow?.emit({
|
|
536
|
+
type: 'work_unit_step',
|
|
537
|
+
unitKey: 'metabase-col-6',
|
|
538
|
+
stepIndex: 7,
|
|
539
|
+
stepBudget: 40,
|
|
540
|
+
});
|
|
541
|
+
input.memoryFlow?.emit({
|
|
542
|
+
type: 'stage_progress',
|
|
543
|
+
stage: 'integration',
|
|
544
|
+
percent: 81,
|
|
545
|
+
message: 'Resolving text conflict for metabase-col-6',
|
|
546
|
+
});
|
|
547
|
+
input.memoryFlow?.emit({ type: 'work_unit_finished', unitKey: 'metabase-col-6', status: 'success' });
|
|
548
|
+
input.memoryFlow?.update({
|
|
549
|
+
plannedWorkUnits: [
|
|
550
|
+
{
|
|
551
|
+
unitKey: 'metabase-col-7',
|
|
552
|
+
rawFiles: ['cards/48.json'],
|
|
553
|
+
peerFileCount: 0,
|
|
554
|
+
dependencyCount: 0,
|
|
555
|
+
},
|
|
556
|
+
],
|
|
557
|
+
});
|
|
558
|
+
input.memoryFlow?.emit({ type: 'chunks_planned', chunkCount: 1, workUnitCount: 1, evictionCount: 0 });
|
|
559
|
+
input.memoryFlow?.emit({
|
|
560
|
+
type: 'work_unit_started',
|
|
561
|
+
unitKey: 'metabase-col-7',
|
|
562
|
+
skills: ['sl_capture'],
|
|
563
|
+
stepBudget: 40,
|
|
564
|
+
});
|
|
565
|
+
input.progress?.onMetabaseChildCompleted?.({
|
|
566
|
+
metabaseConnectionId: 'prod-metabase',
|
|
567
|
+
metabaseDatabaseId: 1,
|
|
568
|
+
targetConnectionId: 'warehouse_a',
|
|
569
|
+
jobId: 'metabase-child-1',
|
|
570
|
+
status: 'done',
|
|
571
|
+
});
|
|
572
|
+
return {
|
|
573
|
+
metabaseConnectionId: 'prod-metabase',
|
|
574
|
+
status: 'all_succeeded',
|
|
575
|
+
totals: { workUnits: 1, failedWorkUnits: 0 },
|
|
576
|
+
children: [],
|
|
577
|
+
};
|
|
578
|
+
},
|
|
579
|
+
})).resolves.toBe(0);
|
|
580
|
+
expect(progressEvents).toEqual(expect.arrayContaining([
|
|
581
|
+
{ percent: 45, message: 'Planned 1 task' },
|
|
582
|
+
{ percent: 55, message: 'Processing 1/1 tasks: metabase-col-6' },
|
|
583
|
+
{
|
|
584
|
+
percent: 60,
|
|
585
|
+
message: 'Processing tasks: 0/1 complete, 1 active; latest metabase-col-6 step 7/40',
|
|
586
|
+
transient: true,
|
|
587
|
+
},
|
|
588
|
+
{ percent: 81, message: 'Resolving text conflict for metabase-col-6' },
|
|
589
|
+
{ percent: 81, message: 'Processing 1/1 tasks: metabase-col-7' },
|
|
590
|
+
]));
|
|
591
|
+
expect(io.stdout()).toContain('"status": "all_succeeded"');
|
|
592
|
+
expect(io.stderr()).not.toContain('Metabase ingest: prod-metabase');
|
|
593
|
+
});
|
|
488
594
|
it('runs Metabase scheduled ingest through the public CLI command path with real fan-out', async () => {
|
|
489
595
|
const projectDir = join(tempDir, 'metabase-cli-project');
|
|
490
596
|
await writeWarehouseConfig(projectDir);
|
|
@@ -759,6 +865,106 @@ describe('runKtxIngest', () => {
|
|
|
759
865
|
expect(io.stderr()).toBe('');
|
|
760
866
|
expect(io.stdout()).toContain('Status: error\n');
|
|
761
867
|
});
|
|
868
|
+
it('prints trace path and error status for stored failed ingest reports', async () => {
|
|
869
|
+
const projectDir = join(tempDir, 'project');
|
|
870
|
+
await writeWarehouseConfig(projectDir);
|
|
871
|
+
const io = makeIo();
|
|
872
|
+
const report = {
|
|
873
|
+
id: 'report-failed',
|
|
874
|
+
runId: 'run-failed',
|
|
875
|
+
jobId: 'job-failed',
|
|
876
|
+
connectionId: 'warehouse',
|
|
877
|
+
sourceKey: 'metabase',
|
|
878
|
+
createdAt: '2026-05-17T12:00:00.000Z',
|
|
879
|
+
body: {
|
|
880
|
+
status: 'failed',
|
|
881
|
+
syncId: 'sync-failed',
|
|
882
|
+
diffSummary: { added: 1, modified: 0, deleted: 0, unchanged: 0 },
|
|
883
|
+
commitSha: null,
|
|
884
|
+
tracePath: '/project/.ktx/ingest-traces/job-failed/trace.jsonl',
|
|
885
|
+
failure: { phase: 'final_gates', message: 'final artifact gates failed' },
|
|
886
|
+
workUnits: [],
|
|
887
|
+
failedWorkUnits: [],
|
|
888
|
+
reconciliationSkipped: true,
|
|
889
|
+
conflictsResolved: [],
|
|
890
|
+
evictionsApplied: [],
|
|
891
|
+
unmappedFallbacks: [],
|
|
892
|
+
evictionInputs: [],
|
|
893
|
+
unresolvedCards: [],
|
|
894
|
+
supersededBy: null,
|
|
895
|
+
overrideOf: null,
|
|
896
|
+
provenanceRows: [],
|
|
897
|
+
toolTranscripts: [],
|
|
898
|
+
},
|
|
899
|
+
};
|
|
900
|
+
await runKtxIngest({
|
|
901
|
+
command: 'status',
|
|
902
|
+
projectDir,
|
|
903
|
+
reportFile: '/project/report-failed.json',
|
|
904
|
+
runId: 'run-failed',
|
|
905
|
+
outputMode: 'plain',
|
|
906
|
+
inputMode: 'disabled',
|
|
907
|
+
}, io.io, {
|
|
908
|
+
readReportFile: vi.fn().mockResolvedValue(report),
|
|
909
|
+
});
|
|
910
|
+
expect(io.stdout()).toContain('Trace: /project/.ktx/ingest-traces/job-failed/trace.jsonl');
|
|
911
|
+
expect(io.stdout()).toContain('Status: error');
|
|
912
|
+
expect(io.stdout()).toContain('Error: final artifact gates failed');
|
|
913
|
+
});
|
|
914
|
+
it('prints a clear first failure reason when query-history work units fail', async () => {
|
|
915
|
+
const projectDir = join(tempDir, 'project');
|
|
916
|
+
await writeWarehouseConfig(projectDir);
|
|
917
|
+
const rawReason = '{"error":"invalid_grant","error_description":"reauth related error (invalid_rapt)","error_uri":"https://support.google.com/a/answer/9368756","error_subtype":"invalid_rapt"}';
|
|
918
|
+
const runLocal = vi.fn(async (input) => {
|
|
919
|
+
const failedWorkUnit = {
|
|
920
|
+
...localFakeBundleReport('query-history-failed').body.workUnits[0],
|
|
921
|
+
unitKey: 'historic-sql-table-orders',
|
|
922
|
+
rawFiles: ['tables/orders.json'],
|
|
923
|
+
status: 'failed',
|
|
924
|
+
reason: rawReason,
|
|
925
|
+
actions: [],
|
|
926
|
+
touchedSlSources: [],
|
|
927
|
+
};
|
|
928
|
+
const report = localFakeBundleReport('query-history-failed', {
|
|
929
|
+
id: 'report-query-history-failed',
|
|
930
|
+
runId: 'run-query-history-failed',
|
|
931
|
+
connectionId: input.connectionId,
|
|
932
|
+
sourceKey: 'historic-sql',
|
|
933
|
+
body: {
|
|
934
|
+
workUnits: [failedWorkUnit],
|
|
935
|
+
failedWorkUnits: [failedWorkUnit.unitKey],
|
|
936
|
+
},
|
|
937
|
+
});
|
|
938
|
+
return {
|
|
939
|
+
result: {
|
|
940
|
+
jobId: 'query-history-failed',
|
|
941
|
+
runId: report.runId,
|
|
942
|
+
syncId: report.body.syncId,
|
|
943
|
+
diffSummary: report.body.diffSummary,
|
|
944
|
+
workUnitCount: report.body.workUnits.length,
|
|
945
|
+
failedWorkUnits: report.body.failedWorkUnits,
|
|
946
|
+
artifactsWritten: report.body.provenanceRows.length,
|
|
947
|
+
commitSha: report.body.commitSha,
|
|
948
|
+
},
|
|
949
|
+
report,
|
|
950
|
+
};
|
|
951
|
+
});
|
|
952
|
+
const io = makeIo();
|
|
953
|
+
await expect(runKtxIngest({
|
|
954
|
+
command: 'run',
|
|
955
|
+
projectDir,
|
|
956
|
+
connectionId: 'warehouse',
|
|
957
|
+
adapter: 'historic-sql',
|
|
958
|
+
outputMode: 'plain',
|
|
959
|
+
}, io.io, {
|
|
960
|
+
runLocalIngest: runLocal,
|
|
961
|
+
jobIdFactory: () => 'query-history-failed',
|
|
962
|
+
})).resolves.toBe(1);
|
|
963
|
+
expect(io.stdout()).toContain('Status: error\n');
|
|
964
|
+
expect(io.stdout()).toContain('Failed tasks: 1\n');
|
|
965
|
+
expect(io.stdout()).toContain('Error: Query history failed for 1 task. First failure: Google Cloud authentication failed while analyzing query history: application-default credentials expired or require reauthentication (invalid_grant / invalid_rapt). Run `gcloud auth application-default login`, then retry.');
|
|
966
|
+
expect(io.stdout()).not.toContain('error_uri');
|
|
967
|
+
});
|
|
762
968
|
it('passes the debug LLM request file to local ingest runs', async () => {
|
|
763
969
|
const projectDir = join(tempDir, 'project');
|
|
764
970
|
await writeWarehouseConfig(projectDir);
|