@kaelio/ktx 0.1.0-rc.5 → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/assets/python/kaelio_ktx-0.1.0-py3-none-any.whl +0 -0
- package/assets/python/manifest.json +2 -2
- package/dist/clack.d.ts +6 -0
- package/dist/clack.js +23 -0
- package/dist/cli-program.js +5 -2
- package/dist/cli-program.test.js +7 -1
- package/dist/cli-runtime.d.ts +4 -0
- package/dist/cli-runtime.js +8 -1
- package/dist/command-schemas.d.ts +1 -1
- package/dist/commands/ingest-commands.js +1 -0
- package/dist/commands/knowledge-commands.js +5 -0
- package/dist/commands/mcp-commands.js +11 -3
- package/dist/commands/mcp-commands.test.js +30 -1
- package/dist/commands/sql-commands.d.ts +3 -0
- package/dist/commands/sql-commands.js +43 -0
- package/dist/commands/sql-commands.test.d.ts +1 -0
- package/dist/commands/sql-commands.test.js +68 -0
- package/dist/context-build-view.js +5 -1
- package/dist/dev.test.js +27 -0
- package/dist/index.d.ts +1 -0
- package/dist/index.js +1 -0
- package/dist/index.test.js +56 -21
- package/dist/ingest.js +123 -18
- package/dist/ingest.test.js +206 -0
- package/dist/io/print-list.d.ts +2 -1
- package/dist/io/print-list.js +7 -0
- package/dist/io/print-list.test.js +13 -11
- package/dist/io/symbols.d.ts +2 -2
- package/dist/knowledge.d.ts +1 -0
- package/dist/knowledge.js +34 -16
- package/dist/knowledge.test.js +27 -0
- package/dist/managed-python-command.d.ts +2 -0
- package/dist/managed-python-command.js +17 -9
- package/dist/managed-python-command.test.js +59 -4
- package/dist/next-steps.js +1 -1
- package/dist/next-steps.test.js +2 -0
- package/dist/print-command-tree.js +7 -1
- package/dist/public-ingest.d.ts +9 -1
- package/dist/public-ingest.js +50 -7
- package/dist/public-ingest.test.js +69 -2
- package/dist/release-version.d.ts +5 -0
- package/dist/release-version.js +44 -0
- package/dist/runtime-requirements.d.ts +23 -0
- package/dist/runtime-requirements.js +99 -0
- package/dist/runtime-requirements.test.d.ts +1 -0
- package/dist/runtime-requirements.test.js +63 -0
- package/dist/setup-agents.d.ts +11 -3
- package/dist/setup-agents.js +397 -134
- package/dist/setup-agents.test.js +359 -61
- package/dist/setup-embeddings.js +3 -6
- package/dist/setup-embeddings.test.js +18 -2
- package/dist/setup-models.js +2 -2
- package/dist/setup-models.test.js +5 -3
- package/dist/setup-ready-menu.d.ts +1 -1
- package/dist/setup-ready-menu.js +2 -0
- package/dist/setup-ready-menu.test.js +3 -0
- package/dist/setup-runtime.d.ts +45 -0
- package/dist/setup-runtime.js +47 -0
- package/dist/setup-runtime.test.d.ts +1 -0
- package/dist/setup-runtime.test.js +110 -0
- package/dist/setup-sources-notion.test.d.ts +1 -0
- package/dist/setup-sources-notion.test.js +107 -0
- package/dist/setup-sources.js +5 -2
- package/dist/setup.d.ts +19 -1
- package/dist/setup.js +104 -29
- package/dist/setup.test.js +221 -57
- package/dist/sl.js +2 -2
- package/dist/sl.test.js +10 -0
- package/dist/source-mapping.js +9 -1
- package/dist/source-mapping.test.d.ts +1 -0
- package/dist/source-mapping.test.js +65 -0
- package/dist/sql.d.ts +22 -0
- package/dist/sql.js +125 -0
- package/dist/sql.test.d.ts +1 -0
- package/dist/sql.test.js +226 -0
- package/node_modules/@ktx/connector-clickhouse/dist/package-exports.test.js +1 -1
- package/node_modules/@ktx/context/dist/connections/connection-type.d.ts +4 -4
- package/node_modules/@ktx/context/dist/core/git.service.d.ts +3 -0
- package/node_modules/@ktx/context/dist/core/git.service.js +47 -1
- package/node_modules/@ktx/context/dist/core/git.service.patch.test.d.ts +1 -0
- package/node_modules/@ktx/context/dist/core/git.service.patch.test.js +40 -0
- package/node_modules/@ktx/context/dist/ingest/adapters/historic-sql/types.d.ts +5 -5
- package/node_modules/@ktx/context/dist/ingest/adapters/looker/looker.adapter.d.ts +2 -2
- package/node_modules/@ktx/context/dist/ingest/adapters/looker/tools/looker-query-to-sl.tool.d.ts +2 -2
- package/node_modules/@ktx/context/dist/ingest/adapters/looker/types.d.ts +16 -16
- package/node_modules/@ktx/context/dist/ingest/adapters/lookml/pull-config.d.ts +1 -1
- package/node_modules/@ktx/context/dist/ingest/adapters/metabase/fetch.js +16 -0
- package/node_modules/@ktx/context/dist/ingest/adapters/metabase/fetch.test.js +41 -0
- package/node_modules/@ktx/context/dist/ingest/adapters/metricflow/metricflow.adapter.d.ts +2 -1
- package/node_modules/@ktx/context/dist/ingest/adapters/metricflow/metricflow.adapter.js +40 -0
- package/node_modules/@ktx/context/dist/ingest/adapters/metricflow/metricflow.adapter.test.js +116 -1
- package/node_modules/@ktx/context/dist/ingest/adapters/metricflow/projection-config.d.ts +29 -0
- package/node_modules/@ktx/context/dist/ingest/adapters/metricflow/projection-config.js +40 -0
- package/node_modules/@ktx/context/dist/ingest/adapters/metricflow/pull-config.d.ts +1 -1
- package/node_modules/@ktx/context/dist/ingest/artifact-gates.d.ts +25 -0
- package/node_modules/@ktx/context/dist/ingest/artifact-gates.js +149 -0
- package/node_modules/@ktx/context/dist/ingest/artifact-gates.test.d.ts +1 -0
- package/node_modules/@ktx/context/dist/ingest/artifact-gates.test.js +167 -0
- package/node_modules/@ktx/context/dist/ingest/final-gate-repair.d.ts +29 -0
- package/node_modules/@ktx/context/dist/ingest/final-gate-repair.js +178 -0
- package/node_modules/@ktx/context/dist/ingest/final-gate-repair.test.d.ts +1 -0
- package/node_modules/@ktx/context/dist/ingest/final-gate-repair.test.js +109 -0
- package/node_modules/@ktx/context/dist/ingest/index.d.ts +8 -1
- package/node_modules/@ktx/context/dist/ingest/index.js +7 -0
- package/node_modules/@ktx/context/dist/ingest/ingest-bundle.runner.d.ts +18 -2
- package/node_modules/@ktx/context/dist/ingest/ingest-bundle.runner.isolated-diff.test.d.ts +1 -0
- package/node_modules/@ktx/context/dist/ingest/ingest-bundle.runner.isolated-diff.test.js +1761 -0
- package/node_modules/@ktx/context/dist/ingest/ingest-bundle.runner.js +1695 -901
- package/node_modules/@ktx/context/dist/ingest/ingest-bundle.runner.test.js +135 -118
- package/node_modules/@ktx/context/dist/ingest/ingest-trace.d.ts +50 -0
- package/node_modules/@ktx/context/dist/ingest/ingest-trace.js +88 -0
- package/node_modules/@ktx/context/dist/ingest/ingest-trace.test.d.ts +1 -0
- package/node_modules/@ktx/context/dist/ingest/ingest-trace.test.js +76 -0
- package/node_modules/@ktx/context/dist/ingest/isolated-diff/git-patch.d.ts +16 -0
- package/node_modules/@ktx/context/dist/ingest/isolated-diff/git-patch.js +78 -0
- package/node_modules/@ktx/context/dist/ingest/isolated-diff/git-patch.test.d.ts +1 -0
- package/node_modules/@ktx/context/dist/ingest/isolated-diff/git-patch.test.js +76 -0
- package/node_modules/@ktx/context/dist/ingest/isolated-diff/patch-integrator.d.ts +58 -0
- package/node_modules/@ktx/context/dist/ingest/isolated-diff/patch-integrator.js +223 -0
- package/node_modules/@ktx/context/dist/ingest/isolated-diff/patch-integrator.test.d.ts +1 -0
- package/node_modules/@ktx/context/dist/ingest/isolated-diff/patch-integrator.test.js +369 -0
- package/node_modules/@ktx/context/dist/ingest/isolated-diff/textual-conflict-resolver.d.ts +23 -0
- package/node_modules/@ktx/context/dist/ingest/isolated-diff/textual-conflict-resolver.js +190 -0
- package/node_modules/@ktx/context/dist/ingest/isolated-diff/textual-conflict-resolver.test.d.ts +1 -0
- package/node_modules/@ktx/context/dist/ingest/isolated-diff/textual-conflict-resolver.test.js +101 -0
- package/node_modules/@ktx/context/dist/ingest/isolated-diff/work-unit-executor.d.ts +15 -0
- package/node_modules/@ktx/context/dist/ingest/isolated-diff/work-unit-executor.js +61 -0
- package/node_modules/@ktx/context/dist/ingest/isolated-diff/work-unit-executor.test.d.ts +1 -0
- package/node_modules/@ktx/context/dist/ingest/isolated-diff/work-unit-executor.test.js +137 -0
- package/node_modules/@ktx/context/dist/ingest/local-bundle-ingest.test.js +7 -0
- package/node_modules/@ktx/context/dist/ingest/local-bundle-runtime.js +54 -10
- package/node_modules/@ktx/context/dist/ingest/local-bundle-runtime.test.js +65 -0
- package/node_modules/@ktx/context/dist/ingest/memory-flow/schema.d.ts +23 -5
- package/node_modules/@ktx/context/dist/ingest/memory-flow/schema.js +17 -0
- package/node_modules/@ktx/context/dist/ingest/memory-flow/schema.test.js +1 -0
- package/node_modules/@ktx/context/dist/ingest/memory-flow/types.d.ts +6 -0
- package/node_modules/@ktx/context/dist/ingest/parsed-target-table.d.ts +1 -1
- package/node_modules/@ktx/context/dist/ingest/ports.d.ts +3 -0
- package/node_modules/@ktx/context/dist/ingest/report-snapshot.d.ts +32 -7
- package/node_modules/@ktx/context/dist/ingest/report-snapshot.js +25 -0
- package/node_modules/@ktx/context/dist/ingest/report-snapshot.test.js +124 -0
- package/node_modules/@ktx/context/dist/ingest/reports.d.ts +23 -0
- package/node_modules/@ktx/context/dist/ingest/semantic-layer-target-policy.d.ts +11 -0
- package/node_modules/@ktx/context/dist/ingest/semantic-layer-target-policy.js +26 -0
- package/node_modules/@ktx/context/dist/ingest/semantic-layer-target-policy.test.d.ts +1 -0
- package/node_modules/@ktx/context/dist/ingest/semantic-layer-target-policy.test.js +25 -0
- package/node_modules/@ktx/context/dist/ingest/stages/stage-3-work-units.d.ts +4 -0
- package/node_modules/@ktx/context/dist/ingest/stages/stage-3-work-units.js +4 -0
- package/node_modules/@ktx/context/dist/ingest/stages/stage-3-work-units.test.js +29 -0
- package/node_modules/@ktx/context/dist/ingest/tools/emit-unmapped-fallback.tool.d.ts +1 -1
- package/node_modules/@ktx/context/dist/ingest/types.d.ts +24 -0
- package/node_modules/@ktx/context/dist/ingest/wiki-body-refs.d.ts +24 -0
- package/node_modules/@ktx/context/dist/ingest/wiki-body-refs.js +111 -0
- package/node_modules/@ktx/context/dist/ingest/wiki-body-refs.test.d.ts +1 -0
- package/node_modules/@ktx/context/dist/ingest/wiki-body-refs.test.js +138 -0
- package/node_modules/@ktx/context/dist/llm/claude-code-runtime.js +19 -2
- package/node_modules/@ktx/context/dist/llm/claude-code-runtime.test.js +33 -0
- package/node_modules/@ktx/context/dist/project/setup-config.d.ts +1 -1
- package/node_modules/@ktx/context/dist/project/setup-config.js +10 -1
- package/node_modules/@ktx/context/dist/project/setup-config.test.js +3 -2
- package/node_modules/@ktx/context/dist/sl/tools/sl-edit-source.tool.js +5 -1
- package/node_modules/@ktx/context/dist/sl/tools/sl-edit-source.tool.test.js +15 -0
- package/node_modules/@ktx/context/dist/sl/tools/sl-write-source.tool.js +5 -1
- package/node_modules/@ktx/context/dist/sl/tools/sl-write-source.tool.test.js +22 -0
- package/node_modules/@ktx/context/dist/tools/action-target-connection.d.ts +9 -0
- package/node_modules/@ktx/context/dist/tools/action-target-connection.js +14 -0
- package/node_modules/@ktx/context/dist/tools/context-candidate-write.tool.d.ts +4 -4
- package/node_modules/@ktx/context/dist/tools/index.d.ts +1 -0
- package/node_modules/@ktx/context/dist/tools/index.js +1 -0
- package/node_modules/@ktx/context/dist/wiki/local-knowledge.js +4 -1
- package/node_modules/@ktx/context/dist/wiki/local-knowledge.test.js +44 -0
- package/node_modules/@ktx/context/dist/wiki/tools/wiki-write.tool.js +3 -48
- package/node_modules/@ktx/context/dist/wiki/tools/wiki-write.tool.test.js +28 -0
- package/node_modules/@ktx/context/dist/wiki/wiki-ref-validation.d.ts +17 -0
- package/node_modules/@ktx/context/dist/wiki/wiki-ref-validation.js +79 -0
- package/node_modules/@ktx/context/dist/wiki/wiki-ref-validation.test.d.ts +1 -0
- package/node_modules/@ktx/context/dist/wiki/wiki-ref-validation.test.js +64 -0
- package/node_modules/@ktx/context/prompts/memory_agent_bundle_ingest_work_unit.md +23 -4
- package/node_modules/@ktx/context/skills/ingest_triage/SKILL.md +7 -3
- package/package.json +4 -4
|
@@ -1,8 +1,7 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises';
|
|
2
2
|
import { tmpdir } from 'node:os';
|
|
3
3
|
import { join } from 'node:path';
|
|
4
4
|
import { beforeEach, describe, expect, it, vi } from 'vitest';
|
|
5
|
-
import { GitService } from '../core/index.js';
|
|
6
5
|
import { addTouchedSlSource } from '../tools/index.js';
|
|
7
6
|
import { IngestBundleRunner } from './ingest-bundle.runner.js';
|
|
8
7
|
import { createMemoryFlowLiveBuffer } from './memory-flow/live-buffer.js';
|
|
@@ -113,9 +112,15 @@ const makeDeps = () => {
|
|
|
113
112
|
};
|
|
114
113
|
const scopedGit = {
|
|
115
114
|
revParseHead: vi.fn().mockResolvedValue('h'),
|
|
116
|
-
commitFiles: vi.fn(),
|
|
115
|
+
commitFiles: vi.fn().mockResolvedValue({ created: true, commitHash: 'h' }),
|
|
116
|
+
commitStaged: vi.fn().mockResolvedValue({ created: false, commitHash: 'h' }),
|
|
117
117
|
resetHardTo: vi.fn(),
|
|
118
118
|
assertWorktreeClean: vi.fn().mockResolvedValue(undefined),
|
|
119
|
+
writeBinaryNoRenamePatch: vi.fn(async (_base, _head, patchPath) => {
|
|
120
|
+
await writeFile(patchPath, '', 'utf-8');
|
|
121
|
+
}),
|
|
122
|
+
applyPatchFile3WayIndex: vi.fn(),
|
|
123
|
+
diffNameStatus: vi.fn().mockResolvedValue([]),
|
|
119
124
|
};
|
|
120
125
|
const sessionWorktreeService = {
|
|
121
126
|
create: vi.fn().mockResolvedValue({
|
|
@@ -157,10 +162,12 @@ const makeDeps = () => {
|
|
|
157
162
|
loadPrompt: vi.fn().mockResolvedValue('base-framing'),
|
|
158
163
|
};
|
|
159
164
|
const wikiService = {
|
|
160
|
-
forWorktree: vi.fn()
|
|
165
|
+
forWorktree: vi.fn(),
|
|
166
|
+
listPageKeys: vi.fn().mockResolvedValue([]),
|
|
161
167
|
readPage: vi.fn().mockResolvedValue(null),
|
|
162
168
|
syncFromCommit: vi.fn().mockResolvedValue(undefined),
|
|
163
169
|
};
|
|
170
|
+
wikiService.forWorktree.mockReturnValue(wikiService);
|
|
164
171
|
const knowledgeSlRefs = {
|
|
165
172
|
syncFromWiki: vi.fn().mockResolvedValue({ inserted: 1, deleted: 0 }),
|
|
166
173
|
};
|
|
@@ -168,7 +175,7 @@ const makeDeps = () => {
|
|
|
168
175
|
listPagesForUser: vi.fn().mockResolvedValue([]),
|
|
169
176
|
};
|
|
170
177
|
const semanticLayerService = {
|
|
171
|
-
forWorktree: vi.fn()
|
|
178
|
+
forWorktree: vi.fn(),
|
|
172
179
|
listFilesForConnection: vi
|
|
173
180
|
.fn()
|
|
174
181
|
.mockImplementation((connectionId) => Promise.resolve(connectionId === 'warehouse-2' ? ['looker__orders.yaml'] : [])),
|
|
@@ -179,6 +186,7 @@ const makeDeps = () => {
|
|
|
179
186
|
loadErrors: [],
|
|
180
187
|
})),
|
|
181
188
|
};
|
|
189
|
+
semanticLayerService.forWorktree.mockReturnValue(semanticLayerService);
|
|
182
190
|
const slSearchService = {
|
|
183
191
|
indexSources: vi.fn().mockResolvedValue(undefined),
|
|
184
192
|
};
|
|
@@ -239,8 +247,12 @@ const buildRunner = (deps = makeDeps(), overrides = {}) => new IngestBundleRunne
|
|
|
239
247
|
resolveUploadDir: (uploadId) => `/tmp/ktx-test/ingest-uploads/${uploadId}`,
|
|
240
248
|
resolvePullDir: (jobId) => `/tmp/ktx-test/ingest-pulls/${jobId}`,
|
|
241
249
|
resolveTranscriptDir: (jobId) => `/tmp/ktx-test/run/wu-transcripts/${jobId}`,
|
|
250
|
+
resolveTracePath: (jobId) => `/tmp/ktx-test/ingest-traces/${jobId}/trace.jsonl`,
|
|
251
|
+
},
|
|
252
|
+
settings: {
|
|
253
|
+
probeRowCount: 1,
|
|
254
|
+
memoryIngestionModel: 'test-model',
|
|
242
255
|
},
|
|
243
|
-
settings: { probeRowCount: 1, memoryIngestionModel: 'test-model' },
|
|
244
256
|
skillsRegistry: deps.skillsRegistry,
|
|
245
257
|
promptService: deps.promptService,
|
|
246
258
|
wikiService: deps.wikiService,
|
|
@@ -377,6 +389,114 @@ describe('IngestBundleRunner — Stages 1 → 7', () => {
|
|
|
377
389
|
// Single touched path → path-scoped diff for the LLM commit-message note.
|
|
378
390
|
expect(deps.configService.enqueueCommitMessageJobForExternalCommit).toHaveBeenCalledWith({ commitHash: 'sq' }, expect.stringContaining('ingest(fake): j1'), 'raw-sources/c1/fake/s/a.yml');
|
|
379
391
|
});
|
|
392
|
+
it('fails before squash when reconciliation leaves a touched wiki page with dangling refs', async () => {
|
|
393
|
+
const deps = makeDeps();
|
|
394
|
+
let currentToolSession = null;
|
|
395
|
+
const scopedWiki = {
|
|
396
|
+
listPageKeys: vi.fn().mockResolvedValue(['page-a']),
|
|
397
|
+
readPage: vi.fn().mockImplementation((_scope, _scopeId, key) => {
|
|
398
|
+
if (key === 'page-a') {
|
|
399
|
+
return Promise.resolve({
|
|
400
|
+
pageKey: 'page-a',
|
|
401
|
+
frontmatter: { summary: 'Page A', usage_mode: 'auto', refs: ['missing-page'] },
|
|
402
|
+
content: 'See [[missing-page]].',
|
|
403
|
+
});
|
|
404
|
+
}
|
|
405
|
+
return Promise.resolve(null);
|
|
406
|
+
}),
|
|
407
|
+
};
|
|
408
|
+
deps.wikiService.forWorktree.mockReturnValue(scopedWiki);
|
|
409
|
+
deps.toolsetFactory.createIngestWuToolset.mockImplementation((toolSession) => {
|
|
410
|
+
currentToolSession = toolSession;
|
|
411
|
+
return {
|
|
412
|
+
toRuntimeTools: vi.fn().mockReturnValue({}),
|
|
413
|
+
getAllTools: vi.fn().mockReturnValue([]),
|
|
414
|
+
getToolNames: vi.fn().mockReturnValue([]),
|
|
415
|
+
};
|
|
416
|
+
});
|
|
417
|
+
deps.agentRunner.runLoop.mockImplementation(async (params) => {
|
|
418
|
+
if (params.telemetryTags.operationName === 'ingest-bundle-wu') {
|
|
419
|
+
currentToolSession.actions.push({ target: 'sl', type: 'updated', key: 'orders', detail: 'Orders source' });
|
|
420
|
+
}
|
|
421
|
+
if (params.telemetryTags.operationName === 'ingest-bundle-reconcile') {
|
|
422
|
+
currentToolSession.actions.push({ target: 'wiki', type: 'created', key: 'page-a', detail: 'Page A' });
|
|
423
|
+
}
|
|
424
|
+
return { stopReason: 'natural' };
|
|
425
|
+
});
|
|
426
|
+
const runner = buildRunner(deps);
|
|
427
|
+
runner.stageRawFilesStage1 = vi.fn().mockResolvedValue({
|
|
428
|
+
currentHashes: new Map([['a.yml', 'h1']]),
|
|
429
|
+
rawDirInWorktree: 'raw-sources/c1/fake/s',
|
|
430
|
+
});
|
|
431
|
+
runner.resolveStagedDir = vi.fn().mockResolvedValue('/tmp/stage/upload-x');
|
|
432
|
+
await expect(runner.run({
|
|
433
|
+
jobId: 'j1',
|
|
434
|
+
connectionId: 'c1',
|
|
435
|
+
sourceKey: 'fake',
|
|
436
|
+
trigger: 'upload',
|
|
437
|
+
bundleRef: { kind: 'upload', uploadId: 'upload-x' },
|
|
438
|
+
})).rejects.toThrow(/wiki references target missing page\(s\): page-a -> missing-page/);
|
|
439
|
+
expect(deps.runsRepo.markFailed).toHaveBeenCalledWith('run-1');
|
|
440
|
+
expect(deps.gitService.squashMergeIntoMain).not.toHaveBeenCalled();
|
|
441
|
+
});
|
|
442
|
+
it('allows reconciliation to save circular wiki refs once both pages exist', async () => {
|
|
443
|
+
const deps = makeDeps();
|
|
444
|
+
let currentToolSession = null;
|
|
445
|
+
const scopedWiki = {
|
|
446
|
+
listPageKeys: vi.fn().mockResolvedValue(['page-a', 'page-b']),
|
|
447
|
+
readPage: vi.fn().mockImplementation((_scope, _scopeId, key) => {
|
|
448
|
+
if (key === 'page-a') {
|
|
449
|
+
return Promise.resolve({
|
|
450
|
+
pageKey: 'page-a',
|
|
451
|
+
frontmatter: { summary: 'Page A', usage_mode: 'auto', refs: ['page-b'] },
|
|
452
|
+
content: 'See [[page-b]].',
|
|
453
|
+
});
|
|
454
|
+
}
|
|
455
|
+
if (key === 'page-b') {
|
|
456
|
+
return Promise.resolve({
|
|
457
|
+
pageKey: 'page-b',
|
|
458
|
+
frontmatter: { summary: 'Page B', usage_mode: 'auto', refs: ['page-a'] },
|
|
459
|
+
content: 'See [[page-a]].',
|
|
460
|
+
});
|
|
461
|
+
}
|
|
462
|
+
return Promise.resolve(null);
|
|
463
|
+
}),
|
|
464
|
+
};
|
|
465
|
+
deps.wikiService.forWorktree.mockReturnValue(scopedWiki);
|
|
466
|
+
deps.toolsetFactory.createIngestWuToolset.mockImplementation((toolSession) => {
|
|
467
|
+
currentToolSession = toolSession;
|
|
468
|
+
return {
|
|
469
|
+
toRuntimeTools: vi.fn().mockReturnValue({}),
|
|
470
|
+
getAllTools: vi.fn().mockReturnValue([]),
|
|
471
|
+
getToolNames: vi.fn().mockReturnValue([]),
|
|
472
|
+
};
|
|
473
|
+
});
|
|
474
|
+
deps.agentRunner.runLoop.mockImplementation(async (params) => {
|
|
475
|
+
if (params.telemetryTags.operationName === 'ingest-bundle-wu') {
|
|
476
|
+
currentToolSession.actions.push({ target: 'sl', type: 'updated', key: 'orders', detail: 'Orders source' });
|
|
477
|
+
}
|
|
478
|
+
if (params.telemetryTags.operationName === 'ingest-bundle-reconcile') {
|
|
479
|
+
currentToolSession.actions.push({ target: 'wiki', type: 'created', key: 'page-a', detail: 'Page A' }, { target: 'wiki', type: 'created', key: 'page-b', detail: 'Page B' });
|
|
480
|
+
}
|
|
481
|
+
return { stopReason: 'natural' };
|
|
482
|
+
});
|
|
483
|
+
const runner = buildRunner(deps);
|
|
484
|
+
runner.stageRawFilesStage1 = vi.fn().mockResolvedValue({
|
|
485
|
+
currentHashes: new Map([['a.yml', 'h1']]),
|
|
486
|
+
rawDirInWorktree: 'raw-sources/c1/fake/s',
|
|
487
|
+
});
|
|
488
|
+
runner.resolveStagedDir = vi.fn().mockResolvedValue('/tmp/stage/upload-x');
|
|
489
|
+
const result = await runner.run({
|
|
490
|
+
jobId: 'j1',
|
|
491
|
+
connectionId: 'c1',
|
|
492
|
+
sourceKey: 'fake',
|
|
493
|
+
trigger: 'upload',
|
|
494
|
+
bundleRef: { kind: 'upload', uploadId: 'upload-x' },
|
|
495
|
+
});
|
|
496
|
+
expect(result.failedWorkUnits).toEqual([]);
|
|
497
|
+
expect(deps.gitService.squashMergeIntoMain).toHaveBeenCalled();
|
|
498
|
+
expect(deps.runsRepo.markFailed).not.toHaveBeenCalled();
|
|
499
|
+
});
|
|
380
500
|
it('threads target warehouse connection names into WorkUnit and reconcile tool sessions', async () => {
|
|
381
501
|
const deps = makeDeps();
|
|
382
502
|
const sessions = [];
|
|
@@ -1174,7 +1294,7 @@ describe('IngestBundleRunner — Stages 1 → 7', () => {
|
|
|
1174
1294
|
});
|
|
1175
1295
|
const runner = buildRunner(deps);
|
|
1176
1296
|
runner.stageRawFilesStage1 = vi.fn().mockResolvedValue({
|
|
1177
|
-
currentHashes: new Map([['
|
|
1297
|
+
currentHashes: new Map([['a.yml', 'h1']]),
|
|
1178
1298
|
rawDirInWorktree: 'raw-sources/looker-run/fake/s',
|
|
1179
1299
|
});
|
|
1180
1300
|
runner.resolveStagedDir = vi.fn().mockResolvedValue('/tmp/stage/upload-x');
|
|
@@ -1232,6 +1352,7 @@ describe('IngestBundleRunner — Stages 1 → 7', () => {
|
|
|
1232
1352
|
workUnits: [{ unitKey: 'u1', rawFiles: ['semantic_models.yml'], peerFileIndex: [], dependencyPaths: [] }],
|
|
1233
1353
|
parseArtifacts: { semanticModels: [{ name: 'orders' }] },
|
|
1234
1354
|
});
|
|
1355
|
+
deps.adapter.listTargetConnectionIds = vi.fn().mockResolvedValue(['warehouse-2']);
|
|
1235
1356
|
deps.semanticLayerService.loadAllSources.mockImplementation((connectionId) => Promise.resolve({ sources: [{ name: `${connectionId}_source` }], loadErrors: [] }));
|
|
1236
1357
|
const postProcessor = {
|
|
1237
1358
|
run: vi.fn().mockResolvedValue({
|
|
@@ -1567,9 +1688,15 @@ describe('IngestBundleRunner — Stages 1 → 7', () => {
|
|
|
1567
1688
|
const assertError = new Error('Worktree has in-progress git operation (sequencer ...); refusing to proceed');
|
|
1568
1689
|
const sessionGit = {
|
|
1569
1690
|
revParseHead: vi.fn().mockResolvedValue('h'),
|
|
1570
|
-
commitFiles: vi.fn(),
|
|
1691
|
+
commitFiles: vi.fn().mockResolvedValue({ created: true, commitHash: 'h' }),
|
|
1692
|
+
commitStaged: vi.fn().mockResolvedValue({ created: false, commitHash: 'h' }),
|
|
1571
1693
|
resetHardTo: vi.fn(),
|
|
1572
1694
|
assertWorktreeClean: vi.fn().mockRejectedValue(assertError),
|
|
1695
|
+
writeBinaryNoRenamePatch: vi.fn(async (_base, _head, patchPath) => {
|
|
1696
|
+
await writeFile(patchPath, '', 'utf-8');
|
|
1697
|
+
}),
|
|
1698
|
+
applyPatchFile3WayIndex: vi.fn(),
|
|
1699
|
+
diffNameStatus: vi.fn().mockResolvedValue([]),
|
|
1573
1700
|
};
|
|
1574
1701
|
deps.sessionWorktreeService.create.mockResolvedValue({
|
|
1575
1702
|
chatId: 'j1',
|
|
@@ -1596,116 +1723,6 @@ describe('IngestBundleRunner — Stages 1 → 7', () => {
|
|
|
1596
1723
|
expect(deps.runsRepo.markFailed).toHaveBeenCalledWith('run-1');
|
|
1597
1724
|
expect(deps.gitService.squashMergeIntoMain).not.toHaveBeenCalled();
|
|
1598
1725
|
});
|
|
1599
|
-
it('squash-merges only successful WUs into main when one WU fails sl_validate', async () => {
|
|
1600
|
-
const homeDir = await mkdtemp(join(tmpdir(), 'ingest-rollback-'));
|
|
1601
|
-
try {
|
|
1602
|
-
const configDir = join(homeDir, 'config');
|
|
1603
|
-
const mainGit = new GitService({
|
|
1604
|
-
storage: { configDir, homeDir },
|
|
1605
|
-
git: {
|
|
1606
|
-
userName: 'System User',
|
|
1607
|
-
userEmail: 'system@example.com',
|
|
1608
|
-
bootstrapMessage: 'Initialize test config repo',
|
|
1609
|
-
bootstrapAuthor: 'test-system',
|
|
1610
|
-
bootstrapAuthorEmail: 'system@example.com',
|
|
1611
|
-
},
|
|
1612
|
-
});
|
|
1613
|
-
await mainGit.onModuleInit();
|
|
1614
|
-
const baseSha = await mainGit.revParseHead();
|
|
1615
|
-
if (!baseSha) {
|
|
1616
|
-
throw new Error('no base sha');
|
|
1617
|
-
}
|
|
1618
|
-
const deps = makeDeps();
|
|
1619
|
-
const sessionDir = join(homeDir, '.worktrees', 'session-j1');
|
|
1620
|
-
const sessionBranch = 'session/j1';
|
|
1621
|
-
let currentToolSession = null;
|
|
1622
|
-
deps.gitService = mainGit;
|
|
1623
|
-
deps.sessionWorktreeService.create.mockImplementation(async (_jobId, startSha) => {
|
|
1624
|
-
await mkdir(join(homeDir, '.worktrees'), { recursive: true });
|
|
1625
|
-
await mainGit.addWorktree(sessionDir, sessionBranch, startSha);
|
|
1626
|
-
return {
|
|
1627
|
-
chatId: 'j1',
|
|
1628
|
-
workdir: sessionDir,
|
|
1629
|
-
branch: sessionBranch,
|
|
1630
|
-
baseSha: startSha,
|
|
1631
|
-
createdAt: new Date(),
|
|
1632
|
-
git: mainGit.forWorktree(sessionDir),
|
|
1633
|
-
config: {},
|
|
1634
|
-
};
|
|
1635
|
-
});
|
|
1636
|
-
deps.sessionWorktreeService.cleanup.mockResolvedValue(undefined);
|
|
1637
|
-
deps.adapter.chunk.mockResolvedValue({
|
|
1638
|
-
workUnits: [
|
|
1639
|
-
{ unitKey: 'wu-good', rawFiles: ['good.raw'], peerFileIndex: [], dependencyPaths: [] },
|
|
1640
|
-
{ unitKey: 'wu-bad', rawFiles: ['bad.raw'], peerFileIndex: [], dependencyPaths: [] },
|
|
1641
|
-
],
|
|
1642
|
-
});
|
|
1643
|
-
deps.toolsetFactory.createIngestWuToolset.mockImplementation((toolSession) => {
|
|
1644
|
-
currentToolSession = toolSession;
|
|
1645
|
-
return {
|
|
1646
|
-
toRuntimeTools: vi.fn().mockReturnValue({}),
|
|
1647
|
-
getAllTools: vi.fn().mockReturnValue([]),
|
|
1648
|
-
getToolNames: vi.fn().mockReturnValue([]),
|
|
1649
|
-
};
|
|
1650
|
-
});
|
|
1651
|
-
deps.slValidator.validateSingleSource.mockImplementation((_validationDeps, _connectionId, sourceName) => ({
|
|
1652
|
-
errors: sourceName === 'bad' ? [{ message: 'bad source rejected' }] : [],
|
|
1653
|
-
warnings: [],
|
|
1654
|
-
}));
|
|
1655
|
-
deps.agentRunner.runLoop.mockImplementation(async (params) => {
|
|
1656
|
-
const unitKey = params.telemetryTags?.unitKey;
|
|
1657
|
-
if (unitKey === 'wu-good') {
|
|
1658
|
-
await mkdir(join(sessionDir, 'semantic-layer', 'c1'), { recursive: true });
|
|
1659
|
-
await writeFile(join(sessionDir, 'semantic-layer', 'c1', 'good.yaml'), 'name: good\n');
|
|
1660
|
-
addTouchedSlSource(currentToolSession.touchedSlSources, 'c1', 'good');
|
|
1661
|
-
currentToolSession.actions.push({ target: 'sl', type: 'created', key: 'good', detail: '' });
|
|
1662
|
-
await currentToolSession.gitService.commitFiles(['semantic-layer/c1/good.yaml'], 'test: add good source', 'KTX Test', 'system@ktx.local');
|
|
1663
|
-
}
|
|
1664
|
-
if (unitKey === 'wu-bad') {
|
|
1665
|
-
await mkdir(join(sessionDir, 'semantic-layer', 'c1'), { recursive: true });
|
|
1666
|
-
await writeFile(join(sessionDir, 'semantic-layer', 'c1', 'bad.yaml'), 'name: bad\n');
|
|
1667
|
-
addTouchedSlSource(currentToolSession.touchedSlSources, 'c1', 'bad');
|
|
1668
|
-
currentToolSession.actions.push({ target: 'sl', type: 'created', key: 'bad', detail: '' });
|
|
1669
|
-
await currentToolSession.gitService.commitFiles(['semantic-layer/c1/bad.yaml'], 'test: add bad source', 'KTX Test', 'system@ktx.local');
|
|
1670
|
-
}
|
|
1671
|
-
return { stopReason: 'natural' };
|
|
1672
|
-
});
|
|
1673
|
-
const runner = buildRunner(deps);
|
|
1674
|
-
runner.stageRawFilesStage1 = vi.fn().mockImplementation(async ({ worktreeRoot }) => {
|
|
1675
|
-
const rawDir = join(worktreeRoot, 'raw-sources', 'c1', 'fake', 's');
|
|
1676
|
-
await mkdir(rawDir, { recursive: true });
|
|
1677
|
-
await writeFile(join(rawDir, 'good.raw'), 'good raw');
|
|
1678
|
-
await writeFile(join(rawDir, 'bad.raw'), 'bad raw');
|
|
1679
|
-
return {
|
|
1680
|
-
currentHashes: new Map([
|
|
1681
|
-
['good.raw', 'good-hash'],
|
|
1682
|
-
['bad.raw', 'bad-hash'],
|
|
1683
|
-
]),
|
|
1684
|
-
rawDirInWorktree: 'raw-sources/c1/fake/s',
|
|
1685
|
-
};
|
|
1686
|
-
});
|
|
1687
|
-
runner.resolveStagedDir = vi.fn().mockResolvedValue('/tmp/stage/upload-x');
|
|
1688
|
-
const result = await runner.run({
|
|
1689
|
-
jobId: 'j1',
|
|
1690
|
-
connectionId: 'c1',
|
|
1691
|
-
sourceKey: 'fake',
|
|
1692
|
-
trigger: 'upload',
|
|
1693
|
-
bundleRef: { kind: 'upload', uploadId: 'upload-x' },
|
|
1694
|
-
});
|
|
1695
|
-
expect(result.failedWorkUnits).toEqual(['wu-bad']);
|
|
1696
|
-
expect(await readFile(join(configDir, 'semantic-layer', 'c1', 'good.yaml'), 'utf-8')).toContain('good');
|
|
1697
|
-
expect(await readFile(join(configDir, 'semantic-layer', 'c1', 'bad.yaml'), 'utf-8').catch(() => null)).toBeNull();
|
|
1698
|
-
expect(deps.reportsRepo.create).toHaveBeenCalledWith(expect.objectContaining({
|
|
1699
|
-
body: expect.objectContaining({
|
|
1700
|
-
failedWorkUnits: ['wu-bad'],
|
|
1701
|
-
}),
|
|
1702
|
-
}));
|
|
1703
|
-
await expect(stat(join(configDir, '.git', 'sequencer'))).rejects.toThrow();
|
|
1704
|
-
}
|
|
1705
|
-
finally {
|
|
1706
|
-
await rm(homeDir, { recursive: true, force: true });
|
|
1707
|
-
}
|
|
1708
|
-
});
|
|
1709
1726
|
it('fails the run and rethrows when the adapter cannot detect the bundle', async () => {
|
|
1710
1727
|
const deps = makeDeps();
|
|
1711
1728
|
deps.adapter.detect.mockResolvedValue(false);
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
export type IngestTraceLevel = 'info' | 'debug' | 'trace' | 'error';
|
|
2
|
+
export interface IngestTraceContext {
|
|
3
|
+
tracePath: string;
|
|
4
|
+
jobId: string;
|
|
5
|
+
connectionId: string;
|
|
6
|
+
sourceKey: string;
|
|
7
|
+
runId?: string;
|
|
8
|
+
syncId?: string;
|
|
9
|
+
level?: IngestTraceLevel;
|
|
10
|
+
}
|
|
11
|
+
export interface IngestTraceEvent {
|
|
12
|
+
schemaVersion: 1;
|
|
13
|
+
at: string;
|
|
14
|
+
level: IngestTraceLevel;
|
|
15
|
+
jobId: string;
|
|
16
|
+
connectionId: string;
|
|
17
|
+
sourceKey: string;
|
|
18
|
+
runId?: string;
|
|
19
|
+
syncId?: string;
|
|
20
|
+
phase: string;
|
|
21
|
+
event: string;
|
|
22
|
+
durationMs?: number;
|
|
23
|
+
data?: Record<string, unknown>;
|
|
24
|
+
error?: {
|
|
25
|
+
name: string;
|
|
26
|
+
message: string;
|
|
27
|
+
stack?: string;
|
|
28
|
+
};
|
|
29
|
+
}
|
|
30
|
+
export interface IngestTraceWriter {
|
|
31
|
+
readonly tracePath: string;
|
|
32
|
+
readonly context: IngestTraceContext;
|
|
33
|
+
withContext(context: Partial<Pick<IngestTraceContext, 'runId' | 'syncId'>>): IngestTraceWriter;
|
|
34
|
+
event(level: IngestTraceLevel, phase: string, event: string, data?: Record<string, unknown>, error?: unknown, durationMs?: number): Promise<void>;
|
|
35
|
+
}
|
|
36
|
+
export declare function ingestTracePathForJob(homeDir: string, jobId: string): string;
|
|
37
|
+
export declare class FileIngestTraceWriter implements IngestTraceWriter {
|
|
38
|
+
readonly tracePath: string;
|
|
39
|
+
readonly context: IngestTraceContext;
|
|
40
|
+
constructor(context: IngestTraceContext);
|
|
41
|
+
withContext(context: Partial<Pick<IngestTraceContext, 'runId' | 'syncId'>>): IngestTraceWriter;
|
|
42
|
+
event(level: IngestTraceLevel, phase: string, event: string, data?: Record<string, unknown>, error?: unknown, durationMs?: number): Promise<void>;
|
|
43
|
+
}
|
|
44
|
+
export declare class NoopIngestTraceWriter implements IngestTraceWriter {
|
|
45
|
+
readonly tracePath = "";
|
|
46
|
+
readonly context: IngestTraceContext;
|
|
47
|
+
withContext(): IngestTraceWriter;
|
|
48
|
+
event(): Promise<void>;
|
|
49
|
+
}
|
|
50
|
+
export declare function traceTimed<T>(trace: IngestTraceWriter, phase: string, event: string, data: Record<string, unknown>, fn: () => Promise<T>): Promise<T>;
|
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
import { appendFile, mkdir } from 'node:fs/promises';
|
|
2
|
+
import { dirname, join } from 'node:path';
|
|
3
|
+
const TRACE_LEVEL_RANK = {
|
|
4
|
+
error: 0,
|
|
5
|
+
info: 1,
|
|
6
|
+
debug: 2,
|
|
7
|
+
trace: 3,
|
|
8
|
+
};
|
|
9
|
+
export function ingestTracePathForJob(homeDir, jobId) {
|
|
10
|
+
return join(homeDir, 'ingest-traces', jobId, 'trace.jsonl');
|
|
11
|
+
}
|
|
12
|
+
function serializeError(error) {
|
|
13
|
+
if (error === undefined || error === null) {
|
|
14
|
+
return undefined;
|
|
15
|
+
}
|
|
16
|
+
if (error instanceof Error) {
|
|
17
|
+
return {
|
|
18
|
+
name: error.name,
|
|
19
|
+
message: error.message,
|
|
20
|
+
...(error.stack ? { stack: error.stack } : {}),
|
|
21
|
+
};
|
|
22
|
+
}
|
|
23
|
+
return { name: 'Error', message: String(error) };
|
|
24
|
+
}
|
|
25
|
+
function shouldWrite(configured, incoming) {
|
|
26
|
+
return TRACE_LEVEL_RANK[incoming] <= TRACE_LEVEL_RANK[configured];
|
|
27
|
+
}
|
|
28
|
+
export class FileIngestTraceWriter {
|
|
29
|
+
tracePath;
|
|
30
|
+
context;
|
|
31
|
+
constructor(context) {
|
|
32
|
+
this.context = { ...context, level: context.level ?? 'debug' };
|
|
33
|
+
this.tracePath = context.tracePath;
|
|
34
|
+
}
|
|
35
|
+
withContext(context) {
|
|
36
|
+
return new FileIngestTraceWriter({ ...this.context, ...context, tracePath: this.tracePath });
|
|
37
|
+
}
|
|
38
|
+
async event(level, phase, event, data, error, durationMs) {
|
|
39
|
+
if (!shouldWrite(this.context.level ?? 'debug', level)) {
|
|
40
|
+
return;
|
|
41
|
+
}
|
|
42
|
+
const serializedError = serializeError(error);
|
|
43
|
+
const payload = {
|
|
44
|
+
schemaVersion: 1,
|
|
45
|
+
at: new Date().toISOString(),
|
|
46
|
+
level,
|
|
47
|
+
jobId: this.context.jobId,
|
|
48
|
+
connectionId: this.context.connectionId,
|
|
49
|
+
sourceKey: this.context.sourceKey,
|
|
50
|
+
...(this.context.runId ? { runId: this.context.runId } : {}),
|
|
51
|
+
...(this.context.syncId ? { syncId: this.context.syncId } : {}),
|
|
52
|
+
phase,
|
|
53
|
+
event,
|
|
54
|
+
...(durationMs !== undefined ? { durationMs } : {}),
|
|
55
|
+
...(data ? { data } : {}),
|
|
56
|
+
...(serializedError ? { error: serializedError } : {}),
|
|
57
|
+
};
|
|
58
|
+
await mkdir(dirname(this.tracePath), { recursive: true });
|
|
59
|
+
await appendFile(this.tracePath, `${JSON.stringify(payload)}\n`, 'utf-8');
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
export class NoopIngestTraceWriter {
|
|
63
|
+
tracePath = '';
|
|
64
|
+
context = {
|
|
65
|
+
tracePath: '',
|
|
66
|
+
jobId: '',
|
|
67
|
+
connectionId: '',
|
|
68
|
+
sourceKey: '',
|
|
69
|
+
level: 'error',
|
|
70
|
+
};
|
|
71
|
+
withContext() {
|
|
72
|
+
return this;
|
|
73
|
+
}
|
|
74
|
+
async event() { }
|
|
75
|
+
}
|
|
76
|
+
export async function traceTimed(trace, phase, event, data, fn) {
|
|
77
|
+
await trace.event('debug', phase, `${event}_started`, data);
|
|
78
|
+
const started = Date.now();
|
|
79
|
+
try {
|
|
80
|
+
const result = await fn();
|
|
81
|
+
await trace.event('debug', phase, `${event}_finished`, data, undefined, Date.now() - started);
|
|
82
|
+
return result;
|
|
83
|
+
}
|
|
84
|
+
catch (error) {
|
|
85
|
+
await trace.event('error', phase, `${event}_failed`, data, error, Date.now() - started);
|
|
86
|
+
throw error;
|
|
87
|
+
}
|
|
88
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
import { mkdtemp, readFile } from 'node:fs/promises';
|
|
2
|
+
import { tmpdir } from 'node:os';
|
|
3
|
+
import { join } from 'node:path';
|
|
4
|
+
import { describe, expect, it, vi } from 'vitest';
|
|
5
|
+
import { FileIngestTraceWriter, ingestTracePathForJob, traceTimed } from './ingest-trace.js';
|
|
6
|
+
describe('FileIngestTraceWriter', () => {
|
|
7
|
+
it('persists structured trace events as JSONL', async () => {
|
|
8
|
+
const root = await mkdtemp(join(tmpdir(), 'ktx-trace-'));
|
|
9
|
+
const tracePath = ingestTracePathForJob(root, 'job-1');
|
|
10
|
+
const trace = new FileIngestTraceWriter({
|
|
11
|
+
tracePath,
|
|
12
|
+
jobId: 'job-1',
|
|
13
|
+
connectionId: 'metabase-main',
|
|
14
|
+
sourceKey: 'metabase',
|
|
15
|
+
level: 'debug',
|
|
16
|
+
});
|
|
17
|
+
await trace.event('debug', 'snapshot', 'input_snapshot', {
|
|
18
|
+
baseSha: 'abc123',
|
|
19
|
+
rawFileCount: 2,
|
|
20
|
+
diffSummary: { added: 1, modified: 1, deleted: 0, unchanged: 3 },
|
|
21
|
+
});
|
|
22
|
+
const lines = (await readFile(tracePath, 'utf-8'))
|
|
23
|
+
.trim()
|
|
24
|
+
.split('\n')
|
|
25
|
+
.map((line) => JSON.parse(line));
|
|
26
|
+
expect(lines).toHaveLength(1);
|
|
27
|
+
expect(lines[0]).toMatchObject({
|
|
28
|
+
schemaVersion: 1,
|
|
29
|
+
jobId: 'job-1',
|
|
30
|
+
connectionId: 'metabase-main',
|
|
31
|
+
sourceKey: 'metabase',
|
|
32
|
+
level: 'debug',
|
|
33
|
+
phase: 'snapshot',
|
|
34
|
+
event: 'input_snapshot',
|
|
35
|
+
data: {
|
|
36
|
+
baseSha: 'abc123',
|
|
37
|
+
rawFileCount: 2,
|
|
38
|
+
diffSummary: { added: 1, modified: 1, deleted: 0, unchanged: 3 },
|
|
39
|
+
},
|
|
40
|
+
});
|
|
41
|
+
expect(typeof lines[0].at).toBe('string');
|
|
42
|
+
});
|
|
43
|
+
it('records timing and error context for postmortem inspection', async () => {
|
|
44
|
+
vi.useFakeTimers();
|
|
45
|
+
vi.setSystemTime(new Date('2026-05-17T12:00:00.000Z'));
|
|
46
|
+
const root = await mkdtemp(join(tmpdir(), 'ktx-trace-'));
|
|
47
|
+
const tracePath = ingestTracePathForJob(root, 'job-2');
|
|
48
|
+
const trace = new FileIngestTraceWriter({
|
|
49
|
+
tracePath,
|
|
50
|
+
jobId: 'job-2',
|
|
51
|
+
connectionId: 'c1',
|
|
52
|
+
sourceKey: 'fake',
|
|
53
|
+
level: 'trace',
|
|
54
|
+
});
|
|
55
|
+
await expect(traceTimed(trace, 'integration', 'apply_patch', { unitKey: 'wu-1' }, async () => {
|
|
56
|
+
vi.advanceTimersByTime(17);
|
|
57
|
+
throw new Error('patch conflict');
|
|
58
|
+
})).rejects.toThrow('patch conflict');
|
|
59
|
+
const lines = (await readFile(tracePath, 'utf-8'))
|
|
60
|
+
.trim()
|
|
61
|
+
.split('\n')
|
|
62
|
+
.map((line) => JSON.parse(line));
|
|
63
|
+
expect(lines.map((line) => line.event)).toEqual(['apply_patch_started', 'apply_patch_failed']);
|
|
64
|
+
expect(lines[1]).toMatchObject({
|
|
65
|
+
level: 'error',
|
|
66
|
+
phase: 'integration',
|
|
67
|
+
data: { unitKey: 'wu-1' },
|
|
68
|
+
error: { name: 'Error', message: 'patch conflict' },
|
|
69
|
+
});
|
|
70
|
+
expect(lines[1].durationMs).toBe(17);
|
|
71
|
+
vi.useRealTimers();
|
|
72
|
+
});
|
|
73
|
+
it('uses the documented trace path layout', () => {
|
|
74
|
+
expect(ingestTracePathForJob('/project/.ktx', 'job-3')).toBe('/project/.ktx/ingest-traces/job-3/trace.jsonl');
|
|
75
|
+
});
|
|
76
|
+
});
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
export declare const textArtifactRoots: readonly ["wiki/", "semantic-layer/"];
|
|
2
|
+
export interface PatchTouchedPath {
|
|
3
|
+
path: string;
|
|
4
|
+
oldPath: string;
|
|
5
|
+
newPath: string;
|
|
6
|
+
mode: string | null;
|
|
7
|
+
binary: boolean;
|
|
8
|
+
}
|
|
9
|
+
export interface PatchPolicyInput {
|
|
10
|
+
unitKey: string;
|
|
11
|
+
patch: string;
|
|
12
|
+
slDisallowed: boolean;
|
|
13
|
+
allowedTargetConnectionIds?: ReadonlySet<string>;
|
|
14
|
+
}
|
|
15
|
+
export declare function parsePatchTouchedPaths(patch: string): PatchTouchedPath[];
|
|
16
|
+
export declare function assertPatchAllowedForWorkUnit(input: PatchPolicyInput): PatchTouchedPath[];
|
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
import { assertSemanticLayerTargetPathsAllowed } from '../semantic-layer-target-policy.js';
|
|
2
|
+
export const textArtifactRoots = ['wiki/', 'semantic-layer/'];
|
|
3
|
+
function stripPrefix(path) {
|
|
4
|
+
return path.replace(/^[ab]\//, '');
|
|
5
|
+
}
|
|
6
|
+
function isTextArtifactPath(path) {
|
|
7
|
+
return textArtifactRoots.some((root) => path.startsWith(root));
|
|
8
|
+
}
|
|
9
|
+
export function parsePatchTouchedPaths(patch) {
|
|
10
|
+
const lines = patch.split('\n');
|
|
11
|
+
const entries = [];
|
|
12
|
+
let current = null;
|
|
13
|
+
const pushCurrent = () => {
|
|
14
|
+
if (current) {
|
|
15
|
+
entries.push(current);
|
|
16
|
+
}
|
|
17
|
+
};
|
|
18
|
+
for (const line of lines) {
|
|
19
|
+
const diffMatch = /^diff --git (.+) (.+)$/.exec(line);
|
|
20
|
+
if (diffMatch) {
|
|
21
|
+
pushCurrent();
|
|
22
|
+
const oldPath = stripPrefix(diffMatch[1] ?? '');
|
|
23
|
+
const newPath = stripPrefix(diffMatch[2] ?? '');
|
|
24
|
+
current = {
|
|
25
|
+
path: newPath === '/dev/null' ? oldPath : newPath,
|
|
26
|
+
oldPath,
|
|
27
|
+
newPath,
|
|
28
|
+
mode: null,
|
|
29
|
+
binary: false,
|
|
30
|
+
};
|
|
31
|
+
continue;
|
|
32
|
+
}
|
|
33
|
+
if (!current) {
|
|
34
|
+
continue;
|
|
35
|
+
}
|
|
36
|
+
const indexMode = /^index [0-9a-f]+\.\.[0-9a-f]+(?: ([0-7]{6}))?$/.exec(line);
|
|
37
|
+
if (indexMode?.[1]) {
|
|
38
|
+
current.mode = indexMode[1];
|
|
39
|
+
}
|
|
40
|
+
const newMode = /^new mode ([0-7]{6})$/.exec(line);
|
|
41
|
+
if (newMode) {
|
|
42
|
+
current.mode = newMode[1] ?? current.mode;
|
|
43
|
+
}
|
|
44
|
+
const newFileMode = /^new file mode ([0-7]{6})$/.exec(line);
|
|
45
|
+
if (newFileMode) {
|
|
46
|
+
current.mode = newFileMode[1] ?? current.mode;
|
|
47
|
+
}
|
|
48
|
+
if (line === 'GIT binary patch' || line.startsWith('Binary files ')) {
|
|
49
|
+
current.binary = true;
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
pushCurrent();
|
|
53
|
+
return entries;
|
|
54
|
+
}
|
|
55
|
+
export function assertPatchAllowedForWorkUnit(input) {
|
|
56
|
+
const touched = parsePatchTouchedPaths(input.patch);
|
|
57
|
+
if (input.allowedTargetConnectionIds) {
|
|
58
|
+
assertSemanticLayerTargetPathsAllowed({
|
|
59
|
+
paths: touched.map((entry) => entry.path),
|
|
60
|
+
allowedConnectionIds: input.allowedTargetConnectionIds,
|
|
61
|
+
});
|
|
62
|
+
}
|
|
63
|
+
for (const entry of touched) {
|
|
64
|
+
if (input.slDisallowed && entry.path.startsWith('semantic-layer/')) {
|
|
65
|
+
throw new Error(`slDisallowed WorkUnit ${input.unitKey} touched ${entry.path}`);
|
|
66
|
+
}
|
|
67
|
+
if (!isTextArtifactPath(entry.path)) {
|
|
68
|
+
continue;
|
|
69
|
+
}
|
|
70
|
+
if (entry.binary) {
|
|
71
|
+
throw new Error(`unexpected binary patch under ${entry.path}`);
|
|
72
|
+
}
|
|
73
|
+
if (entry.mode && entry.mode !== '100644') {
|
|
74
|
+
throw new Error(`unexpected executable mode under ${entry.path}: ${entry.mode}`);
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
return touched;
|
|
78
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|