@kaelio/ktx 0.1.0-rc.5 → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/assets/python/kaelio_ktx-0.1.0-py3-none-any.whl +0 -0
- package/assets/python/manifest.json +2 -2
- package/dist/clack.d.ts +6 -0
- package/dist/clack.js +23 -0
- package/dist/cli-program.js +5 -2
- package/dist/cli-program.test.js +7 -1
- package/dist/cli-runtime.d.ts +4 -0
- package/dist/cli-runtime.js +8 -1
- package/dist/command-schemas.d.ts +1 -1
- package/dist/commands/ingest-commands.js +1 -0
- package/dist/commands/knowledge-commands.js +5 -0
- package/dist/commands/mcp-commands.js +11 -3
- package/dist/commands/mcp-commands.test.js +30 -1
- package/dist/commands/sql-commands.d.ts +3 -0
- package/dist/commands/sql-commands.js +43 -0
- package/dist/commands/sql-commands.test.d.ts +1 -0
- package/dist/commands/sql-commands.test.js +68 -0
- package/dist/context-build-view.js +5 -1
- package/dist/dev.test.js +27 -0
- package/dist/index.d.ts +1 -0
- package/dist/index.js +1 -0
- package/dist/index.test.js +56 -21
- package/dist/ingest.js +123 -18
- package/dist/ingest.test.js +206 -0
- package/dist/io/print-list.d.ts +2 -1
- package/dist/io/print-list.js +7 -0
- package/dist/io/print-list.test.js +13 -11
- package/dist/io/symbols.d.ts +2 -2
- package/dist/knowledge.d.ts +1 -0
- package/dist/knowledge.js +34 -16
- package/dist/knowledge.test.js +27 -0
- package/dist/managed-python-command.d.ts +2 -0
- package/dist/managed-python-command.js +17 -9
- package/dist/managed-python-command.test.js +59 -4
- package/dist/next-steps.js +1 -1
- package/dist/next-steps.test.js +2 -0
- package/dist/print-command-tree.js +7 -1
- package/dist/public-ingest.d.ts +9 -1
- package/dist/public-ingest.js +50 -7
- package/dist/public-ingest.test.js +69 -2
- package/dist/release-version.d.ts +5 -0
- package/dist/release-version.js +44 -0
- package/dist/runtime-requirements.d.ts +23 -0
- package/dist/runtime-requirements.js +99 -0
- package/dist/runtime-requirements.test.d.ts +1 -0
- package/dist/runtime-requirements.test.js +63 -0
- package/dist/setup-agents.d.ts +11 -3
- package/dist/setup-agents.js +397 -134
- package/dist/setup-agents.test.js +359 -61
- package/dist/setup-embeddings.js +3 -6
- package/dist/setup-embeddings.test.js +18 -2
- package/dist/setup-models.js +2 -2
- package/dist/setup-models.test.js +5 -3
- package/dist/setup-ready-menu.d.ts +1 -1
- package/dist/setup-ready-menu.js +2 -0
- package/dist/setup-ready-menu.test.js +3 -0
- package/dist/setup-runtime.d.ts +45 -0
- package/dist/setup-runtime.js +47 -0
- package/dist/setup-runtime.test.d.ts +1 -0
- package/dist/setup-runtime.test.js +110 -0
- package/dist/setup-sources-notion.test.d.ts +1 -0
- package/dist/setup-sources-notion.test.js +107 -0
- package/dist/setup-sources.js +5 -2
- package/dist/setup.d.ts +19 -1
- package/dist/setup.js +104 -29
- package/dist/setup.test.js +221 -57
- package/dist/sl.js +2 -2
- package/dist/sl.test.js +10 -0
- package/dist/source-mapping.js +9 -1
- package/dist/source-mapping.test.d.ts +1 -0
- package/dist/source-mapping.test.js +65 -0
- package/dist/sql.d.ts +22 -0
- package/dist/sql.js +125 -0
- package/dist/sql.test.d.ts +1 -0
- package/dist/sql.test.js +226 -0
- package/node_modules/@ktx/connector-clickhouse/dist/package-exports.test.js +1 -1
- package/node_modules/@ktx/context/dist/connections/connection-type.d.ts +4 -4
- package/node_modules/@ktx/context/dist/core/git.service.d.ts +3 -0
- package/node_modules/@ktx/context/dist/core/git.service.js +47 -1
- package/node_modules/@ktx/context/dist/core/git.service.patch.test.d.ts +1 -0
- package/node_modules/@ktx/context/dist/core/git.service.patch.test.js +40 -0
- package/node_modules/@ktx/context/dist/ingest/adapters/historic-sql/types.d.ts +5 -5
- package/node_modules/@ktx/context/dist/ingest/adapters/looker/looker.adapter.d.ts +2 -2
- package/node_modules/@ktx/context/dist/ingest/adapters/looker/tools/looker-query-to-sl.tool.d.ts +2 -2
- package/node_modules/@ktx/context/dist/ingest/adapters/looker/types.d.ts +16 -16
- package/node_modules/@ktx/context/dist/ingest/adapters/lookml/pull-config.d.ts +1 -1
- package/node_modules/@ktx/context/dist/ingest/adapters/metabase/fetch.js +16 -0
- package/node_modules/@ktx/context/dist/ingest/adapters/metabase/fetch.test.js +41 -0
- package/node_modules/@ktx/context/dist/ingest/adapters/metricflow/metricflow.adapter.d.ts +2 -1
- package/node_modules/@ktx/context/dist/ingest/adapters/metricflow/metricflow.adapter.js +40 -0
- package/node_modules/@ktx/context/dist/ingest/adapters/metricflow/metricflow.adapter.test.js +116 -1
- package/node_modules/@ktx/context/dist/ingest/adapters/metricflow/projection-config.d.ts +29 -0
- package/node_modules/@ktx/context/dist/ingest/adapters/metricflow/projection-config.js +40 -0
- package/node_modules/@ktx/context/dist/ingest/adapters/metricflow/pull-config.d.ts +1 -1
- package/node_modules/@ktx/context/dist/ingest/artifact-gates.d.ts +25 -0
- package/node_modules/@ktx/context/dist/ingest/artifact-gates.js +149 -0
- package/node_modules/@ktx/context/dist/ingest/artifact-gates.test.d.ts +1 -0
- package/node_modules/@ktx/context/dist/ingest/artifact-gates.test.js +167 -0
- package/node_modules/@ktx/context/dist/ingest/final-gate-repair.d.ts +29 -0
- package/node_modules/@ktx/context/dist/ingest/final-gate-repair.js +178 -0
- package/node_modules/@ktx/context/dist/ingest/final-gate-repair.test.d.ts +1 -0
- package/node_modules/@ktx/context/dist/ingest/final-gate-repair.test.js +109 -0
- package/node_modules/@ktx/context/dist/ingest/index.d.ts +8 -1
- package/node_modules/@ktx/context/dist/ingest/index.js +7 -0
- package/node_modules/@ktx/context/dist/ingest/ingest-bundle.runner.d.ts +18 -2
- package/node_modules/@ktx/context/dist/ingest/ingest-bundle.runner.isolated-diff.test.d.ts +1 -0
- package/node_modules/@ktx/context/dist/ingest/ingest-bundle.runner.isolated-diff.test.js +1761 -0
- package/node_modules/@ktx/context/dist/ingest/ingest-bundle.runner.js +1695 -901
- package/node_modules/@ktx/context/dist/ingest/ingest-bundle.runner.test.js +135 -118
- package/node_modules/@ktx/context/dist/ingest/ingest-trace.d.ts +50 -0
- package/node_modules/@ktx/context/dist/ingest/ingest-trace.js +88 -0
- package/node_modules/@ktx/context/dist/ingest/ingest-trace.test.d.ts +1 -0
- package/node_modules/@ktx/context/dist/ingest/ingest-trace.test.js +76 -0
- package/node_modules/@ktx/context/dist/ingest/isolated-diff/git-patch.d.ts +16 -0
- package/node_modules/@ktx/context/dist/ingest/isolated-diff/git-patch.js +78 -0
- package/node_modules/@ktx/context/dist/ingest/isolated-diff/git-patch.test.d.ts +1 -0
- package/node_modules/@ktx/context/dist/ingest/isolated-diff/git-patch.test.js +76 -0
- package/node_modules/@ktx/context/dist/ingest/isolated-diff/patch-integrator.d.ts +58 -0
- package/node_modules/@ktx/context/dist/ingest/isolated-diff/patch-integrator.js +223 -0
- package/node_modules/@ktx/context/dist/ingest/isolated-diff/patch-integrator.test.d.ts +1 -0
- package/node_modules/@ktx/context/dist/ingest/isolated-diff/patch-integrator.test.js +369 -0
- package/node_modules/@ktx/context/dist/ingest/isolated-diff/textual-conflict-resolver.d.ts +23 -0
- package/node_modules/@ktx/context/dist/ingest/isolated-diff/textual-conflict-resolver.js +190 -0
- package/node_modules/@ktx/context/dist/ingest/isolated-diff/textual-conflict-resolver.test.d.ts +1 -0
- package/node_modules/@ktx/context/dist/ingest/isolated-diff/textual-conflict-resolver.test.js +101 -0
- package/node_modules/@ktx/context/dist/ingest/isolated-diff/work-unit-executor.d.ts +15 -0
- package/node_modules/@ktx/context/dist/ingest/isolated-diff/work-unit-executor.js +61 -0
- package/node_modules/@ktx/context/dist/ingest/isolated-diff/work-unit-executor.test.d.ts +1 -0
- package/node_modules/@ktx/context/dist/ingest/isolated-diff/work-unit-executor.test.js +137 -0
- package/node_modules/@ktx/context/dist/ingest/local-bundle-ingest.test.js +7 -0
- package/node_modules/@ktx/context/dist/ingest/local-bundle-runtime.js +54 -10
- package/node_modules/@ktx/context/dist/ingest/local-bundle-runtime.test.js +65 -0
- package/node_modules/@ktx/context/dist/ingest/memory-flow/schema.d.ts +23 -5
- package/node_modules/@ktx/context/dist/ingest/memory-flow/schema.js +17 -0
- package/node_modules/@ktx/context/dist/ingest/memory-flow/schema.test.js +1 -0
- package/node_modules/@ktx/context/dist/ingest/memory-flow/types.d.ts +6 -0
- package/node_modules/@ktx/context/dist/ingest/parsed-target-table.d.ts +1 -1
- package/node_modules/@ktx/context/dist/ingest/ports.d.ts +3 -0
- package/node_modules/@ktx/context/dist/ingest/report-snapshot.d.ts +32 -7
- package/node_modules/@ktx/context/dist/ingest/report-snapshot.js +25 -0
- package/node_modules/@ktx/context/dist/ingest/report-snapshot.test.js +124 -0
- package/node_modules/@ktx/context/dist/ingest/reports.d.ts +23 -0
- package/node_modules/@ktx/context/dist/ingest/semantic-layer-target-policy.d.ts +11 -0
- package/node_modules/@ktx/context/dist/ingest/semantic-layer-target-policy.js +26 -0
- package/node_modules/@ktx/context/dist/ingest/semantic-layer-target-policy.test.d.ts +1 -0
- package/node_modules/@ktx/context/dist/ingest/semantic-layer-target-policy.test.js +25 -0
- package/node_modules/@ktx/context/dist/ingest/stages/stage-3-work-units.d.ts +4 -0
- package/node_modules/@ktx/context/dist/ingest/stages/stage-3-work-units.js +4 -0
- package/node_modules/@ktx/context/dist/ingest/stages/stage-3-work-units.test.js +29 -0
- package/node_modules/@ktx/context/dist/ingest/tools/emit-unmapped-fallback.tool.d.ts +1 -1
- package/node_modules/@ktx/context/dist/ingest/types.d.ts +24 -0
- package/node_modules/@ktx/context/dist/ingest/wiki-body-refs.d.ts +24 -0
- package/node_modules/@ktx/context/dist/ingest/wiki-body-refs.js +111 -0
- package/node_modules/@ktx/context/dist/ingest/wiki-body-refs.test.d.ts +1 -0
- package/node_modules/@ktx/context/dist/ingest/wiki-body-refs.test.js +138 -0
- package/node_modules/@ktx/context/dist/llm/claude-code-runtime.js +19 -2
- package/node_modules/@ktx/context/dist/llm/claude-code-runtime.test.js +33 -0
- package/node_modules/@ktx/context/dist/project/setup-config.d.ts +1 -1
- package/node_modules/@ktx/context/dist/project/setup-config.js +10 -1
- package/node_modules/@ktx/context/dist/project/setup-config.test.js +3 -2
- package/node_modules/@ktx/context/dist/sl/tools/sl-edit-source.tool.js +5 -1
- package/node_modules/@ktx/context/dist/sl/tools/sl-edit-source.tool.test.js +15 -0
- package/node_modules/@ktx/context/dist/sl/tools/sl-write-source.tool.js +5 -1
- package/node_modules/@ktx/context/dist/sl/tools/sl-write-source.tool.test.js +22 -0
- package/node_modules/@ktx/context/dist/tools/action-target-connection.d.ts +9 -0
- package/node_modules/@ktx/context/dist/tools/action-target-connection.js +14 -0
- package/node_modules/@ktx/context/dist/tools/context-candidate-write.tool.d.ts +4 -4
- package/node_modules/@ktx/context/dist/tools/index.d.ts +1 -0
- package/node_modules/@ktx/context/dist/tools/index.js +1 -0
- package/node_modules/@ktx/context/dist/wiki/local-knowledge.js +4 -1
- package/node_modules/@ktx/context/dist/wiki/local-knowledge.test.js +44 -0
- package/node_modules/@ktx/context/dist/wiki/tools/wiki-write.tool.js +3 -48
- package/node_modules/@ktx/context/dist/wiki/tools/wiki-write.tool.test.js +28 -0
- package/node_modules/@ktx/context/dist/wiki/wiki-ref-validation.d.ts +17 -0
- package/node_modules/@ktx/context/dist/wiki/wiki-ref-validation.js +79 -0
- package/node_modules/@ktx/context/dist/wiki/wiki-ref-validation.test.d.ts +1 -0
- package/node_modules/@ktx/context/dist/wiki/wiki-ref-validation.test.js +64 -0
- package/node_modules/@ktx/context/prompts/memory_agent_bundle_ingest_work_unit.md +23 -4
- package/node_modules/@ktx/context/skills/ingest_triage/SKILL.md +7 -3
- package/package.json +4 -4
|
@@ -0,0 +1,1761 @@
|
|
|
1
|
+
import { mkdir, mkdtemp, readFile, readdir, rm, writeFile } from 'node:fs/promises';
|
|
2
|
+
import { tmpdir } from 'node:os';
|
|
3
|
+
import { join } from 'node:path';
|
|
4
|
+
import { describe, expect, it, vi } from 'vitest';
|
|
5
|
+
import { GitService, SessionWorktreeService } from '../core/index.js';
|
|
6
|
+
import { LocalGitFileStore } from '../project/local-git-file-store.js';
|
|
7
|
+
import { addTouchedSlSource } from '../tools/index.js';
|
|
8
|
+
import { IngestBundleRunner } from './ingest-bundle.runner.js';
|
|
9
|
+
async function makeRealGitRuntime() {
|
|
10
|
+
const homeDir = await mkdtemp(join(tmpdir(), 'ktx-isolated-runner-'));
|
|
11
|
+
const configDir = join(homeDir, 'config');
|
|
12
|
+
const git = new GitService({
|
|
13
|
+
storage: { configDir, homeDir },
|
|
14
|
+
git: {
|
|
15
|
+
userName: 'System User',
|
|
16
|
+
userEmail: 'system@example.com',
|
|
17
|
+
bootstrapMessage: 'init',
|
|
18
|
+
bootstrapAuthor: 'system',
|
|
19
|
+
bootstrapAuthorEmail: 'system@example.com',
|
|
20
|
+
},
|
|
21
|
+
});
|
|
22
|
+
await git.onModuleInit();
|
|
23
|
+
const configService = new LocalGitFileStore({ rootDir: configDir, git });
|
|
24
|
+
const sessionWorktreeService = new SessionWorktreeService({
|
|
25
|
+
coreConfig: {
|
|
26
|
+
storage: { configDir, homeDir },
|
|
27
|
+
git: {
|
|
28
|
+
userName: 'System User',
|
|
29
|
+
userEmail: 'system@example.com',
|
|
30
|
+
bootstrapMessage: 'init',
|
|
31
|
+
bootstrapAuthor: 'system',
|
|
32
|
+
bootstrapAuthorEmail: 'system@example.com',
|
|
33
|
+
},
|
|
34
|
+
},
|
|
35
|
+
gitService: git,
|
|
36
|
+
configService,
|
|
37
|
+
});
|
|
38
|
+
return { homeDir, configDir, git, configService, sessionWorktreeService };
|
|
39
|
+
}
|
|
40
|
+
function rootOfConfig(configService, fallback) {
|
|
41
|
+
const rootDir = configService.rootDir;
|
|
42
|
+
return typeof rootDir === 'string' ? rootDir : fallback;
|
|
43
|
+
}
|
|
44
|
+
async function loadSourcesFromRoot(root) {
|
|
45
|
+
const raw = await readFile(join(root, 'semantic-layer/warehouse/mart_account_segments.yaml'), 'utf-8').catch(() => '');
|
|
46
|
+
const hasCents = raw.includes('total_contract_arr_cents');
|
|
47
|
+
const hasDollars = raw.includes('total_contract_arr');
|
|
48
|
+
return {
|
|
49
|
+
sources: hasCents || hasDollars
|
|
50
|
+
? [
|
|
51
|
+
{
|
|
52
|
+
name: 'mart_account_segments',
|
|
53
|
+
grain: ['account_id'],
|
|
54
|
+
columns: [{ name: 'account_id', type: 'string' }],
|
|
55
|
+
joins: [],
|
|
56
|
+
measures: [{ name: hasCents ? 'total_contract_arr_cents' : 'total_contract_arr', expr: 'sum(contract_arr)' }],
|
|
57
|
+
table: 'analytics.mart_account_segments',
|
|
58
|
+
},
|
|
59
|
+
]
|
|
60
|
+
: [],
|
|
61
|
+
loadErrors: [],
|
|
62
|
+
};
|
|
63
|
+
}
|
|
64
|
+
async function listGlobalWikiPageKeys(root) {
|
|
65
|
+
const dir = join(root, 'wiki/global');
|
|
66
|
+
const entries = await readdir(dir).catch(() => []);
|
|
67
|
+
return entries
|
|
68
|
+
.filter((entry) => entry.endsWith('.md'))
|
|
69
|
+
.map((entry) => entry.slice(0, -'.md'.length))
|
|
70
|
+
.sort();
|
|
71
|
+
}
|
|
72
|
+
function frontmatterList(yaml, key) {
|
|
73
|
+
const pattern = new RegExp(`(?:^|\\n)${key}:\\n((?: - .+\\n?)*)`);
|
|
74
|
+
return (pattern
|
|
75
|
+
.exec(yaml)?.[1]
|
|
76
|
+
?.split('\n')
|
|
77
|
+
.map((line) => line.trim().replace(/^- /, ''))
|
|
78
|
+
.filter(Boolean) ?? []);
|
|
79
|
+
}
|
|
80
|
+
function legacyFallbackSettingKey() {
|
|
81
|
+
return ['sharedWorktree', 'SourceKeys'].join('');
|
|
82
|
+
}
|
|
83
|
+
function legacySharedTraceEvent() {
|
|
84
|
+
return ['shared', 'worktree', 'path', 'enabled'].join('_');
|
|
85
|
+
}
|
|
86
|
+
function makeWikiService(root) {
|
|
87
|
+
return {
|
|
88
|
+
listPageKeys: vi.fn(async (scope) => (scope === 'GLOBAL' ? listGlobalWikiPageKeys(root) : [])),
|
|
89
|
+
readPage: vi.fn(async (_scope, _scopeId, key) => {
|
|
90
|
+
const path = join(root, 'wiki/global', `${key}.md`);
|
|
91
|
+
const raw = await readFile(path, 'utf-8').catch(() => null);
|
|
92
|
+
if (!raw) {
|
|
93
|
+
return null;
|
|
94
|
+
}
|
|
95
|
+
const [, yaml = '', content = ''] = /^---\n([\s\S]*?)\n---\n?([\s\S]*)$/.exec(raw) ?? [];
|
|
96
|
+
return {
|
|
97
|
+
pageKey: key,
|
|
98
|
+
frontmatter: {
|
|
99
|
+
summary: key,
|
|
100
|
+
usage_mode: 'auto',
|
|
101
|
+
refs: frontmatterList(yaml, 'refs'),
|
|
102
|
+
sl_refs: frontmatterList(yaml, 'sl_refs'),
|
|
103
|
+
},
|
|
104
|
+
content: content.trim(),
|
|
105
|
+
};
|
|
106
|
+
}),
|
|
107
|
+
syncFromCommit: vi.fn(),
|
|
108
|
+
};
|
|
109
|
+
}
|
|
110
|
+
function makeDeps(runtime, sourceKey = 'metabase', settings = {}) {
|
|
111
|
+
const adapter = {
|
|
112
|
+
source: sourceKey,
|
|
113
|
+
skillNames: [],
|
|
114
|
+
detect: vi.fn().mockResolvedValue(true),
|
|
115
|
+
chunk: vi.fn().mockResolvedValue({
|
|
116
|
+
workUnits: [
|
|
117
|
+
{ unitKey: 'card-wiki', rawFiles: ['cards/wiki.json'], peerFileIndex: [], dependencyPaths: [] },
|
|
118
|
+
{ unitKey: 'card-source', rawFiles: ['cards/source.json'], peerFileIndex: [], dependencyPaths: [] },
|
|
119
|
+
],
|
|
120
|
+
}),
|
|
121
|
+
};
|
|
122
|
+
const wikiService = makeWikiService(runtime.configDir);
|
|
123
|
+
const semanticLayerService = {
|
|
124
|
+
loadAllSources: vi.fn(async () => loadSourcesFromRoot(runtime.configDir)),
|
|
125
|
+
listFilesForConnection: vi.fn().mockResolvedValue(['mart_account_segments.yaml']),
|
|
126
|
+
};
|
|
127
|
+
semanticLayerService.forWorktree = vi.fn((workdir) => ({
|
|
128
|
+
...semanticLayerService,
|
|
129
|
+
loadAllSources: vi.fn(async () => loadSourcesFromRoot(workdir)),
|
|
130
|
+
listFilesForConnection: vi.fn().mockResolvedValue(['mart_account_segments.yaml']),
|
|
131
|
+
}));
|
|
132
|
+
const deps = {
|
|
133
|
+
runs: { create: vi.fn().mockResolvedValue({ id: 'run-1' }), markCompleted: vi.fn(), markFailed: vi.fn() },
|
|
134
|
+
provenance: {
|
|
135
|
+
insertMany: vi.fn(),
|
|
136
|
+
findLatestHashesForCompletedSyncs: vi.fn().mockResolvedValue(new Map()),
|
|
137
|
+
findLatestArtifactsForRawPaths: vi.fn().mockResolvedValue(new Map()),
|
|
138
|
+
},
|
|
139
|
+
reports: { create: vi.fn().mockResolvedValue({ id: 'report-1' }), findByJobId: vi.fn().mockResolvedValue(null), markSuperseded: vi.fn() },
|
|
140
|
+
canonicalPins: { listPins: vi.fn().mockResolvedValue([]) },
|
|
141
|
+
registry: { get: vi.fn().mockReturnValue(adapter), register: vi.fn(), has: vi.fn(), list: vi.fn() },
|
|
142
|
+
diffSetService: {
|
|
143
|
+
compute: vi.fn().mockResolvedValue({ added: ['cards/wiki.json', 'cards/source.json'], modified: [], deleted: [], unchanged: [] }),
|
|
144
|
+
},
|
|
145
|
+
sessionWorktreeService: runtime.sessionWorktreeService,
|
|
146
|
+
agentRunner: { runLoop: vi.fn() },
|
|
147
|
+
gitService: runtime.git,
|
|
148
|
+
lockingService: { withLock: vi.fn(async (_key, fn) => fn()) },
|
|
149
|
+
storage: {
|
|
150
|
+
homeDir: join(runtime.configDir, '.ktx'),
|
|
151
|
+
systemGitAuthor: { name: 'KTX Test', email: 'system@ktx.local' },
|
|
152
|
+
resolveUploadDir: (id) => join(runtime.homeDir, 'upload', id),
|
|
153
|
+
resolvePullDir: (id) => join(runtime.homeDir, 'pull', id),
|
|
154
|
+
resolveTranscriptDir: (id) => join(runtime.configDir, '.ktx/ingest-transcripts', id),
|
|
155
|
+
resolveTracePath: (id) => join(runtime.configDir, '.ktx/ingest-traces', id, 'trace.jsonl'),
|
|
156
|
+
},
|
|
157
|
+
settings: {
|
|
158
|
+
memoryIngestionModel: 'test',
|
|
159
|
+
probeRowCount: 1,
|
|
160
|
+
ingestTraceLevel: 'trace',
|
|
161
|
+
...settings,
|
|
162
|
+
},
|
|
163
|
+
skillsRegistry: {
|
|
164
|
+
listSkills: vi.fn().mockResolvedValue([]),
|
|
165
|
+
getSkill: vi.fn().mockResolvedValue(null),
|
|
166
|
+
buildSkillsPrompt: vi.fn().mockReturnValue(''),
|
|
167
|
+
stripFrontmatter: vi.fn((body) => body),
|
|
168
|
+
},
|
|
169
|
+
promptService: { loadPrompt: vi.fn().mockResolvedValue('base') },
|
|
170
|
+
wikiService: { ...wikiService, forWorktree: vi.fn((workdir) => makeWikiService(workdir)) },
|
|
171
|
+
knowledgeIndex: { listPagesForUser: vi.fn().mockResolvedValue([]) },
|
|
172
|
+
knowledgeSlRefs: { syncFromWiki: vi.fn() },
|
|
173
|
+
semanticLayerService,
|
|
174
|
+
slSearchService: { indexSources: vi.fn() },
|
|
175
|
+
slSourcesRepository: {},
|
|
176
|
+
slValidator: { validateSingleSource: vi.fn().mockResolvedValue({ errors: [], warnings: [] }) },
|
|
177
|
+
connections: { listEnabledConnections: vi.fn().mockResolvedValue([]), getConnectionById: vi.fn() },
|
|
178
|
+
toolsetFactory: { createIngestWuToolset: vi.fn(() => ({ toRuntimeTools: vi.fn(() => ({})) })) },
|
|
179
|
+
commitMessages: { enqueueForExternalCommit: vi.fn() },
|
|
180
|
+
embedding: { maxBatchSize: 64, computeEmbedding: vi.fn(), computeEmbeddingsBulk: vi.fn() },
|
|
181
|
+
};
|
|
182
|
+
return { deps, adapter };
|
|
183
|
+
}
|
|
184
|
+
async function mockStageRawFiles(runner, runtime, hashes, sourceKey = 'metabase') {
|
|
185
|
+
runner.resolveStagedDir = vi.fn().mockResolvedValue(join(runtime.homeDir, 'stage'));
|
|
186
|
+
runner.stageRawFilesStage1 = vi.fn(async ({ worktreeRoot }) => {
|
|
187
|
+
const rawDir = join(worktreeRoot, 'raw-sources/warehouse', sourceKey, 's');
|
|
188
|
+
await mkdir(rawDir, { recursive: true });
|
|
189
|
+
for (const [rawPath] of hashes) {
|
|
190
|
+
await mkdir(join(rawDir, rawPath.split('/').slice(0, -1).join('/')), { recursive: true });
|
|
191
|
+
await writeFile(join(rawDir, rawPath), '{}');
|
|
192
|
+
}
|
|
193
|
+
return { currentHashes: new Map(hashes), rawDirInWorktree: `raw-sources/warehouse/${sourceKey}/s` };
|
|
194
|
+
});
|
|
195
|
+
}
|
|
196
|
+
describe('IngestBundleRunner isolated diff path', () => {
|
|
197
|
+
it('routes an unlisted direct-writing source through isolated diffs by default', async () => {
|
|
198
|
+
const runtime = await makeRealGitRuntime();
|
|
199
|
+
try {
|
|
200
|
+
const sourceKey = 'custom-direct-source';
|
|
201
|
+
const { deps, adapter } = makeDeps(runtime, sourceKey);
|
|
202
|
+
adapter.chunk.mockResolvedValue({
|
|
203
|
+
workUnits: [
|
|
204
|
+
{
|
|
205
|
+
unitKey: 'custom-wiki',
|
|
206
|
+
rawFiles: ['custom/page.json'],
|
|
207
|
+
peerFileIndex: [],
|
|
208
|
+
dependencyPaths: [],
|
|
209
|
+
},
|
|
210
|
+
],
|
|
211
|
+
});
|
|
212
|
+
let currentSession = null;
|
|
213
|
+
deps.toolsetFactory.createIngestWuToolset = vi.fn((toolSession) => {
|
|
214
|
+
currentSession = toolSession;
|
|
215
|
+
return { toRuntimeTools: vi.fn(() => ({})) };
|
|
216
|
+
});
|
|
217
|
+
deps.agentRunner.runLoop = vi.fn(async (params) => {
|
|
218
|
+
if (params.telemetryTags.operationName !== 'ingest-bundle-wu') {
|
|
219
|
+
return { stopReason: 'natural' };
|
|
220
|
+
}
|
|
221
|
+
const root = rootOfConfig(currentSession.configService, runtime.configDir);
|
|
222
|
+
await mkdir(join(root, 'wiki/global'), { recursive: true });
|
|
223
|
+
await writeFile(join(root, 'wiki/global/custom-isolated.md'), '---\nsummary: Custom isolated write\nusage_mode: auto\n---\n\nCustom isolated write.\n', 'utf-8');
|
|
224
|
+
currentSession.actions.push({
|
|
225
|
+
target: 'wiki',
|
|
226
|
+
type: 'created',
|
|
227
|
+
key: 'custom-isolated',
|
|
228
|
+
detail: 'Custom isolated write',
|
|
229
|
+
rawPaths: ['custom/page.json'],
|
|
230
|
+
});
|
|
231
|
+
await currentSession.gitService.commitFiles(['wiki/global/custom-isolated.md'], 'custom wiki', 'KTX Test', 'system@ktx.local');
|
|
232
|
+
return { stopReason: 'natural' };
|
|
233
|
+
});
|
|
234
|
+
const runner = new IngestBundleRunner(deps);
|
|
235
|
+
await mockStageRawFiles(runner, runtime, [['custom/page.json', 'h1']], sourceKey);
|
|
236
|
+
await expect(runner.run({
|
|
237
|
+
jobId: 'job-custom-default',
|
|
238
|
+
connectionId: 'warehouse',
|
|
239
|
+
sourceKey,
|
|
240
|
+
trigger: 'upload',
|
|
241
|
+
bundleRef: { kind: 'upload', uploadId: 'upload' },
|
|
242
|
+
})).resolves.toMatchObject({
|
|
243
|
+
jobId: 'job-custom-default',
|
|
244
|
+
failedWorkUnits: [],
|
|
245
|
+
workUnitCount: 1,
|
|
246
|
+
});
|
|
247
|
+
const trace = await readFile(join(runtime.configDir, '.ktx/ingest-traces/job-custom-default/trace.jsonl'), 'utf-8');
|
|
248
|
+
expect(trace).toContain('isolated_diff_enabled');
|
|
249
|
+
expect(trace).toContain('work_unit_child_created');
|
|
250
|
+
expect(trace).not.toContain(legacySharedTraceEvent());
|
|
251
|
+
const reportCreate = vi.mocked(deps.reports.create).mock.calls.at(-1)?.[0];
|
|
252
|
+
const reportBody = reportCreate?.body;
|
|
253
|
+
expect(reportBody?.isolatedDiff).toMatchObject({
|
|
254
|
+
enabled: true,
|
|
255
|
+
acceptedPatches: 1,
|
|
256
|
+
});
|
|
257
|
+
}
|
|
258
|
+
finally {
|
|
259
|
+
await rm(runtime.homeDir, { recursive: true, force: true });
|
|
260
|
+
}
|
|
261
|
+
});
|
|
262
|
+
it('does not support shared-worktree fallback settings', async () => {
|
|
263
|
+
const runtime = await makeRealGitRuntime();
|
|
264
|
+
try {
|
|
265
|
+
const sourceKey = 'legacy-source';
|
|
266
|
+
const staleSettings = {
|
|
267
|
+
[legacyFallbackSettingKey()]: ['legacy-source'],
|
|
268
|
+
};
|
|
269
|
+
const { deps, adapter } = makeDeps(runtime, sourceKey, staleSettings);
|
|
270
|
+
adapter.chunk.mockResolvedValue({
|
|
271
|
+
workUnits: [
|
|
272
|
+
{
|
|
273
|
+
unitKey: 'legacy-wiki',
|
|
274
|
+
rawFiles: ['legacy/page.json'],
|
|
275
|
+
peerFileIndex: [],
|
|
276
|
+
dependencyPaths: [],
|
|
277
|
+
},
|
|
278
|
+
],
|
|
279
|
+
});
|
|
280
|
+
let currentSession = null;
|
|
281
|
+
deps.toolsetFactory.createIngestWuToolset = vi.fn((toolSession) => {
|
|
282
|
+
currentSession = toolSession;
|
|
283
|
+
return { toRuntimeTools: vi.fn(() => ({})) };
|
|
284
|
+
});
|
|
285
|
+
deps.agentRunner.runLoop = vi.fn(async (params) => {
|
|
286
|
+
if (params.telemetryTags.operationName !== 'ingest-bundle-wu') {
|
|
287
|
+
return { stopReason: 'natural' };
|
|
288
|
+
}
|
|
289
|
+
const root = rootOfConfig(currentSession.configService, runtime.configDir);
|
|
290
|
+
await mkdir(join(root, 'wiki/global'), { recursive: true });
|
|
291
|
+
await writeFile(join(root, 'wiki/global/legacy-isolated.md'), '---\nsummary: Legacy isolated write\nusage_mode: auto\n---\n\nLegacy isolated write.\n', 'utf-8');
|
|
292
|
+
currentSession.actions.push({
|
|
293
|
+
target: 'wiki',
|
|
294
|
+
type: 'created',
|
|
295
|
+
key: 'legacy-isolated',
|
|
296
|
+
detail: 'Legacy isolated write',
|
|
297
|
+
rawPaths: ['legacy/page.json'],
|
|
298
|
+
});
|
|
299
|
+
await currentSession.gitService.commitFiles(['wiki/global/legacy-isolated.md'], 'legacy isolated wiki', 'KTX Test', 'system@ktx.local');
|
|
300
|
+
return { stopReason: 'natural' };
|
|
301
|
+
});
|
|
302
|
+
const runner = new IngestBundleRunner(deps);
|
|
303
|
+
await mockStageRawFiles(runner, runtime, [['legacy/page.json', 'h1']], sourceKey);
|
|
304
|
+
await expect(runner.run({
|
|
305
|
+
jobId: 'job-legacy-isolated',
|
|
306
|
+
connectionId: 'warehouse',
|
|
307
|
+
sourceKey,
|
|
308
|
+
trigger: 'upload',
|
|
309
|
+
bundleRef: { kind: 'upload', uploadId: 'upload' },
|
|
310
|
+
})).resolves.toMatchObject({
|
|
311
|
+
jobId: 'job-legacy-isolated',
|
|
312
|
+
failedWorkUnits: [],
|
|
313
|
+
workUnitCount: 1,
|
|
314
|
+
});
|
|
315
|
+
const trace = await readFile(join(runtime.configDir, '.ktx/ingest-traces/job-legacy-isolated/trace.jsonl'), 'utf-8');
|
|
316
|
+
expect(trace).toContain('isolated_diff_enabled');
|
|
317
|
+
expect(trace).toContain('work_unit_child_created');
|
|
318
|
+
expect(trace).not.toContain(legacySharedTraceEvent());
|
|
319
|
+
const reportCreate = vi.mocked(deps.reports.create).mock.calls.at(-1)?.[0];
|
|
320
|
+
const reportBody = reportCreate?.body;
|
|
321
|
+
expect(reportBody?.isolatedDiff).toMatchObject({
|
|
322
|
+
enabled: true,
|
|
323
|
+
acceptedPatches: 1,
|
|
324
|
+
});
|
|
325
|
+
}
|
|
326
|
+
finally {
|
|
327
|
+
await rm(runtime.homeDir, { recursive: true, force: true });
|
|
328
|
+
}
|
|
329
|
+
});
|
|
330
|
+
it('does not integrate failed isolated WorkUnit patches', async () => {
|
|
331
|
+
const runtime = await makeRealGitRuntime();
|
|
332
|
+
try {
|
|
333
|
+
const { deps, adapter } = makeDeps(runtime, 'fake');
|
|
334
|
+
adapter.chunk.mockResolvedValue({
|
|
335
|
+
workUnits: [
|
|
336
|
+
{ unitKey: 'wu-good', rawFiles: ['good.raw'], peerFileIndex: [], dependencyPaths: [] },
|
|
337
|
+
{ unitKey: 'wu-bad', rawFiles: ['bad.raw'], peerFileIndex: [], dependencyPaths: [] },
|
|
338
|
+
],
|
|
339
|
+
});
|
|
340
|
+
deps.diffSetService.compute = vi.fn().mockResolvedValue({
|
|
341
|
+
added: ['good.raw', 'bad.raw'],
|
|
342
|
+
modified: [],
|
|
343
|
+
deleted: [],
|
|
344
|
+
unchanged: [],
|
|
345
|
+
});
|
|
346
|
+
deps.slValidator.validateSingleSource = vi.fn(async (_validationDeps, _connectionId, sourceName) => ({
|
|
347
|
+
errors: sourceName === 'bad' ? [{ message: 'bad source rejected' }] : [],
|
|
348
|
+
warnings: [],
|
|
349
|
+
}));
|
|
350
|
+
let currentSession = null;
|
|
351
|
+
deps.toolsetFactory.createIngestWuToolset = vi.fn((toolSession) => {
|
|
352
|
+
currentSession = toolSession;
|
|
353
|
+
return { toRuntimeTools: vi.fn(() => ({})) };
|
|
354
|
+
});
|
|
355
|
+
deps.agentRunner.runLoop = vi.fn(async (params) => {
|
|
356
|
+
if (params.telemetryTags.operationName !== 'ingest-bundle-wu') {
|
|
357
|
+
return { stopReason: 'natural' };
|
|
358
|
+
}
|
|
359
|
+
const unitKey = params.telemetryTags.unitKey;
|
|
360
|
+
const root = rootOfConfig(currentSession.configService, runtime.configDir);
|
|
361
|
+
await mkdir(join(root, 'semantic-layer/warehouse'), { recursive: true });
|
|
362
|
+
if (unitKey === 'wu-good') {
|
|
363
|
+
await writeFile(join(root, 'semantic-layer/warehouse/good.yaml'), 'name: good\n', 'utf-8');
|
|
364
|
+
addTouchedSlSource(currentSession.touchedSlSources, 'warehouse', 'good');
|
|
365
|
+
currentSession.actions.push({
|
|
366
|
+
target: 'sl',
|
|
367
|
+
type: 'created',
|
|
368
|
+
key: 'good',
|
|
369
|
+
detail: 'good source',
|
|
370
|
+
targetConnectionId: 'warehouse',
|
|
371
|
+
rawPaths: ['good.raw'],
|
|
372
|
+
});
|
|
373
|
+
await currentSession.gitService.commitFiles(['semantic-layer/warehouse/good.yaml'], 'test: add good source', 'KTX Test', 'system@ktx.local');
|
|
374
|
+
}
|
|
375
|
+
if (unitKey === 'wu-bad') {
|
|
376
|
+
await writeFile(join(root, 'semantic-layer/warehouse/bad.yaml'), 'name: bad\n', 'utf-8');
|
|
377
|
+
addTouchedSlSource(currentSession.touchedSlSources, 'warehouse', 'bad');
|
|
378
|
+
currentSession.actions.push({
|
|
379
|
+
target: 'sl',
|
|
380
|
+
type: 'created',
|
|
381
|
+
key: 'bad',
|
|
382
|
+
detail: 'bad source',
|
|
383
|
+
targetConnectionId: 'warehouse',
|
|
384
|
+
rawPaths: ['bad.raw'],
|
|
385
|
+
});
|
|
386
|
+
await currentSession.gitService.commitFiles(['semantic-layer/warehouse/bad.yaml'], 'test: add bad source', 'KTX Test', 'system@ktx.local');
|
|
387
|
+
}
|
|
388
|
+
return { stopReason: 'natural' };
|
|
389
|
+
});
|
|
390
|
+
const runner = new IngestBundleRunner(deps);
|
|
391
|
+
await mockStageRawFiles(runner, runtime, [
|
|
392
|
+
['good.raw', 'good-hash'],
|
|
393
|
+
['bad.raw', 'bad-hash'],
|
|
394
|
+
], 'fake');
|
|
395
|
+
const result = await runner.run({
|
|
396
|
+
jobId: 'job-failed-wu-isolated',
|
|
397
|
+
connectionId: 'warehouse',
|
|
398
|
+
sourceKey: 'fake',
|
|
399
|
+
trigger: 'upload',
|
|
400
|
+
bundleRef: { kind: 'upload', uploadId: 'upload' },
|
|
401
|
+
});
|
|
402
|
+
expect(result.failedWorkUnits).toEqual(['wu-bad']);
|
|
403
|
+
await expect(readFile(join(runtime.configDir, 'semantic-layer/warehouse/good.yaml'), 'utf-8')).resolves.toContain('good');
|
|
404
|
+
await expect(readFile(join(runtime.configDir, 'semantic-layer/warehouse/bad.yaml'), 'utf-8')).rejects.toThrow();
|
|
405
|
+
const reportCreate = vi.mocked(deps.reports.create).mock.calls.at(-1)?.[0];
|
|
406
|
+
const reportBody = reportCreate?.body;
|
|
407
|
+
expect(reportBody.failedWorkUnits).toEqual(['wu-bad']);
|
|
408
|
+
expect(reportBody.isolatedDiff).toMatchObject({ enabled: true, acceptedPatches: 1 });
|
|
409
|
+
const trace = await readFile(join(runtime.configDir, '.ktx/ingest-traces/job-failed-wu-isolated/trace.jsonl'), 'utf-8');
|
|
410
|
+
expect(trace).toContain('work_unit_failed_before_patch');
|
|
411
|
+
expect(trace).toContain('patch_accepted');
|
|
412
|
+
expect(trace).not.toContain(legacySharedTraceEvent());
|
|
413
|
+
}
|
|
414
|
+
finally {
|
|
415
|
+
await rm(runtime.homeDir, { recursive: true, force: true });
|
|
416
|
+
}
|
|
417
|
+
});
|
|
418
|
+
it.each(['notion', 'lookml', 'looker', 'dbt', 'metricflow'])('routes %s direct writes through isolated child worktrees', async (sourceKey) => {
|
|
419
|
+
const runtime = await makeRealGitRuntime();
|
|
420
|
+
try {
|
|
421
|
+
const { deps, adapter } = makeDeps(runtime, sourceKey);
|
|
422
|
+
adapter.chunk.mockResolvedValue({
|
|
423
|
+
workUnits: [
|
|
424
|
+
{
|
|
425
|
+
unitKey: `${sourceKey}-wiki`,
|
|
426
|
+
rawFiles: [`${sourceKey}/page.json`],
|
|
427
|
+
peerFileIndex: [],
|
|
428
|
+
dependencyPaths: [],
|
|
429
|
+
},
|
|
430
|
+
],
|
|
431
|
+
});
|
|
432
|
+
let currentSession = null;
|
|
433
|
+
deps.toolsetFactory.createIngestWuToolset = vi.fn((toolSession) => {
|
|
434
|
+
currentSession = toolSession;
|
|
435
|
+
return { toRuntimeTools: vi.fn(() => ({})) };
|
|
436
|
+
});
|
|
437
|
+
deps.agentRunner.runLoop = vi.fn(async (params) => {
|
|
438
|
+
if (params.telemetryTags.operationName !== 'ingest-bundle-wu') {
|
|
439
|
+
return { stopReason: 'natural' };
|
|
440
|
+
}
|
|
441
|
+
expect(params.telemetryTags).toMatchObject({
|
|
442
|
+
operationName: 'ingest-bundle-wu',
|
|
443
|
+
source: sourceKey,
|
|
444
|
+
unitKey: `${sourceKey}-wiki`,
|
|
445
|
+
});
|
|
446
|
+
const root = rootOfConfig(currentSession.configService, runtime.configDir);
|
|
447
|
+
await mkdir(join(root, 'wiki/global'), { recursive: true });
|
|
448
|
+
await writeFile(join(root, 'wiki/global', `${sourceKey}-isolated.md`), `---\nsummary: ${sourceKey} isolated write\nusage_mode: auto\n---\n\nIsolated ${sourceKey} write.\n`, 'utf-8');
|
|
449
|
+
currentSession.actions.push({
|
|
450
|
+
target: 'wiki',
|
|
451
|
+
type: 'created',
|
|
452
|
+
key: `${sourceKey}-isolated`,
|
|
453
|
+
detail: `${sourceKey} isolated write`,
|
|
454
|
+
rawPaths: [`${sourceKey}/page.json`],
|
|
455
|
+
});
|
|
456
|
+
await currentSession.gitService.commitFiles([`wiki/global/${sourceKey}-isolated.md`], `${sourceKey} wiki`, 'KTX Test', 'system@ktx.local');
|
|
457
|
+
return { stopReason: 'natural' };
|
|
458
|
+
});
|
|
459
|
+
const runner = new IngestBundleRunner(deps);
|
|
460
|
+
await mockStageRawFiles(runner, runtime, [[`${sourceKey}/page.json`, 'h1']], sourceKey);
|
|
461
|
+
await expect(runner.run({
|
|
462
|
+
jobId: `job-${sourceKey}`,
|
|
463
|
+
connectionId: 'warehouse',
|
|
464
|
+
sourceKey,
|
|
465
|
+
trigger: 'upload',
|
|
466
|
+
bundleRef: { kind: 'upload', uploadId: 'upload' },
|
|
467
|
+
})).resolves.toMatchObject({
|
|
468
|
+
jobId: `job-${sourceKey}`,
|
|
469
|
+
failedWorkUnits: [],
|
|
470
|
+
workUnitCount: 1,
|
|
471
|
+
});
|
|
472
|
+
const trace = await readFile(join(runtime.configDir, '.ktx/ingest-traces', `job-${sourceKey}`, 'trace.jsonl'), 'utf-8');
|
|
473
|
+
expect(trace).toContain('isolated_diff_enabled');
|
|
474
|
+
expect(trace).toContain('work_unit_child_created');
|
|
475
|
+
expect(trace).toContain('work_unit_patch_collected');
|
|
476
|
+
expect(trace).toContain('patch_apply_started');
|
|
477
|
+
expect(trace).not.toContain(legacySharedTraceEvent());
|
|
478
|
+
const reportCreate = vi.mocked(deps.reports.create).mock.calls.at(-1)?.[0];
|
|
479
|
+
const reportBody = reportCreate?.body;
|
|
480
|
+
expect(reportBody?.isolatedDiff).toMatchObject({
|
|
481
|
+
enabled: true,
|
|
482
|
+
acceptedPatches: 1,
|
|
483
|
+
});
|
|
484
|
+
}
|
|
485
|
+
finally {
|
|
486
|
+
await rm(runtime.homeDir, { recursive: true, force: true });
|
|
487
|
+
}
|
|
488
|
+
});
|
|
489
|
+
it('rejects the Metabase stale-measure wiki body regression before squash', async () => {
|
|
490
|
+
const runtime = await makeRealGitRuntime();
|
|
491
|
+
try {
|
|
492
|
+
const { deps, adapter } = makeDeps(runtime);
|
|
493
|
+
adapter.project = vi.fn(async ({ workdir }) => {
|
|
494
|
+
await mkdir(join(workdir, 'semantic-layer/warehouse'), { recursive: true });
|
|
495
|
+
await writeFile(join(workdir, 'semantic-layer/warehouse/mart_account_segments.yaml'), 'name: mart_account_segments\ngrain: [account_id]\ncolumns: [{name: account_id, type: string}]\njoins: []\nmeasures:\n - name: total_contract_arr_cents\n expr: sum(contract_arr)\n');
|
|
496
|
+
return {
|
|
497
|
+
warnings: [],
|
|
498
|
+
errors: [],
|
|
499
|
+
touchedSources: [{ connectionId: 'warehouse', sourceName: 'mart_account_segments' }],
|
|
500
|
+
changedWikiPageKeys: [],
|
|
501
|
+
};
|
|
502
|
+
});
|
|
503
|
+
let currentSession = null;
|
|
504
|
+
deps.toolsetFactory.createIngestWuToolset = vi.fn((toolSession) => {
|
|
505
|
+
currentSession = toolSession;
|
|
506
|
+
return { toRuntimeTools: vi.fn(() => ({})) };
|
|
507
|
+
});
|
|
508
|
+
deps.agentRunner.runLoop = vi.fn(async (params) => {
|
|
509
|
+
const root = rootOfConfig(currentSession.configService, runtime.configDir);
|
|
510
|
+
if (params.telemetryTags.unitKey === 'card-wiki') {
|
|
511
|
+
await mkdir(join(root, 'wiki/global'), { recursive: true });
|
|
512
|
+
await writeFile(join(root, 'wiki/global/account-segments.md'), '---\nsummary: Account segments\nusage_mode: auto\nsl_refs:\n - mart_account_segments\n---\n\nARR is `mart_account_segments.total_contract_arr_cents`.\n');
|
|
513
|
+
currentSession.actions.push({ target: 'wiki', type: 'created', key: 'account-segments', detail: 'Account segments' });
|
|
514
|
+
await currentSession.gitService.commitFiles(['wiki/global/account-segments.md'], 'wu wiki', 'KTX Test', 'system@ktx.local');
|
|
515
|
+
}
|
|
516
|
+
if (params.telemetryTags.unitKey === 'card-source') {
|
|
517
|
+
await writeFile(join(root, 'semantic-layer/warehouse/mart_account_segments.yaml'), 'name: mart_account_segments\ngrain: [account_id]\ncolumns: [{name: account_id, type: string}]\njoins: []\nmeasures:\n - name: total_contract_arr\n expr: sum(contract_arr)\n');
|
|
518
|
+
addTouchedSlSource(currentSession.touchedSlSources, 'warehouse', 'mart_account_segments');
|
|
519
|
+
currentSession.actions.push({
|
|
520
|
+
target: 'sl',
|
|
521
|
+
type: 'updated',
|
|
522
|
+
key: 'mart_account_segments',
|
|
523
|
+
detail: 'Dollar measure',
|
|
524
|
+
targetConnectionId: 'warehouse',
|
|
525
|
+
});
|
|
526
|
+
await currentSession.gitService.commitFiles(['semantic-layer/warehouse/mart_account_segments.yaml'], 'wu source', 'KTX Test', 'system@ktx.local');
|
|
527
|
+
}
|
|
528
|
+
return { stopReason: 'natural' };
|
|
529
|
+
});
|
|
530
|
+
const runner = new IngestBundleRunner(deps);
|
|
531
|
+
await mockStageRawFiles(runner, runtime, [
|
|
532
|
+
['cards/wiki.json', 'h1'],
|
|
533
|
+
['cards/source.json', 'h2'],
|
|
534
|
+
]);
|
|
535
|
+
await expect(runner.run({ jobId: 'job-1', connectionId: 'warehouse', sourceKey: 'metabase', trigger: 'upload', bundleRef: { kind: 'upload', uploadId: 'upload' } })).rejects.toThrow(/total_contract_arr_cents/);
|
|
536
|
+
const trace = await readFile(join(runtime.configDir, '.ktx/ingest-traces/job-1/trace.jsonl'), 'utf-8');
|
|
537
|
+
expect(trace).toContain('input_snapshot');
|
|
538
|
+
expect(trace).toContain('isolated_diff_enabled');
|
|
539
|
+
expect(trace).toContain('work_unit_child_created');
|
|
540
|
+
expect(trace).toContain('work_unit_patch_collected');
|
|
541
|
+
expect(trace).toContain('patch_apply_started');
|
|
542
|
+
expect(trace).toContain('final_artifact_gates_failed');
|
|
543
|
+
expect(trace).toContain('ingest_failed');
|
|
544
|
+
}
|
|
545
|
+
finally {
|
|
546
|
+
await rm(runtime.homeDir, { recursive: true, force: true });
|
|
547
|
+
}
|
|
548
|
+
});
|
|
549
|
+
it('rejects unchanged wiki body refs made stale by isolated semantic-layer changes', async () => {
|
|
550
|
+
const runtime = await makeRealGitRuntime();
|
|
551
|
+
try {
|
|
552
|
+
await mkdir(join(runtime.configDir, 'semantic-layer/warehouse'), { recursive: true });
|
|
553
|
+
await mkdir(join(runtime.configDir, 'wiki/global'), { recursive: true });
|
|
554
|
+
await writeFile(join(runtime.configDir, 'semantic-layer/warehouse/mart_account_segments.yaml'), 'name: mart_account_segments\ngrain: [account_id]\ncolumns: [{name: account_id, type: string}]\njoins: []\nmeasures:\n - name: total_contract_arr_cents\n expr: sum(contract_arr)\n');
|
|
555
|
+
await writeFile(join(runtime.configDir, 'wiki/global/account-segments.md'), '---\nsummary: Account segments\nusage_mode: auto\n---\n\nExisting ARR uses `mart_account_segments.total_contract_arr_cents`.\n');
|
|
556
|
+
await runtime.git.commitFiles(['semantic-layer/warehouse/mart_account_segments.yaml', 'wiki/global/account-segments.md'], 'seed existing wiki body ref', 'KTX Test', 'system@ktx.local');
|
|
557
|
+
const preRunHead = await runtime.git.revParseHead();
|
|
558
|
+
const { deps, adapter } = makeDeps(runtime);
|
|
559
|
+
adapter.chunk.mockResolvedValue({
|
|
560
|
+
workUnits: [{ unitKey: 'source-only', rawFiles: ['cards/source.json'], peerFileIndex: [], dependencyPaths: [] }],
|
|
561
|
+
});
|
|
562
|
+
let currentSession = null;
|
|
563
|
+
deps.toolsetFactory.createIngestWuToolset = vi.fn((toolSession) => {
|
|
564
|
+
currentSession = toolSession;
|
|
565
|
+
return { toRuntimeTools: vi.fn(() => ({})) };
|
|
566
|
+
});
|
|
567
|
+
deps.agentRunner.runLoop = vi.fn(async () => {
|
|
568
|
+
const root = rootOfConfig(currentSession.configService, runtime.configDir);
|
|
569
|
+
await writeFile(join(root, 'semantic-layer/warehouse/mart_account_segments.yaml'), 'name: mart_account_segments\ngrain: [account_id]\ncolumns: [{name: account_id, type: string}]\njoins: []\nmeasures:\n - name: total_contract_arr\n expr: sum(contract_arr)\n');
|
|
570
|
+
addTouchedSlSource(currentSession.touchedSlSources, 'warehouse', 'mart_account_segments');
|
|
571
|
+
currentSession.actions.push({
|
|
572
|
+
target: 'sl',
|
|
573
|
+
type: 'updated',
|
|
574
|
+
key: 'mart_account_segments',
|
|
575
|
+
detail: 'Rename ARR measure',
|
|
576
|
+
targetConnectionId: 'warehouse',
|
|
577
|
+
rawPaths: ['cards/source.json'],
|
|
578
|
+
});
|
|
579
|
+
await currentSession.gitService.commitFiles(['semantic-layer/warehouse/mart_account_segments.yaml'], 'wu source rename', 'KTX Test', 'system@ktx.local');
|
|
580
|
+
return { stopReason: 'natural' };
|
|
581
|
+
});
|
|
582
|
+
const runner = new IngestBundleRunner(deps);
|
|
583
|
+
await mockStageRawFiles(runner, runtime, [['cards/source.json', 'h1']]);
|
|
584
|
+
await expect(runner.run({
|
|
585
|
+
jobId: 'job-existing-body-stale',
|
|
586
|
+
connectionId: 'warehouse',
|
|
587
|
+
sourceKey: 'metabase',
|
|
588
|
+
trigger: 'upload',
|
|
589
|
+
bundleRef: { kind: 'upload', uploadId: 'upload' },
|
|
590
|
+
})).rejects.toThrow(/total_contract_arr_cents/);
|
|
591
|
+
expect(await runtime.git.revParseHead()).toBe(preRunHead);
|
|
592
|
+
const events = (await readFile(join(runtime.configDir, '.ktx/ingest-traces/job-existing-body-stale/trace.jsonl'), 'utf-8'))
|
|
593
|
+
.trim()
|
|
594
|
+
.split('\n')
|
|
595
|
+
.map((line) => JSON.parse(line));
|
|
596
|
+
expect(events.map((event) => event.event)).toEqual(expect.arrayContaining([
|
|
597
|
+
'final_artifact_gates_started',
|
|
598
|
+
'final_artifact_gates_failed',
|
|
599
|
+
'ingest_failed',
|
|
600
|
+
'failure_report_created',
|
|
601
|
+
]));
|
|
602
|
+
expect(events.map((event) => event.event)).not.toContain('squash_finished');
|
|
603
|
+
const gateFailure = events.find((event) => event.event === 'final_artifact_gates_failed');
|
|
604
|
+
expect(gateFailure).toMatchObject({
|
|
605
|
+
data: {
|
|
606
|
+
wikiReferenceGateScope: {
|
|
607
|
+
global: true,
|
|
608
|
+
reasons: expect.arrayContaining(['semantic_layer_changed']),
|
|
609
|
+
pageKeysValidated: expect.arrayContaining(['account-segments']),
|
|
610
|
+
},
|
|
611
|
+
actionOrigins: expect.arrayContaining([
|
|
612
|
+
expect.objectContaining({
|
|
613
|
+
source: 'work_unit_action',
|
|
614
|
+
unitKey: 'source-only',
|
|
615
|
+
unitRawFiles: ['cards/source.json'],
|
|
616
|
+
action: expect.objectContaining({
|
|
617
|
+
target: 'sl',
|
|
618
|
+
type: 'updated',
|
|
619
|
+
key: 'mart_account_segments',
|
|
620
|
+
rawPaths: ['cards/source.json'],
|
|
621
|
+
targetConnectionId: 'warehouse',
|
|
622
|
+
}),
|
|
623
|
+
}),
|
|
624
|
+
]),
|
|
625
|
+
},
|
|
626
|
+
error: { message: expect.stringContaining('total_contract_arr_cents') },
|
|
627
|
+
});
|
|
628
|
+
const failureReport = deps.reports.create.mock.calls
|
|
629
|
+
.map((call) => call[0])
|
|
630
|
+
.find((report) => report.body.status === 'failed');
|
|
631
|
+
expect(failureReport.body.failure).toMatchObject({
|
|
632
|
+
phase: 'final_gates',
|
|
633
|
+
message: expect.stringContaining('total_contract_arr_cents'),
|
|
634
|
+
details: expect.objectContaining({
|
|
635
|
+
wikiReferenceGateScope: expect.objectContaining({
|
|
636
|
+
global: true,
|
|
637
|
+
reasons: expect.arrayContaining(['semantic_layer_changed']),
|
|
638
|
+
pageKeysValidated: expect.arrayContaining(['account-segments']),
|
|
639
|
+
}),
|
|
640
|
+
touchedSlSources: expect.arrayContaining([
|
|
641
|
+
expect.objectContaining({ connectionId: 'warehouse', sourceName: 'mart_account_segments' }),
|
|
642
|
+
]),
|
|
643
|
+
actionOrigins: expect.arrayContaining([
|
|
644
|
+
expect.objectContaining({
|
|
645
|
+
source: 'work_unit_action',
|
|
646
|
+
unitKey: 'source-only',
|
|
647
|
+
action: expect.objectContaining({
|
|
648
|
+
target: 'sl',
|
|
649
|
+
type: 'updated',
|
|
650
|
+
key: 'mart_account_segments',
|
|
651
|
+
rawPaths: ['cards/source.json'],
|
|
652
|
+
targetConnectionId: 'warehouse',
|
|
653
|
+
}),
|
|
654
|
+
}),
|
|
655
|
+
]),
|
|
656
|
+
}),
|
|
657
|
+
});
|
|
658
|
+
expect(failureReport.body.workUnits).toEqual(expect.arrayContaining([
|
|
659
|
+
expect.objectContaining({
|
|
660
|
+
unitKey: 'source-only',
|
|
661
|
+
actions: expect.arrayContaining([
|
|
662
|
+
expect.objectContaining({
|
|
663
|
+
target: 'sl',
|
|
664
|
+
type: 'updated',
|
|
665
|
+
key: 'mart_account_segments',
|
|
666
|
+
rawPaths: ['cards/source.json'],
|
|
667
|
+
}),
|
|
668
|
+
]),
|
|
669
|
+
}),
|
|
670
|
+
]));
|
|
671
|
+
}
|
|
672
|
+
finally {
|
|
673
|
+
await rm(runtime.homeDir, { recursive: true, force: true });
|
|
674
|
+
}
|
|
675
|
+
});
|
|
676
|
+
it('accepts two isolated work units that edit different wiki pages', async () => {
|
|
677
|
+
const runtime = await makeRealGitRuntime();
|
|
678
|
+
try {
|
|
679
|
+
const { deps, adapter } = makeDeps(runtime);
|
|
680
|
+
adapter.chunk.mockResolvedValue({
|
|
681
|
+
workUnits: [
|
|
682
|
+
{ unitKey: 'page-a', rawFiles: ['pages/a.json'], peerFileIndex: [], dependencyPaths: [] },
|
|
683
|
+
{ unitKey: 'page-b', rawFiles: ['pages/b.json'], peerFileIndex: [], dependencyPaths: [] },
|
|
684
|
+
],
|
|
685
|
+
});
|
|
686
|
+
let currentSession = null;
|
|
687
|
+
deps.toolsetFactory.createIngestWuToolset = vi.fn((toolSession) => {
|
|
688
|
+
currentSession = toolSession;
|
|
689
|
+
return { toRuntimeTools: vi.fn(() => ({})) };
|
|
690
|
+
});
|
|
691
|
+
deps.agentRunner.runLoop = vi.fn(async (params) => {
|
|
692
|
+
const unitKey = params.telemetryTags.unitKey;
|
|
693
|
+
const root = rootOfConfig(currentSession.configService, runtime.configDir);
|
|
694
|
+
await mkdir(join(root, 'wiki/global'), { recursive: true });
|
|
695
|
+
await writeFile(join(root, `wiki/global/${unitKey}.md`), `---\nsummary: ${unitKey}\nusage_mode: auto\n---\n\n${unitKey}\n`);
|
|
696
|
+
currentSession.actions.push({ target: 'wiki', type: 'created', key: unitKey, detail: unitKey });
|
|
697
|
+
await currentSession.gitService.commitFiles([`wiki/global/${unitKey}.md`], `wu ${unitKey}`, 'KTX Test', 'system@ktx.local');
|
|
698
|
+
return { stopReason: 'natural' };
|
|
699
|
+
});
|
|
700
|
+
const runner = new IngestBundleRunner(deps);
|
|
701
|
+
await mockStageRawFiles(runner, runtime, [
|
|
702
|
+
['pages/a.json', 'h1'],
|
|
703
|
+
['pages/b.json', 'h2'],
|
|
704
|
+
]);
|
|
705
|
+
const result = await runner.run({ jobId: 'job-clean', connectionId: 'warehouse', sourceKey: 'metabase', trigger: 'upload', bundleRef: { kind: 'upload', uploadId: 'upload' } });
|
|
706
|
+
expect(result.failedWorkUnits).toEqual([]);
|
|
707
|
+
const trace = await readFile(join(runtime.configDir, '.ktx/ingest-traces/job-clean/trace.jsonl'), 'utf-8');
|
|
708
|
+
expect(trace.match(/patch_accepted/g)).toHaveLength(2);
|
|
709
|
+
expect(trace).toContain('ingest_finished');
|
|
710
|
+
}
|
|
711
|
+
finally {
|
|
712
|
+
await rm(runtime.homeDir, { recursive: true, force: true });
|
|
713
|
+
}
|
|
714
|
+
});
|
|
715
|
+
it('classifies same-source patch application failure as a textual conflict', async () => {
|
|
716
|
+
const runtime = await makeRealGitRuntime();
|
|
717
|
+
try {
|
|
718
|
+
const { deps, adapter } = makeDeps(runtime);
|
|
719
|
+
adapter.chunk.mockResolvedValue({
|
|
720
|
+
workUnits: [
|
|
721
|
+
{ unitKey: 'orders-a', rawFiles: ['orders/a.json'], peerFileIndex: [], dependencyPaths: [] },
|
|
722
|
+
{ unitKey: 'orders-b', rawFiles: ['orders/b.json'], peerFileIndex: [], dependencyPaths: [] },
|
|
723
|
+
],
|
|
724
|
+
});
|
|
725
|
+
let currentSession = null;
|
|
726
|
+
deps.toolsetFactory.createIngestWuToolset = vi.fn((toolSession) => {
|
|
727
|
+
currentSession = toolSession;
|
|
728
|
+
return { toRuntimeTools: vi.fn(() => ({})) };
|
|
729
|
+
});
|
|
730
|
+
deps.agentRunner.runLoop = vi.fn(async (params) => {
|
|
731
|
+
if (params.telemetryTags.operationName === 'ingest-isolated-diff-textual-resolver') {
|
|
732
|
+
return { stopReason: 'natural' };
|
|
733
|
+
}
|
|
734
|
+
const suffix = params.telemetryTags.unitKey === 'orders-a' ? 'a' : 'b';
|
|
735
|
+
const root = rootOfConfig(currentSession.configService, runtime.configDir);
|
|
736
|
+
await mkdir(join(root, 'semantic-layer/warehouse'), { recursive: true });
|
|
737
|
+
await writeFile(join(root, 'semantic-layer/warehouse/orders.yaml'), `name: orders\ngrain: [id]\ncolumns: [{name: id, type: string}]\njoins: []\nmeasures:\n - name: order_count_${suffix}\n expr: count(*)\n`);
|
|
738
|
+
addTouchedSlSource(currentSession.touchedSlSources, 'warehouse', 'orders');
|
|
739
|
+
currentSession.actions.push({ target: 'sl', type: 'updated', key: 'orders', detail: suffix, targetConnectionId: 'warehouse' });
|
|
740
|
+
await currentSession.gitService.commitFiles(['semantic-layer/warehouse/orders.yaml'], `wu ${suffix}`, 'KTX Test', 'system@ktx.local');
|
|
741
|
+
return { stopReason: 'natural' };
|
|
742
|
+
});
|
|
743
|
+
const runner = new IngestBundleRunner(deps);
|
|
744
|
+
await mockStageRawFiles(runner, runtime, [
|
|
745
|
+
['orders/a.json', 'h1'],
|
|
746
|
+
['orders/b.json', 'h2'],
|
|
747
|
+
]);
|
|
748
|
+
await expect(runner.run({ jobId: 'job-text-conflict', connectionId: 'warehouse', sourceKey: 'metabase', trigger: 'upload', bundleRef: { kind: 'upload', uploadId: 'upload' } })).rejects.toThrow(/isolated diff textual conflict/);
|
|
749
|
+
const trace = await readFile(join(runtime.configDir, '.ktx/ingest-traces/job-text-conflict/trace.jsonl'), 'utf-8');
|
|
750
|
+
expect(trace).toContain('patch_textual_conflict');
|
|
751
|
+
expect(trace).toContain('textual_conflict_resolver_failed');
|
|
752
|
+
}
|
|
753
|
+
finally {
|
|
754
|
+
await rm(runtime.homeDir, { recursive: true, force: true });
|
|
755
|
+
}
|
|
756
|
+
});
|
|
757
|
+
it('makes deterministic projection visible to child worktrees before WorkUnit synthesis', async () => {
|
|
758
|
+
const runtime = await makeRealGitRuntime();
|
|
759
|
+
try {
|
|
760
|
+
const { deps, adapter } = makeDeps(runtime);
|
|
761
|
+
adapter.chunk.mockResolvedValue({
|
|
762
|
+
workUnits: [{ unitKey: 'wiki-projected', rawFiles: ['projected/wiki.json'], peerFileIndex: [], dependencyPaths: [] }],
|
|
763
|
+
});
|
|
764
|
+
adapter.project = vi.fn(async ({ workdir }) => {
|
|
765
|
+
await mkdir(join(workdir, 'semantic-layer/warehouse'), { recursive: true });
|
|
766
|
+
await writeFile(join(workdir, 'semantic-layer/warehouse/mart_account_segments.yaml'), 'name: mart_account_segments\ngrain: [account_id]\ncolumns: [{name: account_id, type: string}]\njoins: []\nmeasures:\n - name: total_contract_arr\n expr: sum(contract_arr)\n');
|
|
767
|
+
return {
|
|
768
|
+
warnings: [],
|
|
769
|
+
errors: [],
|
|
770
|
+
touchedSources: [{ connectionId: 'warehouse', sourceName: 'mart_account_segments' }],
|
|
771
|
+
changedWikiPageKeys: [],
|
|
772
|
+
};
|
|
773
|
+
});
|
|
774
|
+
let currentSession = null;
|
|
775
|
+
deps.toolsetFactory.createIngestWuToolset = vi.fn((toolSession) => {
|
|
776
|
+
currentSession = toolSession;
|
|
777
|
+
return { toRuntimeTools: vi.fn(() => ({})) };
|
|
778
|
+
});
|
|
779
|
+
deps.agentRunner.runLoop = vi.fn(async () => {
|
|
780
|
+
const root = rootOfConfig(currentSession.configService, runtime.configDir);
|
|
781
|
+
await expect(readFile(join(root, 'semantic-layer/warehouse/mart_account_segments.yaml'), 'utf-8')).resolves.toContain('total_contract_arr');
|
|
782
|
+
await mkdir(join(root, 'wiki/global'), { recursive: true });
|
|
783
|
+
await writeFile(join(root, 'wiki/global/projected-orders.md'), '---\nsummary: Projected orders\nusage_mode: auto\nsl_refs:\n - mart_account_segments\n---\n\nARR `mart_account_segments.total_contract_arr`.\n');
|
|
784
|
+
currentSession.actions.push({ target: 'wiki', type: 'created', key: 'projected-orders', detail: 'Projected orders' });
|
|
785
|
+
await currentSession.gitService.commitFiles(['wiki/global/projected-orders.md'], 'wu projected wiki', 'KTX Test', 'system@ktx.local');
|
|
786
|
+
return { stopReason: 'natural' };
|
|
787
|
+
});
|
|
788
|
+
const runner = new IngestBundleRunner(deps);
|
|
789
|
+
await mockStageRawFiles(runner, runtime, [['projected/wiki.json', 'h1']]);
|
|
790
|
+
const result = await runner.run({ jobId: 'job-projection', connectionId: 'warehouse', sourceKey: 'metabase', trigger: 'upload', bundleRef: { kind: 'upload', uploadId: 'upload' } });
|
|
791
|
+
expect(result.failedWorkUnits).toEqual([]);
|
|
792
|
+
const trace = await readFile(join(runtime.configDir, '.ktx/ingest-traces/job-projection/trace.jsonl'), 'utf-8');
|
|
793
|
+
expect(trace).toContain('deterministic_projection_finished');
|
|
794
|
+
expect(trace).toContain('deterministic_projection_committed');
|
|
795
|
+
}
|
|
796
|
+
finally {
|
|
797
|
+
await rm(runtime.homeDir, { recursive: true, force: true });
|
|
798
|
+
}
|
|
799
|
+
});
|
|
800
|
+
it('rejects Notion-style changed wiki pages with invalid sl_refs', async () => {
|
|
801
|
+
const runtime = await makeRealGitRuntime();
|
|
802
|
+
try {
|
|
803
|
+
const { deps, adapter } = makeDeps(runtime);
|
|
804
|
+
adapter.chunk.mockResolvedValue({
|
|
805
|
+
workUnits: [{ unitKey: 'notion-page', rawFiles: ['pages/notion.json'], peerFileIndex: [], dependencyPaths: [] }],
|
|
806
|
+
});
|
|
807
|
+
let currentSession = null;
|
|
808
|
+
deps.toolsetFactory.createIngestWuToolset = vi.fn((toolSession) => {
|
|
809
|
+
currentSession = toolSession;
|
|
810
|
+
return { toRuntimeTools: vi.fn(() => ({})) };
|
|
811
|
+
});
|
|
812
|
+
deps.agentRunner.runLoop = vi.fn(async (params) => {
|
|
813
|
+
if (params.telemetryTags.operationName === 'ingest-isolated-diff-gate-repair') {
|
|
814
|
+
return { stopReason: 'natural' };
|
|
815
|
+
}
|
|
816
|
+
const root = rootOfConfig(currentSession.configService, runtime.configDir);
|
|
817
|
+
await mkdir(join(root, 'wiki/global'), { recursive: true });
|
|
818
|
+
await writeFile(join(root, 'wiki/global/notion-page.md'), '---\nsummary: Notion page\nusage_mode: auto\nsl_refs:\n - missing_source\n---\n\nBody\n');
|
|
819
|
+
currentSession.actions.push({ target: 'wiki', type: 'created', key: 'notion-page', detail: 'Notion page' });
|
|
820
|
+
await currentSession.gitService.commitFiles(['wiki/global/notion-page.md'], 'wu notion', 'KTX Test', 'system@ktx.local');
|
|
821
|
+
return { stopReason: 'natural' };
|
|
822
|
+
});
|
|
823
|
+
const runner = new IngestBundleRunner(deps);
|
|
824
|
+
await mockStageRawFiles(runner, runtime, [['pages/notion.json', 'h1']]);
|
|
825
|
+
await expect(runner.run({ jobId: 'job-invalid-slrefs', connectionId: 'warehouse', sourceKey: 'metabase', trigger: 'upload', bundleRef: { kind: 'upload', uploadId: 'upload' } })).rejects.toThrow(/gate repair completed without editing an allowed path/);
|
|
826
|
+
}
|
|
827
|
+
finally {
|
|
828
|
+
await rm(runtime.homeDir, { recursive: true, force: true });
|
|
829
|
+
}
|
|
830
|
+
});
|
|
831
|
+
it('runs final artifact gates after reconciliation mutates the integration tree', async () => {
|
|
832
|
+
const runtime = await makeRealGitRuntime();
|
|
833
|
+
try {
|
|
834
|
+
const { deps, adapter } = makeDeps(runtime);
|
|
835
|
+
adapter.chunk.mockResolvedValue({
|
|
836
|
+
workUnits: [{ unitKey: 'card-source', rawFiles: ['cards/source.json'], peerFileIndex: [], dependencyPaths: [] }],
|
|
837
|
+
});
|
|
838
|
+
let currentSession = null;
|
|
839
|
+
deps.toolsetFactory.createIngestWuToolset = vi.fn((toolSession) => {
|
|
840
|
+
currentSession = toolSession;
|
|
841
|
+
return { toRuntimeTools: vi.fn(() => ({})) };
|
|
842
|
+
});
|
|
843
|
+
deps.agentRunner.runLoop = vi.fn(async (params) => {
|
|
844
|
+
const root = rootOfConfig(currentSession.configService, runtime.configDir);
|
|
845
|
+
if (params.telemetryTags.operationName === 'ingest-bundle-wu') {
|
|
846
|
+
await mkdir(join(root, 'semantic-layer/warehouse'), { recursive: true });
|
|
847
|
+
await writeFile(join(root, 'semantic-layer/warehouse/mart_account_segments.yaml'), 'name: mart_account_segments\ngrain: [account_id]\ncolumns: [{name: account_id, type: string}]\njoins: []\nmeasures:\n - name: total_contract_arr\n expr: sum(contract_arr)\n');
|
|
848
|
+
addTouchedSlSource(currentSession.touchedSlSources, 'warehouse', 'mart_account_segments');
|
|
849
|
+
currentSession.actions.push({
|
|
850
|
+
target: 'sl',
|
|
851
|
+
type: 'created',
|
|
852
|
+
key: 'mart_account_segments',
|
|
853
|
+
detail: 'Source with renamed ARR measure',
|
|
854
|
+
targetConnectionId: 'warehouse',
|
|
855
|
+
rawPaths: ['cards/source.json'],
|
|
856
|
+
});
|
|
857
|
+
await currentSession.gitService.commitFiles(['semantic-layer/warehouse/mart_account_segments.yaml'], 'wu source', 'KTX Test', 'system@ktx.local');
|
|
858
|
+
}
|
|
859
|
+
else {
|
|
860
|
+
await mkdir(join(root, 'wiki/global'), { recursive: true });
|
|
861
|
+
await writeFile(join(root, 'wiki/global/account-segments.md'), '---\nsummary: Account segments\nusage_mode: auto\nsl_refs:\n - mart_account_segments\n---\n\nReconcile wrote stale ARR `mart_account_segments.total_contract_arr_cents`.\n');
|
|
862
|
+
currentSession.actions.push({
|
|
863
|
+
target: 'wiki',
|
|
864
|
+
type: 'created',
|
|
865
|
+
key: 'account-segments',
|
|
866
|
+
detail: 'Stale reconcile wiki page',
|
|
867
|
+
rawPaths: ['cards/source.json'],
|
|
868
|
+
});
|
|
869
|
+
await currentSession.gitService.commitFiles(['wiki/global/account-segments.md'], 'reconcile wiki', 'KTX Test', 'system@ktx.local');
|
|
870
|
+
}
|
|
871
|
+
return { stopReason: 'natural' };
|
|
872
|
+
});
|
|
873
|
+
const runner = new IngestBundleRunner(deps);
|
|
874
|
+
await mockStageRawFiles(runner, runtime, [['cards/source.json', 'h1']]);
|
|
875
|
+
await expect(runner.run({
|
|
876
|
+
jobId: 'job-reconcile-stale',
|
|
877
|
+
connectionId: 'warehouse',
|
|
878
|
+
sourceKey: 'metabase',
|
|
879
|
+
trigger: 'upload',
|
|
880
|
+
bundleRef: { kind: 'upload', uploadId: 'upload' },
|
|
881
|
+
})).rejects.toThrow(/total_contract_arr_cents/);
|
|
882
|
+
const trace = await readFile(join(runtime.configDir, '.ktx/ingest-traces/job-reconcile-stale/trace.jsonl'), 'utf-8');
|
|
883
|
+
expect(trace).toContain('reconciliation_finished');
|
|
884
|
+
expect(trace).toContain('final_artifact_gates_failed');
|
|
885
|
+
expect(trace).toContain('ingest_failed');
|
|
886
|
+
expect(await runtime.git.revParseHead()).not.toContain('reconcile wiki');
|
|
887
|
+
}
|
|
888
|
+
finally {
|
|
889
|
+
await rm(runtime.homeDir, { recursive: true, force: true });
|
|
890
|
+
}
|
|
891
|
+
});
|
|
892
|
+
it('stores a failure report and postmortem trace for final gate failures', async () => {
|
|
893
|
+
const runtime = await makeRealGitRuntime();
|
|
894
|
+
try {
|
|
895
|
+
const { deps, adapter } = makeDeps(runtime);
|
|
896
|
+
const createdReports = [];
|
|
897
|
+
deps.reports.create = vi.fn(async (args) => {
|
|
898
|
+
createdReports.push(args);
|
|
899
|
+
return { id: `report-${createdReports.length}` };
|
|
900
|
+
});
|
|
901
|
+
adapter.chunk.mockResolvedValue({
|
|
902
|
+
workUnits: [
|
|
903
|
+
{ unitKey: 'card-wiki', rawFiles: ['cards/wiki.json'], peerFileIndex: [], dependencyPaths: [] },
|
|
904
|
+
{ unitKey: 'card-source', rawFiles: ['cards/source.json'], peerFileIndex: [], dependencyPaths: [] },
|
|
905
|
+
],
|
|
906
|
+
});
|
|
907
|
+
let currentSession = null;
|
|
908
|
+
deps.toolsetFactory.createIngestWuToolset = vi.fn((toolSession) => {
|
|
909
|
+
currentSession = toolSession;
|
|
910
|
+
return { toRuntimeTools: vi.fn(() => ({})) };
|
|
911
|
+
});
|
|
912
|
+
deps.agentRunner.runLoop = vi.fn(async (params) => {
|
|
913
|
+
const root = rootOfConfig(currentSession.configService, runtime.configDir);
|
|
914
|
+
if (params.telemetryTags.unitKey === 'card-wiki') {
|
|
915
|
+
await mkdir(join(root, 'wiki/global'), { recursive: true });
|
|
916
|
+
await writeFile(join(root, 'wiki/global/account-segments.md'), '---\nsummary: Account segments\nusage_mode: auto\n---\n\nARR is `mart_account_segments.total_contract_arr_cents`.\n');
|
|
917
|
+
currentSession.actions.push({
|
|
918
|
+
target: 'wiki',
|
|
919
|
+
type: 'created',
|
|
920
|
+
key: 'account-segments',
|
|
921
|
+
detail: 'Account segments',
|
|
922
|
+
rawPaths: ['cards/wiki.json'],
|
|
923
|
+
});
|
|
924
|
+
await currentSession.gitService.commitFiles(['wiki/global/account-segments.md'], 'wu wiki', 'KTX Test', 'system@ktx.local');
|
|
925
|
+
}
|
|
926
|
+
if (params.telemetryTags.unitKey === 'card-source') {
|
|
927
|
+
await mkdir(join(root, 'semantic-layer/warehouse'), { recursive: true });
|
|
928
|
+
await writeFile(join(root, 'semantic-layer/warehouse/mart_account_segments.yaml'), 'name: mart_account_segments\ngrain: [account_id]\ncolumns: [{name: account_id, type: string}]\njoins: []\nmeasures:\n - name: total_contract_arr\n expr: sum(contract_arr)\n');
|
|
929
|
+
addTouchedSlSource(currentSession.touchedSlSources, 'warehouse', 'mart_account_segments');
|
|
930
|
+
currentSession.actions.push({
|
|
931
|
+
target: 'sl',
|
|
932
|
+
type: 'created',
|
|
933
|
+
key: 'mart_account_segments',
|
|
934
|
+
detail: 'Dollar measure',
|
|
935
|
+
targetConnectionId: 'warehouse',
|
|
936
|
+
rawPaths: ['cards/source.json'],
|
|
937
|
+
});
|
|
938
|
+
await currentSession.gitService.commitFiles(['semantic-layer/warehouse/mart_account_segments.yaml'], 'wu source', 'KTX Test', 'system@ktx.local');
|
|
939
|
+
}
|
|
940
|
+
return { stopReason: 'natural' };
|
|
941
|
+
});
|
|
942
|
+
const runner = new IngestBundleRunner(deps);
|
|
943
|
+
await mockStageRawFiles(runner, runtime, [
|
|
944
|
+
['cards/wiki.json', 'h1'],
|
|
945
|
+
['cards/source.json', 'h2'],
|
|
946
|
+
]);
|
|
947
|
+
await expect(runner.run({
|
|
948
|
+
jobId: 'job-trace-failure',
|
|
949
|
+
connectionId: 'warehouse',
|
|
950
|
+
sourceKey: 'metabase',
|
|
951
|
+
trigger: 'upload',
|
|
952
|
+
bundleRef: { kind: 'upload', uploadId: 'upload' },
|
|
953
|
+
})).rejects.toThrow(/total_contract_arr_cents/);
|
|
954
|
+
const failureReport = createdReports.find((report) => report.body.status === 'failed');
|
|
955
|
+
expect(failureReport.body.tracePath).toContain('job-trace-failure/trace.jsonl');
|
|
956
|
+
expect(failureReport.body.failure).toMatchObject({ phase: 'final_gates' });
|
|
957
|
+
const events = (await readFile(join(runtime.configDir, '.ktx/ingest-traces/job-trace-failure/trace.jsonl'), 'utf-8'))
|
|
958
|
+
.trim()
|
|
959
|
+
.split('\n')
|
|
960
|
+
.map((line) => JSON.parse(line));
|
|
961
|
+
expect(events.map((event) => event.event)).toEqual(expect.arrayContaining([
|
|
962
|
+
'ingest_started',
|
|
963
|
+
'input_snapshot',
|
|
964
|
+
'work_units_planned',
|
|
965
|
+
'isolated_diff_enabled',
|
|
966
|
+
'work_unit_child_created',
|
|
967
|
+
'work_unit_patch_collected',
|
|
968
|
+
'patch_apply_started',
|
|
969
|
+
'patch_accepted',
|
|
970
|
+
'reconciliation_finished',
|
|
971
|
+
'final_artifact_gates_failed',
|
|
972
|
+
'ingest_failed',
|
|
973
|
+
'failure_report_created',
|
|
974
|
+
]));
|
|
975
|
+
const failed = events.find((event) => event.event === 'ingest_failed');
|
|
976
|
+
expect(failed).toMatchObject({
|
|
977
|
+
runId: 'run-1',
|
|
978
|
+
syncId: expect.any(String),
|
|
979
|
+
data: { phase: 'final_gates', tracePath: expect.stringContaining('trace.jsonl') },
|
|
980
|
+
error: { message: expect.stringContaining('total_contract_arr_cents') },
|
|
981
|
+
});
|
|
982
|
+
}
|
|
983
|
+
finally {
|
|
984
|
+
await rm(runtime.homeDir, { recursive: true, force: true });
|
|
985
|
+
}
|
|
986
|
+
});
|
|
987
|
+
it('rejects invalid provenance raw paths before squash reaches main', async () => {
|
|
988
|
+
const runtime = await makeRealGitRuntime();
|
|
989
|
+
try {
|
|
990
|
+
const { deps, adapter } = makeDeps(runtime);
|
|
991
|
+
const createdReports = [];
|
|
992
|
+
deps.reports.create = vi.fn(async (args) => {
|
|
993
|
+
createdReports.push(args);
|
|
994
|
+
return { id: `report-${createdReports.length}` };
|
|
995
|
+
});
|
|
996
|
+
adapter.chunk.mockResolvedValue({
|
|
997
|
+
workUnits: [
|
|
998
|
+
{
|
|
999
|
+
unitKey: 'card-valid-artifacts',
|
|
1000
|
+
rawFiles: ['cards/source.json'],
|
|
1001
|
+
peerFileIndex: [],
|
|
1002
|
+
dependencyPaths: [],
|
|
1003
|
+
},
|
|
1004
|
+
],
|
|
1005
|
+
});
|
|
1006
|
+
let currentSession = null;
|
|
1007
|
+
deps.toolsetFactory.createIngestWuToolset = vi.fn((toolSession) => {
|
|
1008
|
+
currentSession = toolSession;
|
|
1009
|
+
return { toRuntimeTools: vi.fn(() => ({})) };
|
|
1010
|
+
});
|
|
1011
|
+
deps.agentRunner.runLoop = vi.fn(async () => {
|
|
1012
|
+
const root = rootOfConfig(currentSession.configService, runtime.configDir);
|
|
1013
|
+
await mkdir(join(root, 'semantic-layer/warehouse'), { recursive: true });
|
|
1014
|
+
await mkdir(join(root, 'wiki/global'), { recursive: true });
|
|
1015
|
+
await writeFile(join(root, 'semantic-layer/warehouse/mart_account_segments.yaml'), 'name: mart_account_segments\ngrain: [account_id]\ncolumns: [{name: account_id, type: string}]\njoins: []\nmeasures:\n - name: total_contract_arr\n expr: sum(contract_arr)\n');
|
|
1016
|
+
await writeFile(join(root, 'wiki/global/account-segments.md'), '---\nsummary: Account segments\nusage_mode: auto\nsl_refs:\n - mart_account_segments\n---\n\nARR is `mart_account_segments.total_contract_arr`.\n');
|
|
1017
|
+
addTouchedSlSource(currentSession.touchedSlSources, 'warehouse', 'mart_account_segments');
|
|
1018
|
+
currentSession.actions.push({
|
|
1019
|
+
target: 'sl',
|
|
1020
|
+
type: 'created',
|
|
1021
|
+
key: 'mart_account_segments',
|
|
1022
|
+
detail: 'Valid source',
|
|
1023
|
+
targetConnectionId: 'warehouse',
|
|
1024
|
+
rawPaths: ['cards/source.json'],
|
|
1025
|
+
});
|
|
1026
|
+
currentSession.actions.push({
|
|
1027
|
+
target: 'wiki',
|
|
1028
|
+
type: 'created',
|
|
1029
|
+
key: 'account-segments',
|
|
1030
|
+
detail: 'Valid wiki with invalid provenance raw path',
|
|
1031
|
+
rawPaths: ['cards/missing.json'],
|
|
1032
|
+
});
|
|
1033
|
+
await currentSession.gitService.commitFiles(['semantic-layer/warehouse/mart_account_segments.yaml', 'wiki/global/account-segments.md'], 'valid artifacts with invalid provenance', 'KTX Test', 'system@ktx.local');
|
|
1034
|
+
return { stopReason: 'natural' };
|
|
1035
|
+
});
|
|
1036
|
+
const runner = new IngestBundleRunner(deps);
|
|
1037
|
+
await mockStageRawFiles(runner, runtime, [['cards/source.json', 'h1']]);
|
|
1038
|
+
const preRunHead = await runtime.git.revParseHead();
|
|
1039
|
+
await expect(runner.run({
|
|
1040
|
+
jobId: 'job-invalid-provenance',
|
|
1041
|
+
connectionId: 'warehouse',
|
|
1042
|
+
sourceKey: 'metabase',
|
|
1043
|
+
trigger: 'upload',
|
|
1044
|
+
bundleRef: { kind: 'upload', uploadId: 'upload' },
|
|
1045
|
+
})).rejects.toThrow(/provenance row references raw path outside this snapshot: cards\/missing\.json/);
|
|
1046
|
+
expect(await runtime.git.revParseHead()).toBe(preRunHead);
|
|
1047
|
+
expect(deps.provenance.insertMany).not.toHaveBeenCalled();
|
|
1048
|
+
const failureReport = createdReports.find((report) => report.body.status === 'failed');
|
|
1049
|
+
expect(failureReport.body.tracePath).toContain('job-invalid-provenance/trace.jsonl');
|
|
1050
|
+
expect(failureReport.body.failure).toMatchObject({
|
|
1051
|
+
phase: 'provenance_validation',
|
|
1052
|
+
message: expect.stringContaining('cards/missing.json'),
|
|
1053
|
+
});
|
|
1054
|
+
expect(failureReport.body.failure.details).toMatchObject({
|
|
1055
|
+
invalidRawPaths: ['cards/missing.json'],
|
|
1056
|
+
currentRawPaths: ['cards/source.json'],
|
|
1057
|
+
invalidRows: expect.arrayContaining([
|
|
1058
|
+
expect.objectContaining({
|
|
1059
|
+
row: expect.objectContaining({
|
|
1060
|
+
rawPath: 'cards/missing.json',
|
|
1061
|
+
artifactKind: 'wiki',
|
|
1062
|
+
artifactKey: 'account-segments',
|
|
1063
|
+
actionType: 'wiki_written',
|
|
1064
|
+
}),
|
|
1065
|
+
origin: expect.objectContaining({
|
|
1066
|
+
source: 'work_unit_action',
|
|
1067
|
+
unitKey: 'card-valid-artifacts',
|
|
1068
|
+
actionIndex: 1,
|
|
1069
|
+
unitRawFiles: ['cards/source.json'],
|
|
1070
|
+
action: expect.objectContaining({
|
|
1071
|
+
target: 'wiki',
|
|
1072
|
+
type: 'created',
|
|
1073
|
+
key: 'account-segments',
|
|
1074
|
+
rawPaths: ['cards/missing.json'],
|
|
1075
|
+
}),
|
|
1076
|
+
}),
|
|
1077
|
+
}),
|
|
1078
|
+
]),
|
|
1079
|
+
});
|
|
1080
|
+
expect(failureReport.body.provenanceRows).toEqual(expect.arrayContaining([
|
|
1081
|
+
expect.objectContaining({ rawPath: 'cards/source.json', artifactKind: 'sl', artifactKey: 'mart_account_segments' }),
|
|
1082
|
+
expect.objectContaining({ rawPath: 'cards/missing.json', artifactKind: 'wiki', artifactKey: 'account-segments' }),
|
|
1083
|
+
]));
|
|
1084
|
+
expect(failureReport.body.workUnits).toEqual(expect.arrayContaining([
|
|
1085
|
+
expect.objectContaining({
|
|
1086
|
+
unitKey: 'card-valid-artifacts',
|
|
1087
|
+
rawFiles: ['cards/source.json'],
|
|
1088
|
+
actions: expect.arrayContaining([
|
|
1089
|
+
expect.objectContaining({
|
|
1090
|
+
target: 'wiki',
|
|
1091
|
+
key: 'account-segments',
|
|
1092
|
+
rawPaths: ['cards/missing.json'],
|
|
1093
|
+
}),
|
|
1094
|
+
]),
|
|
1095
|
+
}),
|
|
1096
|
+
]));
|
|
1097
|
+
const events = (await readFile(join(runtime.configDir, '.ktx/ingest-traces/job-invalid-provenance/trace.jsonl'), 'utf-8'))
|
|
1098
|
+
.trim()
|
|
1099
|
+
.split('\n')
|
|
1100
|
+
.map((line) => JSON.parse(line));
|
|
1101
|
+
expect(events.map((event) => event.event)).toEqual(expect.arrayContaining([
|
|
1102
|
+
'final_artifact_gates_finished',
|
|
1103
|
+
'provenance_rows_validation_failed',
|
|
1104
|
+
'ingest_failed',
|
|
1105
|
+
'failure_report_created',
|
|
1106
|
+
]));
|
|
1107
|
+
expect(events.map((event) => event.event)).not.toContain('squash_finished');
|
|
1108
|
+
const validationFailure = events.find((event) => event.event === 'provenance_rows_validation_failed');
|
|
1109
|
+
expect(validationFailure).toMatchObject({
|
|
1110
|
+
phase: 'provenance',
|
|
1111
|
+
data: {
|
|
1112
|
+
invalidRawPaths: ['cards/missing.json'],
|
|
1113
|
+
currentRawPaths: ['cards/source.json'],
|
|
1114
|
+
invalidRows: expect.arrayContaining([
|
|
1115
|
+
expect.objectContaining({
|
|
1116
|
+
row: expect.objectContaining({ rawPath: 'cards/missing.json' }),
|
|
1117
|
+
origin: expect.objectContaining({
|
|
1118
|
+
source: 'work_unit_action',
|
|
1119
|
+
unitKey: 'card-valid-artifacts',
|
|
1120
|
+
actionIndex: 1,
|
|
1121
|
+
}),
|
|
1122
|
+
}),
|
|
1123
|
+
]),
|
|
1124
|
+
},
|
|
1125
|
+
});
|
|
1126
|
+
}
|
|
1127
|
+
finally {
|
|
1128
|
+
await rm(runtime.homeDir, { recursive: true, force: true });
|
|
1129
|
+
}
|
|
1130
|
+
});
|
|
1131
|
+
it('rejects slDisallowed patches that touch semantic-layer files', async () => {
|
|
1132
|
+
const runtime = await makeRealGitRuntime();
|
|
1133
|
+
try {
|
|
1134
|
+
const { deps, adapter } = makeDeps(runtime);
|
|
1135
|
+
adapter.chunk.mockResolvedValue({
|
|
1136
|
+
workUnits: [
|
|
1137
|
+
{
|
|
1138
|
+
unitKey: 'lookml-mismatch',
|
|
1139
|
+
rawFiles: ['views/orders.lkml'],
|
|
1140
|
+
peerFileIndex: [],
|
|
1141
|
+
dependencyPaths: [],
|
|
1142
|
+
slDisallowed: true,
|
|
1143
|
+
slDisallowedReason: 'lookml_connection_mismatch',
|
|
1144
|
+
},
|
|
1145
|
+
],
|
|
1146
|
+
});
|
|
1147
|
+
let currentSession = null;
|
|
1148
|
+
deps.toolsetFactory.createIngestWuToolset = vi.fn((toolSession) => {
|
|
1149
|
+
currentSession = toolSession;
|
|
1150
|
+
return { toRuntimeTools: vi.fn(() => ({})) };
|
|
1151
|
+
});
|
|
1152
|
+
deps.agentRunner.runLoop = vi.fn(async () => {
|
|
1153
|
+
const root = rootOfConfig(currentSession.configService, runtime.configDir);
|
|
1154
|
+
await mkdir(join(root, 'semantic-layer/warehouse'), { recursive: true });
|
|
1155
|
+
await writeFile(join(root, 'semantic-layer/warehouse/orders.yaml'), 'name: orders\ngrain: [id]\ncolumns: [{name: id, type: string}]\njoins: []\nmeasures: []\n');
|
|
1156
|
+
currentSession.actions.push({ target: 'sl', type: 'created', key: 'orders', detail: 'forbidden', targetConnectionId: 'warehouse' });
|
|
1157
|
+
await currentSession.gitService.commitFiles(['semantic-layer/warehouse/orders.yaml'], 'forbidden sl', 'KTX Test', 'system@ktx.local');
|
|
1158
|
+
return { stopReason: 'natural' };
|
|
1159
|
+
});
|
|
1160
|
+
const runner = new IngestBundleRunner(deps);
|
|
1161
|
+
await mockStageRawFiles(runner, runtime, [['views/orders.lkml', 'h1']]);
|
|
1162
|
+
await expect(runner.run({ jobId: 'job-sl-disallowed', connectionId: 'warehouse', sourceKey: 'metabase', trigger: 'upload', bundleRef: { kind: 'upload', uploadId: 'upload' } })).rejects.toThrow(/isolated diff textual conflict/);
|
|
1163
|
+
const trace = await readFile(join(runtime.configDir, '.ktx/ingest-traces/job-sl-disallowed/trace.jsonl'), 'utf-8');
|
|
1164
|
+
expect(trace).toContain('patch_policy_rejected');
|
|
1165
|
+
expect(trace).toContain('slDisallowed WorkUnit lookml-mismatch touched semantic-layer/warehouse/orders.yaml');
|
|
1166
|
+
}
|
|
1167
|
+
finally {
|
|
1168
|
+
await rm(runtime.homeDir, { recursive: true, force: true });
|
|
1169
|
+
}
|
|
1170
|
+
});
|
|
1171
|
+
it('rejects final wiki refs broken by another accepted WorkUnit before squash', async () => {
|
|
1172
|
+
const runtime = await makeRealGitRuntime();
|
|
1173
|
+
try {
|
|
1174
|
+
await mkdir(join(runtime.configDir, 'wiki/global'), { recursive: true });
|
|
1175
|
+
await writeFile(join(runtime.configDir, 'wiki/global/source-page.md'), '---\nsummary: Source page\nusage_mode: auto\n---\n\nSource page\n');
|
|
1176
|
+
await runtime.git.commitFiles(['wiki/global/source-page.md'], 'seed source page', 'KTX Test', 'system@ktx.local');
|
|
1177
|
+
const preRunHead = await runtime.git.revParseHead();
|
|
1178
|
+
const { deps, adapter } = makeDeps(runtime);
|
|
1179
|
+
adapter.chunk.mockResolvedValue({
|
|
1180
|
+
workUnits: [
|
|
1181
|
+
{ unitKey: 'page-ref', rawFiles: ['pages/ref.json'], peerFileIndex: [], dependencyPaths: [] },
|
|
1182
|
+
{ unitKey: 'page-delete', rawFiles: ['pages/delete.json'], peerFileIndex: [], dependencyPaths: [] },
|
|
1183
|
+
],
|
|
1184
|
+
});
|
|
1185
|
+
let currentSession = null;
|
|
1186
|
+
deps.toolsetFactory.createIngestWuToolset = vi.fn((toolSession) => {
|
|
1187
|
+
currentSession = toolSession;
|
|
1188
|
+
return { toRuntimeTools: vi.fn(() => ({})) };
|
|
1189
|
+
});
|
|
1190
|
+
deps.agentRunner.runLoop = vi.fn(async (params) => {
|
|
1191
|
+
const root = rootOfConfig(currentSession.configService, runtime.configDir);
|
|
1192
|
+
if (params.telemetryTags.unitKey === 'page-ref') {
|
|
1193
|
+
await mkdir(join(root, 'wiki/global'), { recursive: true });
|
|
1194
|
+
await writeFile(join(root, 'wiki/global/account-segments.md'), '---\nsummary: Account segments\nusage_mode: auto\nrefs:\n - source-page\n---\n\nSee [[source-page]].\n');
|
|
1195
|
+
currentSession.actions.push({
|
|
1196
|
+
target: 'wiki',
|
|
1197
|
+
type: 'created',
|
|
1198
|
+
key: 'account-segments',
|
|
1199
|
+
detail: 'Page with wiki ref',
|
|
1200
|
+
rawPaths: ['pages/ref.json'],
|
|
1201
|
+
});
|
|
1202
|
+
await currentSession.gitService.commitFiles(['wiki/global/account-segments.md'], 'wu page ref', 'KTX Test', 'system@ktx.local');
|
|
1203
|
+
}
|
|
1204
|
+
if (params.telemetryTags.unitKey === 'page-delete') {
|
|
1205
|
+
await rm(join(root, 'wiki/global/source-page.md'), { force: true });
|
|
1206
|
+
currentSession.actions.push({
|
|
1207
|
+
target: 'wiki',
|
|
1208
|
+
type: 'removed',
|
|
1209
|
+
key: 'source-page',
|
|
1210
|
+
detail: 'Delete referenced page',
|
|
1211
|
+
rawPaths: ['pages/delete.json'],
|
|
1212
|
+
});
|
|
1213
|
+
await currentSession.gitService.commitFiles(['wiki/global/source-page.md'], 'wu delete source page', 'KTX Test', 'system@ktx.local');
|
|
1214
|
+
}
|
|
1215
|
+
return { stopReason: 'natural' };
|
|
1216
|
+
});
|
|
1217
|
+
const runner = new IngestBundleRunner(deps);
|
|
1218
|
+
await mockStageRawFiles(runner, runtime, [
|
|
1219
|
+
['pages/ref.json', 'h1'],
|
|
1220
|
+
['pages/delete.json', 'h2'],
|
|
1221
|
+
]);
|
|
1222
|
+
await expect(runner.run({
|
|
1223
|
+
jobId: 'job-wiki-ref-conflict',
|
|
1224
|
+
connectionId: 'warehouse',
|
|
1225
|
+
sourceKey: 'metabase',
|
|
1226
|
+
trigger: 'upload',
|
|
1227
|
+
bundleRef: { kind: 'upload', uploadId: 'upload' },
|
|
1228
|
+
})).rejects.toThrow(/wiki references target missing page\(s\): account-segments -> source-page/);
|
|
1229
|
+
expect(await runtime.git.revParseHead()).toBe(preRunHead);
|
|
1230
|
+
const trace = await readFile(join(runtime.configDir, '.ktx/ingest-traces/job-wiki-ref-conflict/trace.jsonl'), 'utf-8');
|
|
1231
|
+
expect(trace).toContain('final_artifact_gates_failed');
|
|
1232
|
+
expect(trace).toContain('account-segments -> source-page');
|
|
1233
|
+
expect(trace).toContain('ingest_failed');
|
|
1234
|
+
expect(trace).toContain('failure_report_created');
|
|
1235
|
+
expect(trace).not.toContain('squash_finished');
|
|
1236
|
+
const failureReport = deps.reports.create.mock.calls
|
|
1237
|
+
.map((call) => call[0])
|
|
1238
|
+
.find((report) => report.body.status === 'failed');
|
|
1239
|
+
expect(failureReport.body.failure).toMatchObject({
|
|
1240
|
+
phase: 'final_gates',
|
|
1241
|
+
message: expect.stringContaining('account-segments -> source-page'),
|
|
1242
|
+
details: expect.objectContaining({
|
|
1243
|
+
changedWikiPageKeys: expect.arrayContaining(['account-segments']),
|
|
1244
|
+
workUnitPatchTouchedPaths: expect.arrayContaining([
|
|
1245
|
+
'wiki/global/account-segments.md',
|
|
1246
|
+
'wiki/global/source-page.md',
|
|
1247
|
+
]),
|
|
1248
|
+
}),
|
|
1249
|
+
});
|
|
1250
|
+
}
|
|
1251
|
+
finally {
|
|
1252
|
+
await rm(runtime.homeDir, { recursive: true, force: true });
|
|
1253
|
+
}
|
|
1254
|
+
});
|
|
1255
|
+
it('rejects unchanged inbound wiki refs broken by an isolated wiki deletion', async () => {
|
|
1256
|
+
const runtime = await makeRealGitRuntime();
|
|
1257
|
+
try {
|
|
1258
|
+
await mkdir(join(runtime.configDir, 'wiki/global'), { recursive: true });
|
|
1259
|
+
await writeFile(join(runtime.configDir, 'wiki/global/source-page.md'), '---\nsummary: Source page\nusage_mode: auto\n---\n\nSource page\n');
|
|
1260
|
+
await writeFile(join(runtime.configDir, 'wiki/global/account-segments.md'), '---\nsummary: Account segments\nusage_mode: auto\nrefs:\n - source-page\n---\n\nSee [[source-page]].\n');
|
|
1261
|
+
await runtime.git.commitFiles(['wiki/global/source-page.md', 'wiki/global/account-segments.md'], 'seed inbound wiki refs', 'KTX Test', 'system@ktx.local');
|
|
1262
|
+
const preRunHead = await runtime.git.revParseHead();
|
|
1263
|
+
const { deps, adapter } = makeDeps(runtime);
|
|
1264
|
+
adapter.chunk.mockResolvedValue({
|
|
1265
|
+
workUnits: [{ unitKey: 'delete-target-page', rawFiles: ['pages/delete.json'], peerFileIndex: [], dependencyPaths: [] }],
|
|
1266
|
+
});
|
|
1267
|
+
let currentSession = null;
|
|
1268
|
+
deps.toolsetFactory.createIngestWuToolset = vi.fn((toolSession) => {
|
|
1269
|
+
currentSession = toolSession;
|
|
1270
|
+
return { toRuntimeTools: vi.fn(() => ({})) };
|
|
1271
|
+
});
|
|
1272
|
+
deps.agentRunner.runLoop = vi.fn(async (params) => {
|
|
1273
|
+
if (params.telemetryTags.unitKey !== 'delete-target-page') {
|
|
1274
|
+
return { stopReason: 'natural' };
|
|
1275
|
+
}
|
|
1276
|
+
const root = rootOfConfig(currentSession.configService, runtime.configDir);
|
|
1277
|
+
await rm(join(root, 'wiki/global/source-page.md'), { force: true });
|
|
1278
|
+
currentSession.actions.push({
|
|
1279
|
+
target: 'wiki',
|
|
1280
|
+
type: 'removed',
|
|
1281
|
+
key: 'source-page',
|
|
1282
|
+
detail: 'Delete referenced page',
|
|
1283
|
+
rawPaths: ['pages/delete.json'],
|
|
1284
|
+
});
|
|
1285
|
+
await currentSession.gitService.commitFiles(['wiki/global/source-page.md'], 'wu delete target page', 'KTX Test', 'system@ktx.local');
|
|
1286
|
+
return { stopReason: 'natural' };
|
|
1287
|
+
});
|
|
1288
|
+
const runner = new IngestBundleRunner(deps);
|
|
1289
|
+
await mockStageRawFiles(runner, runtime, [['pages/delete.json', 'h1']]);
|
|
1290
|
+
await expect(runner.run({
|
|
1291
|
+
jobId: 'job-existing-wiki-ref-stale',
|
|
1292
|
+
connectionId: 'warehouse',
|
|
1293
|
+
sourceKey: 'metabase',
|
|
1294
|
+
trigger: 'upload',
|
|
1295
|
+
bundleRef: { kind: 'upload', uploadId: 'upload' },
|
|
1296
|
+
})).rejects.toThrow(/wiki references target missing page\(s\): account-segments -> source-page/);
|
|
1297
|
+
expect(await runtime.git.revParseHead()).toBe(preRunHead);
|
|
1298
|
+
const events = (await readFile(join(runtime.configDir, '.ktx/ingest-traces/job-existing-wiki-ref-stale/trace.jsonl'), 'utf-8'))
|
|
1299
|
+
.trim()
|
|
1300
|
+
.split('\n')
|
|
1301
|
+
.map((line) => JSON.parse(line));
|
|
1302
|
+
expect(events.map((event) => event.event)).toEqual(expect.arrayContaining([
|
|
1303
|
+
'final_artifact_gates_started',
|
|
1304
|
+
'final_artifact_gates_failed',
|
|
1305
|
+
'ingest_failed',
|
|
1306
|
+
'failure_report_created',
|
|
1307
|
+
]));
|
|
1308
|
+
expect(events.map((event) => event.event)).not.toContain('squash_finished');
|
|
1309
|
+
const gateFailure = events.find((event) => event.event === 'final_artifact_gates_failed');
|
|
1310
|
+
expect(gateFailure).toMatchObject({
|
|
1311
|
+
data: {
|
|
1312
|
+
wikiReferenceGateScope: {
|
|
1313
|
+
global: true,
|
|
1314
|
+
reasons: expect.arrayContaining(['wiki_page_removed']),
|
|
1315
|
+
removedWikiPageKeys: expect.arrayContaining(['source-page']),
|
|
1316
|
+
pageKeysValidated: expect.arrayContaining(['account-segments']),
|
|
1317
|
+
},
|
|
1318
|
+
actionOrigins: expect.arrayContaining([
|
|
1319
|
+
expect.objectContaining({
|
|
1320
|
+
source: 'work_unit_action',
|
|
1321
|
+
unitKey: 'delete-target-page',
|
|
1322
|
+
unitRawFiles: ['pages/delete.json'],
|
|
1323
|
+
action: expect.objectContaining({
|
|
1324
|
+
target: 'wiki',
|
|
1325
|
+
type: 'removed',
|
|
1326
|
+
key: 'source-page',
|
|
1327
|
+
rawPaths: ['pages/delete.json'],
|
|
1328
|
+
}),
|
|
1329
|
+
}),
|
|
1330
|
+
]),
|
|
1331
|
+
},
|
|
1332
|
+
error: { message: expect.stringContaining('account-segments -> source-page') },
|
|
1333
|
+
});
|
|
1334
|
+
const failureReport = deps.reports.create.mock.calls
|
|
1335
|
+
.map((call) => call[0])
|
|
1336
|
+
.find((report) => report.body.status === 'failed');
|
|
1337
|
+
expect(failureReport.body.failure).toMatchObject({
|
|
1338
|
+
phase: 'final_gates',
|
|
1339
|
+
message: expect.stringContaining('account-segments -> source-page'),
|
|
1340
|
+
details: expect.objectContaining({
|
|
1341
|
+
wikiReferenceGateScope: expect.objectContaining({
|
|
1342
|
+
global: true,
|
|
1343
|
+
reasons: expect.arrayContaining(['wiki_page_removed']),
|
|
1344
|
+
removedWikiPageKeys: expect.arrayContaining(['source-page']),
|
|
1345
|
+
pageKeysValidated: expect.arrayContaining(['account-segments']),
|
|
1346
|
+
}),
|
|
1347
|
+
changedWikiPageKeys: expect.arrayContaining(['source-page']),
|
|
1348
|
+
actionOrigins: expect.arrayContaining([
|
|
1349
|
+
expect.objectContaining({
|
|
1350
|
+
source: 'work_unit_action',
|
|
1351
|
+
unitKey: 'delete-target-page',
|
|
1352
|
+
action: expect.objectContaining({
|
|
1353
|
+
target: 'wiki',
|
|
1354
|
+
type: 'removed',
|
|
1355
|
+
key: 'source-page',
|
|
1356
|
+
rawPaths: ['pages/delete.json'],
|
|
1357
|
+
}),
|
|
1358
|
+
}),
|
|
1359
|
+
]),
|
|
1360
|
+
}),
|
|
1361
|
+
});
|
|
1362
|
+
expect(failureReport.body.workUnits).toEqual(expect.arrayContaining([
|
|
1363
|
+
expect.objectContaining({
|
|
1364
|
+
unitKey: 'delete-target-page',
|
|
1365
|
+
actions: expect.arrayContaining([
|
|
1366
|
+
expect.objectContaining({
|
|
1367
|
+
target: 'wiki',
|
|
1368
|
+
type: 'removed',
|
|
1369
|
+
key: 'source-page',
|
|
1370
|
+
rawPaths: ['pages/delete.json'],
|
|
1371
|
+
}),
|
|
1372
|
+
]),
|
|
1373
|
+
}),
|
|
1374
|
+
]));
|
|
1375
|
+
}
|
|
1376
|
+
finally {
|
|
1377
|
+
await rm(runtime.homeDir, { recursive: true, force: true });
|
|
1378
|
+
}
|
|
1379
|
+
});
|
|
1380
|
+
it('rejects WorkUnit patches that touch unauthorized semantic-layer target connections', async () => {
|
|
1381
|
+
const runtime = await makeRealGitRuntime();
|
|
1382
|
+
try {
|
|
1383
|
+
const { deps, adapter } = makeDeps(runtime);
|
|
1384
|
+
adapter.chunk.mockResolvedValue({
|
|
1385
|
+
workUnits: [{ unitKey: 'finance-source', rawFiles: ['cards/finance.json'], peerFileIndex: [], dependencyPaths: [] }],
|
|
1386
|
+
});
|
|
1387
|
+
let currentSession = null;
|
|
1388
|
+
deps.toolsetFactory.createIngestWuToolset = vi.fn((toolSession) => {
|
|
1389
|
+
currentSession = toolSession;
|
|
1390
|
+
return { toRuntimeTools: vi.fn(() => ({})) };
|
|
1391
|
+
});
|
|
1392
|
+
deps.agentRunner.runLoop = vi.fn(async () => {
|
|
1393
|
+
const root = rootOfConfig(currentSession.configService, runtime.configDir);
|
|
1394
|
+
await mkdir(join(root, 'semantic-layer/finance'), { recursive: true });
|
|
1395
|
+
await writeFile(join(root, 'semantic-layer/finance/orders.yaml'), 'name: orders\ngrain: [id]\ncolumns: [{name: id, type: string}]\njoins: []\nmeasures: []\n');
|
|
1396
|
+
addTouchedSlSource(currentSession.touchedSlSources, 'finance', 'orders');
|
|
1397
|
+
currentSession.actions.push({
|
|
1398
|
+
target: 'sl',
|
|
1399
|
+
type: 'created',
|
|
1400
|
+
key: 'orders',
|
|
1401
|
+
detail: 'Unauthorized target',
|
|
1402
|
+
targetConnectionId: 'finance',
|
|
1403
|
+
rawPaths: ['cards/finance.json'],
|
|
1404
|
+
});
|
|
1405
|
+
await currentSession.gitService.commitFiles(['semantic-layer/finance/orders.yaml'], 'wu unauthorized target', 'KTX Test', 'system@ktx.local');
|
|
1406
|
+
return { stopReason: 'natural' };
|
|
1407
|
+
});
|
|
1408
|
+
const runner = new IngestBundleRunner(deps);
|
|
1409
|
+
await mockStageRawFiles(runner, runtime, [['cards/finance.json', 'h1']]);
|
|
1410
|
+
const preRunHead = await runtime.git.revParseHead();
|
|
1411
|
+
await expect(runner.run({
|
|
1412
|
+
jobId: 'job-unauthorized-wu-target',
|
|
1413
|
+
connectionId: 'warehouse',
|
|
1414
|
+
sourceKey: 'metabase',
|
|
1415
|
+
trigger: 'upload',
|
|
1416
|
+
bundleRef: { kind: 'upload', uploadId: 'upload' },
|
|
1417
|
+
})).rejects.toThrow(/isolated diff textual conflict.*semantic-layer target connection not allowed/);
|
|
1418
|
+
expect(await runtime.git.revParseHead()).toBe(preRunHead);
|
|
1419
|
+
const trace = await readFile(join(runtime.configDir, '.ktx/ingest-traces/job-unauthorized-wu-target/trace.jsonl'), 'utf-8');
|
|
1420
|
+
expect(trace).toContain('patch_policy_rejected');
|
|
1421
|
+
expect(trace).toContain('semantic-layer/finance/orders.yaml');
|
|
1422
|
+
expect(trace).toContain('allowedTargetConnectionIds');
|
|
1423
|
+
expect(trace).toContain('ingest_failed');
|
|
1424
|
+
expect(trace).toContain('failure_report_created');
|
|
1425
|
+
expect(trace).not.toContain('squash_finished');
|
|
1426
|
+
const failureReport = deps.reports.create.mock.calls
|
|
1427
|
+
.map((call) => call[0])
|
|
1428
|
+
.find((report) => report.body.status === 'failed');
|
|
1429
|
+
expect(failureReport.body.failure).toMatchObject({
|
|
1430
|
+
phase: 'integration',
|
|
1431
|
+
message: expect.stringContaining('semantic-layer target connection not allowed'),
|
|
1432
|
+
});
|
|
1433
|
+
expect(failureReport.body.failure.details).toMatchObject({
|
|
1434
|
+
unitKey: 'finance-source',
|
|
1435
|
+
allowedTargetConnectionIds: ['warehouse'],
|
|
1436
|
+
touchedPaths: ['semantic-layer/finance/orders.yaml'],
|
|
1437
|
+
reason: expect.stringContaining('semantic-layer/finance/orders.yaml (finance)'),
|
|
1438
|
+
});
|
|
1439
|
+
}
|
|
1440
|
+
finally {
|
|
1441
|
+
await rm(runtime.homeDir, { recursive: true, force: true });
|
|
1442
|
+
}
|
|
1443
|
+
});
|
|
1444
|
+
it('rejects reconciliation mutations that touch unauthorized semantic-layer target connections before squash', async () => {
|
|
1445
|
+
const runtime = await makeRealGitRuntime();
|
|
1446
|
+
try {
|
|
1447
|
+
const { deps, adapter } = makeDeps(runtime);
|
|
1448
|
+
adapter.chunk.mockResolvedValue({
|
|
1449
|
+
workUnits: [{ unitKey: 'valid-page', rawFiles: ['pages/source.json'], peerFileIndex: [], dependencyPaths: [] }],
|
|
1450
|
+
});
|
|
1451
|
+
let currentSession = null;
|
|
1452
|
+
deps.toolsetFactory.createIngestWuToolset = vi.fn((toolSession) => {
|
|
1453
|
+
currentSession = toolSession;
|
|
1454
|
+
return { toRuntimeTools: vi.fn(() => ({})) };
|
|
1455
|
+
});
|
|
1456
|
+
deps.agentRunner.runLoop = vi.fn(async (params) => {
|
|
1457
|
+
const root = rootOfConfig(currentSession.configService, runtime.configDir);
|
|
1458
|
+
if (params.telemetryTags.operationName === 'ingest-bundle-wu') {
|
|
1459
|
+
await mkdir(join(root, 'wiki/global'), { recursive: true });
|
|
1460
|
+
await writeFile(join(root, 'wiki/global/valid-page.md'), '---\nsummary: Valid page\nusage_mode: auto\n---\n\nValid\n');
|
|
1461
|
+
currentSession.actions.push({
|
|
1462
|
+
target: 'wiki',
|
|
1463
|
+
type: 'created',
|
|
1464
|
+
key: 'valid-page',
|
|
1465
|
+
detail: 'Valid page',
|
|
1466
|
+
rawPaths: ['pages/source.json'],
|
|
1467
|
+
});
|
|
1468
|
+
await currentSession.gitService.commitFiles(['wiki/global/valid-page.md'], 'wu valid page', 'KTX Test', 'system@ktx.local');
|
|
1469
|
+
}
|
|
1470
|
+
else {
|
|
1471
|
+
await mkdir(join(root, 'semantic-layer/finance'), { recursive: true });
|
|
1472
|
+
await writeFile(join(root, 'semantic-layer/finance/reconcile_orders.yaml'), 'name: reconcile_orders\ngrain: [id]\ncolumns: [{name: id, type: string}]\njoins: []\nmeasures: []\n');
|
|
1473
|
+
addTouchedSlSource(currentSession.touchedSlSources, 'finance', 'reconcile_orders');
|
|
1474
|
+
currentSession.actions.push({
|
|
1475
|
+
target: 'sl',
|
|
1476
|
+
type: 'created',
|
|
1477
|
+
key: 'reconcile_orders',
|
|
1478
|
+
detail: 'Unauthorized reconcile target',
|
|
1479
|
+
targetConnectionId: 'finance',
|
|
1480
|
+
rawPaths: ['pages/source.json'],
|
|
1481
|
+
});
|
|
1482
|
+
await currentSession.gitService.commitFiles(['semantic-layer/finance/reconcile_orders.yaml'], 'reconcile unauthorized target', 'KTX Test', 'system@ktx.local');
|
|
1483
|
+
}
|
|
1484
|
+
return { stopReason: 'natural' };
|
|
1485
|
+
});
|
|
1486
|
+
const runner = new IngestBundleRunner(deps);
|
|
1487
|
+
await mockStageRawFiles(runner, runtime, [['pages/source.json', 'h1']]);
|
|
1488
|
+
const preRunHead = await runtime.git.revParseHead();
|
|
1489
|
+
await expect(runner.run({
|
|
1490
|
+
jobId: 'job-unauthorized-reconcile-target',
|
|
1491
|
+
connectionId: 'warehouse',
|
|
1492
|
+
sourceKey: 'metabase',
|
|
1493
|
+
trigger: 'upload',
|
|
1494
|
+
bundleRef: { kind: 'upload', uploadId: 'upload' },
|
|
1495
|
+
})).rejects.toThrow(/semantic-layer target connection not allowed/);
|
|
1496
|
+
expect(await runtime.git.revParseHead()).toBe(preRunHead);
|
|
1497
|
+
const trace = await readFile(join(runtime.configDir, '.ktx/ingest-traces/job-unauthorized-reconcile-target/trace.jsonl'), 'utf-8');
|
|
1498
|
+
expect(trace).toContain('semantic_layer_target_policy_started');
|
|
1499
|
+
expect(trace).toContain('semantic_layer_target_policy_failed');
|
|
1500
|
+
expect(trace).toContain('allowedTargetConnectionIds');
|
|
1501
|
+
expect(trace).toContain('semantic-layer/finance/reconcile_orders.yaml');
|
|
1502
|
+
expect(trace).toContain('ingest_failed');
|
|
1503
|
+
expect(trace).toContain('failure_report_created');
|
|
1504
|
+
expect(trace).not.toContain('squash_finished');
|
|
1505
|
+
const failureReport = deps.reports.create.mock.calls
|
|
1506
|
+
.map((call) => call[0])
|
|
1507
|
+
.find((report) => report.body.status === 'failed');
|
|
1508
|
+
expect(failureReport.body.failure).toMatchObject({
|
|
1509
|
+
phase: 'target_policy',
|
|
1510
|
+
message: expect.stringContaining('semantic-layer target connection not allowed'),
|
|
1511
|
+
});
|
|
1512
|
+
expect(failureReport.body.failure.details).toMatchObject({
|
|
1513
|
+
allowedTargetConnectionIds: ['warehouse'],
|
|
1514
|
+
touchedPaths: expect.arrayContaining(['semantic-layer/finance/reconcile_orders.yaml']),
|
|
1515
|
+
});
|
|
1516
|
+
}
|
|
1517
|
+
finally {
|
|
1518
|
+
await rm(runtime.homeDir, { recursive: true, force: true });
|
|
1519
|
+
}
|
|
1520
|
+
});
|
|
1521
|
+
it('repairs additive same-source textual conflicts before final gates and squash', async () => {
|
|
1522
|
+
const runtime = await makeRealGitRuntime();
|
|
1523
|
+
try {
|
|
1524
|
+
const { deps } = makeDeps(runtime);
|
|
1525
|
+
let currentSession = null;
|
|
1526
|
+
deps.toolsetFactory.createIngestWuToolset = vi.fn((toolSession) => {
|
|
1527
|
+
currentSession = toolSession;
|
|
1528
|
+
return { toRuntimeTools: vi.fn(() => ({})) };
|
|
1529
|
+
});
|
|
1530
|
+
deps.agentRunner.runLoop = vi.fn(async (params) => {
|
|
1531
|
+
if (params.telemetryTags.operationName === 'ingest-isolated-diff-textual-resolver') {
|
|
1532
|
+
const current = await params.toolSet.read_integration_file.execute({
|
|
1533
|
+
path: 'semantic-layer/warehouse/mart_account_segments.yaml',
|
|
1534
|
+
});
|
|
1535
|
+
expect(current.markdown).toContain('total_contract_arr_cents');
|
|
1536
|
+
const patch = await params.toolSet.read_failed_patch.execute({});
|
|
1537
|
+
expect(patch.markdown).toContain('account_count');
|
|
1538
|
+
await params.toolSet.write_integration_file.execute({
|
|
1539
|
+
path: 'semantic-layer/warehouse/mart_account_segments.yaml',
|
|
1540
|
+
content: 'name: mart_account_segments\n' +
|
|
1541
|
+
'grain: [account_id]\n' +
|
|
1542
|
+
'columns: [{name: account_id, type: string}]\n' +
|
|
1543
|
+
'joins: []\n' +
|
|
1544
|
+
'measures:\n' +
|
|
1545
|
+
' - name: total_contract_arr_cents\n' +
|
|
1546
|
+
' expr: sum(contract_arr)\n' +
|
|
1547
|
+
' - name: account_count\n' +
|
|
1548
|
+
' expr: count_distinct(account_id)\n',
|
|
1549
|
+
});
|
|
1550
|
+
return { stopReason: 'natural' };
|
|
1551
|
+
}
|
|
1552
|
+
const root = rootOfConfig(currentSession.configService, runtime.configDir);
|
|
1553
|
+
await mkdir(join(root, 'semantic-layer/warehouse'), { recursive: true });
|
|
1554
|
+
if (params.telemetryTags.unitKey === 'card-wiki') {
|
|
1555
|
+
await writeFile(join(root, 'semantic-layer/warehouse/mart_account_segments.yaml'), 'name: mart_account_segments\n' +
|
|
1556
|
+
'grain: [account_id]\n' +
|
|
1557
|
+
'columns: [{name: account_id, type: string}]\n' +
|
|
1558
|
+
'joins: []\n' +
|
|
1559
|
+
'measures:\n' +
|
|
1560
|
+
' - name: total_contract_arr_cents\n' +
|
|
1561
|
+
' expr: sum(contract_arr)\n');
|
|
1562
|
+
}
|
|
1563
|
+
else if (params.telemetryTags.unitKey === 'card-source') {
|
|
1564
|
+
await writeFile(join(root, 'semantic-layer/warehouse/mart_account_segments.yaml'), 'name: mart_account_segments\n' +
|
|
1565
|
+
'grain: [account_id]\n' +
|
|
1566
|
+
'columns: [{name: account_id, type: string}]\n' +
|
|
1567
|
+
'joins: []\n' +
|
|
1568
|
+
'measures:\n' +
|
|
1569
|
+
' - name: account_count\n' +
|
|
1570
|
+
' expr: count_distinct(account_id)\n');
|
|
1571
|
+
}
|
|
1572
|
+
addTouchedSlSource(currentSession.touchedSlSources, 'warehouse', 'mart_account_segments');
|
|
1573
|
+
currentSession.actions.push({
|
|
1574
|
+
target: 'sl',
|
|
1575
|
+
type: 'updated',
|
|
1576
|
+
key: 'mart_account_segments',
|
|
1577
|
+
detail: 'Updated account segments source',
|
|
1578
|
+
targetConnectionId: 'warehouse',
|
|
1579
|
+
});
|
|
1580
|
+
await currentSession.gitService.commitFiles(['semantic-layer/warehouse/mart_account_segments.yaml'], `wu ${params.telemetryTags.unitKey}`, 'KTX Test', 'system@ktx.local');
|
|
1581
|
+
return { stopReason: 'natural' };
|
|
1582
|
+
});
|
|
1583
|
+
const runner = new IngestBundleRunner(deps);
|
|
1584
|
+
await mockStageRawFiles(runner, runtime, [
|
|
1585
|
+
['cards/wiki.json', 'hash-a'],
|
|
1586
|
+
['cards/source.json', 'hash-b'],
|
|
1587
|
+
]);
|
|
1588
|
+
const result = await runner.run({
|
|
1589
|
+
jobId: 'job-resolver-e2e',
|
|
1590
|
+
connectionId: 'warehouse',
|
|
1591
|
+
sourceKey: 'metabase',
|
|
1592
|
+
trigger: 'manual_resync',
|
|
1593
|
+
bundleRef: { kind: 'upload', uploadId: 'upload-1' },
|
|
1594
|
+
});
|
|
1595
|
+
expect(result.commitSha).toBeTruthy();
|
|
1596
|
+
const source = await readFile(join(runtime.configDir, 'semantic-layer/warehouse/mart_account_segments.yaml'), 'utf-8');
|
|
1597
|
+
expect(source).toContain('total_contract_arr_cents');
|
|
1598
|
+
expect(source).toContain('account_count');
|
|
1599
|
+
expect(deps.agentRunner.runLoop).toHaveBeenCalledWith(expect.objectContaining({
|
|
1600
|
+
modelRole: 'repair',
|
|
1601
|
+
telemetryTags: expect.objectContaining({
|
|
1602
|
+
operationName: 'ingest-isolated-diff-textual-resolver',
|
|
1603
|
+
unitKey: 'card-source',
|
|
1604
|
+
}),
|
|
1605
|
+
}));
|
|
1606
|
+
const successReport = deps.reports.create.mock.calls.at(-1)?.[0]?.body;
|
|
1607
|
+
expect(successReport.isolatedDiff).toMatchObject({
|
|
1608
|
+
acceptedPatches: 2,
|
|
1609
|
+
textualConflicts: 1,
|
|
1610
|
+
semanticConflicts: 0,
|
|
1611
|
+
resolverAttempts: 1,
|
|
1612
|
+
resolverRepairs: 1,
|
|
1613
|
+
resolverFailures: 0,
|
|
1614
|
+
});
|
|
1615
|
+
const trace = await readFile(join(runtime.configDir, '.ktx/ingest-traces/job-resolver-e2e/trace.jsonl'), 'utf-8');
|
|
1616
|
+
expect(trace).toContain('textual_conflict_resolver_repaired');
|
|
1617
|
+
expect(trace).toContain('patch_accepted_after_textual_resolution');
|
|
1618
|
+
}
|
|
1619
|
+
finally {
|
|
1620
|
+
await rm(runtime.homeDir, { recursive: true, force: true });
|
|
1621
|
+
}
|
|
1622
|
+
});
|
|
1623
|
+
it('repairs final wiki body refs before squash when the repair agent edits the scoped page', async () => {
|
|
1624
|
+
const runtime = await makeRealGitRuntime();
|
|
1625
|
+
try {
|
|
1626
|
+
await mkdir(join(runtime.configDir, 'semantic-layer/warehouse'), { recursive: true });
|
|
1627
|
+
await mkdir(join(runtime.configDir, 'wiki/global'), { recursive: true });
|
|
1628
|
+
await writeFile(join(runtime.configDir, 'semantic-layer/warehouse/mart_account_segments.yaml'), 'name: mart_account_segments\ngrain: [account_id]\ncolumns: [{name: account_id, type: string}]\njoins: []\nmeasures:\n - name: total_contract_arr_cents\n expr: sum(contract_arr)\n');
|
|
1629
|
+
await writeFile(join(runtime.configDir, 'wiki/global/account-segments.md'), '---\nsummary: Account segments\nusage_mode: auto\n---\n\nExisting ARR uses `mart_account_segments.total_contract_arr_cents`.\n');
|
|
1630
|
+
await runtime.git.commitFiles(['semantic-layer/warehouse/mart_account_segments.yaml', 'wiki/global/account-segments.md'], 'seed stale wiki body ref', 'KTX Test', 'system@ktx.local');
|
|
1631
|
+
const { deps, adapter } = makeDeps(runtime);
|
|
1632
|
+
adapter.chunk.mockResolvedValue({
|
|
1633
|
+
workUnits: [{ unitKey: 'source-only', rawFiles: ['cards/source.json'], peerFileIndex: [], dependencyPaths: [] }],
|
|
1634
|
+
});
|
|
1635
|
+
let currentSession = null;
|
|
1636
|
+
deps.toolsetFactory.createIngestWuToolset = vi.fn((toolSession) => {
|
|
1637
|
+
currentSession = toolSession;
|
|
1638
|
+
return { toRuntimeTools: vi.fn(() => ({})) };
|
|
1639
|
+
});
|
|
1640
|
+
deps.agentRunner.runLoop = vi.fn(async (params) => {
|
|
1641
|
+
if (params.telemetryTags.operationName === 'ingest-isolated-diff-gate-repair') {
|
|
1642
|
+
const gateError = await params.toolSet.read_gate_error.execute({});
|
|
1643
|
+
expect(gateError.markdown).toContain('total_contract_arr_cents');
|
|
1644
|
+
const page = await params.toolSet.read_repair_file.execute({
|
|
1645
|
+
path: 'wiki/global/account-segments.md',
|
|
1646
|
+
});
|
|
1647
|
+
await params.toolSet.write_repair_file.execute({
|
|
1648
|
+
path: 'wiki/global/account-segments.md',
|
|
1649
|
+
content: page.markdown.replace('total_contract_arr_cents', 'total_contract_arr'),
|
|
1650
|
+
});
|
|
1651
|
+
return { stopReason: 'natural' };
|
|
1652
|
+
}
|
|
1653
|
+
if (params.modelRole === 'reconcile') {
|
|
1654
|
+
return { stopReason: 'natural' };
|
|
1655
|
+
}
|
|
1656
|
+
const root = rootOfConfig(currentSession.configService, runtime.configDir);
|
|
1657
|
+
await writeFile(join(root, 'semantic-layer/warehouse/mart_account_segments.yaml'), 'name: mart_account_segments\ngrain: [account_id]\ncolumns: [{name: account_id, type: string}]\njoins: []\nmeasures:\n - name: total_contract_arr\n expr: sum(contract_arr)\n');
|
|
1658
|
+
addTouchedSlSource(currentSession.touchedSlSources, 'warehouse', 'mart_account_segments');
|
|
1659
|
+
currentSession.actions.push({
|
|
1660
|
+
target: 'sl',
|
|
1661
|
+
type: 'updated',
|
|
1662
|
+
key: 'mart_account_segments',
|
|
1663
|
+
detail: 'Rename ARR measure',
|
|
1664
|
+
targetConnectionId: 'warehouse',
|
|
1665
|
+
rawPaths: ['cards/source.json'],
|
|
1666
|
+
});
|
|
1667
|
+
await currentSession.gitService.commitFiles(['semantic-layer/warehouse/mart_account_segments.yaml'], 'wu source rename', 'KTX Test', 'system@ktx.local');
|
|
1668
|
+
return { stopReason: 'natural' };
|
|
1669
|
+
});
|
|
1670
|
+
const runner = new IngestBundleRunner(deps);
|
|
1671
|
+
await mockStageRawFiles(runner, runtime, [['cards/source.json', 'h1']]);
|
|
1672
|
+
const result = await runner.run({
|
|
1673
|
+
jobId: 'job-final-gate-repair',
|
|
1674
|
+
connectionId: 'warehouse',
|
|
1675
|
+
sourceKey: 'metabase',
|
|
1676
|
+
trigger: 'upload',
|
|
1677
|
+
bundleRef: { kind: 'upload', uploadId: 'upload' },
|
|
1678
|
+
});
|
|
1679
|
+
expect(result.commitSha).toBeTruthy();
|
|
1680
|
+
await expect(readFile(join(runtime.configDir, 'wiki/global/account-segments.md'), 'utf-8')).resolves.toContain('mart_account_segments.total_contract_arr');
|
|
1681
|
+
await expect(readFile(join(runtime.configDir, 'wiki/global/account-segments.md'), 'utf-8')).resolves.not.toContain('total_contract_arr_cents');
|
|
1682
|
+
const reportCreate = vi.mocked(deps.reports.create).mock.calls.at(-1)?.[0];
|
|
1683
|
+
expect(reportCreate.body.isolatedDiff).toMatchObject({
|
|
1684
|
+
gateRepairAttempts: 1,
|
|
1685
|
+
gateRepairs: 1,
|
|
1686
|
+
gateRepairFailures: 0,
|
|
1687
|
+
});
|
|
1688
|
+
const trace = await readFile(join(runtime.configDir, '.ktx/ingest-traces/job-final-gate-repair/trace.jsonl'), 'utf-8');
|
|
1689
|
+
expect(trace).toContain('gate_repair_repaired');
|
|
1690
|
+
expect(trace).toContain('final_artifact_gates_after_gate_repair_finished');
|
|
1691
|
+
expect(trace).toContain('final_gate_repair_committed');
|
|
1692
|
+
}
|
|
1693
|
+
finally {
|
|
1694
|
+
await rm(runtime.homeDir, { recursive: true, force: true });
|
|
1695
|
+
}
|
|
1696
|
+
});
|
|
1697
|
+
it('fails before squash when final gate repair makes no edit', async () => {
|
|
1698
|
+
const runtime = await makeRealGitRuntime();
|
|
1699
|
+
try {
|
|
1700
|
+
await mkdir(join(runtime.configDir, 'semantic-layer/warehouse'), { recursive: true });
|
|
1701
|
+
await mkdir(join(runtime.configDir, 'wiki/global'), { recursive: true });
|
|
1702
|
+
await writeFile(join(runtime.configDir, 'semantic-layer/warehouse/mart_account_segments.yaml'), 'name: mart_account_segments\ngrain: [account_id]\ncolumns: [{name: account_id, type: string}]\njoins: []\nmeasures:\n - name: total_contract_arr_cents\n expr: sum(contract_arr)\n');
|
|
1703
|
+
await writeFile(join(runtime.configDir, 'wiki/global/account-segments.md'), '---\nsummary: Account segments\nusage_mode: auto\n---\n\nExisting ARR uses `mart_account_segments.total_contract_arr_cents`.\n');
|
|
1704
|
+
await runtime.git.commitFiles(['semantic-layer/warehouse/mart_account_segments.yaml', 'wiki/global/account-segments.md'], 'seed stale wiki body ref', 'KTX Test', 'system@ktx.local');
|
|
1705
|
+
const preRunHead = await runtime.git.revParseHead();
|
|
1706
|
+
const { deps, adapter } = makeDeps(runtime);
|
|
1707
|
+
adapter.chunk.mockResolvedValue({
|
|
1708
|
+
workUnits: [{ unitKey: 'source-only', rawFiles: ['cards/source.json'], peerFileIndex: [], dependencyPaths: [] }],
|
|
1709
|
+
});
|
|
1710
|
+
let currentSession = null;
|
|
1711
|
+
deps.toolsetFactory.createIngestWuToolset = vi.fn((toolSession) => {
|
|
1712
|
+
currentSession = toolSession;
|
|
1713
|
+
return { toRuntimeTools: vi.fn(() => ({})) };
|
|
1714
|
+
});
|
|
1715
|
+
deps.agentRunner.runLoop = vi.fn(async (params) => {
|
|
1716
|
+
if (params.telemetryTags.operationName === 'ingest-isolated-diff-gate-repair') {
|
|
1717
|
+
return { stopReason: 'natural' };
|
|
1718
|
+
}
|
|
1719
|
+
if (params.modelRole === 'reconcile') {
|
|
1720
|
+
return { stopReason: 'natural' };
|
|
1721
|
+
}
|
|
1722
|
+
const root = rootOfConfig(currentSession.configService, runtime.configDir);
|
|
1723
|
+
await writeFile(join(root, 'semantic-layer/warehouse/mart_account_segments.yaml'), 'name: mart_account_segments\ngrain: [account_id]\ncolumns: [{name: account_id, type: string}]\njoins: []\nmeasures:\n - name: total_contract_arr\n expr: sum(contract_arr)\n');
|
|
1724
|
+
addTouchedSlSource(currentSession.touchedSlSources, 'warehouse', 'mart_account_segments');
|
|
1725
|
+
currentSession.actions.push({
|
|
1726
|
+
target: 'sl',
|
|
1727
|
+
type: 'updated',
|
|
1728
|
+
key: 'mart_account_segments',
|
|
1729
|
+
detail: 'Rename ARR measure',
|
|
1730
|
+
targetConnectionId: 'warehouse',
|
|
1731
|
+
rawPaths: ['cards/source.json'],
|
|
1732
|
+
});
|
|
1733
|
+
await currentSession.gitService.commitFiles(['semantic-layer/warehouse/mart_account_segments.yaml'], 'wu source rename', 'KTX Test', 'system@ktx.local');
|
|
1734
|
+
return { stopReason: 'natural' };
|
|
1735
|
+
});
|
|
1736
|
+
const runner = new IngestBundleRunner(deps);
|
|
1737
|
+
await mockStageRawFiles(runner, runtime, [['cards/source.json', 'h1']]);
|
|
1738
|
+
await expect(runner.run({
|
|
1739
|
+
jobId: 'job-final-gate-repair-fails',
|
|
1740
|
+
connectionId: 'warehouse',
|
|
1741
|
+
sourceKey: 'metabase',
|
|
1742
|
+
trigger: 'upload',
|
|
1743
|
+
bundleRef: { kind: 'upload', uploadId: 'upload' },
|
|
1744
|
+
})).rejects.toThrow(/gate repair completed without editing an allowed path/);
|
|
1745
|
+
expect(await runtime.git.revParseHead()).toBe(preRunHead);
|
|
1746
|
+
const reportCreate = vi.mocked(deps.reports.create).mock.calls.at(-1)?.[0];
|
|
1747
|
+
expect(reportCreate.body.status).toBe('failed');
|
|
1748
|
+
expect(reportCreate.body.isolatedDiff).toMatchObject({
|
|
1749
|
+
gateRepairAttempts: 1,
|
|
1750
|
+
gateRepairs: 0,
|
|
1751
|
+
gateRepairFailures: 1,
|
|
1752
|
+
});
|
|
1753
|
+
const trace = await readFile(join(runtime.configDir, '.ktx/ingest-traces/job-final-gate-repair-fails/trace.jsonl'), 'utf-8');
|
|
1754
|
+
expect(trace).toContain('gate_repair_failed');
|
|
1755
|
+
expect(trace).not.toContain('squash_finished');
|
|
1756
|
+
}
|
|
1757
|
+
finally {
|
|
1758
|
+
await rm(runtime.homeDir, { recursive: true, force: true });
|
|
1759
|
+
}
|
|
1760
|
+
});
|
|
1761
|
+
});
|