@kaelio/ktx 0.11.0 → 0.12.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (181) hide show
  1. package/assets/python/{kaelio_ktx-0.11.0-py3-none-any.whl → kaelio_ktx-0.12.0-py3-none-any.whl} +0 -0
  2. package/assets/python/manifest.json +4 -4
  3. package/dist/.tsbuildinfo +1 -1
  4. package/dist/admin.js +1 -1
  5. package/dist/clack.d.ts +16 -0
  6. package/dist/clack.js +37 -6
  7. package/dist/claude-code-prompt-caching.js +1 -1
  8. package/dist/cli-program.js +3 -3
  9. package/dist/cli-runtime.js +2 -2
  10. package/dist/commands/connection-commands.js +1 -1
  11. package/dist/commands/ingest-commands.js +4 -4
  12. package/dist/commands/mcp-commands.js +12 -12
  13. package/dist/commands/runtime-commands.js +4 -4
  14. package/dist/commands/setup-commands.js +6 -5
  15. package/dist/commands/sl-commands.js +1 -1
  16. package/dist/commands/sql-commands.js +1 -1
  17. package/dist/commands/status-commands.js +1 -1
  18. package/dist/connection.js +1 -1
  19. package/dist/connectors/clickhouse/connector.js +1 -1
  20. package/dist/connectors/mysql/connector.js +1 -1
  21. package/dist/connectors/snowflake/connector.d.ts +1 -1
  22. package/dist/connectors/sqlite/connector.js +2 -25
  23. package/dist/connectors/sqlserver/connector.js +3 -3
  24. package/dist/context/connections/connection-type.d.ts +1 -1
  25. package/dist/context/connections/read-only-sql.d.ts +1 -0
  26. package/dist/context/connections/read-only-sql.js +116 -2
  27. package/dist/context/core/git.service.d.ts +23 -0
  28. package/dist/context/core/git.service.js +71 -8
  29. package/dist/context/ingest/adapters/historic-sql/projection.js +2 -1
  30. package/dist/context/ingest/adapters/looker/client.js +7 -2
  31. package/dist/context/ingest/adapters/looker/factory.d.ts +8 -1
  32. package/dist/context/ingest/adapters/looker/factory.js +9 -0
  33. package/dist/context/ingest/adapters/looker/mapping.js +1 -1
  34. package/dist/context/ingest/adapters/looker/types.d.ts +1 -1
  35. package/dist/context/ingest/adapters/metabase/client.d.ts +1 -1
  36. package/dist/context/ingest/adapters/metabase/client.js +1 -1
  37. package/dist/context/ingest/adapters/metabase/local-metabase.adapter.js +1 -1
  38. package/dist/context/ingest/adapters/metabase/mapping.js +6 -6
  39. package/dist/context/ingest/artifact-gates.d.ts +2 -6
  40. package/dist/context/ingest/artifact-gates.js +5 -47
  41. package/dist/context/ingest/constrained-repair.d.ts +55 -0
  42. package/dist/context/ingest/constrained-repair.js +167 -0
  43. package/dist/context/ingest/final-gate-repair.d.ts +9 -11
  44. package/dist/context/ingest/final-gate-repair.js +40 -128
  45. package/dist/context/ingest/finalization-scope.d.ts +1 -1
  46. package/dist/context/ingest/finalization-scope.js +15 -15
  47. package/dist/context/ingest/ingest-bundle.runner.d.ts +1 -0
  48. package/dist/context/ingest/ingest-bundle.runner.js +101 -67
  49. package/dist/context/ingest/isolated-diff/patch-integrator.d.ts +6 -13
  50. package/dist/context/ingest/isolated-diff/patch-integrator.js +32 -109
  51. package/dist/context/ingest/isolated-diff/textual-conflict-resolver.d.ts +8 -9
  52. package/dist/context/ingest/isolated-diff/textual-conflict-resolver.js +63 -141
  53. package/dist/context/ingest/local-bundle-runtime.d.ts +2 -0
  54. package/dist/context/ingest/local-bundle-runtime.js +9 -10
  55. package/dist/context/ingest/local-ingest.d.ts +2 -0
  56. package/dist/context/ingest/local-ingest.js +2 -0
  57. package/dist/context/ingest/memory-flow/view-model.js +1 -1
  58. package/dist/context/ingest/stages/stage-3-work-units.d.ts +2 -6
  59. package/dist/context/ingest/stages/stage-3-work-units.js +2 -1
  60. package/dist/context/ingest/stages/validate-wu-sources.d.ts +7 -1
  61. package/dist/context/ingest/stages/validate-wu-sources.js +109 -4
  62. package/dist/context/ingest/tools/warehouse-verification/create-warehouse-verification-tools.d.ts +2 -0
  63. package/dist/context/ingest/tools/warehouse-verification/create-warehouse-verification-tools.js +1 -1
  64. package/dist/context/ingest/tools/warehouse-verification/discover-data.tool.js +3 -3
  65. package/dist/context/ingest/tools/warehouse-verification/sql-execution.tool.d.ts +3 -1
  66. package/dist/context/ingest/tools/warehouse-verification/sql-execution.tool.js +15 -1
  67. package/dist/context/llm/ai-sdk-runtime.js +2 -2
  68. package/dist/context/llm/claude-code-runtime.js +1 -1
  69. package/dist/context/llm/local-config.js +1 -1
  70. package/dist/context/llm/runtime-tools.js +2 -2
  71. package/dist/context/mcp/context-tools.js +7 -7
  72. package/dist/context/mcp/local-project-ports.js +23 -54
  73. package/dist/context/memory/local-memory.js +4 -1
  74. package/dist/context/memory/memory-agent.service.js +1 -1
  75. package/dist/context/project/config.d.ts +11 -4
  76. package/dist/context/project/config.js +85 -30
  77. package/dist/context/project/driver-schemas.js +1 -1
  78. package/dist/context/project/mappings-yaml-schema.js +2 -2
  79. package/dist/context/project/project.js +12 -4
  80. package/dist/context/scan/description-generation.js +4 -4
  81. package/dist/context/scan/local-enrichment-artifacts.js +2 -1
  82. package/dist/context/scan/local-scan.js +2 -2
  83. package/dist/context/scan/local-structural-artifacts.js +5 -5
  84. package/dist/context/scan/relationship-benchmark-report.js +1 -1
  85. package/dist/context/scan/relationship-discovery.js +3 -3
  86. package/dist/context/scan/relationship-llm-proposal.js +3 -3
  87. package/dist/context/sl/local-query.js +3 -33
  88. package/dist/context/sl/local-sl.d.ts +0 -8
  89. package/dist/context/sl/local-sl.js +44 -69
  90. package/dist/context/sl/semantic-layer.service.d.ts +25 -8
  91. package/dist/context/sl/semantic-layer.service.js +109 -56
  92. package/dist/context/sl/source-files.d.ts +46 -0
  93. package/dist/context/sl/source-files.js +131 -0
  94. package/dist/context/sl/tools/base-semantic-layer.tool.d.ts +2 -2
  95. package/dist/context/sl/tools/base-semantic-layer.tool.js +2 -7
  96. package/dist/context/sl/tools/sl-edit-source.tool.js +10 -8
  97. package/dist/context/sl/tools/sl-warehouse-validation.js +55 -27
  98. package/dist/context/sl/tools/sl-write-source.tool.js +12 -9
  99. package/dist/context/sql-analysis/dialect.d.ts +2 -0
  100. package/dist/context/sql-analysis/dialect.js +20 -0
  101. package/dist/context/tools/base-tool.d.ts +6 -19
  102. package/dist/context/tools/base-tool.js +0 -14
  103. package/dist/context-build-view.js +5 -5
  104. package/dist/database-tree-picker.js +18 -3
  105. package/dist/demo-assets.js +0 -1
  106. package/dist/doctor.d.ts +1 -1
  107. package/dist/doctor.js +31 -23
  108. package/dist/errors.d.ts +31 -0
  109. package/dist/errors.js +44 -0
  110. package/dist/ingest.d.ts +1 -1
  111. package/dist/ingest.js +8 -2
  112. package/dist/io/symbols.d.ts +2 -0
  113. package/dist/io/symbols.js +2 -0
  114. package/dist/io/tty.d.ts +8 -0
  115. package/dist/io/tty.js +16 -0
  116. package/dist/llm/embedding-health.js +1 -1
  117. package/dist/llm/embedding-provider.js +3 -3
  118. package/dist/llm/model-provider.js +1 -1
  119. package/dist/local-adapters.d.ts +1 -0
  120. package/dist/local-adapters.js +2 -2
  121. package/dist/local-scan-connectors.js +1 -1
  122. package/dist/managed-local-embeddings.js +17 -8
  123. package/dist/managed-mcp-daemon.js +3 -3
  124. package/dist/managed-python-command.d.ts +7 -0
  125. package/dist/managed-python-command.js +34 -8
  126. package/dist/managed-python-daemon.js +2 -2
  127. package/dist/managed-python-http.js +3 -3
  128. package/dist/managed-python-runtime.d.ts +30 -1
  129. package/dist/managed-python-runtime.js +134 -18
  130. package/dist/managed-uv-release.d.ts +7 -0
  131. package/dist/managed-uv-release.js +11 -0
  132. package/dist/mcp-http-server.js +4 -4
  133. package/dist/mcp-server-factory.js +3 -3
  134. package/dist/mcp-stdio-server.js +1 -1
  135. package/dist/memory-flow-hud.js +2 -2
  136. package/dist/next-steps.js +2 -2
  137. package/dist/prompt-navigation.d.ts +17 -0
  138. package/dist/prompt-navigation.js +49 -3
  139. package/dist/prompts/memory_agent_bundle_ingest_work_unit.md +2 -2
  140. package/dist/prompts/memory_agent_external_ingest.md +2 -2
  141. package/dist/public-ingest-copy.js +1 -1
  142. package/dist/public-ingest.js +3 -3
  143. package/dist/release-version.js +1 -1
  144. package/dist/runtime-requirements.js +1 -1
  145. package/dist/runtime.js +9 -9
  146. package/dist/scan.js +1 -1
  147. package/dist/setup-agents.js +21 -30
  148. package/dist/setup-banner.d.ts +20 -0
  149. package/dist/setup-banner.js +39 -0
  150. package/dist/setup-context.js +24 -15
  151. package/dist/setup-databases.js +31 -59
  152. package/dist/setup-demo-tour.js +12 -8
  153. package/dist/setup-embeddings.js +9 -9
  154. package/dist/setup-interrupt.js +1 -1
  155. package/dist/setup-models.d.ts +4 -1
  156. package/dist/setup-models.js +54 -28
  157. package/dist/setup-project.js +29 -5
  158. package/dist/setup-prompts.js +16 -1
  159. package/dist/setup-ready-menu.js +1 -1
  160. package/dist/setup-sources.js +27 -7
  161. package/dist/setup.js +13 -13
  162. package/dist/skills/analytics/SKILL.md +3 -3
  163. package/dist/skills/dbt_ingest/SKILL.md +3 -3
  164. package/dist/skills/looker_ingest/SKILL.md +3 -3
  165. package/dist/skills/lookml_ingest/SKILL.md +7 -7
  166. package/dist/skills/metabase_ingest/SKILL.md +4 -4
  167. package/dist/skills/metricflow_ingest/SKILL.md +15 -15
  168. package/dist/skills/notion_synthesize/SKILL.md +1 -1
  169. package/dist/skills/sl/SKILL.md +3 -3
  170. package/dist/skills/sl_capture/SKILL.md +1 -1
  171. package/dist/skills/wiki_capture/SKILL.md +1 -1
  172. package/dist/source-mapping.js +1 -1
  173. package/dist/startup-profile.js +1 -1
  174. package/dist/status-project.d.ts +0 -2
  175. package/dist/status-project.js +4 -6
  176. package/dist/telemetry/events.d.ts +1 -1
  177. package/dist/telemetry/exception.js +14 -0
  178. package/dist/text-ingest.js +1 -1
  179. package/dist/tree-picker-tui.d.ts +0 -1
  180. package/dist/tree-picker-tui.js +2 -3
  181. package/package.json +1 -1
@@ -1,16 +1,8 @@
1
1
  import type { AgentRunnerPort } from '../../context/llm/runtime-port.js';
2
- import type { TouchedSlSource } from '../../context/tools/touched-sl-sources.js';
2
+ import type { ConstrainedRepairResult, RepairVerification } from './constrained-repair.js';
3
3
  import type { IngestTraceWriter } from './ingest-trace.js';
4
4
  type FinalGateRepairKind = 'patch_semantic_gate' | 'final_artifact_gate';
5
- export type FinalGateRepairResult = {
6
- status: 'repaired';
7
- attempts: number;
8
- changedPaths: string[];
9
- } | {
10
- status: 'failed';
11
- attempts: number;
12
- reason: string;
13
- };
5
+ export type FinalGateRepairResult = ConstrainedRepairResult;
14
6
  export interface RepairFinalGateFailureInput {
15
7
  agentRunner: AgentRunnerPort;
16
8
  workdir: string;
@@ -18,13 +10,19 @@ export interface RepairFinalGateFailureInput {
18
10
  allowedPaths: string[];
19
11
  trace: IngestTraceWriter;
20
12
  repairKind: FinalGateRepairKind;
13
+ /**
14
+ * Re-runs the failed gate against the current worktree. The repair counts
15
+ * as successful only when this passes — editing files is not the success
16
+ * signal.
17
+ */
18
+ verify(changedPaths: string[]): Promise<RepairVerification>;
21
19
  maxAttempts?: number;
22
20
  stepBudget?: number;
23
21
  abortSignal?: AbortSignal;
24
22
  }
25
23
  export declare function finalGateRepairPaths(input: {
26
24
  changedWikiPageKeys: string[];
27
- touchedSlSources: TouchedSlSource[];
25
+ touchedSlSourcePaths: string[];
28
26
  }): string[];
29
27
  export declare function repairFinalGateFailure(input: RepairFinalGateFailureInput): Promise<FinalGateRepairResult>;
30
28
  export {};
@@ -1,43 +1,8 @@
1
- import { mkdir, readFile, writeFile } from 'node:fs/promises';
2
- import { dirname, join } from 'node:path';
3
1
  import { z } from 'zod';
4
- import { traceTimed } from './ingest-trace.js';
5
- const readRepairFileSchema = z.object({
6
- path: z.string().min(1),
7
- });
8
- const writeRepairFileSchema = z.object({
9
- path: z.string().min(1),
10
- content: z.string(),
11
- });
12
- function normalizeRepoPath(path) {
13
- const normalized = path.replace(/\\/g, '/').replace(/^\/+/, '');
14
- const parts = normalized.split('/').filter((part) => part.length > 0);
15
- if (parts.length === 0 || parts.some((part) => part === '.' || part === '..')) {
16
- throw new Error(`gate repair path must be a repository-relative path: ${path}`);
17
- }
18
- return parts.join('/');
19
- }
20
- function assertAllowedPath(path, allowedPaths) {
21
- const normalized = normalizeRepoPath(path);
22
- if (!allowedPaths.has(normalized)) {
23
- throw new Error(`gate repair path not allowed: ${normalized}`);
24
- }
25
- return normalized;
26
- }
27
- async function readOptionalFile(path) {
28
- try {
29
- return { exists: true, content: await readFile(path, 'utf-8') };
30
- }
31
- catch (error) {
32
- if (error && typeof error === 'object' && 'code' in error && error.code === 'ENOENT') {
33
- return { exists: false, content: '' };
34
- }
35
- throw error;
36
- }
37
- }
2
+ import { runConstrainedRepairLoop } from './constrained-repair.js';
38
3
  function buildGateRepairSystemPrompt() {
39
4
  return `<role>
40
- You repair one KTX isolated-diff artifact gate failure inside the integration worktree.
5
+ You repair one ktx isolated-diff artifact gate failure inside the integration worktree.
41
6
  </role>
42
7
 
43
8
  <rules>
@@ -51,6 +16,9 @@ You repair one KTX isolated-diff artifact gate failure inside the integration wo
51
16
  </rules>`;
52
17
  }
53
18
  function buildGateRepairUserPrompt(input) {
19
+ const previousFailureBlock = input.previousFailure
20
+ ? `\nPrevious attempt did not pass the gate:\n${input.previousFailure}\n`
21
+ : '';
54
22
  return `Repair isolated-diff artifact gates.
55
23
 
56
24
  Repair kind: ${input.repairKind}
@@ -61,119 +29,63 @@ ${input.allowedPaths.map((path) => `- ${path}`).join('\n')}
61
29
 
62
30
  Gate error:
63
31
  ${input.gateError}
64
-
32
+ ${previousFailureBlock}
65
33
  Use read_gate_error first. Then inspect only the allowed files, write the
66
34
  minimal repaired content, and stop.`;
67
35
  }
68
- function buildToolSet(input) {
36
+ function buildReadGateErrorTool(gateError) {
69
37
  return {
70
38
  read_gate_error: {
71
39
  name: 'read_gate_error',
72
40
  description: 'Read the artifact gate failure that must be repaired.',
73
41
  inputSchema: z.object({}),
74
42
  execute: async () => ({
75
- markdown: input.gateError,
76
- structured: { gateError: input.gateError },
43
+ markdown: gateError,
44
+ structured: { gateError },
77
45
  }),
78
46
  },
79
- read_repair_file: {
80
- name: 'read_repair_file',
81
- description: 'Read one allowed file from the integration worktree.',
82
- inputSchema: readRepairFileSchema,
83
- execute: async ({ path }) => {
84
- const normalized = assertAllowedPath(path, input.allowedPaths);
85
- const file = await readOptionalFile(join(input.workdir, normalized));
86
- return {
87
- markdown: file.exists ? file.content : `(missing file: ${normalized})`,
88
- structured: { path: normalized, exists: file.exists },
89
- };
90
- },
91
- },
92
- write_repair_file: {
93
- name: 'write_repair_file',
94
- description: 'Replace one allowed integration worktree file with repaired text content.',
95
- inputSchema: writeRepairFileSchema,
96
- execute: async ({ path, content }) => {
97
- const normalized = assertAllowedPath(path, input.allowedPaths);
98
- const fullPath = join(input.workdir, normalized);
99
- await mkdir(dirname(fullPath), { recursive: true });
100
- await writeFile(fullPath, content, 'utf-8');
101
- input.editedPaths.add(normalized);
102
- return {
103
- markdown: `Wrote ${normalized}`,
104
- structured: { path: normalized, bytes: Buffer.byteLength(content) },
105
- };
106
- },
107
- },
108
47
  };
109
48
  }
110
49
  export function finalGateRepairPaths(input) {
111
50
  return [
112
51
  ...new Set([
113
- ...input.touchedSlSources.map((source) => `semantic-layer/${source.connectionId}/${source.sourceName}.yaml`),
52
+ ...input.touchedSlSourcePaths,
114
53
  ...input.changedWikiPageKeys.map((pageKey) => `wiki/global/${pageKey}.md`),
115
54
  ]),
116
55
  ].sort();
117
56
  }
118
57
  export async function repairFinalGateFailure(input) {
119
- const allowedPaths = new Set(input.allowedPaths.map(normalizeRepoPath));
120
- const maxAttempts = input.maxAttempts ?? 1;
121
- const stepBudget = input.stepBudget ?? 16;
122
- let lastFailure = 'gate repair did not run';
123
- for (let attempt = 1; attempt <= maxAttempts; attempt += 1) {
124
- const editedPaths = new Set();
125
- const sortedAllowedPaths = [...allowedPaths].sort();
126
- const traceData = {
58
+ return runConstrainedRepairLoop({
59
+ agentRunner: input.agentRunner,
60
+ workdir: input.workdir,
61
+ allowedPaths: input.allowedPaths,
62
+ trace: input.trace,
63
+ tracePhase: 'gate_repair',
64
+ traceEventName: 'gate_repair',
65
+ traceData: {
66
+ repairKind: input.repairKind,
67
+ gateError: input.gateError,
68
+ },
69
+ systemPrompt: buildGateRepairSystemPrompt(),
70
+ buildUserPrompt: ({ attempt, maxAttempts, previousFailure }) => buildGateRepairUserPrompt({
71
+ gateError: input.gateError,
72
+ allowedPaths: [...input.allowedPaths].sort(),
127
73
  repairKind: input.repairKind,
128
74
  attempt,
129
75
  maxAttempts,
130
- allowedPaths: sortedAllowedPaths,
131
- gateError: input.gateError,
132
- };
133
- const result = await traceTimed(input.trace, 'gate_repair', 'gate_repair', traceData, async () => input.agentRunner.runLoop({
134
- modelRole: 'repair',
135
- systemPrompt: buildGateRepairSystemPrompt(),
136
- userPrompt: buildGateRepairUserPrompt({
137
- gateError: input.gateError,
138
- allowedPaths: sortedAllowedPaths,
139
- repairKind: input.repairKind,
140
- attempt,
141
- maxAttempts,
142
- }),
143
- toolSet: buildToolSet({
144
- workdir: input.workdir,
145
- gateError: input.gateError,
146
- allowedPaths,
147
- editedPaths,
148
- }),
149
- stepBudget,
150
- telemetryTags: {
151
- operationName: 'ingest-isolated-diff-gate-repair',
152
- source: input.trace.context.sourceKey,
153
- jobId: input.trace.context.jobId,
154
- repairKind: input.repairKind,
155
- },
156
- abortSignal: input.abortSignal,
157
- }));
158
- if (result.stopReason === 'error') {
159
- lastFailure = result.error?.message ?? 'gate repair agent loop errored';
160
- await input.trace.event('error', 'gate_repair', 'gate_repair_failed', traceData, result.error);
161
- continue;
162
- }
163
- const changedPaths = [...editedPaths].sort();
164
- if (changedPaths.length === 0) {
165
- lastFailure = 'gate repair completed without editing an allowed path';
166
- await input.trace.event('error', 'gate_repair', 'gate_repair_failed', {
167
- ...traceData,
168
- reason: lastFailure,
169
- });
170
- continue;
171
- }
172
- await input.trace.event('debug', 'gate_repair', 'gate_repair_repaired', {
173
- ...traceData,
174
- changedPaths,
175
- });
176
- return { status: 'repaired', attempts: attempt, changedPaths };
177
- }
178
- return { status: 'failed', attempts: maxAttempts, reason: lastFailure };
76
+ previousFailure,
77
+ }),
78
+ buildExtraTools: () => buildReadGateErrorTool(input.gateError),
79
+ verify: input.verify,
80
+ noChangeFailureReason: 'gate repair completed without editing an allowed path',
81
+ telemetryTags: {
82
+ operationName: 'ingest-isolated-diff-gate-repair',
83
+ source: input.trace.context.sourceKey,
84
+ jobId: input.trace.context.jobId,
85
+ repairKind: input.repairKind,
86
+ },
87
+ maxAttempts: input.maxAttempts,
88
+ stepBudget: input.stepBudget ?? 16,
89
+ abortSignal: input.abortSignal,
90
+ });
179
91
  }
@@ -17,6 +17,6 @@ interface CompareFinalizationDeclarationsInput {
17
17
  derivedChangedWikiPageKeys: string[];
18
18
  }
19
19
  export declare function deriveFinalizationWikiPageKeys(paths: string[]): string[];
20
- export declare function deriveFinalizationTouchedSources(input: DeriveTouchedSourcesInput): Promise<DeriveTouchedSourcesResult>;
20
+ export declare function deriveFinalizationTouchedSources(input: DeriveTouchedSourcesInput): DeriveTouchedSourcesResult;
21
21
  export declare function compareFinalizationDeclarations(input: CompareFinalizationDeclarationsInput): IngestReportFinalizationMismatch[];
22
22
  export {};
@@ -1,3 +1,4 @@
1
+ import { isSlYamlPath } from '../../context/sl/source-files.js';
1
2
  function uniqueSorted(values) {
2
3
  return [...new Set(values.filter((value) => value.length > 0))].sort();
3
4
  }
@@ -28,32 +29,31 @@ export function deriveFinalizationWikiPageKeys(paths) {
28
29
  .filter((path) => !path.slice('wiki/global/'.length, -'.md'.length).includes('/'))
29
30
  .map((path) => path.slice('wiki/global/'.length, -'.md'.length)));
30
31
  }
31
- export async function deriveFinalizationTouchedSources(input) {
32
+ // Source identity is the in-file `name:`; filenames are derived labels (see
33
+ // source-files.ts), so a changed path — manifest shard or standalone file —
34
+ // cannot be mapped to a source by parsing its filename. Instead, every changed
35
+ // semantic-layer file is attributed through the before/after diff of its
36
+ // connection's composed sources. A changed file whose connection diff is empty
37
+ // cannot be attributed to any source and is surfaced as unresolved.
38
+ export function deriveFinalizationTouchedSources(input) {
32
39
  const touched = new Map();
33
40
  const unresolvedPaths = [];
41
+ const pathsByConnection = new Map();
34
42
  for (const path of input.changedPaths) {
35
- if (!path.startsWith('semantic-layer/') || !(path.endsWith('.yaml') || path.endsWith('.yml'))) {
43
+ if (!path.startsWith('semantic-layer/') || !isSlYamlPath(path)) {
36
44
  continue;
37
45
  }
38
- const parts = path.split('/');
39
- const connectionId = parts[1] ?? '';
46
+ const connectionId = path.split('/')[1] ?? '';
40
47
  if (!connectionId) {
41
48
  unresolvedPaths.push(path);
42
49
  continue;
43
50
  }
44
- if (parts[2] !== '_schema') {
45
- const fileName = parts.at(-1) ?? '';
46
- const sourceName = fileName.replace(/\.ya?ml$/, '');
47
- if (!sourceName) {
48
- unresolvedPaths.push(path);
49
- continue;
50
- }
51
- touched.set(`${connectionId}:${sourceName}`, { connectionId, sourceName });
52
- continue;
53
- }
51
+ pathsByConnection.set(connectionId, [...(pathsByConnection.get(connectionId) ?? []), path]);
52
+ }
53
+ for (const [connectionId, paths] of pathsByConnection) {
54
54
  const changedNames = changedSourceNames(input.beforeSourcesByConnection.get(connectionId) ?? [], input.afterSourcesByConnection.get(connectionId) ?? []);
55
55
  if (changedNames.length === 0) {
56
- unresolvedPaths.push(path);
56
+ unresolvedPaths.push(...paths);
57
57
  continue;
58
58
  }
59
59
  for (const sourceName of changedNames) {
@@ -55,6 +55,7 @@ export declare class IngestBundleRunner {
55
55
  private provenanceValidationTraceData;
56
56
  private wikiPageKeysFromPaths;
57
57
  private touchedSlSourcesFromPaths;
58
+ private touchedSlSourcePaths;
58
59
  private touchedSlSourcesFromActions;
59
60
  private wikiPageKeysFromActions;
60
61
  private uniqueWikiPageKeys;
@@ -4,6 +4,7 @@ import pLimit from 'p-limit';
4
4
  import { z } from 'zod';
5
5
  import { noopLogger } from '../../context/core/config.js';
6
6
  import { createRuntimeToolDescriptorFromAiTool } from '../../context/llm/runtime-tools.js';
7
+ import { isSlYamlPath, slSourceFilePath, slSourceNameForFile, sourceNameFromPath } from '../../context/sl/source-files.js';
7
8
  import { createTouchedSlSources } from '../../context/tools/touched-sl-sources.js';
8
9
  import { findDanglingWikiRefsForActions } from '../wiki/wiki-ref-validation.js';
9
10
  import { actionTargetConnectionId } from './action-identity.js';
@@ -351,7 +352,7 @@ export class IngestBundleRunner {
351
352
  const files = await this.deps.semanticLayerService.listFilesForConnection(connectionId);
352
353
  const names = files
353
354
  .filter((f) => !f.startsWith('_schema/'))
354
- .map((f) => f.replace(/\.yaml$/, ''))
355
+ .map((f) => sourceNameFromPath(f))
355
356
  .sort((left, right) => left.localeCompare(right));
356
357
  const body = names.length > 0 ? names.join('\n') : '(no sources yet)';
357
358
  return `## ${connectionId}\n${body}`;
@@ -584,14 +585,48 @@ export class IngestBundleRunner {
584
585
  .map((path) => path.slice('wiki/global/'.length, -'.md'.length))),
585
586
  ].sort();
586
587
  }
587
- touchedSlSourcesFromPaths(paths) {
588
- return paths
589
- .filter((path) => path.startsWith('semantic-layer/') && path.endsWith('.yaml') && !path.includes('/_schema/'))
590
- .map((path) => {
591
- const [, connectionId, fileName] = path.split('/');
592
- return { connectionId: connectionId ?? '', sourceName: (fileName ?? '').replace(/\.yaml$/, '') };
593
- })
594
- .filter((source) => source.connectionId.length > 0 && source.sourceName.length > 0);
588
+ async touchedSlSourcesFromPaths(worktree, paths, deletedFileSha) {
589
+ const sources = [];
590
+ for (const path of paths) {
591
+ if (!path.startsWith('semantic-layer/') || !isSlYamlPath(path) || path.includes('/_schema/')) {
592
+ continue;
593
+ }
594
+ const [, connectionId] = path.split('/');
595
+ if (!connectionId) {
596
+ continue;
597
+ }
598
+ // Source identity is the in-file `name:`, never the filename — an uppercase
599
+ // warehouse source like `WIDGET_SALES` lives in a hash-derived
600
+ // `widget_sales-<hash>.yaml`, so parsing the basename yields a phantom name.
601
+ // Read the live file; when it was deleted this run, recover its declared
602
+ // name from the pre-change commit the way `revertSourceToPreHead` resolves a
603
+ // gone file from history. The filename is a last resort only when the content
604
+ // is unrecoverable from both.
605
+ let content;
606
+ try {
607
+ content = await readFile(join(worktree.workdir, path), 'utf-8');
608
+ }
609
+ catch {
610
+ content = await worktree.git.getFileAtCommit(path, deletedFileSha).catch(() => null);
611
+ }
612
+ const sourceName = content === null ? sourceNameFromPath(path) : slSourceNameForFile(path, content);
613
+ if (sourceName.length > 0) {
614
+ sources.push({ connectionId, sourceName });
615
+ }
616
+ }
617
+ return sources;
618
+ }
619
+ // Inverse direction for commits and repair allowlists: resolve each touched
620
+ // source to its real on-disk path, falling back to the writer's derived
621
+ // filename when the file was deleted in this run.
622
+ async touchedSlSourcePaths(workdir, touched) {
623
+ const service = this.deps.semanticLayerService.forWorktree(workdir);
624
+ const paths = [];
625
+ for (const source of touched) {
626
+ const file = await service.readSourceFile(source.connectionId, source.sourceName);
627
+ paths.push(file?.path ?? slSourceFilePath(source.connectionId, source.sourceName));
628
+ }
629
+ return paths;
595
630
  }
596
631
  touchedSlSourcesFromActions(actions, fallbackConnectionId) {
597
632
  return actions
@@ -1153,7 +1188,7 @@ export class IngestBundleRunner {
1153
1188
  projectionTouchedSources = projection.touchedSources;
1154
1189
  projectionChangedWikiPageKeys = projection.changedWikiPageKeys;
1155
1190
  const projectionPaths = [
1156
- ...projection.touchedSources.map((source) => `semantic-layer/${source.connectionId}/${source.sourceName}.yaml`),
1191
+ ...(await this.touchedSlSourcePaths(sessionWorktree.workdir, projection.touchedSources)),
1157
1192
  ...projection.changedWikiPageKeys.map((pageKey) => `wiki/global/${pageKey}.md`),
1158
1193
  ];
1159
1194
  projectionTouchedPaths = projectionPaths;
@@ -1297,7 +1332,7 @@ export class IngestBundleRunner {
1297
1332
  await validateFinalIngestArtifacts({
1298
1333
  connectionIds: slConnectionIds,
1299
1334
  changedWikiPageKeys: this.wikiPageKeysFromPaths(touchedPaths),
1300
- touchedSlSources: this.touchedSlSourcesFromPaths(touchedPaths),
1335
+ touchedSlSources: await this.touchedSlSourcesFromPaths(sessionWorktree, touchedPaths, await sessionWorktree.git.revParseHead()),
1301
1336
  wikiService: this.deps.wikiService.forWorktree(sessionWorktree.workdir),
1302
1337
  semanticLayerService: this.deps.semanticLayerService.forWorktree(sessionWorktree.workdir),
1303
1338
  validateTouchedSources: (touched) => validateWuTouchedSources({
@@ -1322,7 +1357,8 @@ export class IngestBundleRunner {
1322
1357
  touchedPaths: context.touchedPaths,
1323
1358
  trace: runTrace,
1324
1359
  reason: context.reason,
1325
- maxAttempts: 1,
1360
+ verify: context.verify,
1361
+ maxAttempts: 2,
1326
1362
  stepBudget: 12,
1327
1363
  abortSignal: ctx?.abortSignal,
1328
1364
  });
@@ -1340,7 +1376,8 @@ export class IngestBundleRunner {
1340
1376
  allowedPaths: context.touchedPaths,
1341
1377
  trace: runTrace,
1342
1378
  repairKind: 'patch_semantic_gate',
1343
- maxAttempts: 1,
1379
+ verify: context.verify,
1380
+ maxAttempts: 2,
1344
1381
  stepBudget: 16,
1345
1382
  abortSignal: ctx?.abortSignal,
1346
1383
  });
@@ -1748,17 +1785,24 @@ export class IngestBundleRunner {
1748
1785
  preFinalizationSha !== postFinalizationSha
1749
1786
  ? (await sessionWorktree.git.diffNameStatus(preFinalizationSha, postFinalizationSha)).map((entry) => entry.path)
1750
1787
  : [];
1751
- const changedConnectionIds = [
1752
- ...new Set([
1753
- ...slConnectionIds,
1754
- ...finalizationTouchedPaths
1755
- .filter((path) => path.startsWith('semantic-layer/'))
1756
- .map((path) => path.split('/')[1])
1757
- .filter((connectionId) => Boolean(connectionId)),
1758
- ]),
1759
- ].sort();
1760
- const postFinalizationSourcesByConnection = await this.loadSourcesByConnection(sessionWorktree.workdir, changedConnectionIds);
1761
- const scope = await deriveFinalizationTouchedSources({
1788
+ // Validate the write scope before deriving touched sources: attribution
1789
+ // by before/after diff is only defined for connections whose
1790
+ // pre-finalization snapshot was loaded (slConnectionIds), and an
1791
+ // out-of-scope write would otherwise surface downstream as a bogus
1792
+ // unresolved-path or declaration-mismatch failure instead of the real
1793
+ // policy violation.
1794
+ await traceTimed(runTrace, 'finalization', 'semantic_layer_target_policy', {
1795
+ sourceKey: job.sourceKey,
1796
+ allowedTargetConnectionIds: slConnectionIds,
1797
+ touchedPaths: [...new Set(finalizationTouchedPaths)].sort(),
1798
+ }, async () => {
1799
+ assertSemanticLayerTargetPathsAllowed({
1800
+ paths: finalizationTouchedPaths,
1801
+ allowedConnectionIds: new Set(slConnectionIds),
1802
+ });
1803
+ });
1804
+ const postFinalizationSourcesByConnection = await this.loadSourcesByConnection(sessionWorktree.workdir, slConnectionIds);
1805
+ const scope = deriveFinalizationTouchedSources({
1762
1806
  changedPaths: finalizationTouchedPaths,
1763
1807
  beforeSourcesByConnection: preFinalizationSourcesByConnection,
1764
1808
  afterSourcesByConnection: postFinalizationSourcesByConnection,
@@ -1876,7 +1920,7 @@ export class IngestBundleRunner {
1876
1920
  ...(isolatedDiffEnabled ? projectionTouchedSources : []),
1877
1921
  ...workUnitOutcomes.flatMap((outcome) => outcome.touchedSlSources),
1878
1922
  ...this.touchedSlSourcesFromActions(reconcileActions, job.connectionId),
1879
- ...this.touchedSlSourcesFromPaths(postReconciliationPaths),
1923
+ ...(await this.touchedSlSourcesFromPaths(sessionWorktree, postReconciliationPaths, preReconciliationSha)),
1880
1924
  ...finalizationTouchedSources,
1881
1925
  ]);
1882
1926
  const finalWikiGateScope = await this.wikiPageKeysForFinalGates({
@@ -1926,32 +1970,33 @@ export class IngestBundleRunner {
1926
1970
  activePhase = 'final_gates';
1927
1971
  activeFailureDetails = finalArtifactGateTraceData;
1928
1972
  emitStageProgress('final_gates', 89, 'Running final artifact gates');
1929
- try {
1930
- await traceTimed(runTrace, 'final_gates', 'final_artifact_gates', finalArtifactGateTraceData, async () => {
1931
- await validateFinalIngestArtifacts({
1932
- connectionIds: repairConnectionIds,
1933
- changedWikiPageKeys: finalChangedWikiPageKeys,
1934
- touchedSlSources: finalTouchedSlSources,
1935
- wikiService: this.deps.wikiService.forWorktree(sessionWorktree.workdir),
1973
+ const runFinalArtifactGates = async () => {
1974
+ await validateFinalIngestArtifacts({
1975
+ connectionIds: repairConnectionIds,
1976
+ changedWikiPageKeys: finalChangedWikiPageKeys,
1977
+ touchedSlSources: finalTouchedSlSources,
1978
+ wikiService: this.deps.wikiService.forWorktree(sessionWorktree.workdir),
1979
+ semanticLayerService: this.deps.semanticLayerService.forWorktree(sessionWorktree.workdir),
1980
+ validateTouchedSources: (touched) => validateWuTouchedSources({
1936
1981
  semanticLayerService: this.deps.semanticLayerService.forWorktree(sessionWorktree.workdir),
1937
- validateTouchedSources: (touched) => validateWuTouchedSources({
1938
- semanticLayerService: this.deps.semanticLayerService.forWorktree(sessionWorktree.workdir),
1939
- connections: this.deps.connections,
1940
- configService: sessionWorktree.config,
1941
- gitService: sessionWorktree.git,
1942
- slSourcesRepository: this.deps.slSourcesRepository,
1943
- probeRowCount: this.deps.settings.probeRowCount,
1944
- slValidator: this.deps.slValidator,
1945
- }, touched),
1946
- tableExists: (connectionId, tableRef) => this.tableRefExistsInSemanticLayer(this.deps.semanticLayerService.forWorktree(sessionWorktree.workdir), [connectionId], tableRef),
1947
- });
1982
+ connections: this.deps.connections,
1983
+ configService: sessionWorktree.config,
1984
+ gitService: sessionWorktree.git,
1985
+ slSourcesRepository: this.deps.slSourcesRepository,
1986
+ probeRowCount: this.deps.settings.probeRowCount,
1987
+ slValidator: this.deps.slValidator,
1988
+ }, touched),
1989
+ tableExists: (connectionId, tableRef) => this.tableRefExistsInSemanticLayer(this.deps.semanticLayerService.forWorktree(sessionWorktree.workdir), [connectionId], tableRef),
1948
1990
  });
1991
+ };
1992
+ try {
1993
+ await traceTimed(runTrace, 'final_gates', 'final_artifact_gates', finalArtifactGateTraceData, runFinalArtifactGates);
1949
1994
  }
1950
1995
  catch (error) {
1951
1996
  const gateError = this.errorMessage(error);
1952
1997
  const repairPaths = finalGateRepairPaths({
1953
1998
  changedWikiPageKeys: finalChangedWikiPageKeys,
1954
- touchedSlSources: finalTouchedSlSources,
1999
+ touchedSlSourcePaths: await this.touchedSlSourcePaths(sessionWorktree.workdir, finalTouchedSlSources),
1955
2000
  });
1956
2001
  emitStageProgress('final_gates', 89, 'Repairing final artifact gates');
1957
2002
  const gateRepair = await repairFinalGateFailure({
@@ -1961,7 +2006,16 @@ export class IngestBundleRunner {
1961
2006
  allowedPaths: repairPaths,
1962
2007
  trace: runTrace,
1963
2008
  repairKind: 'final_artifact_gate',
1964
- maxAttempts: 1,
2009
+ verify: async () => {
2010
+ try {
2011
+ await runFinalArtifactGates();
2012
+ return { ok: true };
2013
+ }
2014
+ catch (verifyError) {
2015
+ return { ok: false, reason: this.errorMessage(verifyError) };
2016
+ }
2017
+ },
2018
+ maxAttempts: 2,
1965
2019
  stepBudget: 16,
1966
2020
  abortSignal: ctx?.abortSignal,
1967
2021
  });
@@ -1975,29 +2029,9 @@ export class IngestBundleRunner {
1975
2029
  };
1976
2030
  throw new Error(`${gateError}\ngate repair failed: ${gateRepair.reason}`);
1977
2031
  }
2032
+ // The repair loop re-ran the gates via `verify` before reporting
2033
+ // success, so a repaired status here means the tree already passed.
1978
2034
  isolatedDiffSummary.gateRepairs += 1;
1979
- await traceTimed(runTrace, 'final_gates', 'final_artifact_gates_after_gate_repair', {
1980
- ...finalArtifactGateTraceData,
1981
- repairedPaths: gateRepair.changedPaths,
1982
- }, async () => {
1983
- await validateFinalIngestArtifacts({
1984
- connectionIds: repairConnectionIds,
1985
- changedWikiPageKeys: finalChangedWikiPageKeys,
1986
- touchedSlSources: finalTouchedSlSources,
1987
- wikiService: this.deps.wikiService.forWorktree(sessionWorktree.workdir),
1988
- semanticLayerService: this.deps.semanticLayerService.forWorktree(sessionWorktree.workdir),
1989
- validateTouchedSources: (touched) => validateWuTouchedSources({
1990
- semanticLayerService: this.deps.semanticLayerService.forWorktree(sessionWorktree.workdir),
1991
- connections: this.deps.connections,
1992
- configService: sessionWorktree.config,
1993
- gitService: sessionWorktree.git,
1994
- slSourcesRepository: this.deps.slSourcesRepository,
1995
- probeRowCount: this.deps.settings.probeRowCount,
1996
- slValidator: this.deps.slValidator,
1997
- }, touched),
1998
- tableExists: (connectionId, tableRef) => this.tableRefExistsInSemanticLayer(this.deps.semanticLayerService.forWorktree(sessionWorktree.workdir), [connectionId], tableRef),
1999
- });
2000
- });
2001
2035
  const repairCommit = await sessionWorktree.git.commitFiles(gateRepair.changedPaths, `ingest(${job.sourceKey}): repair final gates syncId=${syncId}`, this.deps.storage.systemGitAuthor.name, this.deps.storage.systemGitAuthor.email);
2002
2036
  if (!repairCommit.created) {
2003
2037
  isolatedDiffSummary.gateRepairFailures += 1;
@@ -1,33 +1,25 @@
1
1
  import type { GitService } from '../../../context/core/git.service.js';
2
+ import type { RepairVerification } from '../constrained-repair.js';
2
3
  import type { FinalGateRepairResult } from '../final-gate-repair.js';
3
4
  import type { IngestTraceWriter } from '../ingest-trace.js';
4
5
  import type { TextualConflictResolutionResult } from './textual-conflict-resolver.js';
5
- type PatchIntegrationTextualResolution = {
6
- status: 'repaired';
7
- attempts: number;
8
- changedPaths: string[];
9
- } | {
10
- status: 'failed';
11
- attempts: number;
12
- reason: string;
13
- };
14
6
  export type PatchIntegrationResult = {
15
7
  status: 'accepted';
16
8
  commitSha: string;
17
9
  touchedPaths: string[];
18
- textualResolution?: PatchIntegrationTextualResolution;
10
+ textualResolution?: TextualConflictResolutionResult;
19
11
  gateRepair?: FinalGateRepairResult;
20
12
  } | {
21
13
  status: 'textual_conflict';
22
14
  reason: string;
23
15
  touchedPaths: string[];
24
- textualResolution?: PatchIntegrationTextualResolution;
16
+ textualResolution?: TextualConflictResolutionResult;
25
17
  gateRepair?: FinalGateRepairResult;
26
18
  } | {
27
19
  status: 'semantic_conflict';
28
20
  reason: string;
29
21
  touchedPaths: string[];
30
- textualResolution?: PatchIntegrationTextualResolution;
22
+ textualResolution?: TextualConflictResolutionResult;
31
23
  gateRepair?: FinalGateRepairResult;
32
24
  };
33
25
  export interface IntegrateWorkUnitPatchInput {
@@ -47,13 +39,14 @@ export interface IntegrateWorkUnitPatchInput {
47
39
  patchPath: string;
48
40
  touchedPaths: string[];
49
41
  reason: string;
42
+ verify(changedPaths: string[]): Promise<RepairVerification>;
50
43
  }): Promise<TextualConflictResolutionResult>;
51
44
  repairGateFailure?(input: {
52
45
  unitKey: string;
53
46
  patchPath: string;
54
47
  touchedPaths: string[];
55
48
  reason: string;
49
+ verify(changedPaths: string[]): Promise<RepairVerification>;
56
50
  }): Promise<FinalGateRepairResult>;
57
51
  }
58
52
  export declare function integrateWorkUnitPatch(input: IntegrateWorkUnitPatchInput): Promise<PatchIntegrationResult>;
59
- export {};