@kaelio/ktx 0.11.0 → 0.13.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (212) hide show
  1. package/assets/python/kaelio_ktx-0.13.0-py3-none-any.whl +0 -0
  2. package/assets/python/manifest.json +4 -4
  3. package/dist/.tsbuildinfo +1 -1
  4. package/dist/admin.js +1 -1
  5. package/dist/clack.d.ts +16 -0
  6. package/dist/clack.js +37 -6
  7. package/dist/claude-code-prompt-caching.js +1 -1
  8. package/dist/cli-program.js +3 -3
  9. package/dist/cli-runtime.js +2 -2
  10. package/dist/commands/connection-commands.js +1 -1
  11. package/dist/commands/ingest-commands.js +4 -4
  12. package/dist/commands/mcp-commands.js +12 -12
  13. package/dist/commands/runtime-commands.js +4 -4
  14. package/dist/commands/setup-commands.js +19 -5
  15. package/dist/commands/sl-commands.js +1 -1
  16. package/dist/commands/sql-commands.js +1 -1
  17. package/dist/commands/status-commands.js +1 -1
  18. package/dist/connection.js +15 -3
  19. package/dist/connectors/bigquery/connector.js +1 -14
  20. package/dist/connectors/clickhouse/connector.js +2 -16
  21. package/dist/connectors/duckdb/federated-attach.d.ts +7 -0
  22. package/dist/connectors/duckdb/federated-attach.js +86 -0
  23. package/dist/connectors/duckdb/federated-executor.d.ts +5 -0
  24. package/dist/connectors/duckdb/federated-executor.js +59 -0
  25. package/dist/connectors/mysql/connector.js +2 -16
  26. package/dist/connectors/postgres/connector.js +1 -14
  27. package/dist/connectors/shared/string-reference.d.ts +6 -0
  28. package/dist/connectors/shared/string-reference.js +19 -0
  29. package/dist/connectors/snowflake/connector.d.ts +1 -1
  30. package/dist/connectors/snowflake/connector.js +1 -14
  31. package/dist/connectors/sqlite/connector.js +2 -25
  32. package/dist/connectors/sqlserver/connector.js +4 -17
  33. package/dist/context/connections/connection-type.d.ts +1 -1
  34. package/dist/context/connections/federation.d.ts +33 -0
  35. package/dist/context/connections/federation.js +51 -0
  36. package/dist/context/connections/local-warehouse-descriptor.d.ts +2 -0
  37. package/dist/context/connections/project-sql-executor.d.ts +18 -0
  38. package/dist/context/connections/project-sql-executor.js +39 -0
  39. package/dist/context/connections/query-executor.d.ts +2 -2
  40. package/dist/context/connections/read-only-sql.d.ts +1 -0
  41. package/dist/context/connections/read-only-sql.js +119 -4
  42. package/dist/context/connections/resolve-connection.d.ts +12 -0
  43. package/dist/context/connections/resolve-connection.js +37 -0
  44. package/dist/context/core/git-env.d.ts +4 -0
  45. package/dist/context/core/git-env.js +5 -1
  46. package/dist/context/core/git.service.d.ts +23 -0
  47. package/dist/context/core/git.service.js +71 -8
  48. package/dist/context/ingest/adapters/historic-sql/projection.js +2 -1
  49. package/dist/context/ingest/adapters/live-database/manifest.d.ts +3 -0
  50. package/dist/context/ingest/adapters/live-database/manifest.js +19 -11
  51. package/dist/context/ingest/adapters/looker/client.js +7 -2
  52. package/dist/context/ingest/adapters/looker/factory.d.ts +8 -1
  53. package/dist/context/ingest/adapters/looker/factory.js +9 -0
  54. package/dist/context/ingest/adapters/looker/mapping.js +1 -1
  55. package/dist/context/ingest/adapters/looker/types.d.ts +1 -1
  56. package/dist/context/ingest/adapters/metabase/client.d.ts +1 -1
  57. package/dist/context/ingest/adapters/metabase/client.js +1 -1
  58. package/dist/context/ingest/adapters/metabase/local-metabase.adapter.js +1 -1
  59. package/dist/context/ingest/adapters/metabase/mapping.js +6 -6
  60. package/dist/context/ingest/artifact-gates.d.ts +2 -6
  61. package/dist/context/ingest/artifact-gates.js +5 -47
  62. package/dist/context/ingest/constrained-repair.d.ts +55 -0
  63. package/dist/context/ingest/constrained-repair.js +167 -0
  64. package/dist/context/ingest/final-gate-repair.d.ts +9 -11
  65. package/dist/context/ingest/final-gate-repair.js +40 -128
  66. package/dist/context/ingest/finalization-scope.d.ts +1 -1
  67. package/dist/context/ingest/finalization-scope.js +15 -15
  68. package/dist/context/ingest/ingest-bundle.runner.d.ts +1 -0
  69. package/dist/context/ingest/ingest-bundle.runner.js +101 -67
  70. package/dist/context/ingest/isolated-diff/patch-integrator.d.ts +6 -13
  71. package/dist/context/ingest/isolated-diff/patch-integrator.js +32 -109
  72. package/dist/context/ingest/isolated-diff/textual-conflict-resolver.d.ts +8 -9
  73. package/dist/context/ingest/isolated-diff/textual-conflict-resolver.js +63 -141
  74. package/dist/context/ingest/local-bundle-runtime.d.ts +2 -0
  75. package/dist/context/ingest/local-bundle-runtime.js +9 -10
  76. package/dist/context/ingest/local-ingest.d.ts +2 -0
  77. package/dist/context/ingest/local-ingest.js +2 -0
  78. package/dist/context/ingest/memory-flow/view-model.js +1 -1
  79. package/dist/context/ingest/stages/stage-3-work-units.d.ts +2 -6
  80. package/dist/context/ingest/stages/stage-3-work-units.js +2 -1
  81. package/dist/context/ingest/stages/validate-wu-sources.d.ts +7 -1
  82. package/dist/context/ingest/stages/validate-wu-sources.js +109 -4
  83. package/dist/context/ingest/tools/warehouse-verification/create-warehouse-verification-tools.d.ts +2 -0
  84. package/dist/context/ingest/tools/warehouse-verification/create-warehouse-verification-tools.js +1 -1
  85. package/dist/context/ingest/tools/warehouse-verification/discover-data.tool.js +3 -3
  86. package/dist/context/ingest/tools/warehouse-verification/sql-execution.tool.d.ts +3 -1
  87. package/dist/context/ingest/tools/warehouse-verification/sql-execution.tool.js +15 -1
  88. package/dist/context/llm/ai-sdk-runtime.js +2 -2
  89. package/dist/context/llm/claude-code-runtime.js +19 -3
  90. package/dist/context/llm/local-config.js +1 -1
  91. package/dist/context/llm/runtime-tools.js +2 -2
  92. package/dist/context/mcp/context-tools.js +33 -8
  93. package/dist/context/mcp/local-project-ports.js +63 -89
  94. package/dist/context/mcp/types.d.ts +2 -0
  95. package/dist/context/memory/local-memory.js +4 -1
  96. package/dist/context/memory/memory-agent.service.js +1 -1
  97. package/dist/context/project/config.d.ts +11 -4
  98. package/dist/context/project/config.js +85 -30
  99. package/dist/context/project/driver-schemas.js +1 -1
  100. package/dist/context/project/mappings-yaml-schema.js +2 -2
  101. package/dist/context/project/project.js +12 -4
  102. package/dist/context/scan/description-generation.js +4 -4
  103. package/dist/context/scan/local-enrichment-artifacts.js +33 -4
  104. package/dist/context/scan/local-scan.js +2 -2
  105. package/dist/context/scan/local-structural-artifacts.js +5 -5
  106. package/dist/context/scan/relationship-benchmark-report.js +1 -1
  107. package/dist/context/scan/relationship-discovery.js +3 -3
  108. package/dist/context/scan/relationship-llm-proposal.js +3 -3
  109. package/dist/context/sl/local-query.js +31 -44
  110. package/dist/context/sl/local-sl.d.ts +0 -8
  111. package/dist/context/sl/local-sl.js +71 -70
  112. package/dist/context/sl/semantic-layer.service.d.ts +25 -8
  113. package/dist/context/sl/semantic-layer.service.js +109 -56
  114. package/dist/context/sl/source-files.d.ts +48 -0
  115. package/dist/context/sl/source-files.js +138 -0
  116. package/dist/context/sl/tools/base-semantic-layer.tool.d.ts +2 -2
  117. package/dist/context/sl/tools/base-semantic-layer.tool.js +2 -7
  118. package/dist/context/sl/tools/sl-edit-source.tool.js +10 -8
  119. package/dist/context/sl/tools/sl-warehouse-validation.js +55 -27
  120. package/dist/context/sl/tools/sl-write-source.tool.js +12 -9
  121. package/dist/context/sql-analysis/dialect.d.ts +2 -0
  122. package/dist/context/sql-analysis/dialect.js +20 -0
  123. package/dist/context/tools/base-tool.d.ts +6 -19
  124. package/dist/context/tools/base-tool.js +0 -14
  125. package/dist/context-build-view.js +5 -5
  126. package/dist/database-tree-picker.js +18 -3
  127. package/dist/demo-assets.js +0 -1
  128. package/dist/doctor.d.ts +1 -1
  129. package/dist/doctor.js +31 -23
  130. package/dist/errors.d.ts +31 -0
  131. package/dist/errors.js +44 -0
  132. package/dist/ingest-query-executor.d.ts +2 -0
  133. package/dist/ingest-query-executor.js +8 -22
  134. package/dist/ingest.d.ts +1 -1
  135. package/dist/ingest.js +8 -2
  136. package/dist/io/symbols.d.ts +2 -0
  137. package/dist/io/symbols.js +2 -0
  138. package/dist/io/tty.d.ts +8 -0
  139. package/dist/io/tty.js +16 -0
  140. package/dist/llm/embedding-health.js +1 -1
  141. package/dist/llm/embedding-provider.js +3 -3
  142. package/dist/llm/model-provider.js +1 -1
  143. package/dist/local-adapters.d.ts +1 -0
  144. package/dist/local-adapters.js +2 -2
  145. package/dist/local-scan-connectors.js +1 -1
  146. package/dist/managed-local-embeddings.js +17 -8
  147. package/dist/managed-mcp-daemon.js +3 -3
  148. package/dist/managed-python-command.d.ts +7 -0
  149. package/dist/managed-python-command.js +34 -8
  150. package/dist/managed-python-daemon.js +2 -2
  151. package/dist/managed-python-http.js +3 -3
  152. package/dist/managed-python-runtime.d.ts +30 -1
  153. package/dist/managed-python-runtime.js +134 -18
  154. package/dist/managed-uv-release.d.ts +7 -0
  155. package/dist/managed-uv-release.js +11 -0
  156. package/dist/mcp-http-server.js +4 -4
  157. package/dist/mcp-server-factory.js +3 -3
  158. package/dist/mcp-stdio-server.js +1 -1
  159. package/dist/memory-flow-hud.js +2 -2
  160. package/dist/next-steps.js +2 -2
  161. package/dist/prompt-navigation.d.ts +17 -0
  162. package/dist/prompt-navigation.js +49 -3
  163. package/dist/prompts/memory_agent_bundle_ingest_work_unit.md +2 -2
  164. package/dist/prompts/memory_agent_external_ingest.md +2 -2
  165. package/dist/public-ingest-copy.js +1 -1
  166. package/dist/public-ingest.js +3 -3
  167. package/dist/release-version.js +1 -1
  168. package/dist/runtime-requirements.js +1 -1
  169. package/dist/runtime.js +9 -9
  170. package/dist/scan.js +1 -1
  171. package/dist/setup-agents.d.ts +21 -15
  172. package/dist/setup-agents.js +143 -66
  173. package/dist/setup-banner.d.ts +20 -0
  174. package/dist/setup-banner.js +39 -0
  175. package/dist/setup-context.js +24 -15
  176. package/dist/setup-databases.d.ts +3 -0
  177. package/dist/setup-databases.js +47 -59
  178. package/dist/setup-demo-tour.js +12 -8
  179. package/dist/setup-embeddings.js +9 -9
  180. package/dist/setup-interrupt.js +1 -1
  181. package/dist/setup-models.d.ts +4 -1
  182. package/dist/setup-models.js +54 -28
  183. package/dist/setup-project.js +29 -5
  184. package/dist/setup-prompts.js +16 -1
  185. package/dist/setup-ready-menu.js +1 -1
  186. package/dist/setup-sources.js +28 -12
  187. package/dist/setup.d.ts +1 -0
  188. package/dist/setup.js +14 -13
  189. package/dist/skills/analytics/SKILL.md +3 -3
  190. package/dist/skills/dbt_ingest/SKILL.md +3 -3
  191. package/dist/skills/looker_ingest/SKILL.md +3 -3
  192. package/dist/skills/lookml_ingest/SKILL.md +7 -7
  193. package/dist/skills/metabase_ingest/SKILL.md +4 -4
  194. package/dist/skills/metricflow_ingest/SKILL.md +15 -15
  195. package/dist/skills/notion_synthesize/SKILL.md +1 -1
  196. package/dist/skills/sl/SKILL.md +3 -3
  197. package/dist/skills/sl_capture/SKILL.md +1 -1
  198. package/dist/skills/wiki_capture/SKILL.md +1 -1
  199. package/dist/source-mapping.js +1 -1
  200. package/dist/sql.d.ts +2 -0
  201. package/dist/sql.js +35 -53
  202. package/dist/startup-profile.js +1 -1
  203. package/dist/status-project.d.ts +0 -2
  204. package/dist/status-project.js +4 -6
  205. package/dist/telemetry/events.d.ts +3 -2
  206. package/dist/telemetry/events.js +11 -1
  207. package/dist/telemetry/exception.js +14 -0
  208. package/dist/text-ingest.js +1 -1
  209. package/dist/tree-picker-tui.d.ts +0 -1
  210. package/dist/tree-picker-tui.js +2 -3
  211. package/package.json +2 -1
  212. package/assets/python/kaelio_ktx-0.11.0-py3-none-any.whl +0 -0
@@ -1,58 +1,26 @@
1
- import { mkdir, readFile, rm, writeFile } from 'node:fs/promises';
2
- import { dirname, join } from 'node:path';
1
+ import { readFile } from 'node:fs/promises';
3
2
  import { z } from 'zod';
4
- import { traceTimed } from '../ingest-trace.js';
5
- const readIntegrationFileSchema = z.object({
6
- path: z.string().min(1),
7
- });
8
- const writeIntegrationFileSchema = z.object({
9
- path: z.string().min(1),
10
- content: z.string(),
11
- });
12
- const deleteIntegrationFileSchema = z.object({
13
- path: z.string().min(1),
14
- });
15
- function normalizeRepoPath(path) {
16
- const normalized = path.replace(/\\/g, '/').replace(/^\/+/, '');
17
- const parts = normalized.split('/').filter((part) => part.length > 0);
18
- if (parts.length === 0 || parts.some((part) => part === '.' || part === '..')) {
19
- throw new Error(`resolver path must be a repository-relative path: ${path}`);
20
- }
21
- return parts.join('/');
22
- }
23
- function assertAllowedPath(path, allowedPaths) {
24
- const normalized = normalizeRepoPath(path);
25
- if (!allowedPaths.has(normalized)) {
26
- throw new Error(`resolver path not allowed: ${normalized}`);
27
- }
28
- return normalized;
29
- }
30
- async function readOptionalFile(path) {
31
- try {
32
- return { exists: true, content: await readFile(path, 'utf-8') };
33
- }
34
- catch (error) {
35
- if (error && typeof error === 'object' && 'code' in error && error.code === 'ENOENT') {
36
- return { exists: false, content: '' };
37
- }
38
- throw error;
39
- }
40
- }
3
+ import { buildDeleteRepairFileTool, runConstrainedRepairLoop } from '../constrained-repair.js';
41
4
  function buildResolverSystemPrompt() {
42
5
  return `<role>
43
- You repair one failed KTX isolated-diff patch inside the integration worktree.
6
+ You repair one failed ktx isolated-diff patch inside the integration worktree.
44
7
  </role>
45
8
 
46
9
  <rules>
47
10
  - Preserve accepted integration content that is unrelated to the failed patch.
48
11
  - Incorporate the failed patch only when the patch evidence is compatible with the current file.
12
+ - If the current file already represents everything the failed patch contributes (for example a
13
+ duplicate page created by another work unit), call declare_patch_redundant instead of editing.
49
14
  - Edit only paths exposed by the resolver tools.
50
15
  - Prefer the smallest text edit that makes the composed artifact coherent.
51
16
  - Do not create new facts that are absent from the current file or failed patch.
52
- - Stop after writing the repaired file content.
17
+ - Stop after writing the repaired file content or declaring the patch redundant.
53
18
  </rules>`;
54
19
  }
55
20
  function buildResolverUserPrompt(input) {
21
+ const previousFailureBlock = input.previousFailure
22
+ ? `\nPrevious attempt did not pass the artifact gates:\n${input.previousFailure}\n`
23
+ : '';
56
24
  return `Repair isolated-diff textual conflict.
57
25
 
58
26
  WorkUnit: ${input.unitKey}
@@ -63,11 +31,18 @@ ${input.touchedPaths.map((path) => `- ${path}`).join('\n')}
63
31
 
64
32
  Git apply failure:
65
33
  ${input.reason}
66
-
67
- Use read_failed_patch first. Then read the touched integration files, write the
68
- repaired content, and stop.`;
34
+ ${previousFailureBlock}
35
+ Use read_failed_patch first. Then read the touched integration files and either
36
+ write the repaired content or, when the patch adds nothing the current files do
37
+ not already cover, call declare_patch_redundant. Then stop.`;
69
38
  }
70
- function buildToolSet(input) {
39
+ function buildResolverExtraTools(input) {
40
+ const declareSchema = z.object({
41
+ reason: z
42
+ .string()
43
+ .min(1)
44
+ .describe('Why the integration tree already represents everything this patch contributes.'),
45
+ });
71
46
  return {
72
47
  read_failed_patch: {
73
48
  name: 'read_failed_patch',
@@ -81,111 +56,58 @@ function buildToolSet(input) {
81
56
  };
82
57
  },
83
58
  },
84
- read_integration_file: {
85
- name: 'read_integration_file',
86
- description: 'Read one allowed file from the current integration worktree.',
87
- inputSchema: readIntegrationFileSchema,
88
- execute: async ({ path }) => {
89
- const normalized = assertAllowedPath(path, input.allowedPaths);
90
- const file = await readOptionalFile(join(input.workdir, normalized));
59
+ ...buildDeleteRepairFileTool(input.context),
60
+ declare_patch_redundant: {
61
+ name: 'declare_patch_redundant',
62
+ description: 'Declare that the failed patch needs no integration because the current worktree already ' +
63
+ 'represents its content (for example a duplicate page created by another work unit).',
64
+ inputSchema: declareSchema,
65
+ execute: async ({ reason }) => {
66
+ input.context.declareNoChange(reason);
91
67
  return {
92
- markdown: file.exists ? file.content : `(missing file: ${normalized})`,
93
- structured: { path: normalized, exists: file.exists },
94
- };
95
- },
96
- },
97
- write_integration_file: {
98
- name: 'write_integration_file',
99
- description: 'Replace one allowed integration worktree file with repaired text content.',
100
- inputSchema: writeIntegrationFileSchema,
101
- execute: async ({ path, content }) => {
102
- const normalized = assertAllowedPath(path, input.allowedPaths);
103
- const fullPath = join(input.workdir, normalized);
104
- await mkdir(dirname(fullPath), { recursive: true });
105
- await writeFile(fullPath, content, 'utf-8');
106
- input.editedPaths.add(normalized);
107
- return {
108
- markdown: `Wrote ${normalized}`,
109
- structured: { path: normalized, bytes: Buffer.byteLength(content) },
110
- };
111
- },
112
- },
113
- delete_integration_file: {
114
- name: 'delete_integration_file',
115
- description: 'Delete one allowed integration worktree file when the failed patch proves the deletion is correct.',
116
- inputSchema: deleteIntegrationFileSchema,
117
- execute: async ({ path }) => {
118
- const normalized = assertAllowedPath(path, input.allowedPaths);
119
- await rm(join(input.workdir, normalized), { force: true });
120
- input.editedPaths.add(normalized);
121
- return {
122
- markdown: `Deleted ${normalized}`,
123
- structured: { path: normalized },
68
+ markdown: `Declared patch redundant: ${reason}`,
69
+ structured: { reason },
124
70
  };
125
71
  },
126
72
  },
127
73
  };
128
74
  }
129
75
  export async function resolveTextualConflict(input) {
130
- const allowedPaths = new Set(input.touchedPaths.map(normalizeRepoPath));
131
- const maxAttempts = input.maxAttempts ?? 1;
132
- const stepBudget = input.stepBudget ?? 12;
133
- let lastFailure = 'resolver did not run';
134
- for (let attempt = 1; attempt <= maxAttempts; attempt += 1) {
135
- const editedPaths = new Set();
136
- const traceData = {
76
+ const sortedTouchedPaths = [...input.touchedPaths].sort();
77
+ return runConstrainedRepairLoop({
78
+ agentRunner: input.agentRunner,
79
+ workdir: input.workdir,
80
+ allowedPaths: input.touchedPaths,
81
+ trace: input.trace,
82
+ tracePhase: 'resolver',
83
+ traceEventName: 'textual_conflict_resolver',
84
+ traceData: {
85
+ unitKey: input.unitKey,
86
+ patchPath: input.patchPath,
87
+ touchedPaths: sortedTouchedPaths,
88
+ reason: input.reason,
89
+ },
90
+ systemPrompt: buildResolverSystemPrompt(),
91
+ buildUserPrompt: ({ attempt, maxAttempts, previousFailure }) => buildResolverUserPrompt({
137
92
  unitKey: input.unitKey,
138
93
  patchPath: input.patchPath,
139
- touchedPaths: [...allowedPaths].sort(),
94
+ touchedPaths: sortedTouchedPaths,
95
+ reason: input.reason,
140
96
  attempt,
141
97
  maxAttempts,
142
- reason: input.reason,
143
- };
144
- const result = await traceTimed(input.trace, 'resolver', 'textual_conflict_resolver', traceData, async () => input.agentRunner.runLoop({
145
- modelRole: 'repair',
146
- systemPrompt: buildResolverSystemPrompt(),
147
- userPrompt: buildResolverUserPrompt({
148
- unitKey: input.unitKey,
149
- patchPath: input.patchPath,
150
- touchedPaths: [...allowedPaths].sort(),
151
- reason: input.reason,
152
- attempt,
153
- maxAttempts,
154
- }),
155
- toolSet: buildToolSet({
156
- workdir: input.workdir,
157
- patchPath: input.patchPath,
158
- allowedPaths,
159
- editedPaths,
160
- }),
161
- stepBudget,
162
- telemetryTags: {
163
- operationName: 'ingest-isolated-diff-textual-resolver',
164
- source: input.trace.context.sourceKey,
165
- jobId: input.trace.context.jobId,
166
- unitKey: input.unitKey,
167
- },
168
- abortSignal: input.abortSignal,
169
- }));
170
- if (result.stopReason === 'error') {
171
- lastFailure = result.error?.message ?? 'resolver agent loop errored';
172
- await input.trace.event('error', 'resolver', 'textual_conflict_resolver_failed', traceData, result.error);
173
- continue;
174
- }
175
- const changedPaths = [...editedPaths].sort();
176
- if (changedPaths.length === 0) {
177
- lastFailure = 'resolver completed without editing an allowed path';
178
- await input.trace.event('error', 'resolver', 'textual_conflict_resolver_failed', {
179
- ...traceData,
180
- reason: lastFailure,
181
- });
182
- continue;
183
- }
184
- await input.trace.event('debug', 'resolver', 'textual_conflict_resolver_repaired', {
185
- ...traceData,
186
- changedPaths,
187
- });
188
- return { status: 'repaired', attempts: attempt, changedPaths };
189
- }
190
- return { status: 'failed', attempts: maxAttempts, reason: lastFailure };
98
+ previousFailure,
99
+ }),
100
+ buildExtraTools: (context) => buildResolverExtraTools({ patchPath: input.patchPath, context }),
101
+ verify: input.verify,
102
+ noChangeFailureReason: 'resolver completed without editing an allowed path or declaring the patch redundant',
103
+ telemetryTags: {
104
+ operationName: 'ingest-isolated-diff-textual-resolver',
105
+ source: input.trace.context.sourceKey,
106
+ jobId: input.trace.context.jobId,
107
+ unitKey: input.unitKey,
108
+ },
109
+ maxAttempts: input.maxAttempts,
110
+ stepBudget: input.stepBudget ?? 12,
111
+ abortSignal: input.abortSignal,
112
+ });
191
113
  }
@@ -1,4 +1,5 @@
1
1
  import type { KtxSqlQueryExecutorPort } from '../../context/connections/query-executor.js';
2
+ import type { SqlAnalysisPort } from '../../context/sql-analysis/ports.js';
2
3
  import type { KtxLogger } from '../../context/core/config.js';
3
4
  import type { KtxSemanticLayerComputePort } from '../../context/daemon/semantic-layer-compute.js';
4
5
  import { createLocalKtxLlmRuntimeFromConfig } from '../../context/llm/local-config.js';
@@ -20,6 +21,7 @@ export interface CreateLocalBundleIngestRuntimeOptions {
20
21
  memoryModel?: string;
21
22
  semanticLayerCompute?: KtxSemanticLayerComputePort;
22
23
  queryExecutor?: KtxSqlQueryExecutorPort;
24
+ sqlAnalysis?: SqlAnalysisPort;
23
25
  jobIdFactory?: () => string;
24
26
  logger?: KtxLogger;
25
27
  embeddingProvider?: KtxEmbeddingProvider | null;
@@ -52,7 +52,7 @@ import { SourceAdapterRegistry } from './source-adapter-registry.js';
52
52
  import { SqliteBundleIngestStore } from './sqlite-bundle-ingest-store.js';
53
53
  const promptsDir = fileURLToPath(new URL('../../prompts', import.meta.url));
54
54
  const skillsDir = fileURLToPath(new URL('../../skills', import.meta.url));
55
- const LOCAL_AUTHOR = { name: 'KTX Local', email: 'local@ktx.local' };
55
+ const LOCAL_AUTHOR = { name: 'ktx Local', email: 'local@ktx.local' };
56
56
  const LOCAL_SHAPE_WARNING = 'Local ingest validates semantic-layer YAML shape only.';
57
57
  const INGEST_TRACE_LEVELS = new Set(['error', 'info', 'debug', 'trace']);
58
58
  function ingestTraceLevelFromEnv(env = process.env) {
@@ -169,7 +169,8 @@ class LocalSlPythonPort {
169
169
  }
170
170
  class LocalShapeOnlySlValidator {
171
171
  validateParsedSource(sourceName, parsed) {
172
- const isOverlay = parsed.table == null && parsed.sql == null;
172
+ const fields = (parsed ?? {});
173
+ const isOverlay = fields.table == null && fields.sql == null;
173
174
  const result = (isOverlay ? sourceOverlaySchema : sourceDefinitionSchema).safeParse(parsed);
174
175
  return result.success
175
176
  ? { errors: [], warnings: [LOCAL_SHAPE_WARNING] }
@@ -200,16 +201,12 @@ class LocalShapeOnlySlValidator {
200
201
  }
201
202
  }
202
203
  async validateSingleSource(deps, connectionId, sourceName) {
203
- let content;
204
- try {
205
- const file = await deps.semanticLayerService.readSourceFile(connectionId, sourceName);
206
- content = file.content;
207
- }
208
- catch (error) {
209
- return this.validateComposedSource(deps, connectionId, sourceName, error);
204
+ const file = await deps.semanticLayerService.readSourceFile(connectionId, sourceName);
205
+ if (!file) {
206
+ return this.validateComposedSource(deps, connectionId, sourceName, 'no standalone or overlay file found');
210
207
  }
211
208
  try {
212
- const parsed = YAML.parse(content);
209
+ const parsed = YAML.parse(file.content);
213
210
  return this.validateParsedSource(sourceName, parsed);
214
211
  }
215
212
  catch (error) {
@@ -439,6 +436,7 @@ class LocalIngestToolsetFactory {
439
436
  const slDiscoverTool = new SlDiscoverTool(slDeps, { maxSources: 25, minRrfScore: 0, maxDetailedSources: 5 });
440
437
  const warehouseVerificationTools = createWarehouseVerificationTools({
441
438
  connections: deps.connections,
439
+ ...(deps.sqlAnalysis ? { sqlAnalysis: deps.sqlAnalysis } : {}),
442
440
  fallbackFileStore: deps.project.fileStore,
443
441
  wikiSearchTool,
444
442
  slDiscoverTool,
@@ -556,6 +554,7 @@ export function createLocalBundleIngestRuntime(options) {
556
554
  authorResolver: new LocalAuthorResolver(),
557
555
  slSourcesRepository,
558
556
  connections,
557
+ ...(options.sqlAnalysis ? { sqlAnalysis: options.sqlAnalysis } : {}),
559
558
  contextStore,
560
559
  embedding,
561
560
  });
@@ -1,4 +1,5 @@
1
1
  import type { KtxSqlQueryExecutorPort } from '../../context/connections/query-executor.js';
2
+ import type { SqlAnalysisPort } from '../../context/sql-analysis/ports.js';
2
3
  import type { KtxLogger } from '../../context/core/config.js';
3
4
  import type { KtxSemanticLayerComputePort } from '../../context/daemon/semantic-layer-compute.js';
4
5
  import type { AgentRunnerPort, KtxLlmRuntimePort } from '../../context/llm/runtime-port.js';
@@ -23,6 +24,7 @@ export interface RunLocalIngestOptions {
23
24
  memoryModel?: string;
24
25
  semanticLayerCompute?: KtxSemanticLayerComputePort;
25
26
  queryExecutor?: KtxSqlQueryExecutorPort;
27
+ sqlAnalysis?: SqlAnalysisPort;
26
28
  logger?: KtxLogger;
27
29
  embeddingProvider?: import('../../llm/types.js').KtxEmbeddingProvider | null;
28
30
  abortSignal?: AbortSignal;
@@ -102,6 +102,7 @@ export async function runLocalIngest(options) {
102
102
  memoryModel: options.memoryModel,
103
103
  semanticLayerCompute: options.semanticLayerCompute,
104
104
  queryExecutor: options.queryExecutor,
105
+ sqlAnalysis: options.sqlAnalysis,
105
106
  logger: options.logger,
106
107
  embeddingProvider: options.embeddingProvider,
107
108
  abortSignal: options.abortSignal,
@@ -259,6 +260,7 @@ export async function runLocalMetabaseIngest(options) {
259
260
  memoryModel: options.memoryModel,
260
261
  semanticLayerCompute: options.semanticLayerCompute,
261
262
  queryExecutor: options.queryExecutor,
263
+ sqlAnalysis: options.sqlAnalysis,
262
264
  logger: options.logger,
263
265
  embeddingProvider: options.embeddingProvider,
264
266
  abortSignal: options.abortSignal,
@@ -436,7 +436,7 @@ export function buildMemoryFlowViewModel(input) {
436
436
  ? [...new Set(sources.map((s) => humanizeAdapter(s.adapter)))].join(' + ')
437
437
  : `${input.connectionId}/${input.adapter}`;
438
438
  return {
439
- title: `KTX memory flow ${titleSources} ${input.status}`,
439
+ title: `ktx memory flow ${titleSources} ${input.status}`,
440
440
  subtitle: `Run ${input.runId} Sync ${input.syncId}`,
441
441
  status: input.status,
442
442
  activeLine: activeLine(input),
@@ -2,18 +2,15 @@ import type { KtxModelRole } from '../../../llm/types.js';
2
2
  import type { AgentRunnerPort, KtxRuntimeToolSet, RunLoopMetrics } from '../../../context/llm/runtime-port.js';
3
3
  import type { CaptureSession, MemoryAction } from '../../../context/memory/types.js';
4
4
  import { type TouchedSlSource } from '../../../context/tools/touched-sl-sources.js';
5
+ import { type WuValidationResult } from './validate-wu-sources.js';
5
6
  import type { WorkUnit } from '../types.js';
6
- interface TouchedValidationResult {
7
- invalidSources: string[];
8
- validSources: string[];
9
- }
10
7
  export interface WorkUnitExecutionDeps {
11
8
  sessionWorktreeGit: {
12
9
  revParseHead(): Promise<string | null>;
13
10
  };
14
11
  agentRunner: AgentRunnerPort;
15
12
  validateWikiRefs?: (actions: MemoryAction[]) => Promise<string[]>;
16
- validateTouchedSources: (touched: TouchedSlSource[]) => Promise<TouchedValidationResult>;
13
+ validateTouchedSources: (touched: TouchedSlSource[]) => Promise<WuValidationResult>;
17
14
  resetHardTo: (targetSha: string) => Promise<void>;
18
15
  buildSystemPrompt: (wu: WorkUnit) => string;
19
16
  buildUserPrompt: (wu: WorkUnit) => string;
@@ -45,4 +42,3 @@ export interface WorkUnitOutcome {
45
42
  metrics?: RunLoopMetrics;
46
43
  }
47
44
  export declare function executeWorkUnit(deps: WorkUnitExecutionDeps, wu: WorkUnit): Promise<WorkUnitOutcome>;
48
- export {};
@@ -1,5 +1,6 @@
1
1
  import { isAbortError } from '../../core/abort.js';
2
2
  import { listTouchedSlSources } from '../../../context/tools/touched-sl-sources.js';
3
+ import { formatInvalidWuSources } from './validate-wu-sources.js';
3
4
  const MAX_WORK_UNIT_PROMPT_CHARS = 240_000;
4
5
  export async function executeWorkUnit(deps, wu) {
5
6
  const preSha = (await deps.sessionWorktreeGit.revParseHead()) ?? '';
@@ -97,7 +98,7 @@ export async function executeWorkUnit(deps, wu) {
97
98
  // Spec: invalid SL writes reset the session worktree to the WU's pre-state, WU is marked failed,
98
99
  // its files are absent from the Stage Index. Per-source surgical revert is the
99
100
  // memory-agent pattern — NOT the bundle-ingest pattern.
100
- return failWithReset(`sl_validate failed for: ${validation.invalidSources.join(', ')}`);
101
+ return failWithReset(`sl_validate failed for: ${formatInvalidWuSources(validation.invalidSources)}`);
101
102
  }
102
103
  }
103
104
  return {
@@ -1,10 +1,16 @@
1
1
  import type { SlValidationDeps } from '../../../context/sl/tools/sl-warehouse-validation.js';
2
2
  import type { SlValidatorPort } from '../../../context/sl/sl-validator.port.js';
3
3
  import type { TouchedSlSource } from '../../../context/tools/touched-sl-sources.js';
4
+ export interface InvalidWuSource {
5
+ /** `${connectionId}:${sourceName}` */
6
+ source: string;
7
+ errors: string[];
8
+ }
4
9
  export interface WuValidationResult {
5
10
  validSources: string[];
6
- invalidSources: string[];
11
+ invalidSources: InvalidWuSource[];
7
12
  }
13
+ export declare function formatInvalidWuSources(invalid: InvalidWuSource[]): string;
8
14
  export declare function validateWuTouchedSources(deps: SlValidationDeps & {
9
15
  slValidator: SlValidatorPort<SlValidationDeps>;
10
16
  }, touched: TouchedSlSource[]): Promise<WuValidationResult>;
@@ -1,13 +1,118 @@
1
+ import { findMissingJoinTargets, formatMissingJoinTarget } from '../../../context/sl/semantic-layer.service.js';
2
+ export function formatInvalidWuSources(invalid) {
3
+ return invalid.map((entry) => `${entry.source} (${entry.errors.join('; ')})`).join(', ');
4
+ }
5
+ function uniqueTouchedSources(sources) {
6
+ const seen = new Set();
7
+ const unique = [];
8
+ for (const source of sources) {
9
+ const key = `${source.connectionId}:${source.sourceName}`;
10
+ if (seen.has(key)) {
11
+ continue;
12
+ }
13
+ seen.add(key);
14
+ unique.push(source);
15
+ }
16
+ return unique.sort((left, right) => {
17
+ const byConnection = left.connectionId.localeCompare(right.connectionId);
18
+ return byConnection === 0 ? left.sourceName.localeCompare(right.sourceName) : byConnection;
19
+ });
20
+ }
21
+ /**
22
+ * Expand the touched set with direct join neighbors that exist: targets the
23
+ * touched sources join to, and existing sources that join to a touched one.
24
+ * Missing targets are not added here — they are reported as join-target
25
+ * errors on the source that declares them, so the failure names the file
26
+ * that must change instead of the phantom neighbor.
27
+ */
28
+ function expandWithExistingJoinNeighbors(touched, sourcesByConnection) {
29
+ const expanded = [...touched];
30
+ const touchedByConnection = new Map();
31
+ for (const source of touched) {
32
+ const bucket = touchedByConnection.get(source.connectionId) ?? new Set();
33
+ bucket.add(source.sourceName);
34
+ touchedByConnection.set(source.connectionId, bucket);
35
+ }
36
+ for (const [connectionId, sources] of sourcesByConnection) {
37
+ const touchedNames = touchedByConnection.get(connectionId);
38
+ if (!touchedNames || touchedNames.size === 0) {
39
+ continue;
40
+ }
41
+ const existingNames = new Set(sources.map((source) => source.name));
42
+ for (const source of sources) {
43
+ if (touchedNames.has(source.name)) {
44
+ for (const join of source.joins ?? []) {
45
+ if (existingNames.has(join.to)) {
46
+ expanded.push({ connectionId, sourceName: join.to });
47
+ }
48
+ }
49
+ }
50
+ if ((source.joins ?? []).some((join) => touchedNames.has(join.to))) {
51
+ expanded.push({ connectionId, sourceName: source.name });
52
+ }
53
+ }
54
+ }
55
+ return uniqueTouchedSources(expanded);
56
+ }
57
+ /**
58
+ * Join-target errors attributable to this change set: every join declared by
59
+ * a touched source must resolve, and no source may be left joining to a name
60
+ * this change set removed. Pre-existing dangling joins on untouched sources
61
+ * are out of scope — they must not block unrelated work. Resolution is the
62
+ * Python engine's: exact source-name match within the connection.
63
+ */
64
+ function findJoinTargetErrors(touched, sourcesByConnection) {
65
+ const errorsBySource = new Map();
66
+ const touchedByConnection = new Map();
67
+ for (const source of touched) {
68
+ const bucket = touchedByConnection.get(source.connectionId) ?? new Set();
69
+ bucket.add(source.sourceName);
70
+ touchedByConnection.set(source.connectionId, bucket);
71
+ }
72
+ for (const [connectionId, sources] of sourcesByConnection) {
73
+ const touchedNames = touchedByConnection.get(connectionId);
74
+ if (!touchedNames || touchedNames.size === 0) {
75
+ continue;
76
+ }
77
+ const existingNames = sources.map((source) => source.name);
78
+ for (const source of sources) {
79
+ const sourceIsTouched = touchedNames.has(source.name);
80
+ const candidateJoins = sourceIsTouched
81
+ ? source.joins
82
+ : (source.joins ?? []).filter((join) => touchedNames.has(join.to));
83
+ const missing = findMissingJoinTargets(candidateJoins, existingNames);
84
+ if (missing.length === 0) {
85
+ continue;
86
+ }
87
+ const key = `${connectionId}:${source.name}`;
88
+ const messages = missing.map(formatMissingJoinTarget);
89
+ errorsBySource.set(key, [...(errorsBySource.get(key) ?? []), ...messages]);
90
+ }
91
+ }
92
+ return errorsBySource;
93
+ }
1
94
  export async function validateWuTouchedSources(deps, touched) {
95
+ if (touched.length === 0) {
96
+ return { validSources: [], invalidSources: [] };
97
+ }
98
+ const sourcesByConnection = new Map();
99
+ for (const connectionId of new Set(touched.map((source) => source.connectionId))) {
100
+ const { sources } = await deps.semanticLayerService.loadAllSources(connectionId);
101
+ sourcesByConnection.set(connectionId, sources);
102
+ }
103
+ const expanded = expandWithExistingJoinNeighbors(touched, sourcesByConnection);
104
+ const joinTargetErrors = findJoinTargetErrors(touched, sourcesByConnection);
2
105
  const valid = [];
3
106
  const invalid = [];
4
- for (const source of touched) {
107
+ for (const source of expanded) {
108
+ const key = `${source.connectionId}:${source.sourceName}`;
5
109
  const result = await deps.slValidator.validateSingleSource(deps, source.connectionId, source.sourceName);
6
- if (result.errors.length === 0) {
7
- valid.push(`${source.connectionId}:${source.sourceName}`);
110
+ const errors = [...result.errors, ...(joinTargetErrors.get(key) ?? [])];
111
+ if (errors.length === 0) {
112
+ valid.push(key);
8
113
  }
9
114
  else {
10
- invalid.push(`${source.connectionId}:${source.sourceName}`);
115
+ invalid.push({ source: key, errors });
11
116
  }
12
117
  }
13
118
  return { validSources: valid, invalidSources: invalid };
@@ -1,8 +1,10 @@
1
1
  import type { KtxFileStorePort } from '../../../core/file-store.js';
2
2
  import type { SlConnectionCatalogPort } from '../../../sl/ports.js';
3
+ import type { SqlAnalysisPort } from '../../../sql-analysis/ports.js';
3
4
  import type { BaseTool } from '../../../tools/base-tool.js';
4
5
  export declare function createWarehouseVerificationTools(deps: {
5
6
  connections: SlConnectionCatalogPort;
7
+ sqlAnalysis?: SqlAnalysisPort;
6
8
  fallbackFileStore: KtxFileStorePort;
7
9
  wikiSearchTool: BaseTool;
8
10
  slDiscoverTool: BaseTool;
@@ -8,7 +8,7 @@ export function createWarehouseVerificationTools(deps) {
8
8
  });
9
9
  return [
10
10
  new EntityDetailsTool(catalogFactory),
11
- new SqlExecutionTool(deps.connections),
11
+ new SqlExecutionTool(deps.connections, deps.sqlAnalysis),
12
12
  new DiscoverDataTool({
13
13
  wikiSearchTool: deps.wikiSearchTool,
14
14
  slDiscoverTool: deps.slDiscoverTool,
@@ -47,7 +47,7 @@ export class DiscoverDataTool extends BaseTool {
47
47
  };
48
48
  }
49
49
  if (input.sourceName) {
50
- const sl = await this.deps.slDiscoverTool.call({ sourceName: input.sourceName, connectionId: input.connectionId }, context);
50
+ const sl = (await this.deps.slDiscoverTool.call({ sourceName: input.sourceName, connectionId: input.connectionId }, context));
51
51
  return { markdown: sl.markdown, structured: { wiki: null, sl: sl.structured, raw: null } };
52
52
  }
53
53
  const query = input.query?.trim() || '';
@@ -57,13 +57,13 @@ export class DiscoverDataTool extends BaseTool {
57
57
  let sl = null;
58
58
  let raw = null;
59
59
  if (query) {
60
- const wikiResult = await this.deps.wikiSearchTool.call({ query, limit }, context);
60
+ const wikiResult = (await this.deps.wikiSearchTool.call({ query, limit }, context));
61
61
  if (totalFound(wikiResult.structured) > 0) {
62
62
  parts.push('## Wiki Pages', '> use `wiki_read(blockKey)` for full content', wikiResult.markdown, '');
63
63
  wiki = wikiResult.structured;
64
64
  }
65
65
  }
66
- const slResult = await this.deps.slDiscoverTool.call({ query: query || undefined, connectionId: input.connectionId }, context);
66
+ const slResult = (await this.deps.slDiscoverTool.call({ query: query || undefined, connectionId: input.connectionId }, context));
67
67
  if (totalSources(slResult.structured) > 0) {
68
68
  parts.push('## Semantic Layer Sources', '> use `sl_read_source(sourceName)` for the YAML, or `entity_details` for warehouse-shape details', slResult.markdown, '');
69
69
  sl = slResult.structured;
@@ -1,5 +1,6 @@
1
1
  import { z } from 'zod';
2
2
  import type { SlConnectionCatalogPort } from '../../../../context/sl/ports.js';
3
+ import type { SqlAnalysisPort } from '../../../../context/sql-analysis/ports.js';
3
4
  import { BaseTool, type ToolContext, type ToolOutput } from '../../../../context/tools/base-tool.js';
4
5
  declare const sqlExecutionInputSchema: z.ZodObject<{
5
6
  connectionId: z.ZodString;
@@ -18,8 +19,9 @@ export interface SqlExecutionStructured {
18
19
  }
19
20
  export declare class SqlExecutionTool extends BaseTool<typeof sqlExecutionInputSchema> {
20
21
  private readonly connections;
22
+ private readonly sqlAnalysis?;
21
23
  readonly name = "sql_execution";
22
- constructor(connections: SlConnectionCatalogPort);
24
+ constructor(connections: SlConnectionCatalogPort, sqlAnalysis?: SqlAnalysisPort | undefined);
23
25
  get description(): string;
24
26
  get inputSchema(): z.ZodObject<{
25
27
  connectionId: z.ZodString;
@@ -1,5 +1,6 @@
1
1
  import { z } from 'zod';
2
2
  import { assertReadOnlySql, limitSqlForExecution } from '../../../../context/connections/read-only-sql.js';
3
+ import { sqlAnalysisDialectForDriver } from '../../../../context/sql-analysis/dialect.js';
3
4
  import { BaseTool } from '../../../../context/tools/base-tool.js';
4
5
  const sqlExecutionInputSchema = z.object({
5
6
  connectionId: z.string().regex(/^[a-zA-Z0-9][a-zA-Z0-9_-]*$/),
@@ -23,10 +24,12 @@ function markdownTable(headers, rows, totalRows) {
23
24
  }
24
25
  export class SqlExecutionTool extends BaseTool {
25
26
  connections;
27
+ sqlAnalysis;
26
28
  name = 'sql_execution';
27
- constructor(connections) {
29
+ constructor(connections, sqlAnalysis) {
28
30
  super();
29
31
  this.connections = connections;
32
+ this.sqlAnalysis = sqlAnalysis;
30
33
  }
31
34
  get description() {
32
35
  return 'Run a single read-only SELECT or WITH probe against an allowed warehouse connection and return a capped markdown table or the warehouse error.';
@@ -50,9 +53,20 @@ export class SqlExecutionTool extends BaseTool {
50
53
  },
51
54
  };
52
55
  }
56
+ if (!this.sqlAnalysis) {
57
+ throw new Error('sql_execution requires parser-backed SQL validation.');
58
+ }
53
59
  let sql;
54
60
  let wrappedSql;
55
61
  try {
62
+ const connection = await this.connections.getConnectionById(input.connectionId);
63
+ if (!connection) {
64
+ throw new Error(`Connection not found: ${input.connectionId}`);
65
+ }
66
+ const validation = await this.sqlAnalysis.validateReadOnly(input.sql, sqlAnalysisDialectForDriver(connection.connectionType));
67
+ if (!validation.ok) {
68
+ throw new Error(validation.error ?? 'SQL is not read-only.');
69
+ }
56
70
  sql = assertReadOnlySql(input.sql);
57
71
  wrappedSql = limitSqlForExecution(sql, input.rowLimit);
58
72
  }