@kaelio/ktx 0.11.0 → 0.13.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (212) hide show
  1. package/assets/python/kaelio_ktx-0.13.0-py3-none-any.whl +0 -0
  2. package/assets/python/manifest.json +4 -4
  3. package/dist/.tsbuildinfo +1 -1
  4. package/dist/admin.js +1 -1
  5. package/dist/clack.d.ts +16 -0
  6. package/dist/clack.js +37 -6
  7. package/dist/claude-code-prompt-caching.js +1 -1
  8. package/dist/cli-program.js +3 -3
  9. package/dist/cli-runtime.js +2 -2
  10. package/dist/commands/connection-commands.js +1 -1
  11. package/dist/commands/ingest-commands.js +4 -4
  12. package/dist/commands/mcp-commands.js +12 -12
  13. package/dist/commands/runtime-commands.js +4 -4
  14. package/dist/commands/setup-commands.js +19 -5
  15. package/dist/commands/sl-commands.js +1 -1
  16. package/dist/commands/sql-commands.js +1 -1
  17. package/dist/commands/status-commands.js +1 -1
  18. package/dist/connection.js +15 -3
  19. package/dist/connectors/bigquery/connector.js +1 -14
  20. package/dist/connectors/clickhouse/connector.js +2 -16
  21. package/dist/connectors/duckdb/federated-attach.d.ts +7 -0
  22. package/dist/connectors/duckdb/federated-attach.js +86 -0
  23. package/dist/connectors/duckdb/federated-executor.d.ts +5 -0
  24. package/dist/connectors/duckdb/federated-executor.js +59 -0
  25. package/dist/connectors/mysql/connector.js +2 -16
  26. package/dist/connectors/postgres/connector.js +1 -14
  27. package/dist/connectors/shared/string-reference.d.ts +6 -0
  28. package/dist/connectors/shared/string-reference.js +19 -0
  29. package/dist/connectors/snowflake/connector.d.ts +1 -1
  30. package/dist/connectors/snowflake/connector.js +1 -14
  31. package/dist/connectors/sqlite/connector.js +2 -25
  32. package/dist/connectors/sqlserver/connector.js +4 -17
  33. package/dist/context/connections/connection-type.d.ts +1 -1
  34. package/dist/context/connections/federation.d.ts +33 -0
  35. package/dist/context/connections/federation.js +51 -0
  36. package/dist/context/connections/local-warehouse-descriptor.d.ts +2 -0
  37. package/dist/context/connections/project-sql-executor.d.ts +18 -0
  38. package/dist/context/connections/project-sql-executor.js +39 -0
  39. package/dist/context/connections/query-executor.d.ts +2 -2
  40. package/dist/context/connections/read-only-sql.d.ts +1 -0
  41. package/dist/context/connections/read-only-sql.js +119 -4
  42. package/dist/context/connections/resolve-connection.d.ts +12 -0
  43. package/dist/context/connections/resolve-connection.js +37 -0
  44. package/dist/context/core/git-env.d.ts +4 -0
  45. package/dist/context/core/git-env.js +5 -1
  46. package/dist/context/core/git.service.d.ts +23 -0
  47. package/dist/context/core/git.service.js +71 -8
  48. package/dist/context/ingest/adapters/historic-sql/projection.js +2 -1
  49. package/dist/context/ingest/adapters/live-database/manifest.d.ts +3 -0
  50. package/dist/context/ingest/adapters/live-database/manifest.js +19 -11
  51. package/dist/context/ingest/adapters/looker/client.js +7 -2
  52. package/dist/context/ingest/adapters/looker/factory.d.ts +8 -1
  53. package/dist/context/ingest/adapters/looker/factory.js +9 -0
  54. package/dist/context/ingest/adapters/looker/mapping.js +1 -1
  55. package/dist/context/ingest/adapters/looker/types.d.ts +1 -1
  56. package/dist/context/ingest/adapters/metabase/client.d.ts +1 -1
  57. package/dist/context/ingest/adapters/metabase/client.js +1 -1
  58. package/dist/context/ingest/adapters/metabase/local-metabase.adapter.js +1 -1
  59. package/dist/context/ingest/adapters/metabase/mapping.js +6 -6
  60. package/dist/context/ingest/artifact-gates.d.ts +2 -6
  61. package/dist/context/ingest/artifact-gates.js +5 -47
  62. package/dist/context/ingest/constrained-repair.d.ts +55 -0
  63. package/dist/context/ingest/constrained-repair.js +167 -0
  64. package/dist/context/ingest/final-gate-repair.d.ts +9 -11
  65. package/dist/context/ingest/final-gate-repair.js +40 -128
  66. package/dist/context/ingest/finalization-scope.d.ts +1 -1
  67. package/dist/context/ingest/finalization-scope.js +15 -15
  68. package/dist/context/ingest/ingest-bundle.runner.d.ts +1 -0
  69. package/dist/context/ingest/ingest-bundle.runner.js +101 -67
  70. package/dist/context/ingest/isolated-diff/patch-integrator.d.ts +6 -13
  71. package/dist/context/ingest/isolated-diff/patch-integrator.js +32 -109
  72. package/dist/context/ingest/isolated-diff/textual-conflict-resolver.d.ts +8 -9
  73. package/dist/context/ingest/isolated-diff/textual-conflict-resolver.js +63 -141
  74. package/dist/context/ingest/local-bundle-runtime.d.ts +2 -0
  75. package/dist/context/ingest/local-bundle-runtime.js +9 -10
  76. package/dist/context/ingest/local-ingest.d.ts +2 -0
  77. package/dist/context/ingest/local-ingest.js +2 -0
  78. package/dist/context/ingest/memory-flow/view-model.js +1 -1
  79. package/dist/context/ingest/stages/stage-3-work-units.d.ts +2 -6
  80. package/dist/context/ingest/stages/stage-3-work-units.js +2 -1
  81. package/dist/context/ingest/stages/validate-wu-sources.d.ts +7 -1
  82. package/dist/context/ingest/stages/validate-wu-sources.js +109 -4
  83. package/dist/context/ingest/tools/warehouse-verification/create-warehouse-verification-tools.d.ts +2 -0
  84. package/dist/context/ingest/tools/warehouse-verification/create-warehouse-verification-tools.js +1 -1
  85. package/dist/context/ingest/tools/warehouse-verification/discover-data.tool.js +3 -3
  86. package/dist/context/ingest/tools/warehouse-verification/sql-execution.tool.d.ts +3 -1
  87. package/dist/context/ingest/tools/warehouse-verification/sql-execution.tool.js +15 -1
  88. package/dist/context/llm/ai-sdk-runtime.js +2 -2
  89. package/dist/context/llm/claude-code-runtime.js +19 -3
  90. package/dist/context/llm/local-config.js +1 -1
  91. package/dist/context/llm/runtime-tools.js +2 -2
  92. package/dist/context/mcp/context-tools.js +33 -8
  93. package/dist/context/mcp/local-project-ports.js +63 -89
  94. package/dist/context/mcp/types.d.ts +2 -0
  95. package/dist/context/memory/local-memory.js +4 -1
  96. package/dist/context/memory/memory-agent.service.js +1 -1
  97. package/dist/context/project/config.d.ts +11 -4
  98. package/dist/context/project/config.js +85 -30
  99. package/dist/context/project/driver-schemas.js +1 -1
  100. package/dist/context/project/mappings-yaml-schema.js +2 -2
  101. package/dist/context/project/project.js +12 -4
  102. package/dist/context/scan/description-generation.js +4 -4
  103. package/dist/context/scan/local-enrichment-artifacts.js +33 -4
  104. package/dist/context/scan/local-scan.js +2 -2
  105. package/dist/context/scan/local-structural-artifacts.js +5 -5
  106. package/dist/context/scan/relationship-benchmark-report.js +1 -1
  107. package/dist/context/scan/relationship-discovery.js +3 -3
  108. package/dist/context/scan/relationship-llm-proposal.js +3 -3
  109. package/dist/context/sl/local-query.js +31 -44
  110. package/dist/context/sl/local-sl.d.ts +0 -8
  111. package/dist/context/sl/local-sl.js +71 -70
  112. package/dist/context/sl/semantic-layer.service.d.ts +25 -8
  113. package/dist/context/sl/semantic-layer.service.js +109 -56
  114. package/dist/context/sl/source-files.d.ts +48 -0
  115. package/dist/context/sl/source-files.js +138 -0
  116. package/dist/context/sl/tools/base-semantic-layer.tool.d.ts +2 -2
  117. package/dist/context/sl/tools/base-semantic-layer.tool.js +2 -7
  118. package/dist/context/sl/tools/sl-edit-source.tool.js +10 -8
  119. package/dist/context/sl/tools/sl-warehouse-validation.js +55 -27
  120. package/dist/context/sl/tools/sl-write-source.tool.js +12 -9
  121. package/dist/context/sql-analysis/dialect.d.ts +2 -0
  122. package/dist/context/sql-analysis/dialect.js +20 -0
  123. package/dist/context/tools/base-tool.d.ts +6 -19
  124. package/dist/context/tools/base-tool.js +0 -14
  125. package/dist/context-build-view.js +5 -5
  126. package/dist/database-tree-picker.js +18 -3
  127. package/dist/demo-assets.js +0 -1
  128. package/dist/doctor.d.ts +1 -1
  129. package/dist/doctor.js +31 -23
  130. package/dist/errors.d.ts +31 -0
  131. package/dist/errors.js +44 -0
  132. package/dist/ingest-query-executor.d.ts +2 -0
  133. package/dist/ingest-query-executor.js +8 -22
  134. package/dist/ingest.d.ts +1 -1
  135. package/dist/ingest.js +8 -2
  136. package/dist/io/symbols.d.ts +2 -0
  137. package/dist/io/symbols.js +2 -0
  138. package/dist/io/tty.d.ts +8 -0
  139. package/dist/io/tty.js +16 -0
  140. package/dist/llm/embedding-health.js +1 -1
  141. package/dist/llm/embedding-provider.js +3 -3
  142. package/dist/llm/model-provider.js +1 -1
  143. package/dist/local-adapters.d.ts +1 -0
  144. package/dist/local-adapters.js +2 -2
  145. package/dist/local-scan-connectors.js +1 -1
  146. package/dist/managed-local-embeddings.js +17 -8
  147. package/dist/managed-mcp-daemon.js +3 -3
  148. package/dist/managed-python-command.d.ts +7 -0
  149. package/dist/managed-python-command.js +34 -8
  150. package/dist/managed-python-daemon.js +2 -2
  151. package/dist/managed-python-http.js +3 -3
  152. package/dist/managed-python-runtime.d.ts +30 -1
  153. package/dist/managed-python-runtime.js +134 -18
  154. package/dist/managed-uv-release.d.ts +7 -0
  155. package/dist/managed-uv-release.js +11 -0
  156. package/dist/mcp-http-server.js +4 -4
  157. package/dist/mcp-server-factory.js +3 -3
  158. package/dist/mcp-stdio-server.js +1 -1
  159. package/dist/memory-flow-hud.js +2 -2
  160. package/dist/next-steps.js +2 -2
  161. package/dist/prompt-navigation.d.ts +17 -0
  162. package/dist/prompt-navigation.js +49 -3
  163. package/dist/prompts/memory_agent_bundle_ingest_work_unit.md +2 -2
  164. package/dist/prompts/memory_agent_external_ingest.md +2 -2
  165. package/dist/public-ingest-copy.js +1 -1
  166. package/dist/public-ingest.js +3 -3
  167. package/dist/release-version.js +1 -1
  168. package/dist/runtime-requirements.js +1 -1
  169. package/dist/runtime.js +9 -9
  170. package/dist/scan.js +1 -1
  171. package/dist/setup-agents.d.ts +21 -15
  172. package/dist/setup-agents.js +143 -66
  173. package/dist/setup-banner.d.ts +20 -0
  174. package/dist/setup-banner.js +39 -0
  175. package/dist/setup-context.js +24 -15
  176. package/dist/setup-databases.d.ts +3 -0
  177. package/dist/setup-databases.js +47 -59
  178. package/dist/setup-demo-tour.js +12 -8
  179. package/dist/setup-embeddings.js +9 -9
  180. package/dist/setup-interrupt.js +1 -1
  181. package/dist/setup-models.d.ts +4 -1
  182. package/dist/setup-models.js +54 -28
  183. package/dist/setup-project.js +29 -5
  184. package/dist/setup-prompts.js +16 -1
  185. package/dist/setup-ready-menu.js +1 -1
  186. package/dist/setup-sources.js +28 -12
  187. package/dist/setup.d.ts +1 -0
  188. package/dist/setup.js +14 -13
  189. package/dist/skills/analytics/SKILL.md +3 -3
  190. package/dist/skills/dbt_ingest/SKILL.md +3 -3
  191. package/dist/skills/looker_ingest/SKILL.md +3 -3
  192. package/dist/skills/lookml_ingest/SKILL.md +7 -7
  193. package/dist/skills/metabase_ingest/SKILL.md +4 -4
  194. package/dist/skills/metricflow_ingest/SKILL.md +15 -15
  195. package/dist/skills/notion_synthesize/SKILL.md +1 -1
  196. package/dist/skills/sl/SKILL.md +3 -3
  197. package/dist/skills/sl_capture/SKILL.md +1 -1
  198. package/dist/skills/wiki_capture/SKILL.md +1 -1
  199. package/dist/source-mapping.js +1 -1
  200. package/dist/sql.d.ts +2 -0
  201. package/dist/sql.js +35 -53
  202. package/dist/startup-profile.js +1 -1
  203. package/dist/status-project.d.ts +0 -2
  204. package/dist/status-project.js +4 -6
  205. package/dist/telemetry/events.d.ts +3 -2
  206. package/dist/telemetry/events.js +11 -1
  207. package/dist/telemetry/exception.js +14 -0
  208. package/dist/text-ingest.js +1 -1
  209. package/dist/tree-picker-tui.d.ts +0 -1
  210. package/dist/tree-picker-tui.js +2 -3
  211. package/package.json +2 -1
  212. package/assets/python/kaelio_ktx-0.11.0-py3-none-any.whl +0 -0
@@ -1,54 +1,19 @@
1
1
  import { join } from 'node:path';
2
2
  import YAML from 'yaml';
3
3
  import { z } from 'zod';
4
+ import { deriveFederatedConnection, FEDERATED_CONNECTION_ID } from '../connections/federation.js';
4
5
  import { HybridSearchCore } from '../../context/search/hybrid-search-core.js';
5
6
  import { DEFAULT_PRIORITY, resolveDescription } from './descriptions.js';
6
7
  import { normalizeSemanticLayerDescriptions } from './description-normalization.js';
7
8
  import { sourceDefinitionSchema, sourceOverlaySchema } from './schemas.js';
8
9
  import { composeOverlay, projectManifestEntry, SemanticLayerService, toResolvedWire, } from './semantic-layer.service.js';
9
10
  import { loadLatestSlDictionaryEntries } from './sl-dictionary-profile.js';
11
+ import { assertSafeConnectionId, isSafeConnectionId, isSlYamlPath, slSourceNameForFile, sourceNameFromPath, } from './source-files.js';
10
12
  import { buildSemanticLayerSourceSearchText, SlSearchService } from './sl-search.service.js';
11
13
  import { SqliteSlSourcesIndex } from './sqlite-sl-sources-index.js';
12
- const LOCAL_AUTHOR = 'ktx';
13
- const LOCAL_AUTHOR_EMAIL = 'ktx@example.com';
14
- function assertSafePathToken(kind, value) {
15
- if (value.trim().length === 0 ||
16
- value.includes('..') ||
17
- value.includes('\\') ||
18
- value.startsWith('/') ||
19
- value.startsWith('.') ||
20
- value.includes('//')) {
21
- throw new Error(`Unsafe ${kind}: ${value}`);
22
- }
23
- return value;
24
- }
25
- function assertSafeConnectionId(connectionId) {
26
- if (!/^[a-zA-Z0-9][a-zA-Z0-9_-]*$/.test(connectionId)) {
27
- throw new Error(`Unsafe connection id: ${connectionId}`);
28
- }
29
- return assertSafePathToken('connection id', connectionId);
30
- }
31
- function isSafeConnectionId(connectionId) {
32
- return typeof connectionId === 'string' && /^[a-zA-Z0-9][a-zA-Z0-9_-]*$/.test(connectionId);
33
- }
34
- function assertSafeSourceName(sourceName) {
35
- if (!/^[a-z0-9][a-z0-9_]*$/.test(sourceName)) {
36
- throw new Error(`Unsafe semantic-layer source name: ${sourceName}`);
37
- }
38
- return assertSafePathToken('semantic-layer source name', sourceName);
39
- }
40
14
  function isRecord(value) {
41
15
  return typeof value === 'object' && value !== null && !Array.isArray(value);
42
16
  }
43
- function slPath(connectionId, sourceName) {
44
- return `semantic-layer/${assertSafeConnectionId(connectionId)}/${assertSafeSourceName(sourceName)}.yaml`;
45
- }
46
- function sourceNameFromPath(path) {
47
- return (path
48
- .split('/')
49
- .at(-1)
50
- ?.replace(/\.ya?ml$/, '') ?? path);
51
- }
52
17
  function parseYamlRecord(raw) {
53
18
  const parsed = YAML.parse(raw);
54
19
  if (!isRecord(parsed)) {
@@ -122,15 +87,46 @@ function parsedStandaloneSource(parsed, name) {
122
87
  });
123
88
  }
124
89
  export async function loadLocalSlSourceRecords(project, input) {
125
- const connectionId = assertSafeConnectionId(input.connectionId);
90
+ if (input.connectionId === FEDERATED_CONNECTION_ID) {
91
+ const descriptor = deriveFederatedConnection(project.config.connections, project.projectDir);
92
+ if (!descriptor) {
93
+ return [];
94
+ }
95
+ const perMember = await Promise.all(descriptor.members.map(async (member) => {
96
+ const records = await loadSingleConnectionSourceRecords(project, member.connectionId);
97
+ return records.map((record) => {
98
+ // The federated view is one virtual connection: rows carry its id and a
99
+ // member-prefixed name, so a listing/search row round-trips to
100
+ // `ktx sl -c _ktx_federated read <name>`. Member origin lives in the name.
101
+ const name = `${member.connectionId}.${record.name}`;
102
+ return {
103
+ ...record,
104
+ connectionId: FEDERATED_CONNECTION_ID,
105
+ name,
106
+ source: { ...record.source, name },
107
+ };
108
+ });
109
+ }));
110
+ return perMember.flat();
111
+ }
112
+ return loadSingleConnectionSourceRecords(project, input.connectionId);
113
+ }
114
+ async function loadSingleConnectionSourceRecords(project, rawConnectionId) {
115
+ const connectionId = assertSafeConnectionId(rawConnectionId);
126
116
  const dir = `semantic-layer/${connectionId}`;
127
117
  const schemaDir = `${dir}/_schema`;
128
118
  const listed = await project.fileStore.listFiles(dir);
129
- const paths = listed.files.filter((file) => file.endsWith('.yaml') || file.endsWith('.yml')).sort();
119
+ const paths = listed.files.filter(isSlYamlPath).sort();
130
120
  const sources = new Map();
131
121
  for (const path of paths.filter((file) => file.startsWith(`${schemaDir}/`))) {
132
122
  const raw = await project.fileStore.readFile(path);
133
- const tables = manifestTables(parseYamlRecord(raw.content));
123
+ let tables;
124
+ try {
125
+ tables = manifestTables(parseYamlRecord(raw.content));
126
+ }
127
+ catch (error) {
128
+ throw new Error(`${path}: ${error instanceof Error ? error.message : String(error)}`);
129
+ }
134
130
  if (!tables) {
135
131
  continue;
136
132
  }
@@ -146,7 +142,30 @@ export async function loadLocalSlSourceRecords(project, input) {
146
142
  }
147
143
  for (const path of paths.filter((file) => !file.startsWith(`${schemaDir}/`))) {
148
144
  const raw = await project.fileStore.readFile(path);
149
- const parsed = parseYamlRecord(raw.content);
145
+ let parsed;
146
+ try {
147
+ parsed = parseYamlRecord(raw.content);
148
+ }
149
+ catch {
150
+ // A source mid-edit (e.g. an agent saved half-written YAML) must not take
151
+ // down reads, listings, or search for its siblings. Key it by the same
152
+ // name the writer side uses (the intact top-level `name:`, recovered even
153
+ // when the YAML is broken below it; filename only as a last resort) so a
154
+ // broken uppercase/hashed/human-renamed source stays reachable under its
155
+ // real name, and surface the raw content for repair.
156
+ const brokenName = slSourceNameForFile(path, raw.content);
157
+ sources.set(brokenName, {
158
+ connectionId,
159
+ name: brokenName,
160
+ path,
161
+ columnCount: 0,
162
+ measureCount: 0,
163
+ joinCount: 0,
164
+ yaml: raw.content,
165
+ source: { name: brokenName, grain: [], columns: [], joins: [], measures: [] },
166
+ });
167
+ continue;
168
+ }
150
169
  const name = typeof parsed.name === 'string' && parsed.name.length > 0 ? parsed.name : sourceNameFromPath(path);
151
170
  if (parsed.table || parsed.sql) {
152
171
  const source = parsedStandaloneSource(parsed, name);
@@ -191,36 +210,18 @@ export async function validateLocalSlSource(rawYaml, options) {
191
210
  return { valid: false, errors: validationErrors(error) };
192
211
  }
193
212
  }
194
- /** @internal */
195
- export async function writeLocalSlSource(project, input) {
196
- const validation = await validateLocalSlSource(input.yaml, { project, connectionId: input.connectionId });
197
- if (!validation.valid) {
198
- throw new Error(`Invalid semantic-layer source: ${validation.errors.join('; ')}`);
199
- }
200
- const parsed = parseYamlRecord(input.yaml);
201
- if (typeof parsed.name === 'string' && parsed.name !== input.sourceName) {
202
- throw new Error(`Semantic-layer source name "${parsed.name}" does not match requested path "${input.sourceName}"`);
203
- }
204
- const path = slPath(input.connectionId, input.sourceName);
205
- return project.fileStore.writeFile(path, input.yaml.endsWith('\n') ? input.yaml : `${input.yaml}\n`, LOCAL_AUTHOR, LOCAL_AUTHOR_EMAIL, `Write semantic-layer source: ${input.connectionId}/${input.sourceName}`);
206
- }
207
- /** @internal */
208
213
  export async function readLocalSlSource(project, input) {
209
- const path = slPath(input.connectionId, input.sourceName);
210
- try {
211
- const result = await project.fileStore.readFile(path);
212
- return {
213
- ...summarizeSource({ connectionId: input.connectionId, path, raw: result.content }),
214
- yaml: result.content,
215
- };
216
- }
217
- catch {
218
- const records = await loadLocalSlSourceRecords(project, {
219
- connectionId: input.connectionId,
220
- });
221
- const record = records.find((source) => source.name === input.sourceName);
222
- return record ? { ...record } : null;
223
- }
214
+ // Source identity is the in-file `name:` (mirroring the warehouse identifier
215
+ // verbatim, e.g. Snowflake's uppercase `WIDGET_SALES`), never the filename. The
216
+ // record loader resolves standalone files, overlays, manifest-backed sources,
217
+ // and mid-edit files whose YAML no longer parses — so readers — `ktx sl read`,
218
+ // `ktx sl validate`, and the `sl_read_source` MCP tool — can surface broken
219
+ // content for repair instead of failing on it.
220
+ const records = await loadLocalSlSourceRecords(project, {
221
+ connectionId: input.connectionId,
222
+ });
223
+ const record = records.find((source) => source.name === input.sourceName);
224
+ return record ? { ...record } : null;
224
225
  }
225
226
  export async function resolveLocalSlSource(project, input) {
226
227
  if (input.connectionId !== undefined) {
@@ -38,17 +38,23 @@ export declare class SemanticLayerService {
38
38
  name: string;
39
39
  connectionType: string;
40
40
  }>>;
41
- private sourcePath;
41
+ private resolveWritePath;
42
42
  writeSource(connectionId: string, source: SemanticLayerSource, author: string, authorEmail: string, commitMessage?: string, options?: WriteSourceOptions & {
43
43
  skipLock?: boolean;
44
44
  }): Promise<{
45
+ path: string;
45
46
  warnings: string[];
46
47
  commitHash?: string | null;
47
48
  }>;
49
+ /**
50
+ * Raw standalone/overlay file for a source, resolved by its in-file `name:`.
51
+ * Returns null when no file declares the name (the source may still exist as
52
+ * a manifest entry under `_schema/`).
53
+ */
48
54
  readSourceFile(connectionId: string, sourceName: string): Promise<{
49
55
  content: string;
50
56
  path: string;
51
- }>;
57
+ } | null>;
52
58
  loadSource(connectionId: string, sourceName: string): Promise<SemanticLayerSource | null>;
53
59
  loadAllSources(connectionId: string): Promise<LoadAllSourcesResult>;
54
60
  /**
@@ -87,14 +93,8 @@ export declare class SemanticLayerService {
87
93
  } | null>;
88
94
  validatePhysicalTableReferences(connectionId: string, sources: SemanticLayerSource[]): Promise<string[]>;
89
95
  getDialectForConnection(connectionId: string): Promise<string>;
90
- listSourceNames(connectionId: string): Promise<string[]>;
91
96
  listFilesForConnection(connectionId: string): Promise<string[]>;
92
- readFileByPath(connectionId: string, relativePath: string): Promise<{
93
- content: string;
94
- readOnly: boolean;
95
- }>;
96
97
  deleteSource(connectionId: string, sourceName: string, author: string, authorEmail: string): Promise<import("../../context/core/file-store.js").KtxFileWriteResult | null>;
97
- getSourceHistory(connectionId: string, sourceName: string): Promise<unknown>;
98
98
  /**
99
99
  * Validate the semantic layer state that *would* exist if `proposedSource`
100
100
  * were written, without persisting anything. Used by write/edit tools to
@@ -203,6 +203,23 @@ export interface ManifestTableEntry {
203
203
  usage?: TableUsageOutput;
204
204
  }
205
205
  export declare function projectManifestEntry(name: string, entry: ManifestTableEntry): SemanticLayerSource;
206
+ export interface MissingJoinTarget {
207
+ to: string;
208
+ /** Source whose name matches only case-insensitively, if any — the usual authoring mistake. */
209
+ caseMismatch: string | null;
210
+ }
211
+ /**
212
+ * Join targets that do not exactly match a known source name. The Python
213
+ * engine resolves `joins[].to` by exact name within one connection's source
214
+ * set (`engine._collect_orphan_join_target_errors`) and `query()` raises on a
215
+ * miss, so anything looser here — case-insensitive matches, table refs,
216
+ * sources in other connections — would pass this gate and then fail
217
+ * query/validation as an orphan join target.
218
+ */
219
+ export declare function findMissingJoinTargets(joins: Array<{
220
+ to: string;
221
+ }> | undefined, knownSourceNames: Iterable<string>): MissingJoinTarget[];
222
+ export declare function formatMissingJoinTarget(missing: MissingJoinTarget): string;
206
223
  /**
207
224
  * Returns one message per measure-level segment reference that doesn't resolve to
208
225
  * a segment defined on the source. Array is empty when every reference checks out.
@@ -2,6 +2,7 @@ import YAML from 'yaml';
2
2
  import { noopLogger } from '../../context/core/config.js';
3
3
  import { normalizeSemanticLayerDescriptions } from './description-normalization.js';
4
4
  import { isOverlaySource, resolvedSourceSchema, sourceDefinitionSchema, sourceOverlaySchema } from './schemas.js';
5
+ import { isSlYamlPath, resolveSlSourceFile, slDeclaredSourceName, slSourceFilePath } from './source-files.js';
5
6
  const SL_DIR_PREFIX = 'semantic-layer';
6
7
  const CONNECTION_ID_PATTERN = /^[a-zA-Z0-9][a-zA-Z0-9_-]*$/;
7
8
  /** @internal */
@@ -85,7 +86,7 @@ export class SemanticLayerService {
85
86
  async listConnectionIds() {
86
87
  try {
87
88
  const result = await this.configService.listFiles(SL_DIR_PREFIX);
88
- // Directories under semantic-layer/ are connectionIds. Local KTX projects use
89
+ // Directories under semantic-layer/ are connectionIds. Local ktx projects use
89
90
  // readable ids like "warehouse" and "dbt-main", not only UUIDs.
90
91
  return result.files
91
92
  .map((f) => f.replace(`${SL_DIR_PREFIX}/`, '').split('/')[0])
@@ -104,8 +105,31 @@ export class SemanticLayerService {
104
105
  return this.connections.listEnabledConnections(ids);
105
106
  }
106
107
  // ── YAML File Operations ────────────────────────────────
107
- sourcePath(connectionId, sourceName) {
108
- return `${SL_DIR_PREFIX}/${connectionId}/${sourceName}.yaml`;
108
+ // The in-file `name:` is the source's identity; the filename is only a derived
109
+ // label. Rewrites land on the file that already declares the name (humans may
110
+ // rename files freely); new sources get a derived filename. A file already
111
+ // sitting at the derived path that declares a name declares a *different* one
112
+ // (the resolver would have matched it otherwise) — fail instead of clobbering
113
+ // it. A nameless/unparseable file there is the broken remains of this very
114
+ // source (the derived path is a function of the name), so overwriting it is
115
+ // the repair path, not data loss.
116
+ async resolveWritePath(connectionId, sourceName) {
117
+ const existing = await resolveSlSourceFile(this.configService, connectionId, sourceName);
118
+ if (existing) {
119
+ return existing.path;
120
+ }
121
+ const path = slSourceFilePath(connectionId, sourceName);
122
+ let occupant = null;
123
+ try {
124
+ occupant = slDeclaredSourceName((await this.configService.readFile(path)).content);
125
+ }
126
+ catch {
127
+ return path;
128
+ }
129
+ if (occupant !== null) {
130
+ throw new Error(`Cannot write source '${sourceName}': ${path} already defines source '${occupant}'`);
131
+ }
132
+ return path;
109
133
  }
110
134
  async writeSource(connectionId, source, author, authorEmail, commitMessage, options) {
111
135
  // Writes are intentionally permissive — the agent must be able to save broken files so
@@ -140,38 +164,40 @@ export class SemanticLayerService {
140
164
  this.logger.warn(`[writeSource] '${source.name}': ${danglingRefs.join('; ')}. Saving anyway.`);
141
165
  }
142
166
  }
143
- const path = this.sourcePath(connectionId, source.name);
167
+ const path = await this.resolveWritePath(connectionId, source.name);
144
168
  const normalizedSource = normalizeSemanticLayerDescriptions(source);
145
169
  const content = YAML.stringify(normalizedSource, { indent: 2, lineWidth: 0, version: '1.1' });
146
170
  const message = commitMessage ?? `Update semantic layer source: ${source.name}`;
147
171
  const result = await this.configService.writeFile(path, content, author, authorEmail, message, {
148
172
  skipLock: options?.skipLock,
149
173
  });
150
- return { ...result, warnings };
174
+ // The filename is derived from (or resolved by) the source name — surface
175
+ // the actual path so callers don't have to re-resolve it.
176
+ return { ...result, path, warnings };
151
177
  }
178
+ /**
179
+ * Raw standalone/overlay file for a source, resolved by its in-file `name:`.
180
+ * Returns null when no file declares the name (the source may still exist as
181
+ * a manifest entry under `_schema/`).
182
+ */
152
183
  async readSourceFile(connectionId, sourceName) {
153
- const path = this.sourcePath(connectionId, sourceName);
154
- const result = await this.configService.readFile(path);
155
- return { content: result.content, path };
184
+ const file = await resolveSlSourceFile(this.configService, connectionId, sourceName);
185
+ return file ? { content: file.content, path: file.path } : null;
156
186
  }
157
187
  async loadSource(connectionId, sourceName) {
158
- let content;
159
- try {
160
- const result = await this.readSourceFile(connectionId, sourceName);
161
- content = result.content;
162
- }
163
- catch {
188
+ const file = await this.readSourceFile(connectionId, sourceName);
189
+ if (!file) {
164
190
  return null;
165
191
  }
166
192
  try {
167
- return YAML.parse(content);
193
+ return YAML.parse(file.content);
168
194
  }
169
195
  catch (error) {
170
196
  // Distinguish a YAML parse failure from a missing file. The file exists but
171
197
  // its contents are unparseable — callers that treat null as "does not exist"
172
198
  // could otherwise overwrite the broken file. Surface the parse failure via
173
199
  // the service logger so the broken source is at least visible.
174
- this.logger.warn(`[loadSource] ${connectionId}/${sourceName}.yaml: YAML parse failed: ${error instanceof Error ? error.message : String(error)}`);
200
+ this.logger.warn(`[loadSource] ${file.path}: YAML parse failed: ${error instanceof Error ? error.message : String(error)}`);
175
201
  return null;
176
202
  }
177
203
  }
@@ -182,7 +208,7 @@ export class SemanticLayerService {
182
208
  let allFiles;
183
209
  try {
184
210
  const result = await this.configService.listFiles(dir);
185
- allFiles = result.files.filter((f) => f.endsWith('.yaml'));
211
+ allFiles = result.files.filter((f) => isSlYamlPath(f));
186
212
  }
187
213
  catch (e) {
188
214
  const message = `Failed to list semantic-layer files under ${dir}: ${e instanceof Error ? e.message : String(e)}`;
@@ -283,7 +309,7 @@ export class SemanticLayerService {
283
309
  let allFiles;
284
310
  try {
285
311
  const listing = await this.configService.listFiles(dir);
286
- allFiles = listing.files.filter((f) => f.endsWith('.yaml'));
312
+ allFiles = listing.files.filter((f) => isSlYamlPath(f));
287
313
  }
288
314
  catch {
289
315
  return result;
@@ -350,7 +376,7 @@ export class SemanticLayerService {
350
376
  const schemaDir = `${SL_DIR_PREFIX}/${connectionId}/_schema`;
351
377
  try {
352
378
  const result = await this.configService.listFiles(schemaDir);
353
- const yamlFiles = result.files.filter((f) => f.endsWith('.yaml'));
379
+ const yamlFiles = result.files.filter((f) => isSlYamlPath(f));
354
380
  for (const filePath of yamlFiles) {
355
381
  try {
356
382
  const { content } = await this.configService.readFile(filePath);
@@ -390,7 +416,7 @@ export class SemanticLayerService {
390
416
  let yamlFiles;
391
417
  try {
392
418
  const result = await this.configService.listFiles(schemaDir);
393
- yamlFiles = result.files.filter((f) => f.endsWith('.yaml'));
419
+ yamlFiles = result.files.filter((f) => isSlYamlPath(f));
394
420
  }
395
421
  catch {
396
422
  return null;
@@ -457,7 +483,7 @@ export class SemanticLayerService {
457
483
  .filter((c) => !c.expr && !manifestColumns.has(c.name.toLowerCase()))
458
484
  .map((c) => c.name);
459
485
  if (absentDeclaredColumns.length > 0) {
460
- errors.push(`${source.name}.yaml: table "${source.table}" matched manifest ${manifestLabel}, ` +
486
+ errors.push(`${source.name}: table "${source.table}" matched manifest ${manifestLabel}, ` +
461
487
  `but declared column(s) absent from physical table: ${absentDeclaredColumns.join(', ')}. ` +
462
488
  `Available columns: ${[...manifestColumns.values()].join(', ')}`);
463
489
  }
@@ -466,7 +492,7 @@ export class SemanticLayerService {
466
492
  return !declared || (!declared.expr && !manifestColumns.has(grain.toLowerCase()));
467
493
  });
468
494
  if (missingGrainColumns.length > 0) {
469
- errors.push(`${source.name}.yaml: grain column(s) absent from physical table "${source.table}": ${missingGrainColumns.join(', ')}`);
495
+ errors.push(`${source.name}: grain column(s) absent from physical table "${source.table}": ${missingGrainColumns.join(', ')}`);
470
496
  }
471
497
  for (const column of declaredColumns) {
472
498
  if (!column.expr) {
@@ -480,7 +506,7 @@ export class SemanticLayerService {
480
506
  validMeasures: new Set(),
481
507
  });
482
508
  if (missing.length > 0) {
483
- errors.push(`${source.name}.yaml: computed column "${column.name}" references unknown column(s): ${missing.join(', ')}`);
509
+ errors.push(`${source.name}: computed column "${column.name}" references unknown column(s): ${missing.join(', ')}`);
484
510
  }
485
511
  }
486
512
  for (const segment of source.segments ?? []) {
@@ -492,7 +518,7 @@ export class SemanticLayerService {
492
518
  validMeasures: new Set(),
493
519
  });
494
520
  if (missing.length > 0) {
495
- errors.push(`${source.name}.yaml: segment "${segment.name}" references unknown column(s): ${missing.join(', ')}`);
521
+ errors.push(`${source.name}: segment "${segment.name}" references unknown column(s): ${missing.join(', ')}`);
496
522
  }
497
523
  }
498
524
  for (const measure of source.measures ?? []) {
@@ -504,7 +530,7 @@ export class SemanticLayerService {
504
530
  validMeasures: measureNames,
505
531
  });
506
532
  if (exprMissing.length > 0) {
507
- errors.push(`${source.name}.yaml: measure "${measure.name}" references unknown column(s): ${exprMissing.join(', ')}`);
533
+ errors.push(`${source.name}: measure "${measure.name}" references unknown column(s): ${exprMissing.join(', ')}`);
508
534
  }
509
535
  if (measure.filter) {
510
536
  const filterMissing = missingLocalExpressionRefs({
@@ -515,7 +541,7 @@ export class SemanticLayerService {
515
541
  validMeasures: new Set(),
516
542
  });
517
543
  if (filterMissing.length > 0) {
518
- errors.push(`${source.name}.yaml: measure "${measure.name}" filter references unknown column(s): ${filterMissing.join(', ')}`);
544
+ errors.push(`${source.name}: measure "${measure.name}" filter references unknown column(s): ${filterMissing.join(', ')}`);
519
545
  }
520
546
  }
521
547
  }
@@ -525,7 +551,7 @@ export class SemanticLayerService {
525
551
  continue;
526
552
  }
527
553
  if (!validOutputColumns.has(parsed.localColumn.toLowerCase())) {
528
- errors.push(`${source.name}.yaml: join to "${join.to}" references local column ` +
554
+ errors.push(`${source.name}: join to "${join.to}" references local column ` +
529
555
  `"${parsed.localColumn}" that is not a valid output column`);
530
556
  }
531
557
  const targetSource = sourcesByName.get(join.to.toLowerCase()) ??
@@ -533,7 +559,7 @@ export class SemanticLayerService {
533
559
  if (targetSource) {
534
560
  const targetColumns = new Set(targetSource.columns.map((c) => c.name.toLowerCase()));
535
561
  if (!targetColumns.has(parsed.targetColumn.toLowerCase())) {
536
- errors.push(`${source.name}.yaml: join to "${join.to}" references target column ` +
562
+ errors.push(`${source.name}: join to "${join.to}" references target column ` +
537
563
  `"${parsed.targetColumn}" that does not exist on the target source`);
538
564
  }
539
565
  }
@@ -548,41 +574,28 @@ export class SemanticLayerService {
548
574
  }
549
575
  return SemanticLayerService.mapDialect(connection.connectionType);
550
576
  }
551
- async listSourceNames(connectionId) {
552
- const dir = `${SL_DIR_PREFIX}/${connectionId}`;
553
- try {
554
- const result = await this.configService.listFiles(dir);
555
- return result.files.filter((f) => f.endsWith('.yaml')).map((f) => f.replace(`${dir}/`, '').replace('.yaml', ''));
556
- }
557
- catch {
558
- return [];
559
- }
560
- }
561
577
  async listFilesForConnection(connectionId) {
562
578
  const dir = `${SL_DIR_PREFIX}/${connectionId}`;
563
579
  try {
564
580
  const result = await this.configService.listFiles(dir, true);
565
- return result.files.filter((f) => f.endsWith('.yaml'));
581
+ return result.files.filter((f) => isSlYamlPath(f));
566
582
  }
567
583
  catch {
568
584
  return [];
569
585
  }
570
586
  }
571
- async readFileByPath(connectionId, relativePath) {
572
- const fullPath = `${SL_DIR_PREFIX}/${connectionId}/${relativePath}`;
573
- const result = await this.configService.readFile(fullPath);
574
- return {
575
- content: result.content,
576
- readOnly: relativePath.startsWith('_schema/'),
577
- };
578
- }
579
587
  async deleteSource(connectionId, sourceName, author, authorEmail) {
580
- const path = this.sourcePath(connectionId, sourceName);
581
- return this.configService.deleteFile(path, author, authorEmail, `Delete semantic layer source: ${sourceName}`);
582
- }
583
- async getSourceHistory(connectionId, sourceName) {
584
- const path = this.sourcePath(connectionId, sourceName);
585
- return this.configService.getFileHistory(path);
588
+ const file = await resolveSlSourceFile(this.configService, connectionId, sourceName);
589
+ if (!file) {
590
+ // `deleteFile` returns null for a missing path, which would let a no-op
591
+ // delete read as success. Distinguish the two real cases instead.
592
+ if (await this.isManifestBacked(connectionId, sourceName)) {
593
+ throw new Error(`Source '${sourceName}' is defined by the scan manifest (_schema/) and has no overlay file to delete. ` +
594
+ `Rescan the connection to remove it from the manifest.`);
595
+ }
596
+ throw new Error(`Semantic-layer source not found: ${connectionId}/${sourceName}`);
597
+ }
598
+ return this.configService.deleteFile(file.path, author, authorEmail, `Delete semantic layer source: ${sourceName}`);
586
599
  }
587
600
  /**
588
601
  * Validate the semantic layer state that *would* exist if `proposedSource`
@@ -639,6 +652,16 @@ export class SemanticLayerService {
639
652
  // any column-without-type errors via the warehouse probe.
640
653
  }
641
654
  merged.push(toPush);
655
+ // A join target the engine cannot resolve fails every downstream gate and
656
+ // query with the error attributed to the phantom target. Reject it here,
657
+ // on the source that declares it, while the writing agent can still fix it.
658
+ const missingJoinTargets = findMissingJoinTargets(toPush.joins, merged.map((s) => s.name));
659
+ const joinTargetErrors = missingJoinTargets.map((missing) => `${toPush.name}: ${formatMissingJoinTarget(missing)}. Declare joins only to existing ` +
660
+ `semantic-layer sources in this connection, or drop the join and keep the relationship ` +
661
+ `in a column description.`);
662
+ if (joinTargetErrors.length > 0) {
663
+ return { errors: [...loadErrors, ...joinTargetErrors], warnings: [], perSourceWarnings: {} };
664
+ }
642
665
  const validatable = merged.filter((s) => s.table != null || s.sql != null);
643
666
  if (validatable.length === 0) {
644
667
  return { errors: loadErrors, warnings: [], perSourceWarnings: {} };
@@ -704,7 +727,7 @@ export class SemanticLayerService {
704
727
  catch {
705
728
  return [];
706
729
  }
707
- const schemaFiles = files.filter((file) => /^semantic-layer\/[^/]+\/_schema\/.+\.ya?ml$/.test(file));
730
+ const schemaFiles = files.filter((file) => /^semantic-layer\/[^/]+\/_schema\//.test(file) && isSlYamlPath(file));
708
731
  const entries = [];
709
732
  for (const filePath of schemaFiles) {
710
733
  const connectionId = filePath.split('/')[1];
@@ -732,7 +755,7 @@ export class SemanticLayerService {
732
755
  let allFiles;
733
756
  try {
734
757
  const result = await this.configService.listFiles(dir);
735
- allFiles = result.files.filter((f) => f.endsWith('.yaml'));
758
+ allFiles = result.files.filter((f) => isSlYamlPath(f));
736
759
  }
737
760
  catch {
738
761
  return warnings;
@@ -883,7 +906,7 @@ export class SemanticLayerService {
883
906
  const tables = new Map();
884
907
  try {
885
908
  const result = await this.configService.listFiles(dir);
886
- const yamlFiles = result.files.filter((f) => f.endsWith('.yaml'));
909
+ const yamlFiles = result.files.filter((f) => isSlYamlPath(f));
887
910
  for (const filePath of yamlFiles) {
888
911
  try {
889
912
  const { content } = await this.configService.readFile(filePath);
@@ -1182,6 +1205,36 @@ function parseJoinColumns(on, sourceName, targetName) {
1182
1205
  }
1183
1206
  return { localColumn: left.column, targetColumn: right.column };
1184
1207
  }
1208
+ /**
1209
+ * Join targets that do not exactly match a known source name. The Python
1210
+ * engine resolves `joins[].to` by exact name within one connection's source
1211
+ * set (`engine._collect_orphan_join_target_errors`) and `query()` raises on a
1212
+ * miss, so anything looser here — case-insensitive matches, table refs,
1213
+ * sources in other connections — would pass this gate and then fail
1214
+ * query/validation as an orphan join target.
1215
+ */
1216
+ export function findMissingJoinTargets(joins, knownSourceNames) {
1217
+ const known = new Set();
1218
+ const canonicalByLower = new Map();
1219
+ for (const name of knownSourceNames) {
1220
+ known.add(name);
1221
+ canonicalByLower.set(name.toLowerCase(), name);
1222
+ }
1223
+ const missing = [];
1224
+ for (const join of joins ?? []) {
1225
+ if (known.has(join.to)) {
1226
+ continue;
1227
+ }
1228
+ missing.push({ to: join.to, caseMismatch: canonicalByLower.get(join.to.toLowerCase()) ?? null });
1229
+ }
1230
+ return missing;
1231
+ }
1232
+ export function formatMissingJoinTarget(missing) {
1233
+ const hint = missing.caseMismatch
1234
+ ? `; join targets are case-sensitive — the source is named "${missing.caseMismatch}"`
1235
+ : '';
1236
+ return `join target "${missing.to}" does not exist${hint}`;
1237
+ }
1185
1238
  /**
1186
1239
  * Returns one message per measure-level segment reference that doesn't resolve to
1187
1240
  * a segment defined on the source. Array is empty when every reference checks out.
@@ -0,0 +1,48 @@
1
+ import type { KtxFileStorePort } from '../../context/core/file-store.js';
2
+ /** @internal */
3
+ export declare function isReservedConnectionId(connectionId: string): boolean;
4
+ export declare function assertSafeConnectionId(connectionId: string): string;
5
+ export declare function isSafeConnectionId(connectionId: string | undefined): connectionId is string;
6
+ export declare function sourceNameFromPath(path: string): string;
7
+ export declare function isSlYamlPath(path: string): boolean;
8
+ /**
9
+ * Derive the filename for a semantic-layer source. Total over all possible
10
+ * source names — never throws.
11
+ *
12
+ * Names that are already safe lowercase snake_case become `<name>.yaml`;
13
+ * anything else becomes `<slug>-<8 hex of sha256(name)>.yaml`. The two ranges
14
+ * are disjoint and the mapping is injective: safe filenames contain no `-`,
15
+ * hashed filenames always end in `-<8 hex>`, and slugs are lowercased so names
16
+ * differing only by case get distinct hashes instead of colliding paths on
17
+ * case-insensitive filesystems (macOS APFS, Windows).
18
+ *
19
+ * @internal
20
+ */
21
+ export declare function slSourceFileName(sourceName: string): string;
22
+ export declare function slSourceFilePath(connectionId: string, sourceName: string): string;
23
+ export interface SlSourceFile {
24
+ path: string;
25
+ content: string;
26
+ }
27
+ export declare function slSourceNameForFile(path: string, content: string): string;
28
+ /**
29
+ * The `name:` a semantic-layer YAML file declares, or null when the file is
30
+ * nameless or so broken even the name is unrecoverable. Null is how
31
+ * `writeSource` tells a genuine name conflict at a derived path apart from the
32
+ * broken remains of the source being written, which a rewrite must repair
33
+ * rather than refuse.
34
+ *
35
+ * Uses `parseDocument`, not `parse`: a file with a syntax error below the
36
+ * `name:` line still parses into a partial tree whose top-level `name:` is
37
+ * intact. `parse` would throw on the same input and drop the source to its
38
+ * filename — wrong for human-renamed files, whose filename is not the name.
39
+ */
40
+ export declare function slDeclaredSourceName(content: string): string | null;
41
+ /**
42
+ * Find the standalone/overlay file that defines `sourceName` for a connection.
43
+ * Returns null when no file declares the name (the source may still exist as a
44
+ * manifest entry under `_schema/`). Throws when more than one file declares the
45
+ * same name — that breaks the one-file-per-name invariant and must be repaired
46
+ * by hand rather than silently picking one.
47
+ */
48
+ export declare function resolveSlSourceFile(fileStore: Pick<KtxFileStorePort, 'listFiles' | 'readFile'>, connectionId: string, sourceName: string): Promise<SlSourceFile | null>;