@kaelio/ktx 0.11.0 → 0.13.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (212) hide show
  1. package/assets/python/kaelio_ktx-0.13.0-py3-none-any.whl +0 -0
  2. package/assets/python/manifest.json +4 -4
  3. package/dist/.tsbuildinfo +1 -1
  4. package/dist/admin.js +1 -1
  5. package/dist/clack.d.ts +16 -0
  6. package/dist/clack.js +37 -6
  7. package/dist/claude-code-prompt-caching.js +1 -1
  8. package/dist/cli-program.js +3 -3
  9. package/dist/cli-runtime.js +2 -2
  10. package/dist/commands/connection-commands.js +1 -1
  11. package/dist/commands/ingest-commands.js +4 -4
  12. package/dist/commands/mcp-commands.js +12 -12
  13. package/dist/commands/runtime-commands.js +4 -4
  14. package/dist/commands/setup-commands.js +19 -5
  15. package/dist/commands/sl-commands.js +1 -1
  16. package/dist/commands/sql-commands.js +1 -1
  17. package/dist/commands/status-commands.js +1 -1
  18. package/dist/connection.js +15 -3
  19. package/dist/connectors/bigquery/connector.js +1 -14
  20. package/dist/connectors/clickhouse/connector.js +2 -16
  21. package/dist/connectors/duckdb/federated-attach.d.ts +7 -0
  22. package/dist/connectors/duckdb/federated-attach.js +86 -0
  23. package/dist/connectors/duckdb/federated-executor.d.ts +5 -0
  24. package/dist/connectors/duckdb/federated-executor.js +59 -0
  25. package/dist/connectors/mysql/connector.js +2 -16
  26. package/dist/connectors/postgres/connector.js +1 -14
  27. package/dist/connectors/shared/string-reference.d.ts +6 -0
  28. package/dist/connectors/shared/string-reference.js +19 -0
  29. package/dist/connectors/snowflake/connector.d.ts +1 -1
  30. package/dist/connectors/snowflake/connector.js +1 -14
  31. package/dist/connectors/sqlite/connector.js +2 -25
  32. package/dist/connectors/sqlserver/connector.js +4 -17
  33. package/dist/context/connections/connection-type.d.ts +1 -1
  34. package/dist/context/connections/federation.d.ts +33 -0
  35. package/dist/context/connections/federation.js +51 -0
  36. package/dist/context/connections/local-warehouse-descriptor.d.ts +2 -0
  37. package/dist/context/connections/project-sql-executor.d.ts +18 -0
  38. package/dist/context/connections/project-sql-executor.js +39 -0
  39. package/dist/context/connections/query-executor.d.ts +2 -2
  40. package/dist/context/connections/read-only-sql.d.ts +1 -0
  41. package/dist/context/connections/read-only-sql.js +119 -4
  42. package/dist/context/connections/resolve-connection.d.ts +12 -0
  43. package/dist/context/connections/resolve-connection.js +37 -0
  44. package/dist/context/core/git-env.d.ts +4 -0
  45. package/dist/context/core/git-env.js +5 -1
  46. package/dist/context/core/git.service.d.ts +23 -0
  47. package/dist/context/core/git.service.js +71 -8
  48. package/dist/context/ingest/adapters/historic-sql/projection.js +2 -1
  49. package/dist/context/ingest/adapters/live-database/manifest.d.ts +3 -0
  50. package/dist/context/ingest/adapters/live-database/manifest.js +19 -11
  51. package/dist/context/ingest/adapters/looker/client.js +7 -2
  52. package/dist/context/ingest/adapters/looker/factory.d.ts +8 -1
  53. package/dist/context/ingest/adapters/looker/factory.js +9 -0
  54. package/dist/context/ingest/adapters/looker/mapping.js +1 -1
  55. package/dist/context/ingest/adapters/looker/types.d.ts +1 -1
  56. package/dist/context/ingest/adapters/metabase/client.d.ts +1 -1
  57. package/dist/context/ingest/adapters/metabase/client.js +1 -1
  58. package/dist/context/ingest/adapters/metabase/local-metabase.adapter.js +1 -1
  59. package/dist/context/ingest/adapters/metabase/mapping.js +6 -6
  60. package/dist/context/ingest/artifact-gates.d.ts +2 -6
  61. package/dist/context/ingest/artifact-gates.js +5 -47
  62. package/dist/context/ingest/constrained-repair.d.ts +55 -0
  63. package/dist/context/ingest/constrained-repair.js +167 -0
  64. package/dist/context/ingest/final-gate-repair.d.ts +9 -11
  65. package/dist/context/ingest/final-gate-repair.js +40 -128
  66. package/dist/context/ingest/finalization-scope.d.ts +1 -1
  67. package/dist/context/ingest/finalization-scope.js +15 -15
  68. package/dist/context/ingest/ingest-bundle.runner.d.ts +1 -0
  69. package/dist/context/ingest/ingest-bundle.runner.js +101 -67
  70. package/dist/context/ingest/isolated-diff/patch-integrator.d.ts +6 -13
  71. package/dist/context/ingest/isolated-diff/patch-integrator.js +32 -109
  72. package/dist/context/ingest/isolated-diff/textual-conflict-resolver.d.ts +8 -9
  73. package/dist/context/ingest/isolated-diff/textual-conflict-resolver.js +63 -141
  74. package/dist/context/ingest/local-bundle-runtime.d.ts +2 -0
  75. package/dist/context/ingest/local-bundle-runtime.js +9 -10
  76. package/dist/context/ingest/local-ingest.d.ts +2 -0
  77. package/dist/context/ingest/local-ingest.js +2 -0
  78. package/dist/context/ingest/memory-flow/view-model.js +1 -1
  79. package/dist/context/ingest/stages/stage-3-work-units.d.ts +2 -6
  80. package/dist/context/ingest/stages/stage-3-work-units.js +2 -1
  81. package/dist/context/ingest/stages/validate-wu-sources.d.ts +7 -1
  82. package/dist/context/ingest/stages/validate-wu-sources.js +109 -4
  83. package/dist/context/ingest/tools/warehouse-verification/create-warehouse-verification-tools.d.ts +2 -0
  84. package/dist/context/ingest/tools/warehouse-verification/create-warehouse-verification-tools.js +1 -1
  85. package/dist/context/ingest/tools/warehouse-verification/discover-data.tool.js +3 -3
  86. package/dist/context/ingest/tools/warehouse-verification/sql-execution.tool.d.ts +3 -1
  87. package/dist/context/ingest/tools/warehouse-verification/sql-execution.tool.js +15 -1
  88. package/dist/context/llm/ai-sdk-runtime.js +2 -2
  89. package/dist/context/llm/claude-code-runtime.js +19 -3
  90. package/dist/context/llm/local-config.js +1 -1
  91. package/dist/context/llm/runtime-tools.js +2 -2
  92. package/dist/context/mcp/context-tools.js +33 -8
  93. package/dist/context/mcp/local-project-ports.js +63 -89
  94. package/dist/context/mcp/types.d.ts +2 -0
  95. package/dist/context/memory/local-memory.js +4 -1
  96. package/dist/context/memory/memory-agent.service.js +1 -1
  97. package/dist/context/project/config.d.ts +11 -4
  98. package/dist/context/project/config.js +85 -30
  99. package/dist/context/project/driver-schemas.js +1 -1
  100. package/dist/context/project/mappings-yaml-schema.js +2 -2
  101. package/dist/context/project/project.js +12 -4
  102. package/dist/context/scan/description-generation.js +4 -4
  103. package/dist/context/scan/local-enrichment-artifacts.js +33 -4
  104. package/dist/context/scan/local-scan.js +2 -2
  105. package/dist/context/scan/local-structural-artifacts.js +5 -5
  106. package/dist/context/scan/relationship-benchmark-report.js +1 -1
  107. package/dist/context/scan/relationship-discovery.js +3 -3
  108. package/dist/context/scan/relationship-llm-proposal.js +3 -3
  109. package/dist/context/sl/local-query.js +31 -44
  110. package/dist/context/sl/local-sl.d.ts +0 -8
  111. package/dist/context/sl/local-sl.js +71 -70
  112. package/dist/context/sl/semantic-layer.service.d.ts +25 -8
  113. package/dist/context/sl/semantic-layer.service.js +109 -56
  114. package/dist/context/sl/source-files.d.ts +48 -0
  115. package/dist/context/sl/source-files.js +138 -0
  116. package/dist/context/sl/tools/base-semantic-layer.tool.d.ts +2 -2
  117. package/dist/context/sl/tools/base-semantic-layer.tool.js +2 -7
  118. package/dist/context/sl/tools/sl-edit-source.tool.js +10 -8
  119. package/dist/context/sl/tools/sl-warehouse-validation.js +55 -27
  120. package/dist/context/sl/tools/sl-write-source.tool.js +12 -9
  121. package/dist/context/sql-analysis/dialect.d.ts +2 -0
  122. package/dist/context/sql-analysis/dialect.js +20 -0
  123. package/dist/context/tools/base-tool.d.ts +6 -19
  124. package/dist/context/tools/base-tool.js +0 -14
  125. package/dist/context-build-view.js +5 -5
  126. package/dist/database-tree-picker.js +18 -3
  127. package/dist/demo-assets.js +0 -1
  128. package/dist/doctor.d.ts +1 -1
  129. package/dist/doctor.js +31 -23
  130. package/dist/errors.d.ts +31 -0
  131. package/dist/errors.js +44 -0
  132. package/dist/ingest-query-executor.d.ts +2 -0
  133. package/dist/ingest-query-executor.js +8 -22
  134. package/dist/ingest.d.ts +1 -1
  135. package/dist/ingest.js +8 -2
  136. package/dist/io/symbols.d.ts +2 -0
  137. package/dist/io/symbols.js +2 -0
  138. package/dist/io/tty.d.ts +8 -0
  139. package/dist/io/tty.js +16 -0
  140. package/dist/llm/embedding-health.js +1 -1
  141. package/dist/llm/embedding-provider.js +3 -3
  142. package/dist/llm/model-provider.js +1 -1
  143. package/dist/local-adapters.d.ts +1 -0
  144. package/dist/local-adapters.js +2 -2
  145. package/dist/local-scan-connectors.js +1 -1
  146. package/dist/managed-local-embeddings.js +17 -8
  147. package/dist/managed-mcp-daemon.js +3 -3
  148. package/dist/managed-python-command.d.ts +7 -0
  149. package/dist/managed-python-command.js +34 -8
  150. package/dist/managed-python-daemon.js +2 -2
  151. package/dist/managed-python-http.js +3 -3
  152. package/dist/managed-python-runtime.d.ts +30 -1
  153. package/dist/managed-python-runtime.js +134 -18
  154. package/dist/managed-uv-release.d.ts +7 -0
  155. package/dist/managed-uv-release.js +11 -0
  156. package/dist/mcp-http-server.js +4 -4
  157. package/dist/mcp-server-factory.js +3 -3
  158. package/dist/mcp-stdio-server.js +1 -1
  159. package/dist/memory-flow-hud.js +2 -2
  160. package/dist/next-steps.js +2 -2
  161. package/dist/prompt-navigation.d.ts +17 -0
  162. package/dist/prompt-navigation.js +49 -3
  163. package/dist/prompts/memory_agent_bundle_ingest_work_unit.md +2 -2
  164. package/dist/prompts/memory_agent_external_ingest.md +2 -2
  165. package/dist/public-ingest-copy.js +1 -1
  166. package/dist/public-ingest.js +3 -3
  167. package/dist/release-version.js +1 -1
  168. package/dist/runtime-requirements.js +1 -1
  169. package/dist/runtime.js +9 -9
  170. package/dist/scan.js +1 -1
  171. package/dist/setup-agents.d.ts +21 -15
  172. package/dist/setup-agents.js +143 -66
  173. package/dist/setup-banner.d.ts +20 -0
  174. package/dist/setup-banner.js +39 -0
  175. package/dist/setup-context.js +24 -15
  176. package/dist/setup-databases.d.ts +3 -0
  177. package/dist/setup-databases.js +47 -59
  178. package/dist/setup-demo-tour.js +12 -8
  179. package/dist/setup-embeddings.js +9 -9
  180. package/dist/setup-interrupt.js +1 -1
  181. package/dist/setup-models.d.ts +4 -1
  182. package/dist/setup-models.js +54 -28
  183. package/dist/setup-project.js +29 -5
  184. package/dist/setup-prompts.js +16 -1
  185. package/dist/setup-ready-menu.js +1 -1
  186. package/dist/setup-sources.js +28 -12
  187. package/dist/setup.d.ts +1 -0
  188. package/dist/setup.js +14 -13
  189. package/dist/skills/analytics/SKILL.md +3 -3
  190. package/dist/skills/dbt_ingest/SKILL.md +3 -3
  191. package/dist/skills/looker_ingest/SKILL.md +3 -3
  192. package/dist/skills/lookml_ingest/SKILL.md +7 -7
  193. package/dist/skills/metabase_ingest/SKILL.md +4 -4
  194. package/dist/skills/metricflow_ingest/SKILL.md +15 -15
  195. package/dist/skills/notion_synthesize/SKILL.md +1 -1
  196. package/dist/skills/sl/SKILL.md +3 -3
  197. package/dist/skills/sl_capture/SKILL.md +1 -1
  198. package/dist/skills/wiki_capture/SKILL.md +1 -1
  199. package/dist/source-mapping.js +1 -1
  200. package/dist/sql.d.ts +2 -0
  201. package/dist/sql.js +35 -53
  202. package/dist/startup-profile.js +1 -1
  203. package/dist/status-project.d.ts +0 -2
  204. package/dist/status-project.js +4 -6
  205. package/dist/telemetry/events.d.ts +3 -2
  206. package/dist/telemetry/events.js +11 -1
  207. package/dist/telemetry/exception.js +14 -0
  208. package/dist/text-ingest.js +1 -1
  209. package/dist/tree-picker-tui.d.ts +0 -1
  210. package/dist/tree-picker-tui.js +2 -3
  211. package/package.json +2 -1
  212. package/assets/python/kaelio_ktx-0.11.0-py3-none-any.whl +0 -0
@@ -0,0 +1,37 @@
1
+ import { KtxExpectedError } from '../../errors.js';
2
+ function configuredConnectionIds(config) {
3
+ return Object.keys(config.connections).sort();
4
+ }
5
+ function availableConnectionsHint(config) {
6
+ const ids = configuredConnectionIds(config);
7
+ return ids.length === 0
8
+ ? 'No connections are configured in ktx.yaml.'
9
+ : `Configured connections: ${ids.join(', ')}.`;
10
+ }
11
+ /**
12
+ * Look up a connection by id, throwing an expected (caller-driven) error that
13
+ * names the configured connections so an agent or CLI user can self-correct.
14
+ */
15
+ export function resolveConfiguredConnection(config, connectionId) {
16
+ const connection = config.connections[connectionId];
17
+ if (!connection) {
18
+ throw new KtxExpectedError(`Connection "${connectionId}" is not configured in ktx.yaml. ${availableConnectionsHint(config)}`);
19
+ }
20
+ return connection;
21
+ }
22
+ /**
23
+ * Resolve the connection id to run against: validate a requested id against the
24
+ * configured connections, or default to the sole connection when none is given.
25
+ * Throws an expected error that lists the configured connections otherwise.
26
+ */
27
+ export function resolveRequiredConnectionId(config, requested) {
28
+ if (requested !== undefined) {
29
+ resolveConfiguredConnection(config, requested);
30
+ return requested;
31
+ }
32
+ const ids = configuredConnectionIds(config);
33
+ if (ids.length === 1) {
34
+ return ids[0];
35
+ }
36
+ throw new KtxExpectedError(`connectionId is required. ${availableConnectionsHint(config)}`);
37
+ }
@@ -6,6 +6,10 @@ import { type SimpleGit } from 'simple-git';
6
6
  * directory is an existing repo ktx did not create and the machine has no configured git
7
7
  * identity (e.g. a fresh Mac with no ~/.gitconfig), without mutating the user's repo config.
8
8
  * Explicit `--author` flags on individual commits still take precedence over GIT_AUTHOR_NAME.
9
+ *
10
+ * `commit.gpgsign=false` is injected as a per-invocation `-c` override so ktx's commits never
11
+ * attempt GPG signing: ktx commits under a synthetic identity that can never own a secret key, so
12
+ * a user's `commit.gpgsign=true` would otherwise fail every commit with "No secret key".
9
13
  */
10
14
  export declare function createSimpleGit(baseDir: string, identity?: {
11
15
  name: string;
@@ -28,6 +28,10 @@ function sanitizedGitEnv(env = process.env) {
28
28
  * directory is an existing repo ktx did not create and the machine has no configured git
29
29
  * identity (e.g. a fresh Mac with no ~/.gitconfig), without mutating the user's repo config.
30
30
  * Explicit `--author` flags on individual commits still take precedence over GIT_AUTHOR_NAME.
31
+ *
32
+ * `commit.gpgsign=false` is injected as a per-invocation `-c` override so ktx's commits never
33
+ * attempt GPG signing: ktx commits under a synthetic identity that can never own a secret key, so
34
+ * a user's `commit.gpgsign=true` would otherwise fail every commit with "No secret key".
31
35
  */
32
36
  export function createSimpleGit(baseDir, identity) {
33
37
  const env = sanitizedGitEnv();
@@ -37,5 +41,5 @@ export function createSimpleGit(baseDir, identity) {
37
41
  env.GIT_COMMITTER_NAME = identity.name;
38
42
  env.GIT_COMMITTER_EMAIL = identity.email;
39
43
  }
40
- return simpleGit({ baseDir, unsafe: { allowUnsafeAskPass: true } }).env(env);
44
+ return simpleGit({ baseDir, config: ['commit.gpgsign=false'], unsafe: { allowUnsafeAskPass: true } }).env(env);
41
45
  }
@@ -20,6 +20,23 @@ export interface WorktreeEntry {
20
20
  branch: string | null;
21
21
  head: string | null;
22
22
  }
23
+ export type KtxRepoOwnership = 'unowned' | 'ktx-managed' | 'foreign';
24
+ export declare class KtxForeignGitRepositoryError extends Error {
25
+ constructor(configDir: string);
26
+ }
27
+ /**
28
+ * Classify whether ktx may own a git repository rooted exactly at `dir`. A root
29
+ * `ktx.yaml` is the ownership signal; the working tree decides, not git history,
30
+ * because older ktx versions left `ktx.yaml` uncommitted (it holds secret refs).
31
+ *
32
+ * - `unowned`: no repo here (including a missing or non-directory path) → ktx may `git init`.
33
+ * - `ktx-managed`: `<dir>/.git` is a directory and `ktx.yaml` sits at the root.
34
+ * - `foreign`: any other repo — no root `ktx.yaml`, or a `.git` *file* (a linked
35
+ * worktree). ktx must never adopt or mutate it.
36
+ *
37
+ * Reads only `<dir>` itself; never walks up, so a parent repo cannot change the answer.
38
+ */
39
+ export declare function classifyKtxRepoOwnership(dir: string): Promise<KtxRepoOwnership>;
23
40
  export type SquashMergeResult = {
24
41
  ok: true;
25
42
  squashSha: string;
@@ -102,6 +119,12 @@ export declare class GitService {
102
119
  path: string;
103
120
  }>>;
104
121
  changedPaths(): Promise<string[]>;
122
+ /**
123
+ * List all paths matching `pathSpec` as they exist at `commitHash`. Reads from
124
+ * git object storage, so it's safe against concurrent working-tree mutations
125
+ * and can recover paths (e.g. a human-renamed file) that no longer exist on disk.
126
+ */
127
+ listFilesAtCommit(pathSpec: string, commitHash: string): Promise<string[]>;
105
128
  /**
106
129
  * List all paths under the working tree that match `pathSpec`, scoped to HEAD.
107
130
  * Used for the reconciler's first-ever run when there's no watermark to diff from.
@@ -2,6 +2,53 @@ import { promises as fs } from 'node:fs';
2
2
  import { dirname, join } from 'node:path';
3
3
  import { noopLogger, resolveConfigDir } from './config.js';
4
4
  import { createSimpleGit } from './git-env.js';
5
+ export class KtxForeignGitRepositoryError extends Error {
6
+ constructor(configDir) {
7
+ super(`${configDir} is already a git repository that ktx did not create. ` +
8
+ 'ktx maintains its context in a repository it owns; run ktx in a dedicated directory or move the existing repository aside.');
9
+ this.name = 'KtxForeignGitRepositoryError';
10
+ }
11
+ }
12
+ function isNodeErrnoException(error) {
13
+ return error instanceof Error && 'code' in error;
14
+ }
15
+ /**
16
+ * Classify whether ktx may own a git repository rooted exactly at `dir`. A root
17
+ * `ktx.yaml` is the ownership signal; the working tree decides, not git history,
18
+ * because older ktx versions left `ktx.yaml` uncommitted (it holds secret refs).
19
+ *
20
+ * - `unowned`: no repo here (including a missing or non-directory path) → ktx may `git init`.
21
+ * - `ktx-managed`: `<dir>/.git` is a directory and `ktx.yaml` sits at the root.
22
+ * - `foreign`: any other repo — no root `ktx.yaml`, or a `.git` *file* (a linked
23
+ * worktree). ktx must never adopt or mutate it.
24
+ *
25
+ * Reads only `<dir>` itself; never walks up, so a parent repo cannot change the answer.
26
+ */
27
+ export async function classifyKtxRepoOwnership(dir) {
28
+ let dotGitIsDirectory;
29
+ try {
30
+ dotGitIsDirectory = (await fs.lstat(join(dir, '.git'))).isDirectory();
31
+ }
32
+ catch (error) {
33
+ // ENOENT: `<dir>/.git` is absent. ENOTDIR: `<dir>` itself is a file, so it
34
+ // can hold no repo. Either way there is nothing for ktx to avoid here.
35
+ if (isNodeErrnoException(error) && (error.code === 'ENOENT' || error.code === 'ENOTDIR')) {
36
+ return 'unowned';
37
+ }
38
+ throw error;
39
+ }
40
+ if (!dotGitIsDirectory) {
41
+ return 'foreign';
42
+ }
43
+ try {
44
+ // stat (not lstat): follow symlinks, matching what `loadKtxProject`'s
45
+ // readFile accepts — a dir that loads as a ktx project classifies as one.
46
+ return (await fs.stat(join(dir, 'ktx.yaml'))).isFile() ? 'ktx-managed' : 'foreign';
47
+ }
48
+ catch {
49
+ return 'foreign';
50
+ }
51
+ }
5
52
  function mergeErrorMessage(error) {
6
53
  if (error instanceof Error) {
7
54
  return error.message;
@@ -58,12 +105,15 @@ export class GitService {
58
105
  }
59
106
  async initialize() {
60
107
  try {
61
- // Check if already initialized
62
- const isRepo = await this.git.checkIsRepo();
63
- if (!isRepo) {
108
+ const ownership = await classifyKtxRepoOwnership(this.configDir);
109
+ if (ownership === 'foreign') {
110
+ throw new KtxForeignGitRepositoryError(this.configDir);
111
+ }
112
+ if (ownership === 'unowned') {
64
113
  await this.git.init();
65
- this.logger.log('Initialized git repository');
114
+ this.logger.log('Initialized ktx-managed git repository');
66
115
  }
116
+ // ownership === 'ktx-managed' → ktx's own repo; proceed with the normal re-run path.
67
117
  // Keep any auto-maintenance triggered by writes in-process. Detached maintenance can
68
118
  // keep object-pack directories alive briefly after awaited git commands complete,
69
119
  // which makes temp-project cleanup flaky in CI.
@@ -81,6 +131,11 @@ export class GitService {
81
131
  }
82
132
  }
83
133
  catch (error) {
134
+ // The foreign-repo error is already typed and actionable; surface it verbatim so every
135
+ // command that loads the project shows the same clear guidance instead of a generic wrapper.
136
+ if (error instanceof KtxForeignGitRepositoryError) {
137
+ throw error;
138
+ }
84
139
  this.logger.error('Failed to initialize git repository', error);
85
140
  // Preserve the underlying git error: the generic message alone is undiagnosable in
86
141
  // telemetry and unactionable for the user. The exception reporter walks `cause` and
@@ -442,12 +497,13 @@ export class GitService {
442
497
  return [...new Set(paths)].sort();
443
498
  }
444
499
  /**
445
- * List all paths under the working tree that match `pathSpec`, scoped to HEAD.
446
- * Used for the reconciler's first-ever run when there's no watermark to diff from.
500
+ * List all paths matching `pathSpec` as they exist at `commitHash`. Reads from
501
+ * git object storage, so it's safe against concurrent working-tree mutations
502
+ * and can recover paths (e.g. a human-renamed file) that no longer exist on disk.
447
503
  */
448
- async listFilesAtHead(pathSpec) {
504
+ async listFilesAtCommit(pathSpec, commitHash) {
449
505
  try {
450
- const raw = await this.git.raw(['ls-tree', '-r', '-z', '--name-only', 'HEAD', '--', pathSpec]);
506
+ const raw = await this.git.raw(['ls-tree', '-r', '-z', '--name-only', commitHash, '--', pathSpec]);
451
507
  if (!raw) {
452
508
  return [];
453
509
  }
@@ -457,6 +513,13 @@ export class GitService {
457
513
  return [];
458
514
  }
459
515
  }
516
+ /**
517
+ * List all paths under the working tree that match `pathSpec`, scoped to HEAD.
518
+ * Used for the reconciler's first-ever run when there's no watermark to diff from.
519
+ */
520
+ async listFilesAtHead(pathSpec) {
521
+ return this.listFilesAtCommit(pathSpec, 'HEAD');
522
+ }
460
523
  /**
461
524
  * Collapse all commits between `preHead` and current HEAD into a single commit with the given
462
525
  * message. Used by the memory agent to squash N per-tool-call commits into one ingest commit.
@@ -2,6 +2,7 @@ import { access, mkdir, readdir, readFile, rename, writeFile } from 'node:fs/pro
2
2
  import { dirname, join, relative } from 'node:path';
3
3
  import YAML from 'yaml';
4
4
  import { rawSourcesDirForSync } from '../../raw-sources-paths.js';
5
+ import { isSlYamlPath } from '../../../sl/source-files.js';
5
6
  import { mergeUsagePreservingExternal } from '../live-database/manifest.js';
6
7
  import { historicSqlEvidenceEnvelopeSchema } from './evidence.js';
7
8
  import { stagedManifestSchema } from './types.js';
@@ -197,7 +198,7 @@ export async function projectHistoricSqlEvidence(input) {
197
198
  const tableEvidence = evidence.filter((entry) => entry.kind === 'table_usage');
198
199
  const patternEvidence = evidence.filter((entry) => entry.kind === 'pattern');
199
200
  const schemaRoot = join(input.workdir, 'semantic-layer', input.connectionId, '_schema');
200
- for (const file of (await walkFiles(schemaRoot)).filter((candidate) => candidate.endsWith('.yaml') || candidate.endsWith('.yml'))) {
201
+ for (const file of (await walkFiles(schemaRoot)).filter(isSlYamlPath)) {
201
202
  const path = join(schemaRoot, file);
202
203
  const before = await readFile(path, 'utf-8');
203
204
  const shard = (YAML.parse(before) ?? {});
@@ -56,11 +56,14 @@ export interface BuildLiveDatabaseManifestShardsInput {
56
56
  existingPreservedJoins?: Map<string, LiveDatabaseManifestJoinEntry[]>;
57
57
  existingDescriptions?: Map<string, LiveDatabaseManifestExistingDescriptions>;
58
58
  existingUsage?: Map<string, TableUsageOutput>;
59
+ federatedSiblingTargets?: Set<string>;
59
60
  }
60
61
  export interface BuildLiveDatabaseManifestShardsResult {
61
62
  shards: Map<string, LiveDatabaseManifestShard>;
62
63
  tablesProcessed: number;
63
64
  }
64
65
  export declare function mergeUsagePreservingExternal(existing: TableUsageOutput | undefined, incoming: TableUsageOutput | undefined): TableUsageOutput | undefined;
66
+ /** @internal */
67
+ export declare function buildJoinsByTable(tableNames: Set<string>, joins: LiveDatabaseManifestJoinData[], preservedJoins: Map<string, LiveDatabaseManifestJoinEntry[]>, federatedSiblingTargets?: Set<string>): Map<string, LiveDatabaseManifestJoinEntry[]>;
65
68
  export declare function buildLiveDatabaseManifestShards(input: BuildLiveDatabaseManifestShardsInput): BuildLiveDatabaseManifestShardsResult;
66
69
  export {};
@@ -106,10 +106,14 @@ function joinCondition(leftTable, leftColumns, rightTable, rightColumns) {
106
106
  })
107
107
  .join(' AND ');
108
108
  }
109
- function buildJoinsByTable(tableNames, joins, preservedJoins) {
109
+ /** @internal */
110
+ export function buildJoinsByTable(tableNames, joins, preservedJoins, federatedSiblingTargets = new Set()) {
110
111
  const joinsByTable = new Map();
111
112
  for (const join of joins) {
112
- if (!tableNames.has(join.fromTable) || !tableNames.has(join.toTable)) {
113
+ const fromLocal = tableNames.has(join.fromTable);
114
+ const toLocal = tableNames.has(join.toTable);
115
+ const toSibling = federatedSiblingTargets.has(join.toTable);
116
+ if (!fromLocal || (!toLocal && !toSibling)) {
113
117
  continue;
114
118
  }
115
119
  const relationship = RELATIONSHIP_MAP[join.relationship] ?? join.relationship;
@@ -119,20 +123,24 @@ function buildJoinsByTable(tableNames, joins, preservedJoins) {
119
123
  relationship,
120
124
  source: join.source,
121
125
  });
122
- const reverseRelationship = RELATIONSHIP_INVERSE[relationship] ?? 'one_to_many';
123
- addJoinOnce(joinsByTable, join.toTable, {
124
- to: join.fromTable,
125
- on: joinCondition(join.toTable, join.toColumns, join.fromTable, join.fromColumns),
126
- relationship: reverseRelationship,
127
- source: join.source,
128
- });
126
+ // Reverse direction only when the target is a local table in THIS snapshot;
127
+ // a federated sibling has no shard here, so it gets no reverse entry.
128
+ if (toLocal) {
129
+ const reverseRelationship = RELATIONSHIP_INVERSE[relationship] ?? 'one_to_many';
130
+ addJoinOnce(joinsByTable, join.toTable, {
131
+ to: join.fromTable,
132
+ on: joinCondition(join.toTable, join.toColumns, join.fromTable, join.fromColumns),
133
+ relationship: reverseRelationship,
134
+ source: join.source,
135
+ });
136
+ }
129
137
  }
130
138
  for (const [tableName, tableJoins] of preservedJoins) {
131
139
  if (!tableNames.has(tableName)) {
132
140
  continue;
133
141
  }
134
142
  for (const join of tableJoins) {
135
- if (tableNames.has(join.to)) {
143
+ if (tableNames.has(join.to) || federatedSiblingTargets.has(join.to)) {
136
144
  addJoinOnce(joinsByTable, tableName, join);
137
145
  }
138
146
  }
@@ -141,7 +149,7 @@ function buildJoinsByTable(tableNames, joins, preservedJoins) {
141
149
  }
142
150
  export function buildLiveDatabaseManifestShards(input) {
143
151
  const tableNames = new Set(input.tables.map((table) => table.name));
144
- const joinsByTable = buildJoinsByTable(tableNames, input.joins, input.existingPreservedJoins ?? new Map());
152
+ const joinsByTable = buildJoinsByTable(tableNames, input.joins, input.existingPreservedJoins ?? new Map(), input.federatedSiblingTargets ?? new Set());
145
153
  const shards = new Map();
146
154
  for (const table of input.tables) {
147
155
  const shardKey = getShardKey(input.connectionType, table.catalog, table.db);
@@ -12,13 +12,18 @@ const defaultLogger = {
12
12
  class InlineLookerSettings extends NodeSettings {
13
13
  params;
14
14
  constructor(params) {
15
- super('', {
15
+ // @looker/sdk-rtl boundary: NodeSettings consumes a string-valued config
16
+ // section (read back via the readConfig override below), but its constructor
17
+ // is typed to accept a fully-realized IApiSettings. The string record is the
18
+ // shape the library actually reads, so narrow to IApiSection first.
19
+ const settings = {
16
20
  base_url: normalizeBaseUrl(params.base_url),
17
21
  client_id: params.client_id,
18
22
  client_secret: params.client_secret, // pragma: allowlist secret
19
23
  verify_ssl: 'true',
20
24
  timeout: '120',
21
- });
25
+ };
26
+ super('', settings);
22
27
  this.params = params;
23
28
  }
24
29
  readConfig(_section) {
@@ -1,5 +1,5 @@
1
1
  import type { FetchContext } from '../../types.js';
2
- import { type LookerClientDeps, type LookerConnectionParams } from './client.js';
2
+ import { LookerClient, type LookerClientDeps, type LookerConnectionParams } from './client.js';
3
3
  import type { LookerClientFactory, LookerRuntimeClient } from './fetch.js';
4
4
  import type { LookerPullConfig } from './types.js';
5
5
  export interface LookerCredentialResolver {
@@ -14,6 +14,13 @@ export declare class DefaultLookerConnectionClientFactory implements LookerConne
14
14
  private readonly deps;
15
15
  constructor(resolver: LookerCredentialResolver, deps?: LookerClientDeps);
16
16
  createClient(lookerConnectionId: string): Promise<LookerRuntimeClient>;
17
+ /**
18
+ * Like {@link createClient} but preserves the concrete {@link LookerClient}
19
+ * type, so callers that need methods outside the `LookerRuntimeClient`
20
+ * contract (e.g. `listLookerConnections`, `testConnection`) keep them without
21
+ * a cast.
22
+ */
23
+ createLookerClient(lookerConnectionId: string): Promise<LookerClient>;
17
24
  }
18
25
  /** @internal */
19
26
  export declare class DefaultLookerClientFactory implements LookerClientFactory {
@@ -7,6 +7,15 @@ export class DefaultLookerConnectionClientFactory {
7
7
  this.deps = deps;
8
8
  }
9
9
  async createClient(lookerConnectionId) {
10
+ return this.createLookerClient(lookerConnectionId);
11
+ }
12
+ /**
13
+ * Like {@link createClient} but preserves the concrete {@link LookerClient}
14
+ * type, so callers that need methods outside the `LookerRuntimeClient`
15
+ * contract (e.g. `listLookerConnections`, `testConnection`) keep them without
16
+ * a cast.
17
+ */
18
+ async createLookerClient(lookerConnectionId) {
10
19
  const credentials = await this.resolver.resolve(lookerConnectionId);
11
20
  return new LookerClient(credentials, this.deps);
12
21
  }
@@ -120,7 +120,7 @@ export function validateLookerMappings(args) {
120
120
  if (!args.knownKtxConnectionIds.has(mapping.ktxConnectionId)) {
121
121
  errors.push({
122
122
  key: mapping.lookerConnectionName,
123
- reason: `KTX connection ${mapping.ktxConnectionId} does not exist`,
123
+ reason: `ktx connection ${mapping.ktxConnectionId} does not exist`,
124
124
  });
125
125
  continue;
126
126
  }
@@ -12,6 +12,7 @@ export declare const lookerPullConfigSchema: z.ZodObject<{
12
12
  lookUpdatedSince: z.ZodOptional<z.ZodNullable<z.ZodISODateTime>>;
13
13
  connectionMappings: z.ZodDefault<z.ZodRecord<z.ZodString, z.ZodString>>;
14
14
  connectionTypes: z.ZodDefault<z.ZodRecord<z.ZodString, z.ZodEnum<{
15
+ PLAIN: "PLAIN";
15
16
  BIGQUERY: "BIGQUERY";
16
17
  SNOWFLAKE: "SNOWFLAKE";
17
18
  MYSQL: "MYSQL";
@@ -31,7 +32,6 @@ export declare const lookerPullConfigSchema: z.ZodObject<{
31
32
  METABASE: "METABASE";
32
33
  LOOKER: "LOOKER";
33
34
  NOTION: "NOTION";
34
- PLAIN: "PLAIN";
35
35
  BETTERSTACK: "BETTERSTACK";
36
36
  }>>>;
37
37
  parsedTargetTables: z.ZodDefault<z.ZodRecord<z.ZodString, z.ZodDiscriminatedUnion<[z.ZodObject<{
@@ -18,7 +18,7 @@ export declare const DEFAULT_METABASE_CLIENT_CONFIG: MetabaseClientConfig;
18
18
  * Strip Metabase `[[ ... {{ var }} ... ]]` optional-clause blocks from native SQL.
19
19
  *
20
20
  * The bracketed blocks are emitted only when the embedded `{{ var }}` is supplied at
21
- * Metabase query time. For KTX semantic-layer ingest there's no such runtime
21
+ * Metabase query time. For ktx semantic-layer ingest there's no such runtime
22
22
  * parameter — chat-time filters are composed by the SL query planner — so the optional
23
23
  * block must be removed before the SQL becomes a permanent SL source. Substituting a
24
24
  * dummy value (the alternative) bakes a placeholder filter into the source and silently
@@ -30,7 +30,7 @@ class MetabaseApiError extends Error {
30
30
  * Strip Metabase `[[ ... {{ var }} ... ]]` optional-clause blocks from native SQL.
31
31
  *
32
32
  * The bracketed blocks are emitted only when the embedded `{{ var }}` is supplied at
33
- * Metabase query time. For KTX semantic-layer ingest there's no such runtime
33
+ * Metabase query time. For ktx semantic-layer ingest there's no such runtime
34
34
  * parameter — chat-time filters are composed by the SL query planner — so the optional
35
35
  * block must be removed before the SQL becomes a permanent SL source. Substituting a
36
36
  * dummy value (the alternative) bakes a placeholder filter into the source and silently
@@ -15,7 +15,7 @@ export function metabaseRuntimeConfigFromLocalConnection(connectionId, connectio
15
15
  throw new Error(`Connection "${connectionId}" is not a Metabase connection`);
16
16
  }
17
17
  if (hasNetworkProxy(connection)) {
18
- throw new Error(`Standalone KTX does not support proxy-bearing Metabase connections yet. Use hosted Metabase ingest for "${connectionId}" until the KTX Metabase proxy support spec lands.`);
18
+ throw new Error(`Standalone ktx does not support proxy-bearing Metabase connections yet. Use hosted Metabase ingest for "${connectionId}" until the ktx Metabase proxy support spec lands.`);
19
19
  }
20
20
  const apiUrl = stringField(connection.api_url);
21
21
  const literalApiKey = stringField(connection.api_key);
@@ -93,7 +93,7 @@ export function validateMetabaseMappings(args) {
93
93
  continue;
94
94
  }
95
95
  if (!args.knownKtxConnectionIds.has(connectionId)) {
96
- errors.push({ key, reason: `KTX connection ${connectionId} does not exist` });
96
+ errors.push({ key, reason: `ktx connection ${connectionId} does not exist` });
97
97
  }
98
98
  }
99
99
  return errors.length === 0 ? { ok: true } : { ok: false, errors };
@@ -108,13 +108,13 @@ export function validateMappingPhysicalMatch(mapping, target) {
108
108
  return null;
109
109
  }
110
110
  if (target.connection_type !== expectedType) {
111
- return `Metabase database engine '${engine}' does not match KTX connection type '${target.connection_type}'`;
111
+ return `Metabase database engine '${engine}' does not match ktx connection type '${target.connection_type}'`;
112
112
  }
113
113
  const metabaseDb = normalizeName(mapping.metabaseDbName);
114
114
  const targetDb = normalizeName(getTargetDatabase(target));
115
115
  if (engine === 'snowflake' || engine === 'bigquery' || engine === 'bigquery-cloud-sdk') {
116
116
  if (metabaseDb && targetDb && metabaseDb !== targetDb) {
117
- return `Metabase database '${mapping.metabaseDbName}' does not match KTX connection database '${displayValue(getTargetDatabase(target))}'`;
117
+ return `Metabase database '${mapping.metabaseDbName}' does not match ktx connection database '${displayValue(getTargetDatabase(target))}'`;
118
118
  }
119
119
  return null;
120
120
  }
@@ -122,10 +122,10 @@ export function validateMappingPhysicalMatch(mapping, target) {
122
122
  const metabaseHost = normalizeHost(mapping.metabaseHost);
123
123
  const targetHost = normalizeHost(target.host);
124
124
  if (metabaseHost && targetHost && metabaseHost !== targetHost) {
125
- return `Metabase host '${mapping.metabaseHost}' does not match KTX connection host '${displayValue(target.host)}'`;
125
+ return `Metabase host '${mapping.metabaseHost}' does not match ktx connection host '${displayValue(target.host)}'`;
126
126
  }
127
127
  if (metabaseDb && targetDb && metabaseDb !== targetDb) {
128
- return `Metabase database '${mapping.metabaseDbName}' does not match KTX connection database '${displayValue(getTargetDatabase(target))}'`;
128
+ return `Metabase database '${mapping.metabaseDbName}' does not match ktx connection database '${displayValue(getTargetDatabase(target))}'`;
129
129
  }
130
130
  return null;
131
131
  }
@@ -157,7 +157,7 @@ export async function refreshMetabaseMapping(args) {
157
157
  if (!target) {
158
158
  physicalMismatches.push({
159
159
  mappingId: String(mapping.id),
160
- reason: `KTX connection ${mapping.ktxConnectionId} does not exist`,
160
+ reason: `ktx connection ${mapping.ktxConnectionId} does not exist`,
161
161
  });
162
162
  continue;
163
163
  }
@@ -1,17 +1,14 @@
1
1
  import type { SemanticLayerService } from '../../context/sl/semantic-layer.service.js';
2
2
  import type { TouchedSlSource } from '../../context/tools/touched-sl-sources.js';
3
3
  import type { KnowledgeWikiService } from '../../context/wiki/knowledge-wiki.service.js';
4
- interface TouchedValidationResult {
5
- invalidSources: string[];
6
- validSources: string[];
7
- }
4
+ import type { WuValidationResult } from './stages/validate-wu-sources.js';
8
5
  export interface FinalArtifactGateInput {
9
6
  connectionIds: string[];
10
7
  changedWikiPageKeys: string[];
11
8
  touchedSlSources: TouchedSlSource[];
12
9
  wikiService: KnowledgeWikiService;
13
10
  semanticLayerService: SemanticLayerService;
14
- validateTouchedSources(touched: TouchedSlSource[]): Promise<TouchedValidationResult>;
11
+ validateTouchedSources(touched: TouchedSlSource[]): Promise<WuValidationResult>;
15
12
  tableExists(connectionId: string, tableRef: string): Promise<boolean>;
16
13
  }
17
14
  export interface ProvenanceRawPathValidationInput {
@@ -23,4 +20,3 @@ export interface ProvenanceRawPathValidationInput {
23
20
  }
24
21
  export declare function validateFinalIngestArtifacts(input: FinalArtifactGateInput): Promise<void>;
25
22
  export declare function validateProvenanceRawPaths(input: ProvenanceRawPathValidationInput): void;
26
- export {};
@@ -13,50 +13,6 @@ function slEntityNames(source) {
13
13
  ...(source.segments ?? []).map((segment) => segment.name),
14
14
  ]);
15
15
  }
16
- function uniqueTouchedSources(sources) {
17
- const seen = new Set();
18
- const unique = [];
19
- for (const source of sources) {
20
- const key = `${source.connectionId}:${source.sourceName}`;
21
- if (seen.has(key)) {
22
- continue;
23
- }
24
- seen.add(key);
25
- unique.push(source);
26
- }
27
- return unique.sort((left, right) => {
28
- const byConnection = left.connectionId.localeCompare(right.connectionId);
29
- return byConnection === 0 ? left.sourceName.localeCompare(right.sourceName) : byConnection;
30
- });
31
- }
32
- async function expandTouchedSlSourcesWithDirectJoinNeighbors(input) {
33
- const expanded = [...input.touchedSlSources];
34
- const touchedByConnection = new Map();
35
- for (const source of input.touchedSlSources) {
36
- const bucket = touchedByConnection.get(source.connectionId) ?? new Set();
37
- bucket.add(source.sourceName);
38
- touchedByConnection.set(source.connectionId, bucket);
39
- }
40
- for (const connectionId of input.connectionIds) {
41
- const touched = touchedByConnection.get(connectionId);
42
- if (!touched || touched.size === 0) {
43
- continue;
44
- }
45
- const { sources } = await input.semanticLayerService.loadAllSources(connectionId);
46
- for (const source of sources) {
47
- const sourceIsTouched = touched.has(source.name);
48
- if (sourceIsTouched) {
49
- for (const join of source.joins ?? []) {
50
- expanded.push({ connectionId, sourceName: join.to });
51
- }
52
- }
53
- if ((source.joins ?? []).some((join) => touched.has(join.to))) {
54
- expanded.push({ connectionId, sourceName: source.name });
55
- }
56
- }
57
- }
58
- return uniqueTouchedSources(expanded);
59
- }
60
16
  async function validateWikiSlRefs(input) {
61
17
  const errors = [];
62
18
  const sourcesByConnection = new Map();
@@ -112,9 +68,11 @@ async function validateWikiRefs(input) {
112
68
  return dangling;
113
69
  }
114
70
  export async function validateFinalIngestArtifacts(input) {
115
- const touchedWithDependencies = await expandTouchedSlSourcesWithDirectJoinNeighbors(input);
116
- const validation = await input.validateTouchedSources(touchedWithDependencies);
117
- const errors = validation.invalidSources.map((source) => `semantic-layer validation failed for ${source}`);
71
+ // Join-neighbor expansion happens inside validateTouchedSources so work-unit
72
+ // validation and this gate check the same set — a source that passes one
73
+ // passes the other.
74
+ const validation = await input.validateTouchedSources(input.touchedSlSources);
75
+ const errors = validation.invalidSources.map((invalid) => `semantic-layer validation failed for ${invalid.source}: ${invalid.errors.join('; ')}`);
118
76
  errors.push(...(await validateWikiSlRefs(input)));
119
77
  const danglingWikiRefs = await validateWikiRefs(input);
120
78
  if (danglingWikiRefs.length > 0) {
@@ -0,0 +1,55 @@
1
+ import type { AgentRunnerPort, KtxRuntimeToolSet } from '../../context/llm/runtime-port.js';
2
+ import type { IngestTraceWriter } from './ingest-trace.js';
3
+ /**
4
+ * Shared loop for the two integration-time repair agents (semantic gate
5
+ * repair, textual conflict resolution). Success is decided by re-running the
6
+ * failed check — `verify` — never by whether the agent edited files: an
7
+ * ineffective edit fails, and an explicit no-change declaration that verifies
8
+ * succeeds.
9
+ */
10
+ export type RepairVerification = {
11
+ ok: true;
12
+ } | {
13
+ ok: false;
14
+ reason: string;
15
+ };
16
+ export type ConstrainedRepairResult = {
17
+ status: 'repaired';
18
+ attempts: number;
19
+ changedPaths: string[];
20
+ } | {
21
+ status: 'failed';
22
+ attempts: number;
23
+ reason: string;
24
+ };
25
+ export interface ConstrainedRepairToolContext {
26
+ workdir: string;
27
+ allowedPaths: ReadonlySet<string>;
28
+ editedPaths: Set<string>;
29
+ declareNoChange(reason: string): void;
30
+ }
31
+ export interface ConstrainedRepairLoopInput {
32
+ agentRunner: AgentRunnerPort;
33
+ workdir: string;
34
+ allowedPaths: string[];
35
+ trace: IngestTraceWriter;
36
+ tracePhase: string;
37
+ traceEventName: string;
38
+ traceData: Record<string, unknown>;
39
+ systemPrompt: string;
40
+ buildUserPrompt(input: {
41
+ attempt: number;
42
+ maxAttempts: number;
43
+ previousFailure: string | null;
44
+ }): string;
45
+ buildExtraTools?(context: ConstrainedRepairToolContext): KtxRuntimeToolSet;
46
+ verify(changedPaths: string[]): Promise<RepairVerification>;
47
+ /** Failure reason when an attempt neither edits nor declares no-change. */
48
+ noChangeFailureReason: string;
49
+ telemetryTags: Record<string, string>;
50
+ maxAttempts?: number;
51
+ stepBudget?: number;
52
+ abortSignal?: AbortSignal;
53
+ }
54
+ export declare function buildDeleteRepairFileTool(context: ConstrainedRepairToolContext): KtxRuntimeToolSet;
55
+ export declare function runConstrainedRepairLoop(input: ConstrainedRepairLoopInput): Promise<ConstrainedRepairResult>;