@kaelio/ktx 0.10.0 → 0.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/assets/python/{kaelio_ktx-0.10.0-py3-none-any.whl → kaelio_ktx-0.12.0-py3-none-any.whl} +0 -0
- package/assets/python/manifest.json +4 -4
- package/dist/.tsbuildinfo +1 -1
- package/dist/admin.js +1 -1
- package/dist/clack.d.ts +16 -0
- package/dist/clack.js +37 -6
- package/dist/claude-code-prompt-caching.js +1 -1
- package/dist/cli-program.js +7 -3
- package/dist/cli-runtime.d.ts +2 -0
- package/dist/cli-runtime.js +14 -8
- package/dist/commands/connection-commands.js +1 -1
- package/dist/commands/ingest-commands.js +4 -4
- package/dist/commands/mcp-commands.js +12 -12
- package/dist/commands/runtime-commands.js +4 -4
- package/dist/commands/setup-commands.js +6 -5
- package/dist/commands/sl-commands.js +1 -1
- package/dist/commands/sql-commands.js +1 -1
- package/dist/commands/status-commands.js +1 -1
- package/dist/community-cta.d.ts +11 -0
- package/dist/community-cta.js +19 -0
- package/dist/connection.js +1 -1
- package/dist/connectors/clickhouse/connector.js +1 -1
- package/dist/connectors/mysql/connector.js +1 -1
- package/dist/connectors/snowflake/connector.d.ts +1 -1
- package/dist/connectors/sqlite/connector.js +2 -25
- package/dist/connectors/sqlserver/connector.js +3 -3
- package/dist/context/connections/connection-type.d.ts +1 -1
- package/dist/context/connections/read-only-sql.d.ts +1 -0
- package/dist/context/connections/read-only-sql.js +116 -2
- package/dist/context/core/git-env.d.ts +12 -1
- package/dist/context/core/git-env.js +17 -2
- package/dist/context/core/git.service.d.ts +23 -0
- package/dist/context/core/git.service.js +86 -15
- package/dist/context/ingest/adapters/historic-sql/projection.js +2 -1
- package/dist/context/ingest/adapters/looker/client.js +7 -2
- package/dist/context/ingest/adapters/looker/factory.d.ts +8 -1
- package/dist/context/ingest/adapters/looker/factory.js +9 -0
- package/dist/context/ingest/adapters/looker/mapping.js +1 -1
- package/dist/context/ingest/adapters/looker/types.d.ts +1 -1
- package/dist/context/ingest/adapters/metabase/client.d.ts +1 -1
- package/dist/context/ingest/adapters/metabase/client.js +1 -1
- package/dist/context/ingest/adapters/metabase/local-metabase.adapter.js +1 -1
- package/dist/context/ingest/adapters/metabase/mapping.js +6 -6
- package/dist/context/ingest/artifact-gates.d.ts +2 -6
- package/dist/context/ingest/artifact-gates.js +5 -47
- package/dist/context/ingest/constrained-repair.d.ts +55 -0
- package/dist/context/ingest/constrained-repair.js +167 -0
- package/dist/context/ingest/final-gate-repair.d.ts +9 -11
- package/dist/context/ingest/final-gate-repair.js +40 -128
- package/dist/context/ingest/finalization-scope.d.ts +1 -1
- package/dist/context/ingest/finalization-scope.js +15 -15
- package/dist/context/ingest/ingest-bundle.runner.d.ts +1 -0
- package/dist/context/ingest/ingest-bundle.runner.js +101 -67
- package/dist/context/ingest/isolated-diff/patch-integrator.d.ts +6 -13
- package/dist/context/ingest/isolated-diff/patch-integrator.js +32 -109
- package/dist/context/ingest/isolated-diff/textual-conflict-resolver.d.ts +8 -9
- package/dist/context/ingest/isolated-diff/textual-conflict-resolver.js +63 -141
- package/dist/context/ingest/local-bundle-runtime.d.ts +2 -0
- package/dist/context/ingest/local-bundle-runtime.js +9 -10
- package/dist/context/ingest/local-ingest.d.ts +2 -0
- package/dist/context/ingest/local-ingest.js +2 -0
- package/dist/context/ingest/memory-flow/view-model.js +1 -1
- package/dist/context/ingest/stages/stage-3-work-units.d.ts +2 -6
- package/dist/context/ingest/stages/stage-3-work-units.js +2 -1
- package/dist/context/ingest/stages/validate-wu-sources.d.ts +7 -1
- package/dist/context/ingest/stages/validate-wu-sources.js +109 -4
- package/dist/context/ingest/tools/warehouse-verification/create-warehouse-verification-tools.d.ts +2 -0
- package/dist/context/ingest/tools/warehouse-verification/create-warehouse-verification-tools.js +1 -1
- package/dist/context/ingest/tools/warehouse-verification/discover-data.tool.js +3 -3
- package/dist/context/ingest/tools/warehouse-verification/sql-execution.tool.d.ts +3 -1
- package/dist/context/ingest/tools/warehouse-verification/sql-execution.tool.js +15 -1
- package/dist/context/llm/ai-sdk-runtime.js +2 -2
- package/dist/context/llm/claude-code-runtime.js +1 -1
- package/dist/context/llm/local-config.js +1 -1
- package/dist/context/llm/runtime-tools.js +2 -2
- package/dist/context/mcp/context-tools.js +7 -7
- package/dist/context/mcp/local-project-ports.js +23 -54
- package/dist/context/memory/local-memory.js +4 -1
- package/dist/context/memory/memory-agent.service.js +1 -1
- package/dist/context/project/config.d.ts +11 -4
- package/dist/context/project/config.js +85 -30
- package/dist/context/project/driver-schemas.js +1 -1
- package/dist/context/project/mappings-yaml-schema.js +2 -2
- package/dist/context/project/project.js +12 -4
- package/dist/context/scan/description-generation.js +4 -4
- package/dist/context/scan/local-enrichment-artifacts.js +2 -1
- package/dist/context/scan/local-scan.js +2 -2
- package/dist/context/scan/local-structural-artifacts.js +5 -5
- package/dist/context/scan/relationship-benchmark-report.js +1 -1
- package/dist/context/scan/relationship-discovery.js +3 -3
- package/dist/context/scan/relationship-llm-proposal.js +3 -3
- package/dist/context/sl/local-query.js +3 -33
- package/dist/context/sl/local-sl.d.ts +0 -8
- package/dist/context/sl/local-sl.js +44 -69
- package/dist/context/sl/semantic-layer.service.d.ts +25 -8
- package/dist/context/sl/semantic-layer.service.js +109 -56
- package/dist/context/sl/source-files.d.ts +46 -0
- package/dist/context/sl/source-files.js +131 -0
- package/dist/context/sl/tools/base-semantic-layer.tool.d.ts +2 -2
- package/dist/context/sl/tools/base-semantic-layer.tool.js +2 -7
- package/dist/context/sl/tools/sl-edit-source.tool.js +10 -8
- package/dist/context/sl/tools/sl-warehouse-validation.js +55 -27
- package/dist/context/sl/tools/sl-write-source.tool.js +12 -9
- package/dist/context/sql-analysis/dialect.d.ts +2 -0
- package/dist/context/sql-analysis/dialect.js +20 -0
- package/dist/context/tools/base-tool.d.ts +6 -19
- package/dist/context/tools/base-tool.js +0 -14
- package/dist/context-build-view.js +5 -5
- package/dist/database-tree-picker.js +18 -3
- package/dist/demo-assets.js +0 -1
- package/dist/doctor.d.ts +1 -1
- package/dist/doctor.js +31 -23
- package/dist/errors.d.ts +31 -0
- package/dist/errors.js +44 -0
- package/dist/ingest.d.ts +1 -1
- package/dist/ingest.js +8 -2
- package/dist/io/symbols.d.ts +2 -0
- package/dist/io/symbols.js +2 -0
- package/dist/io/tty.d.ts +17 -0
- package/dist/io/tty.js +21 -0
- package/dist/links.d.ts +1 -0
- package/dist/links.js +1 -0
- package/dist/llm/embedding-health.js +1 -1
- package/dist/llm/embedding-provider.js +3 -3
- package/dist/llm/model-provider.js +1 -1
- package/dist/local-adapters.d.ts +1 -0
- package/dist/local-adapters.js +2 -2
- package/dist/local-scan-connectors.js +1 -1
- package/dist/managed-local-embeddings.js +17 -8
- package/dist/managed-mcp-daemon.js +3 -3
- package/dist/managed-python-command.d.ts +7 -0
- package/dist/managed-python-command.js +34 -8
- package/dist/managed-python-daemon.js +2 -2
- package/dist/managed-python-http.js +3 -3
- package/dist/managed-python-runtime.d.ts +30 -1
- package/dist/managed-python-runtime.js +134 -18
- package/dist/managed-uv-release.d.ts +7 -0
- package/dist/managed-uv-release.js +11 -0
- package/dist/mcp-http-server.js +4 -4
- package/dist/mcp-server-factory.js +3 -3
- package/dist/mcp-stdio-server.js +1 -1
- package/dist/memory-flow-hud.js +2 -2
- package/dist/next-steps.js +2 -2
- package/dist/prompt-navigation.d.ts +17 -0
- package/dist/prompt-navigation.js +49 -3
- package/dist/prompts/memory_agent_bundle_ingest_work_unit.md +2 -2
- package/dist/prompts/memory_agent_external_ingest.md +2 -2
- package/dist/public-ingest-copy.js +1 -1
- package/dist/public-ingest.js +3 -3
- package/dist/release-version.js +1 -1
- package/dist/runtime-requirements.js +1 -1
- package/dist/runtime.js +9 -9
- package/dist/scan.js +1 -1
- package/dist/setup-agents.js +22 -35
- package/dist/setup-banner.d.ts +20 -0
- package/dist/setup-banner.js +39 -0
- package/dist/setup-context.js +24 -15
- package/dist/setup-databases.js +31 -59
- package/dist/setup-demo-tour.js +12 -8
- package/dist/setup-embeddings.js +9 -9
- package/dist/setup-interrupt.js +1 -1
- package/dist/setup-models.d.ts +4 -1
- package/dist/setup-models.js +54 -28
- package/dist/setup-project.js +29 -5
- package/dist/setup-prompts.js +16 -5
- package/dist/setup-ready-menu.js +1 -1
- package/dist/setup-sources.js +27 -7
- package/dist/setup.d.ts +25 -0
- package/dist/setup.js +90 -19
- package/dist/skills/analytics/SKILL.md +3 -3
- package/dist/skills/dbt_ingest/SKILL.md +3 -3
- package/dist/skills/looker_ingest/SKILL.md +3 -3
- package/dist/skills/lookml_ingest/SKILL.md +7 -7
- package/dist/skills/metabase_ingest/SKILL.md +4 -4
- package/dist/skills/metricflow_ingest/SKILL.md +15 -15
- package/dist/skills/notion_synthesize/SKILL.md +1 -1
- package/dist/skills/sl/SKILL.md +3 -3
- package/dist/skills/sl_capture/SKILL.md +1 -1
- package/dist/skills/wiki_capture/SKILL.md +1 -1
- package/dist/source-mapping.js +1 -1
- package/dist/startup-profile.js +1 -1
- package/dist/status-project.d.ts +0 -2
- package/dist/status-project.js +4 -6
- package/dist/telemetry/command-hook.d.ts +24 -0
- package/dist/telemetry/command-hook.js +37 -3
- package/dist/telemetry/events.d.ts +1 -1
- package/dist/telemetry/exception.js +14 -0
- package/dist/telemetry/index.d.ts +2 -2
- package/dist/telemetry/index.js +2 -2
- package/dist/text-ingest.js +1 -1
- package/dist/tree-picker-tui.d.ts +0 -1
- package/dist/tree-picker-tui.js +2 -3
- package/package.json +1 -1
|
@@ -1,14 +1,128 @@
|
|
|
1
1
|
const MUTATING_SQL = /^\s*(insert|update|delete|merge|alter|drop|create|truncate|grant|revoke|copy|call|do|vacuum|analyze|refresh)\b/i;
|
|
2
2
|
const READ_SQL = /^\s*(select|with)\b/i;
|
|
3
|
+
// Agents (and the daemon's sqlglot validator, which ignores comments) routinely
|
|
4
|
+
// emit read-only queries prefixed with `-- ...` or `/* ... */`. Strip leading
|
|
5
|
+
// comments so the prefix check sees the real statement; otherwise valid SELECT/WITH
|
|
6
|
+
// SQL is rejected here while the parser-backed validator accepts it.
|
|
7
|
+
function stripLeadingSqlComments(sql) {
|
|
8
|
+
let index = 0;
|
|
9
|
+
while (index < sql.length) {
|
|
10
|
+
while (/\s/.test(sql[index] ?? '')) {
|
|
11
|
+
index += 1;
|
|
12
|
+
}
|
|
13
|
+
if (sql.startsWith('--', index)) {
|
|
14
|
+
const end = sql.indexOf('\n', index + 2);
|
|
15
|
+
index = end === -1 ? sql.length : end + 1;
|
|
16
|
+
continue;
|
|
17
|
+
}
|
|
18
|
+
if (sql.startsWith('/*', index)) {
|
|
19
|
+
const end = sql.indexOf('*/', index + 2);
|
|
20
|
+
if (end === -1) {
|
|
21
|
+
return sql.slice(index);
|
|
22
|
+
}
|
|
23
|
+
index = end + 2;
|
|
24
|
+
continue;
|
|
25
|
+
}
|
|
26
|
+
break;
|
|
27
|
+
}
|
|
28
|
+
return sql.slice(index);
|
|
29
|
+
}
|
|
30
|
+
// Lexes past one string literal, quoted identifier, or comment starting at
|
|
31
|
+
// `index`, using standard-SQL rules ('' and "" escapes; no dialect extensions
|
|
32
|
+
// such as backslash escapes or dollar quoting). Returns the index after the
|
|
33
|
+
// token, or `index` unchanged when no quoted/comment token starts there.
|
|
34
|
+
function skipQuotedOrComment(sql, index) {
|
|
35
|
+
const quote = sql[index];
|
|
36
|
+
if (quote === "'" || quote === '"') {
|
|
37
|
+
let i = index + 1;
|
|
38
|
+
while (i < sql.length) {
|
|
39
|
+
if (sql[i] === quote) {
|
|
40
|
+
if (sql[i + 1] === quote) {
|
|
41
|
+
i += 2;
|
|
42
|
+
continue;
|
|
43
|
+
}
|
|
44
|
+
return i + 1;
|
|
45
|
+
}
|
|
46
|
+
i += 1;
|
|
47
|
+
}
|
|
48
|
+
return sql.length;
|
|
49
|
+
}
|
|
50
|
+
if (sql.startsWith('--', index)) {
|
|
51
|
+
const end = sql.indexOf('\n', index + 2);
|
|
52
|
+
return end === -1 ? sql.length : end + 1;
|
|
53
|
+
}
|
|
54
|
+
if (sql.startsWith('/*', index)) {
|
|
55
|
+
const end = sql.indexOf('*/', index + 2);
|
|
56
|
+
return end === -1 ? sql.length : end + 2;
|
|
57
|
+
}
|
|
58
|
+
return index;
|
|
59
|
+
}
|
|
60
|
+
// Backstop against statement smuggling (`select 1; drop table x`): reject any
|
|
61
|
+
// semicolon that is followed by real content. Semicolons inside string
|
|
62
|
+
// literals, quoted identifiers, and comments are fine, as are trailing
|
|
63
|
+
// semicolons (optionally followed by whitespace and comments). This deliberately
|
|
64
|
+
// lexes standard SQL only, so dialect-specific escapes can cause a false
|
|
65
|
+
// reject — never a false accept; the canonical gate is the daemon's
|
|
66
|
+
// sqlglot-backed validateReadOnly.
|
|
67
|
+
function assertSingleSqlStatement(sql) {
|
|
68
|
+
let index = 0;
|
|
69
|
+
let sawSemicolon = false;
|
|
70
|
+
while (index < sql.length) {
|
|
71
|
+
const skipped = skipQuotedOrComment(sql, index);
|
|
72
|
+
if (skipped > index) {
|
|
73
|
+
index = skipped;
|
|
74
|
+
continue;
|
|
75
|
+
}
|
|
76
|
+
if (sql[index] === ';') {
|
|
77
|
+
sawSemicolon = true;
|
|
78
|
+
}
|
|
79
|
+
else if (sawSemicolon && !/\s/.test(sql[index])) {
|
|
80
|
+
throw new Error('Only one SQL statement can be executed.');
|
|
81
|
+
}
|
|
82
|
+
index += 1;
|
|
83
|
+
}
|
|
84
|
+
}
|
|
3
85
|
export function assertReadOnlySql(sql) {
|
|
4
|
-
const trimmed = sql.trim();
|
|
86
|
+
const trimmed = stripLeadingSqlComments(sql).trim();
|
|
5
87
|
if (!READ_SQL.test(trimmed) || MUTATING_SQL.test(trimmed)) {
|
|
6
88
|
throw new Error('Only read-only SELECT/WITH queries can be executed locally.');
|
|
7
89
|
}
|
|
90
|
+
assertSingleSqlStatement(trimmed);
|
|
8
91
|
return trimmed;
|
|
9
92
|
}
|
|
93
|
+
// `assertReadOnlySql` deliberately keeps trailing semicolons, comments, and
|
|
94
|
+
// whitespace (e.g. `select 1; -- done`) — harmless for direct single-statement
|
|
95
|
+
// execution. A row-limit subquery wrapper needs a bare expression instead: a
|
|
96
|
+
// trailing `;` would sit illegally inside the subquery, and a trailing line
|
|
97
|
+
// comment would comment out the closing paren and limit clause. Lex forward with
|
|
98
|
+
// the same standard-SQL rules as the single-statement gate and truncate at the
|
|
99
|
+
// end of the last meaningful token, dropping trailing semicolons, comments, and
|
|
100
|
+
// whitespace. Characters inside string literals and quoted identifiers stay
|
|
101
|
+
// meaningful, so a `;` or `--` within a literal is never mistaken for a
|
|
102
|
+
// terminator (a plain regex cannot make that distinction).
|
|
103
|
+
export function stripTrailingSqlNoise(sql) {
|
|
104
|
+
let index = 0;
|
|
105
|
+
let meaningfulEnd = 0;
|
|
106
|
+
while (index < sql.length) {
|
|
107
|
+
if (sql.startsWith('--', index) || sql.startsWith('/*', index)) {
|
|
108
|
+
index = skipQuotedOrComment(sql, index);
|
|
109
|
+
continue;
|
|
110
|
+
}
|
|
111
|
+
const afterQuoted = skipQuotedOrComment(sql, index);
|
|
112
|
+
if (afterQuoted > index) {
|
|
113
|
+
meaningfulEnd = afterQuoted;
|
|
114
|
+
index = afterQuoted;
|
|
115
|
+
continue;
|
|
116
|
+
}
|
|
117
|
+
if (sql[index] !== ';' && !/\s/.test(sql[index] ?? '')) {
|
|
118
|
+
meaningfulEnd = index + 1;
|
|
119
|
+
}
|
|
120
|
+
index += 1;
|
|
121
|
+
}
|
|
122
|
+
return sql.slice(0, meaningfulEnd);
|
|
123
|
+
}
|
|
10
124
|
export function limitSqlForExecution(sql, maxRows) {
|
|
11
|
-
const trimmed = assertReadOnlySql(sql)
|
|
125
|
+
const trimmed = stripTrailingSqlNoise(assertReadOnlySql(sql));
|
|
12
126
|
if (!maxRows) {
|
|
13
127
|
return trimmed;
|
|
14
128
|
}
|
|
@@ -1,2 +1,13 @@
|
|
|
1
1
|
import { type SimpleGit } from 'simple-git';
|
|
2
|
-
|
|
2
|
+
/**
|
|
3
|
+
* Create a simple-git client scoped to `baseDir`. When an identity is provided, ktx's own
|
|
4
|
+
* commits carry it through the GIT_AUTHOR and GIT_COMMITTER environment variables instead of
|
|
5
|
+
* relying on repo-local or global git config. This keeps commits working when the project
|
|
6
|
+
* directory is an existing repo ktx did not create and the machine has no configured git
|
|
7
|
+
* identity (e.g. a fresh Mac with no ~/.gitconfig), without mutating the user's repo config.
|
|
8
|
+
* Explicit `--author` flags on individual commits still take precedence over GIT_AUTHOR_NAME.
|
|
9
|
+
*/
|
|
10
|
+
export declare function createSimpleGit(baseDir: string, identity?: {
|
|
11
|
+
name: string;
|
|
12
|
+
email: string;
|
|
13
|
+
}): SimpleGit;
|
|
@@ -21,6 +21,21 @@ function sanitizedGitEnv(env = process.env) {
|
|
|
21
21
|
}
|
|
22
22
|
return sanitized;
|
|
23
23
|
}
|
|
24
|
-
|
|
25
|
-
|
|
24
|
+
/**
|
|
25
|
+
* Create a simple-git client scoped to `baseDir`. When an identity is provided, ktx's own
|
|
26
|
+
* commits carry it through the GIT_AUTHOR and GIT_COMMITTER environment variables instead of
|
|
27
|
+
* relying on repo-local or global git config. This keeps commits working when the project
|
|
28
|
+
* directory is an existing repo ktx did not create and the machine has no configured git
|
|
29
|
+
* identity (e.g. a fresh Mac with no ~/.gitconfig), without mutating the user's repo config.
|
|
30
|
+
* Explicit `--author` flags on individual commits still take precedence over GIT_AUTHOR_NAME.
|
|
31
|
+
*/
|
|
32
|
+
export function createSimpleGit(baseDir, identity) {
|
|
33
|
+
const env = sanitizedGitEnv();
|
|
34
|
+
if (identity?.name && identity.email) {
|
|
35
|
+
env.GIT_AUTHOR_NAME = identity.name;
|
|
36
|
+
env.GIT_AUTHOR_EMAIL = identity.email;
|
|
37
|
+
env.GIT_COMMITTER_NAME = identity.name;
|
|
38
|
+
env.GIT_COMMITTER_EMAIL = identity.email;
|
|
39
|
+
}
|
|
40
|
+
return simpleGit({ baseDir, unsafe: { allowUnsafeAskPass: true } }).env(env);
|
|
26
41
|
}
|
|
@@ -20,6 +20,23 @@ export interface WorktreeEntry {
|
|
|
20
20
|
branch: string | null;
|
|
21
21
|
head: string | null;
|
|
22
22
|
}
|
|
23
|
+
export type KtxRepoOwnership = 'unowned' | 'ktx-managed' | 'foreign';
|
|
24
|
+
export declare class KtxForeignGitRepositoryError extends Error {
|
|
25
|
+
constructor(configDir: string);
|
|
26
|
+
}
|
|
27
|
+
/**
|
|
28
|
+
* Classify whether ktx may own a git repository rooted exactly at `dir`. A root
|
|
29
|
+
* `ktx.yaml` is the ownership signal; the working tree decides, not git history,
|
|
30
|
+
* because older ktx versions left `ktx.yaml` uncommitted (it holds secret refs).
|
|
31
|
+
*
|
|
32
|
+
* - `unowned`: no repo here (including a missing or non-directory path) → ktx may `git init`.
|
|
33
|
+
* - `ktx-managed`: `<dir>/.git` is a directory and `ktx.yaml` sits at the root.
|
|
34
|
+
* - `foreign`: any other repo — no root `ktx.yaml`, or a `.git` *file* (a linked
|
|
35
|
+
* worktree). ktx must never adopt or mutate it.
|
|
36
|
+
*
|
|
37
|
+
* Reads only `<dir>` itself; never walks up, so a parent repo cannot change the answer.
|
|
38
|
+
*/
|
|
39
|
+
export declare function classifyKtxRepoOwnership(dir: string): Promise<KtxRepoOwnership>;
|
|
23
40
|
export type SquashMergeResult = {
|
|
24
41
|
ok: true;
|
|
25
42
|
squashSha: string;
|
|
@@ -102,6 +119,12 @@ export declare class GitService {
|
|
|
102
119
|
path: string;
|
|
103
120
|
}>>;
|
|
104
121
|
changedPaths(): Promise<string[]>;
|
|
122
|
+
/**
|
|
123
|
+
* List all paths matching `pathSpec` as they exist at `commitHash`. Reads from
|
|
124
|
+
* git object storage, so it's safe against concurrent working-tree mutations
|
|
125
|
+
* and can recover paths (e.g. a human-renamed file) that no longer exist on disk.
|
|
126
|
+
*/
|
|
127
|
+
listFilesAtCommit(pathSpec: string, commitHash: string): Promise<string[]>;
|
|
105
128
|
/**
|
|
106
129
|
* List all paths under the working tree that match `pathSpec`, scoped to HEAD.
|
|
107
130
|
* Used for the reconciler's first-ever run when there's no watermark to diff from.
|
|
@@ -2,6 +2,53 @@ import { promises as fs } from 'node:fs';
|
|
|
2
2
|
import { dirname, join } from 'node:path';
|
|
3
3
|
import { noopLogger, resolveConfigDir } from './config.js';
|
|
4
4
|
import { createSimpleGit } from './git-env.js';
|
|
5
|
+
export class KtxForeignGitRepositoryError extends Error {
|
|
6
|
+
constructor(configDir) {
|
|
7
|
+
super(`${configDir} is already a git repository that ktx did not create. ` +
|
|
8
|
+
'ktx maintains its context in a repository it owns; run ktx in a dedicated directory or move the existing repository aside.');
|
|
9
|
+
this.name = 'KtxForeignGitRepositoryError';
|
|
10
|
+
}
|
|
11
|
+
}
|
|
12
|
+
function isNodeErrnoException(error) {
|
|
13
|
+
return error instanceof Error && 'code' in error;
|
|
14
|
+
}
|
|
15
|
+
/**
|
|
16
|
+
* Classify whether ktx may own a git repository rooted exactly at `dir`. A root
|
|
17
|
+
* `ktx.yaml` is the ownership signal; the working tree decides, not git history,
|
|
18
|
+
* because older ktx versions left `ktx.yaml` uncommitted (it holds secret refs).
|
|
19
|
+
*
|
|
20
|
+
* - `unowned`: no repo here (including a missing or non-directory path) → ktx may `git init`.
|
|
21
|
+
* - `ktx-managed`: `<dir>/.git` is a directory and `ktx.yaml` sits at the root.
|
|
22
|
+
* - `foreign`: any other repo — no root `ktx.yaml`, or a `.git` *file* (a linked
|
|
23
|
+
* worktree). ktx must never adopt or mutate it.
|
|
24
|
+
*
|
|
25
|
+
* Reads only `<dir>` itself; never walks up, so a parent repo cannot change the answer.
|
|
26
|
+
*/
|
|
27
|
+
export async function classifyKtxRepoOwnership(dir) {
|
|
28
|
+
let dotGitIsDirectory;
|
|
29
|
+
try {
|
|
30
|
+
dotGitIsDirectory = (await fs.lstat(join(dir, '.git'))).isDirectory();
|
|
31
|
+
}
|
|
32
|
+
catch (error) {
|
|
33
|
+
// ENOENT: `<dir>/.git` is absent. ENOTDIR: `<dir>` itself is a file, so it
|
|
34
|
+
// can hold no repo. Either way there is nothing for ktx to avoid here.
|
|
35
|
+
if (isNodeErrnoException(error) && (error.code === 'ENOENT' || error.code === 'ENOTDIR')) {
|
|
36
|
+
return 'unowned';
|
|
37
|
+
}
|
|
38
|
+
throw error;
|
|
39
|
+
}
|
|
40
|
+
if (!dotGitIsDirectory) {
|
|
41
|
+
return 'foreign';
|
|
42
|
+
}
|
|
43
|
+
try {
|
|
44
|
+
// stat (not lstat): follow symlinks, matching what `loadKtxProject`'s
|
|
45
|
+
// readFile accepts — a dir that loads as a ktx project classifies as one.
|
|
46
|
+
return (await fs.stat(join(dir, 'ktx.yaml'))).isFile() ? 'ktx-managed' : 'foreign';
|
|
47
|
+
}
|
|
48
|
+
catch {
|
|
49
|
+
return 'foreign';
|
|
50
|
+
}
|
|
51
|
+
}
|
|
5
52
|
function mergeErrorMessage(error) {
|
|
6
53
|
if (error instanceof Error) {
|
|
7
54
|
return error.message;
|
|
@@ -47,22 +94,26 @@ export class GitService {
|
|
|
47
94
|
// Ensure config directory exists
|
|
48
95
|
await fs.mkdir(this.configDir, { recursive: true });
|
|
49
96
|
this.logger.log(`Config directory ensured at: ${this.configDir}`);
|
|
50
|
-
// Initialize simple-git
|
|
51
|
-
this
|
|
97
|
+
// Initialize simple-git. Carry ktx's identity in the environment so commits succeed even
|
|
98
|
+
// when this repo already exists and the machine has no configured git identity.
|
|
99
|
+
this.git = createSimpleGit(this.configDir, {
|
|
100
|
+
name: this.config.git.userName,
|
|
101
|
+
email: this.config.git.userEmail,
|
|
102
|
+
});
|
|
52
103
|
// Initialize git repository
|
|
53
104
|
await this.initialize();
|
|
54
105
|
}
|
|
55
106
|
async initialize() {
|
|
56
107
|
try {
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
108
|
+
const ownership = await classifyKtxRepoOwnership(this.configDir);
|
|
109
|
+
if (ownership === 'foreign') {
|
|
110
|
+
throw new KtxForeignGitRepositoryError(this.configDir);
|
|
111
|
+
}
|
|
112
|
+
if (ownership === 'unowned') {
|
|
60
113
|
await this.git.init();
|
|
61
|
-
|
|
62
|
-
await this.git.addConfig('user.name', gitConfig.userName);
|
|
63
|
-
await this.git.addConfig('user.email', gitConfig.userEmail);
|
|
64
|
-
this.logger.log('Initialized git repository');
|
|
114
|
+
this.logger.log('Initialized ktx-managed git repository');
|
|
65
115
|
}
|
|
116
|
+
// ownership === 'ktx-managed' → ktx's own repo; proceed with the normal re-run path.
|
|
66
117
|
// Keep any auto-maintenance triggered by writes in-process. Detached maintenance can
|
|
67
118
|
// keep object-pack directories alive briefly after awaited git commands complete,
|
|
68
119
|
// which makes temp-project cleanup flaky in CI.
|
|
@@ -80,8 +131,17 @@ export class GitService {
|
|
|
80
131
|
}
|
|
81
132
|
}
|
|
82
133
|
catch (error) {
|
|
134
|
+
// The foreign-repo error is already typed and actionable; surface it verbatim so every
|
|
135
|
+
// command that loads the project shows the same clear guidance instead of a generic wrapper.
|
|
136
|
+
if (error instanceof KtxForeignGitRepositoryError) {
|
|
137
|
+
throw error;
|
|
138
|
+
}
|
|
83
139
|
this.logger.error('Failed to initialize git repository', error);
|
|
84
|
-
|
|
140
|
+
// Preserve the underlying git error: the generic message alone is undiagnosable in
|
|
141
|
+
// telemetry and unactionable for the user. The exception reporter walks `cause` and
|
|
142
|
+
// redacts secrets before send.
|
|
143
|
+
const detail = error instanceof Error ? error.message : String(error);
|
|
144
|
+
throw new Error(`Failed to initialize git repository: ${detail}`, { cause: error });
|
|
85
145
|
}
|
|
86
146
|
}
|
|
87
147
|
async commitFile(filePath, commitMessage, author, authorEmail) {
|
|
@@ -437,12 +497,13 @@ export class GitService {
|
|
|
437
497
|
return [...new Set(paths)].sort();
|
|
438
498
|
}
|
|
439
499
|
/**
|
|
440
|
-
* List all paths
|
|
441
|
-
*
|
|
500
|
+
* List all paths matching `pathSpec` as they exist at `commitHash`. Reads from
|
|
501
|
+
* git object storage, so it's safe against concurrent working-tree mutations
|
|
502
|
+
* and can recover paths (e.g. a human-renamed file) that no longer exist on disk.
|
|
442
503
|
*/
|
|
443
|
-
async
|
|
504
|
+
async listFilesAtCommit(pathSpec, commitHash) {
|
|
444
505
|
try {
|
|
445
|
-
const raw = await this.git.raw(['ls-tree', '-r', '-z', '--name-only',
|
|
506
|
+
const raw = await this.git.raw(['ls-tree', '-r', '-z', '--name-only', commitHash, '--', pathSpec]);
|
|
446
507
|
if (!raw) {
|
|
447
508
|
return [];
|
|
448
509
|
}
|
|
@@ -452,6 +513,13 @@ export class GitService {
|
|
|
452
513
|
return [];
|
|
453
514
|
}
|
|
454
515
|
}
|
|
516
|
+
/**
|
|
517
|
+
* List all paths under the working tree that match `pathSpec`, scoped to HEAD.
|
|
518
|
+
* Used for the reconciler's first-ever run when there's no watermark to diff from.
|
|
519
|
+
*/
|
|
520
|
+
async listFilesAtHead(pathSpec) {
|
|
521
|
+
return this.listFilesAtCommit(pathSpec, 'HEAD');
|
|
522
|
+
}
|
|
455
523
|
/**
|
|
456
524
|
* Collapse all commits between `preHead` and current HEAD into a single commit with the given
|
|
457
525
|
* message. Used by the memory agent to squash N per-tool-call commits into one ingest commit.
|
|
@@ -740,7 +808,10 @@ export class GitService {
|
|
|
740
808
|
*/
|
|
741
809
|
forWorktree(workdir) {
|
|
742
810
|
const scoped = new GitService(this.config, this.logger);
|
|
743
|
-
scoped.git = createSimpleGit(workdir
|
|
811
|
+
scoped.git = createSimpleGit(workdir, {
|
|
812
|
+
name: this.config.git.userName,
|
|
813
|
+
email: this.config.git.userEmail,
|
|
814
|
+
});
|
|
744
815
|
scoped.configDir = workdir;
|
|
745
816
|
return scoped;
|
|
746
817
|
}
|
|
@@ -2,6 +2,7 @@ import { access, mkdir, readdir, readFile, rename, writeFile } from 'node:fs/pro
|
|
|
2
2
|
import { dirname, join, relative } from 'node:path';
|
|
3
3
|
import YAML from 'yaml';
|
|
4
4
|
import { rawSourcesDirForSync } from '../../raw-sources-paths.js';
|
|
5
|
+
import { isSlYamlPath } from '../../../sl/source-files.js';
|
|
5
6
|
import { mergeUsagePreservingExternal } from '../live-database/manifest.js';
|
|
6
7
|
import { historicSqlEvidenceEnvelopeSchema } from './evidence.js';
|
|
7
8
|
import { stagedManifestSchema } from './types.js';
|
|
@@ -197,7 +198,7 @@ export async function projectHistoricSqlEvidence(input) {
|
|
|
197
198
|
const tableEvidence = evidence.filter((entry) => entry.kind === 'table_usage');
|
|
198
199
|
const patternEvidence = evidence.filter((entry) => entry.kind === 'pattern');
|
|
199
200
|
const schemaRoot = join(input.workdir, 'semantic-layer', input.connectionId, '_schema');
|
|
200
|
-
for (const file of (await walkFiles(schemaRoot)).filter(
|
|
201
|
+
for (const file of (await walkFiles(schemaRoot)).filter(isSlYamlPath)) {
|
|
201
202
|
const path = join(schemaRoot, file);
|
|
202
203
|
const before = await readFile(path, 'utf-8');
|
|
203
204
|
const shard = (YAML.parse(before) ?? {});
|
|
@@ -12,13 +12,18 @@ const defaultLogger = {
|
|
|
12
12
|
class InlineLookerSettings extends NodeSettings {
|
|
13
13
|
params;
|
|
14
14
|
constructor(params) {
|
|
15
|
-
|
|
15
|
+
// @looker/sdk-rtl boundary: NodeSettings consumes a string-valued config
|
|
16
|
+
// section (read back via the readConfig override below), but its constructor
|
|
17
|
+
// is typed to accept a fully-realized IApiSettings. The string record is the
|
|
18
|
+
// shape the library actually reads, so narrow to IApiSection first.
|
|
19
|
+
const settings = {
|
|
16
20
|
base_url: normalizeBaseUrl(params.base_url),
|
|
17
21
|
client_id: params.client_id,
|
|
18
22
|
client_secret: params.client_secret, // pragma: allowlist secret
|
|
19
23
|
verify_ssl: 'true',
|
|
20
24
|
timeout: '120',
|
|
21
|
-
}
|
|
25
|
+
};
|
|
26
|
+
super('', settings);
|
|
22
27
|
this.params = params;
|
|
23
28
|
}
|
|
24
29
|
readConfig(_section) {
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import type { FetchContext } from '../../types.js';
|
|
2
|
-
import { type LookerClientDeps, type LookerConnectionParams } from './client.js';
|
|
2
|
+
import { LookerClient, type LookerClientDeps, type LookerConnectionParams } from './client.js';
|
|
3
3
|
import type { LookerClientFactory, LookerRuntimeClient } from './fetch.js';
|
|
4
4
|
import type { LookerPullConfig } from './types.js';
|
|
5
5
|
export interface LookerCredentialResolver {
|
|
@@ -14,6 +14,13 @@ export declare class DefaultLookerConnectionClientFactory implements LookerConne
|
|
|
14
14
|
private readonly deps;
|
|
15
15
|
constructor(resolver: LookerCredentialResolver, deps?: LookerClientDeps);
|
|
16
16
|
createClient(lookerConnectionId: string): Promise<LookerRuntimeClient>;
|
|
17
|
+
/**
|
|
18
|
+
* Like {@link createClient} but preserves the concrete {@link LookerClient}
|
|
19
|
+
* type, so callers that need methods outside the `LookerRuntimeClient`
|
|
20
|
+
* contract (e.g. `listLookerConnections`, `testConnection`) keep them without
|
|
21
|
+
* a cast.
|
|
22
|
+
*/
|
|
23
|
+
createLookerClient(lookerConnectionId: string): Promise<LookerClient>;
|
|
17
24
|
}
|
|
18
25
|
/** @internal */
|
|
19
26
|
export declare class DefaultLookerClientFactory implements LookerClientFactory {
|
|
@@ -7,6 +7,15 @@ export class DefaultLookerConnectionClientFactory {
|
|
|
7
7
|
this.deps = deps;
|
|
8
8
|
}
|
|
9
9
|
async createClient(lookerConnectionId) {
|
|
10
|
+
return this.createLookerClient(lookerConnectionId);
|
|
11
|
+
}
|
|
12
|
+
/**
|
|
13
|
+
* Like {@link createClient} but preserves the concrete {@link LookerClient}
|
|
14
|
+
* type, so callers that need methods outside the `LookerRuntimeClient`
|
|
15
|
+
* contract (e.g. `listLookerConnections`, `testConnection`) keep them without
|
|
16
|
+
* a cast.
|
|
17
|
+
*/
|
|
18
|
+
async createLookerClient(lookerConnectionId) {
|
|
10
19
|
const credentials = await this.resolver.resolve(lookerConnectionId);
|
|
11
20
|
return new LookerClient(credentials, this.deps);
|
|
12
21
|
}
|
|
@@ -120,7 +120,7 @@ export function validateLookerMappings(args) {
|
|
|
120
120
|
if (!args.knownKtxConnectionIds.has(mapping.ktxConnectionId)) {
|
|
121
121
|
errors.push({
|
|
122
122
|
key: mapping.lookerConnectionName,
|
|
123
|
-
reason: `
|
|
123
|
+
reason: `ktx connection ${mapping.ktxConnectionId} does not exist`,
|
|
124
124
|
});
|
|
125
125
|
continue;
|
|
126
126
|
}
|
|
@@ -12,6 +12,7 @@ export declare const lookerPullConfigSchema: z.ZodObject<{
|
|
|
12
12
|
lookUpdatedSince: z.ZodOptional<z.ZodNullable<z.ZodISODateTime>>;
|
|
13
13
|
connectionMappings: z.ZodDefault<z.ZodRecord<z.ZodString, z.ZodString>>;
|
|
14
14
|
connectionTypes: z.ZodDefault<z.ZodRecord<z.ZodString, z.ZodEnum<{
|
|
15
|
+
PLAIN: "PLAIN";
|
|
15
16
|
BIGQUERY: "BIGQUERY";
|
|
16
17
|
SNOWFLAKE: "SNOWFLAKE";
|
|
17
18
|
MYSQL: "MYSQL";
|
|
@@ -31,7 +32,6 @@ export declare const lookerPullConfigSchema: z.ZodObject<{
|
|
|
31
32
|
METABASE: "METABASE";
|
|
32
33
|
LOOKER: "LOOKER";
|
|
33
34
|
NOTION: "NOTION";
|
|
34
|
-
PLAIN: "PLAIN";
|
|
35
35
|
BETTERSTACK: "BETTERSTACK";
|
|
36
36
|
}>>>;
|
|
37
37
|
parsedTargetTables: z.ZodDefault<z.ZodRecord<z.ZodString, z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
@@ -18,7 +18,7 @@ export declare const DEFAULT_METABASE_CLIENT_CONFIG: MetabaseClientConfig;
|
|
|
18
18
|
* Strip Metabase `[[ ... {{ var }} ... ]]` optional-clause blocks from native SQL.
|
|
19
19
|
*
|
|
20
20
|
* The bracketed blocks are emitted only when the embedded `{{ var }}` is supplied at
|
|
21
|
-
* Metabase query time. For
|
|
21
|
+
* Metabase query time. For ktx semantic-layer ingest there's no such runtime
|
|
22
22
|
* parameter — chat-time filters are composed by the SL query planner — so the optional
|
|
23
23
|
* block must be removed before the SQL becomes a permanent SL source. Substituting a
|
|
24
24
|
* dummy value (the alternative) bakes a placeholder filter into the source and silently
|
|
@@ -30,7 +30,7 @@ class MetabaseApiError extends Error {
|
|
|
30
30
|
* Strip Metabase `[[ ... {{ var }} ... ]]` optional-clause blocks from native SQL.
|
|
31
31
|
*
|
|
32
32
|
* The bracketed blocks are emitted only when the embedded `{{ var }}` is supplied at
|
|
33
|
-
* Metabase query time. For
|
|
33
|
+
* Metabase query time. For ktx semantic-layer ingest there's no such runtime
|
|
34
34
|
* parameter — chat-time filters are composed by the SL query planner — so the optional
|
|
35
35
|
* block must be removed before the SQL becomes a permanent SL source. Substituting a
|
|
36
36
|
* dummy value (the alternative) bakes a placeholder filter into the source and silently
|
|
@@ -15,7 +15,7 @@ export function metabaseRuntimeConfigFromLocalConnection(connectionId, connectio
|
|
|
15
15
|
throw new Error(`Connection "${connectionId}" is not a Metabase connection`);
|
|
16
16
|
}
|
|
17
17
|
if (hasNetworkProxy(connection)) {
|
|
18
|
-
throw new Error(`Standalone
|
|
18
|
+
throw new Error(`Standalone ktx does not support proxy-bearing Metabase connections yet. Use hosted Metabase ingest for "${connectionId}" until the ktx Metabase proxy support spec lands.`);
|
|
19
19
|
}
|
|
20
20
|
const apiUrl = stringField(connection.api_url);
|
|
21
21
|
const literalApiKey = stringField(connection.api_key);
|
|
@@ -93,7 +93,7 @@ export function validateMetabaseMappings(args) {
|
|
|
93
93
|
continue;
|
|
94
94
|
}
|
|
95
95
|
if (!args.knownKtxConnectionIds.has(connectionId)) {
|
|
96
|
-
errors.push({ key, reason: `
|
|
96
|
+
errors.push({ key, reason: `ktx connection ${connectionId} does not exist` });
|
|
97
97
|
}
|
|
98
98
|
}
|
|
99
99
|
return errors.length === 0 ? { ok: true } : { ok: false, errors };
|
|
@@ -108,13 +108,13 @@ export function validateMappingPhysicalMatch(mapping, target) {
|
|
|
108
108
|
return null;
|
|
109
109
|
}
|
|
110
110
|
if (target.connection_type !== expectedType) {
|
|
111
|
-
return `Metabase database engine '${engine}' does not match
|
|
111
|
+
return `Metabase database engine '${engine}' does not match ktx connection type '${target.connection_type}'`;
|
|
112
112
|
}
|
|
113
113
|
const metabaseDb = normalizeName(mapping.metabaseDbName);
|
|
114
114
|
const targetDb = normalizeName(getTargetDatabase(target));
|
|
115
115
|
if (engine === 'snowflake' || engine === 'bigquery' || engine === 'bigquery-cloud-sdk') {
|
|
116
116
|
if (metabaseDb && targetDb && metabaseDb !== targetDb) {
|
|
117
|
-
return `Metabase database '${mapping.metabaseDbName}' does not match
|
|
117
|
+
return `Metabase database '${mapping.metabaseDbName}' does not match ktx connection database '${displayValue(getTargetDatabase(target))}'`;
|
|
118
118
|
}
|
|
119
119
|
return null;
|
|
120
120
|
}
|
|
@@ -122,10 +122,10 @@ export function validateMappingPhysicalMatch(mapping, target) {
|
|
|
122
122
|
const metabaseHost = normalizeHost(mapping.metabaseHost);
|
|
123
123
|
const targetHost = normalizeHost(target.host);
|
|
124
124
|
if (metabaseHost && targetHost && metabaseHost !== targetHost) {
|
|
125
|
-
return `Metabase host '${mapping.metabaseHost}' does not match
|
|
125
|
+
return `Metabase host '${mapping.metabaseHost}' does not match ktx connection host '${displayValue(target.host)}'`;
|
|
126
126
|
}
|
|
127
127
|
if (metabaseDb && targetDb && metabaseDb !== targetDb) {
|
|
128
|
-
return `Metabase database '${mapping.metabaseDbName}' does not match
|
|
128
|
+
return `Metabase database '${mapping.metabaseDbName}' does not match ktx connection database '${displayValue(getTargetDatabase(target))}'`;
|
|
129
129
|
}
|
|
130
130
|
return null;
|
|
131
131
|
}
|
|
@@ -157,7 +157,7 @@ export async function refreshMetabaseMapping(args) {
|
|
|
157
157
|
if (!target) {
|
|
158
158
|
physicalMismatches.push({
|
|
159
159
|
mappingId: String(mapping.id),
|
|
160
|
-
reason: `
|
|
160
|
+
reason: `ktx connection ${mapping.ktxConnectionId} does not exist`,
|
|
161
161
|
});
|
|
162
162
|
continue;
|
|
163
163
|
}
|
|
@@ -1,17 +1,14 @@
|
|
|
1
1
|
import type { SemanticLayerService } from '../../context/sl/semantic-layer.service.js';
|
|
2
2
|
import type { TouchedSlSource } from '../../context/tools/touched-sl-sources.js';
|
|
3
3
|
import type { KnowledgeWikiService } from '../../context/wiki/knowledge-wiki.service.js';
|
|
4
|
-
|
|
5
|
-
invalidSources: string[];
|
|
6
|
-
validSources: string[];
|
|
7
|
-
}
|
|
4
|
+
import type { WuValidationResult } from './stages/validate-wu-sources.js';
|
|
8
5
|
export interface FinalArtifactGateInput {
|
|
9
6
|
connectionIds: string[];
|
|
10
7
|
changedWikiPageKeys: string[];
|
|
11
8
|
touchedSlSources: TouchedSlSource[];
|
|
12
9
|
wikiService: KnowledgeWikiService;
|
|
13
10
|
semanticLayerService: SemanticLayerService;
|
|
14
|
-
validateTouchedSources(touched: TouchedSlSource[]): Promise<
|
|
11
|
+
validateTouchedSources(touched: TouchedSlSource[]): Promise<WuValidationResult>;
|
|
15
12
|
tableExists(connectionId: string, tableRef: string): Promise<boolean>;
|
|
16
13
|
}
|
|
17
14
|
export interface ProvenanceRawPathValidationInput {
|
|
@@ -23,4 +20,3 @@ export interface ProvenanceRawPathValidationInput {
|
|
|
23
20
|
}
|
|
24
21
|
export declare function validateFinalIngestArtifacts(input: FinalArtifactGateInput): Promise<void>;
|
|
25
22
|
export declare function validateProvenanceRawPaths(input: ProvenanceRawPathValidationInput): void;
|
|
26
|
-
export {};
|
|
@@ -13,50 +13,6 @@ function slEntityNames(source) {
|
|
|
13
13
|
...(source.segments ?? []).map((segment) => segment.name),
|
|
14
14
|
]);
|
|
15
15
|
}
|
|
16
|
-
function uniqueTouchedSources(sources) {
|
|
17
|
-
const seen = new Set();
|
|
18
|
-
const unique = [];
|
|
19
|
-
for (const source of sources) {
|
|
20
|
-
const key = `${source.connectionId}:${source.sourceName}`;
|
|
21
|
-
if (seen.has(key)) {
|
|
22
|
-
continue;
|
|
23
|
-
}
|
|
24
|
-
seen.add(key);
|
|
25
|
-
unique.push(source);
|
|
26
|
-
}
|
|
27
|
-
return unique.sort((left, right) => {
|
|
28
|
-
const byConnection = left.connectionId.localeCompare(right.connectionId);
|
|
29
|
-
return byConnection === 0 ? left.sourceName.localeCompare(right.sourceName) : byConnection;
|
|
30
|
-
});
|
|
31
|
-
}
|
|
32
|
-
async function expandTouchedSlSourcesWithDirectJoinNeighbors(input) {
|
|
33
|
-
const expanded = [...input.touchedSlSources];
|
|
34
|
-
const touchedByConnection = new Map();
|
|
35
|
-
for (const source of input.touchedSlSources) {
|
|
36
|
-
const bucket = touchedByConnection.get(source.connectionId) ?? new Set();
|
|
37
|
-
bucket.add(source.sourceName);
|
|
38
|
-
touchedByConnection.set(source.connectionId, bucket);
|
|
39
|
-
}
|
|
40
|
-
for (const connectionId of input.connectionIds) {
|
|
41
|
-
const touched = touchedByConnection.get(connectionId);
|
|
42
|
-
if (!touched || touched.size === 0) {
|
|
43
|
-
continue;
|
|
44
|
-
}
|
|
45
|
-
const { sources } = await input.semanticLayerService.loadAllSources(connectionId);
|
|
46
|
-
for (const source of sources) {
|
|
47
|
-
const sourceIsTouched = touched.has(source.name);
|
|
48
|
-
if (sourceIsTouched) {
|
|
49
|
-
for (const join of source.joins ?? []) {
|
|
50
|
-
expanded.push({ connectionId, sourceName: join.to });
|
|
51
|
-
}
|
|
52
|
-
}
|
|
53
|
-
if ((source.joins ?? []).some((join) => touched.has(join.to))) {
|
|
54
|
-
expanded.push({ connectionId, sourceName: source.name });
|
|
55
|
-
}
|
|
56
|
-
}
|
|
57
|
-
}
|
|
58
|
-
return uniqueTouchedSources(expanded);
|
|
59
|
-
}
|
|
60
16
|
async function validateWikiSlRefs(input) {
|
|
61
17
|
const errors = [];
|
|
62
18
|
const sourcesByConnection = new Map();
|
|
@@ -112,9 +68,11 @@ async function validateWikiRefs(input) {
|
|
|
112
68
|
return dangling;
|
|
113
69
|
}
|
|
114
70
|
export async function validateFinalIngestArtifacts(input) {
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
71
|
+
// Join-neighbor expansion happens inside validateTouchedSources so work-unit
|
|
72
|
+
// validation and this gate check the same set — a source that passes one
|
|
73
|
+
// passes the other.
|
|
74
|
+
const validation = await input.validateTouchedSources(input.touchedSlSources);
|
|
75
|
+
const errors = validation.invalidSources.map((invalid) => `semantic-layer validation failed for ${invalid.source}: ${invalid.errors.join('; ')}`);
|
|
118
76
|
errors.push(...(await validateWikiSlRefs(input)));
|
|
119
77
|
const danglingWikiRefs = await validateWikiRefs(input);
|
|
120
78
|
if (danglingWikiRefs.length > 0) {
|