@kaelio/ktx 0.13.0 → 0.14.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/assets/python/{kaelio_ktx-0.13.0-py3-none-any.whl → kaelio_ktx-0.14.0-py3-none-any.whl} +0 -0
- package/assets/python/manifest.json +4 -4
- package/dist/.tsbuildinfo +1 -1
- package/dist/cli-program.js +1 -1
- package/dist/commands/ingest-commands.d.ts +9 -0
- package/dist/commands/ingest-commands.js +37 -1
- package/dist/commands/knowledge-commands.js +3 -0
- package/dist/commands/setup-commands.js +14 -1
- package/dist/connection-drivers.d.ts +2 -0
- package/dist/connection-drivers.js +7 -3
- package/dist/connection.d.ts +3 -0
- package/dist/connection.js +27 -0
- package/dist/connectors/bigquery/connector.d.ts +17 -6
- package/dist/connectors/bigquery/connector.js +152 -76
- package/dist/connectors/bigquery/dialect.d.ts +2 -2
- package/dist/connectors/clickhouse/connector.d.ts +1 -0
- package/dist/connectors/clickhouse/connector.js +45 -16
- package/dist/connectors/clickhouse/dialect.d.ts +2 -2
- package/dist/connectors/mongodb/connector.d.ts +69 -0
- package/dist/connectors/mongodb/connector.js +307 -0
- package/dist/connectors/mongodb/dialect.d.ts +17 -0
- package/dist/connectors/mongodb/dialect.js +50 -0
- package/dist/connectors/mongodb/live-database-introspection.d.ts +10 -0
- package/dist/connectors/mongodb/live-database-introspection.js +24 -0
- package/dist/connectors/mongodb/schema-inference.d.ts +20 -0
- package/dist/connectors/mongodb/schema-inference.js +110 -0
- package/dist/connectors/mysql/connector.d.ts +1 -0
- package/dist/connectors/mysql/connector.js +18 -2
- package/dist/connectors/mysql/dialect.d.ts +2 -2
- package/dist/connectors/postgres/connector.d.ts +1 -0
- package/dist/connectors/postgres/connector.js +20 -3
- package/dist/connectors/postgres/dialect.d.ts +2 -2
- package/dist/connectors/snowflake/connector.d.ts +1 -0
- package/dist/connectors/snowflake/connector.js +27 -3
- package/dist/connectors/snowflake/dialect.d.ts +2 -2
- package/dist/connectors/sqlite/connector.d.ts +11 -1
- package/dist/connectors/sqlite/connector.js +120 -17
- package/dist/connectors/sqlite/dialect.d.ts +2 -2
- package/dist/connectors/sqlite/read-query-child.d.ts +1 -0
- package/dist/connectors/sqlite/read-query-child.js +23 -0
- package/dist/connectors/sqlserver/connector.d.ts +2 -0
- package/dist/connectors/sqlserver/connector.js +23 -5
- package/dist/connectors/sqlserver/dialect.d.ts +2 -2
- package/dist/context/cache/content-result-cache.d.ts +36 -0
- package/dist/context/cache/content-result-cache.js +14 -0
- package/dist/context/cache/sqlite-content-result-cache.d.ts +18 -0
- package/dist/context/cache/sqlite-content-result-cache.js +223 -0
- package/dist/context/connections/bigquery-identifiers.d.ts +1 -0
- package/dist/context/connections/bigquery-identifiers.js +7 -0
- package/dist/context/connections/configured-connections.d.ts +9 -0
- package/dist/context/connections/configured-connections.js +18 -0
- package/dist/context/connections/dialects.d.ts +30 -4
- package/dist/context/connections/dialects.js +38 -11
- package/dist/context/connections/drivers.js +21 -0
- package/dist/context/connections/gdrive-config.d.ts +19 -0
- package/dist/context/connections/gdrive-config.js +57 -0
- package/dist/context/connections/query-deadline.d.ts +31 -0
- package/dist/context/connections/query-deadline.js +37 -0
- package/dist/context/connections/read-only-sql.d.ts +5 -0
- package/dist/context/connections/read-only-sql.js +139 -1
- package/dist/context/ingest/adapters/gdrive/chunk.d.ts +3 -0
- package/dist/context/ingest/adapters/gdrive/chunk.js +74 -0
- package/dist/context/ingest/adapters/gdrive/detect.d.ts +1 -0
- package/dist/context/ingest/adapters/gdrive/detect.js +20 -0
- package/dist/context/ingest/adapters/gdrive/fetch.d.ts +6 -0
- package/dist/context/ingest/adapters/gdrive/fetch.js +95 -0
- package/dist/context/ingest/adapters/gdrive/gdrive-client.d.ts +30 -0
- package/dist/context/ingest/adapters/gdrive/gdrive-client.js +132 -0
- package/dist/context/ingest/adapters/gdrive/gdrive.adapter.d.ts +11 -0
- package/dist/context/ingest/adapters/gdrive/gdrive.adapter.js +27 -0
- package/dist/context/ingest/adapters/gdrive/normalize.d.ts +2 -0
- package/dist/context/ingest/adapters/gdrive/normalize.js +256 -0
- package/dist/context/ingest/adapters/gdrive/types.d.ts +153 -0
- package/dist/context/ingest/adapters/gdrive/types.js +36 -0
- package/dist/context/ingest/adapters/live-database/daemon-introspection.js +24 -0
- package/dist/context/ingest/adapters/live-database/fetch-report.d.ts +8 -0
- package/dist/context/ingest/adapters/live-database/fetch-report.js +37 -0
- package/dist/context/ingest/adapters/live-database/live-database.adapter.d.ts +2 -1
- package/dist/context/ingest/adapters/live-database/live-database.adapter.js +9 -2
- package/dist/context/ingest/adapters/live-database/manifest.d.ts +2 -0
- package/dist/context/ingest/adapters/live-database/manifest.js +8 -4
- package/dist/context/ingest/adapters/live-database/scan-outcome.d.ts +17 -0
- package/dist/context/ingest/adapters/live-database/scan-outcome.js +40 -0
- package/dist/context/ingest/adapters/live-database/stage.js +5 -5
- package/dist/context/ingest/adapters/metabase/types.d.ts +1 -1
- package/dist/context/ingest/adapters/metabase/types.js +1 -1
- package/dist/context/ingest/artifact-gates.d.ts +32 -1
- package/dist/context/ingest/artifact-gates.js +85 -18
- package/dist/context/ingest/final-gate-prune.d.ts +35 -0
- package/dist/context/ingest/final-gate-prune.js +229 -0
- package/dist/context/ingest/ingest-bundle.runner.d.ts +3 -0
- package/dist/context/ingest/ingest-bundle.runner.js +459 -240
- package/dist/context/ingest/isolated-diff/patch-integrator.d.ts +0 -11
- package/dist/context/ingest/isolated-diff/patch-integrator.js +0 -44
- package/dist/context/ingest/isolated-diff/work-unit-executor.d.ts +1 -0
- package/dist/context/ingest/isolated-diff/work-unit-executor.js +13 -4
- package/dist/context/ingest/local-adapters.js +6 -0
- package/dist/context/ingest/local-bundle-runtime.js +7 -2
- package/dist/context/ingest/local-ingest.js +1 -1
- package/dist/context/ingest/local-stage-ingest.d.ts +3 -1
- package/dist/context/ingest/local-stage-ingest.js +3 -1
- package/dist/context/ingest/ports.d.ts +3 -0
- package/dist/context/ingest/report-snapshot.js +13 -3
- package/dist/context/ingest/reports.d.ts +3 -3
- package/dist/context/ingest/reports.js +3 -1
- package/dist/context/ingest/stages/build-wu-context.d.ts +1 -0
- package/dist/context/ingest/stages/build-wu-context.js +2 -1
- package/dist/context/ingest/stages/stage-3-work-units.d.ts +2 -1
- package/dist/context/ingest/stages/stage-3-work-units.js +4 -10
- package/dist/context/ingest/stages/validate-wu-sources.d.ts +12 -0
- package/dist/context/ingest/stages/validate-wu-sources.js +23 -8
- package/dist/context/ingest/tools/read-raw-file.tool.js +10 -5
- package/dist/context/ingest/tools/read-raw-span.tool.js +10 -5
- package/dist/context/ingest/tools/warehouse-verification/discover-data.tool.js +1 -1
- package/dist/context/ingest/tools/warehouse-verification/entity-details.tool.js +1 -1
- package/dist/context/ingest/tools/warehouse-verification/sql-execution.tool.js +1 -1
- package/dist/context/ingest/types.d.ts +3 -0
- package/dist/context/ingest/wiki-body-refs.d.ts +28 -0
- package/dist/context/ingest/wiki-body-refs.js +37 -8
- package/dist/context/ingest/wiki-sl-ref-repair.d.ts +1 -0
- package/dist/context/ingest/wiki-sl-ref-repair.js +4 -0
- package/dist/context/ingest/work-unit-cache.d.ts +67 -0
- package/dist/context/ingest/work-unit-cache.js +230 -0
- package/dist/context/llm/ai-sdk-runtime.d.ts +1 -0
- package/dist/context/llm/ai-sdk-runtime.js +5 -0
- package/dist/context/llm/claude-code-runtime.d.ts +4 -1
- package/dist/context/llm/claude-code-runtime.js +38 -8
- package/dist/context/llm/codex-runtime.d.ts +4 -1
- package/dist/context/llm/codex-runtime.js +48 -35
- package/dist/context/llm/runtime-port.d.ts +26 -0
- package/dist/context/llm/subprocess-generate-object-child.d.ts +1 -0
- package/dist/context/llm/subprocess-generate-object-child.js +34 -0
- package/dist/context/llm/subprocess-generate-object.d.ts +42 -0
- package/dist/context/llm/subprocess-generate-object.js +122 -0
- package/dist/context/mcp/context-tools.d.ts +2 -0
- package/dist/context/mcp/context-tools.js +111 -17
- package/dist/context/mcp/local-project-ports.d.ts +7 -0
- package/dist/context/mcp/local-project-ports.js +29 -0
- package/dist/context/mcp/logger.d.ts +24 -0
- package/dist/context/mcp/logger.js +49 -0
- package/dist/context/mcp/server.js +2 -0
- package/dist/context/mcp/types.d.ts +17 -0
- package/dist/context/memory/memory-agent.service.js +1 -1
- package/dist/context/project/config.d.ts +42 -0
- package/dist/context/project/config.js +5 -0
- package/dist/context/project/driver-schemas.d.ts +20 -0
- package/dist/context/project/driver-schemas.js +50 -1
- package/dist/context/project/project.js +1 -1
- package/dist/context/project/setup-config.js +1 -0
- package/dist/context/scan/description-generation.d.ts +4 -0
- package/dist/context/scan/description-generation.js +100 -13
- package/dist/context/scan/enabled-tables.d.ts +5 -0
- package/dist/context/scan/enabled-tables.js +13 -1
- package/dist/context/scan/enrichment-state.d.ts +50 -7
- package/dist/context/scan/enrichment-state.js +30 -13
- package/dist/context/scan/local-enrichment-artifacts.d.ts +41 -0
- package/dist/context/scan/local-enrichment-artifacts.js +155 -32
- package/dist/context/scan/local-enrichment.d.ts +39 -4
- package/dist/context/scan/local-enrichment.js +345 -92
- package/dist/context/scan/local-scan.d.ts +4 -1
- package/dist/context/scan/local-scan.js +54 -13
- package/dist/context/scan/local-structural-artifacts.d.ts +3 -1
- package/dist/context/scan/local-structural-artifacts.js +5 -0
- package/dist/context/scan/object-introspection.d.ts +21 -0
- package/dist/context/scan/object-introspection.js +35 -0
- package/dist/context/scan/relationship-benchmarks.js +5 -3
- package/dist/context/scan/relationship-composite-candidates.d.ts +6 -3
- package/dist/context/scan/relationship-composite-candidates.js +13 -1
- package/dist/context/scan/relationship-detection-budget.d.ts +35 -0
- package/dist/context/scan/relationship-detection-budget.js +53 -0
- package/dist/context/scan/relationship-diagnostics.d.ts +5 -0
- package/dist/context/scan/relationship-diagnostics.js +2 -0
- package/dist/context/scan/relationship-discovery.d.ts +9 -3
- package/dist/context/scan/relationship-discovery.js +27 -4
- package/dist/context/scan/relationship-llm-proposal.js +36 -27
- package/dist/context/scan/relationship-profiling.d.ts +7 -3
- package/dist/context/scan/relationship-profiling.js +53 -41
- package/dist/context/scan/relationship-validation.d.ts +6 -4
- package/dist/context/scan/relationship-validation.js +46 -32
- package/dist/context/scan/sqlite-local-enrichment-state-store.d.ts +8 -1
- package/dist/context/scan/sqlite-local-enrichment-state-store.js +71 -159
- package/dist/context/scan/types.d.ts +5 -3
- package/dist/context/scan/types.js +2 -0
- package/dist/context/sl/local-query.js +5 -0
- package/dist/context/sl/pglite-sl-search-prototype.js +1 -1
- package/dist/context/sl/semantic-layer.service.js +1 -1
- package/dist/context/sl/source-files.d.ts +5 -0
- package/dist/context/sl/source-files.js +17 -11
- package/dist/context/sl/tools/connection-id-schema.js +1 -1
- package/dist/context/sql-analysis/dialect-notes.d.ts +9 -0
- package/dist/context/sql-analysis/dialect-notes.js +40 -0
- package/dist/context/sql-analysis/dialects/bigquery.md +13 -0
- package/dist/context/sql-analysis/dialects/clickhouse.md +9 -0
- package/dist/context/sql-analysis/dialects/mysql.md +9 -0
- package/dist/context/sql-analysis/dialects/postgres.md +10 -0
- package/dist/context/sql-analysis/dialects/snowflake.md +10 -0
- package/dist/context/sql-analysis/dialects/sqlite.md +11 -0
- package/dist/context/sql-analysis/dialects/tsql.md +10 -0
- package/dist/context/wiki/keys.js +1 -1
- package/dist/context/wiki/knowledge-wiki.service.d.ts +4 -4
- package/dist/context/wiki/knowledge-wiki.service.js +2 -1
- package/dist/context/wiki/local-knowledge.d.ts +13 -0
- package/dist/context/wiki/local-knowledge.js +50 -6
- package/dist/context/wiki/sqlite-knowledge-index.d.ts +2 -0
- package/dist/context/wiki/sqlite-knowledge-index.js +21 -5
- package/dist/context/wiki/tools/wiki-write.tool.d.ts +2 -0
- package/dist/context/wiki/tools/wiki-write.tool.js +27 -0
- package/dist/context/wiki/types.d.ts +6 -0
- package/dist/context-build-view.d.ts +3 -0
- package/dist/context-build-view.js +1 -0
- package/dist/knowledge.d.ts +2 -0
- package/dist/knowledge.js +12 -1
- package/dist/local-adapters.js +11 -0
- package/dist/mcp-http-server.js +15 -1
- package/dist/mcp-server-factory.d.ts +2 -0
- package/dist/mcp-server-factory.js +15 -1
- package/dist/mcp-stdio-server.js +10 -2
- package/dist/notion-page-picker.js +1 -1
- package/dist/prompts/memory_agent_external_ingest.md +2 -0
- package/dist/public-ingest.d.ts +3 -1
- package/dist/public-ingest.js +3 -0
- package/dist/scan.d.ts +3 -1
- package/dist/scan.js +7 -0
- package/dist/setup-databases.d.ts +1 -1
- package/dist/setup-databases.js +43 -1
- package/dist/setup-sources.d.ts +5 -1
- package/dist/setup-sources.js +124 -48
- package/dist/setup.d.ts +3 -0
- package/dist/setup.js +6 -1
- package/dist/skills/analytics/SKILL.md +200 -2
- package/dist/skills/gdrive_synthesize/SKILL.md +97 -0
- package/dist/skills/wiki_capture/SKILL.md +24 -0
- package/dist/sql.js +4 -0
- package/dist/status-project.d.ts +4 -0
- package/dist/status-project.js +54 -2
- package/dist/telemetry/events.d.ts +2 -2
- package/dist/text-ingest.d.ts +4 -0
- package/dist/text-ingest.js +58 -29
- package/dist/verbatim-ingest.d.ts +46 -0
- package/dist/verbatim-ingest.js +193 -0
- package/package.json +5 -1
- package/dist/context/ingest/final-gate-repair.d.ts +0 -28
- package/dist/context/ingest/final-gate-repair.js +0 -91
|
@@ -1,6 +1,5 @@
|
|
|
1
1
|
import type { GitService } from '../../../context/core/git.service.js';
|
|
2
2
|
import type { RepairVerification } from '../constrained-repair.js';
|
|
3
|
-
import type { FinalGateRepairResult } from '../final-gate-repair.js';
|
|
4
3
|
import type { IngestTraceWriter } from '../ingest-trace.js';
|
|
5
4
|
import type { TextualConflictResolutionResult } from './textual-conflict-resolver.js';
|
|
6
5
|
export type PatchIntegrationResult = {
|
|
@@ -8,19 +7,16 @@ export type PatchIntegrationResult = {
|
|
|
8
7
|
commitSha: string;
|
|
9
8
|
touchedPaths: string[];
|
|
10
9
|
textualResolution?: TextualConflictResolutionResult;
|
|
11
|
-
gateRepair?: FinalGateRepairResult;
|
|
12
10
|
} | {
|
|
13
11
|
status: 'textual_conflict';
|
|
14
12
|
reason: string;
|
|
15
13
|
touchedPaths: string[];
|
|
16
14
|
textualResolution?: TextualConflictResolutionResult;
|
|
17
|
-
gateRepair?: FinalGateRepairResult;
|
|
18
15
|
} | {
|
|
19
16
|
status: 'semantic_conflict';
|
|
20
17
|
reason: string;
|
|
21
18
|
touchedPaths: string[];
|
|
22
19
|
textualResolution?: TextualConflictResolutionResult;
|
|
23
|
-
gateRepair?: FinalGateRepairResult;
|
|
24
20
|
};
|
|
25
21
|
export interface IntegrateWorkUnitPatchInput {
|
|
26
22
|
unitKey: string;
|
|
@@ -41,12 +37,5 @@ export interface IntegrateWorkUnitPatchInput {
|
|
|
41
37
|
reason: string;
|
|
42
38
|
verify(changedPaths: string[]): Promise<RepairVerification>;
|
|
43
39
|
}): Promise<TextualConflictResolutionResult>;
|
|
44
|
-
repairGateFailure?(input: {
|
|
45
|
-
unitKey: string;
|
|
46
|
-
patchPath: string;
|
|
47
|
-
touchedPaths: string[];
|
|
48
|
-
reason: string;
|
|
49
|
-
verify(changedPaths: string[]): Promise<RepairVerification>;
|
|
50
|
-
}): Promise<FinalGateRepairResult>;
|
|
51
40
|
}
|
|
52
41
|
export declare function integrateWorkUnitPatch(input: IntegrateWorkUnitPatchInput): Promise<PatchIntegrationResult>;
|
|
@@ -153,50 +153,6 @@ export async function integrateWorkUnitPatch(input) {
|
|
|
153
153
|
touchedPaths,
|
|
154
154
|
reason,
|
|
155
155
|
});
|
|
156
|
-
if (input.repairGateFailure) {
|
|
157
|
-
const gateRepair = await input.repairGateFailure({
|
|
158
|
-
unitKey: input.unitKey,
|
|
159
|
-
patchPath: input.patchPath,
|
|
160
|
-
touchedPaths,
|
|
161
|
-
reason,
|
|
162
|
-
verify: verifyAppliedTree,
|
|
163
|
-
});
|
|
164
|
-
if (gateRepair.status === 'failed') {
|
|
165
|
-
if (preApplyHead) {
|
|
166
|
-
await input.integrationGit.resetHardTo(preApplyHead);
|
|
167
|
-
}
|
|
168
|
-
return {
|
|
169
|
-
status: 'semantic_conflict',
|
|
170
|
-
reason: gateRepair.reason,
|
|
171
|
-
touchedPaths,
|
|
172
|
-
gateRepair,
|
|
173
|
-
};
|
|
174
|
-
}
|
|
175
|
-
const commit = await input.integrationGit.commitFiles(gateRepair.changedPaths, `ingest: repair WorkUnit ${input.unitKey} gates`, input.author.name, input.author.email);
|
|
176
|
-
if (!commit.created) {
|
|
177
|
-
if (preApplyHead) {
|
|
178
|
-
await input.integrationGit.resetHardTo(preApplyHead);
|
|
179
|
-
}
|
|
180
|
-
return {
|
|
181
|
-
status: 'semantic_conflict',
|
|
182
|
-
reason: 'gate repair produced no committable changes',
|
|
183
|
-
touchedPaths: gateRepair.changedPaths,
|
|
184
|
-
gateRepair,
|
|
185
|
-
};
|
|
186
|
-
}
|
|
187
|
-
await input.trace.event('debug', 'integration', 'patch_accepted_after_gate_repair', {
|
|
188
|
-
unitKey: input.unitKey,
|
|
189
|
-
commitSha: commit.commitHash,
|
|
190
|
-
touchedPaths: gateRepair.changedPaths,
|
|
191
|
-
attempts: gateRepair.attempts,
|
|
192
|
-
});
|
|
193
|
-
return {
|
|
194
|
-
status: 'accepted',
|
|
195
|
-
commitSha: commit.commitHash,
|
|
196
|
-
touchedPaths: gateRepair.changedPaths,
|
|
197
|
-
gateRepair,
|
|
198
|
-
};
|
|
199
|
-
}
|
|
200
156
|
if (preApplyHead) {
|
|
201
157
|
await input.integrationGit.resetHardTo(preApplyHead);
|
|
202
158
|
}
|
|
@@ -13,4 +13,5 @@ export interface RunIsolatedWorkUnitInput {
|
|
|
13
13
|
run(child: IngestSessionWorktree): Promise<WorkUnitOutcome>;
|
|
14
14
|
afterSuccess?(child: IngestSessionWorktree): Promise<void>;
|
|
15
15
|
}
|
|
16
|
+
export declare function workUnitPatchFileName(unitIndex: number, unitKey: string): string;
|
|
16
17
|
export declare function runIsolatedWorkUnit(input: RunIsolatedWorkUnitInput): Promise<WorkUnitOutcome>;
|
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
import { mkdir, readFile } from 'node:fs/promises';
|
|
2
2
|
import { join } from 'node:path';
|
|
3
|
+
import { captureIngestWorkUnitCachedArtifactFiles } from '../work-unit-cache.js';
|
|
3
4
|
import { parsePatchTouchedPaths } from './git-patch.js';
|
|
4
|
-
function
|
|
5
|
+
export function workUnitPatchFileName(unitIndex, unitKey) {
|
|
5
6
|
const safeKey = unitKey.replace(/[^a-zA-Z0-9_.-]+/g, '-');
|
|
6
7
|
return `${String(unitIndex).padStart(4, '0')}-${safeKey}.patch`;
|
|
7
8
|
}
|
|
@@ -49,21 +50,29 @@ export async function runIsolatedWorkUnit(input) {
|
|
|
49
50
|
}
|
|
50
51
|
await input.afterSuccess?.(child);
|
|
51
52
|
await mkdir(input.patchDir, { recursive: true });
|
|
52
|
-
const patchPath = join(input.patchDir,
|
|
53
|
+
const patchPath = join(input.patchDir, workUnitPatchFileName(input.unitIndex, input.workUnit.unitKey));
|
|
53
54
|
await child.git.writeBinaryNoRenamePatch(input.ingestionBaseSha, 'HEAD', patchPath);
|
|
54
55
|
const patch = await readFile(patchPath, 'utf-8');
|
|
55
56
|
const touched = parsePatchTouchedPaths(patch);
|
|
57
|
+
const patchTouchedPaths = touched.map((entry) => entry.path);
|
|
58
|
+
const artifactFiles = await captureIngestWorkUnitCachedArtifactFiles({
|
|
59
|
+
git: child.git,
|
|
60
|
+
workdir: child.workdir,
|
|
61
|
+
baseSha: input.ingestionBaseSha,
|
|
62
|
+
patchTouchedPaths,
|
|
63
|
+
});
|
|
56
64
|
cleanupOutcome = 'success';
|
|
57
65
|
await input.trace.event('debug', 'work_unit', 'work_unit_patch_collected', {
|
|
58
66
|
unitKey: input.workUnit.unitKey,
|
|
59
67
|
patchPath,
|
|
60
|
-
touchedPaths:
|
|
68
|
+
touchedPaths: patchTouchedPaths,
|
|
61
69
|
patchBytes: Buffer.byteLength(patch),
|
|
62
70
|
});
|
|
63
71
|
return {
|
|
64
72
|
...outcome,
|
|
65
73
|
patchPath,
|
|
66
|
-
patchTouchedPaths
|
|
74
|
+
patchTouchedPaths,
|
|
75
|
+
artifactFiles,
|
|
67
76
|
childWorktreePath: child.workdir,
|
|
68
77
|
};
|
|
69
78
|
}
|
|
@@ -1,10 +1,12 @@
|
|
|
1
1
|
import { join } from 'node:path';
|
|
2
|
+
import { gdriveConnectionToPullConfig, parseGdriveConnectionConfig } from '../../context/connections/gdrive-config.js';
|
|
2
3
|
import { localConnectionToWarehouseDescriptor } from '../../context/connections/local-warehouse-descriptor.js';
|
|
3
4
|
import { notionConnectionToPullConfig, parseNotionConnectionConfig } from '../../context/connections/notion-config.js';
|
|
4
5
|
import { resolveKtxConfigReference } from '../core/config-reference.js';
|
|
5
6
|
import { ktxLocalStateDbPath } from '../../context/project/local-state-db.js';
|
|
6
7
|
import { DbtSourceAdapter } from './adapters/dbt/dbt.adapter.js';
|
|
7
8
|
import { FakeSourceAdapter } from './adapters/fake/fake.adapter.js';
|
|
9
|
+
import { GdriveSourceAdapter } from './adapters/gdrive/gdrive.adapter.js';
|
|
8
10
|
import { HistoricSqlSourceAdapter } from './adapters/historic-sql/historic-sql.adapter.js';
|
|
9
11
|
import { PostgresPgssReader } from './adapters/historic-sql/postgres-pgss-reader.js';
|
|
10
12
|
import { resolveQueryHistoryScopeFloor } from './adapters/historic-sql/scope-floor.js';
|
|
@@ -49,6 +51,7 @@ export function createDefaultLocalIngestAdapters(project, options = {}) {
|
|
|
49
51
|
createLocalMetabaseSourceAdapter(project, {
|
|
50
52
|
...(options.logger ? { logger: options.logger } : {}),
|
|
51
53
|
}),
|
|
54
|
+
new GdriveSourceAdapter(),
|
|
52
55
|
new LookerSourceAdapter({
|
|
53
56
|
clientFactory: {
|
|
54
57
|
async createClient(config, ctx) {
|
|
@@ -239,6 +242,9 @@ export async function localPullConfigForAdapter(project, adapter, connectionId,
|
|
|
239
242
|
lastSuccessfulCursor: await localNotionRuntimeStore(project).readCursor(connectionId),
|
|
240
243
|
};
|
|
241
244
|
}
|
|
245
|
+
if (adapter.source === 'gdrive') {
|
|
246
|
+
return await gdriveConnectionToPullConfig(parseGdriveConnectionConfig(connection));
|
|
247
|
+
}
|
|
242
248
|
if (adapter.source === 'metricflow') {
|
|
243
249
|
const metricflow = connection.metricflow;
|
|
244
250
|
const metricflowConfig = typeof metricflow === 'object' && metricflow !== null && !Array.isArray(metricflow)
|
|
@@ -10,6 +10,7 @@ import { createLocalKtxLlmRuntimeFromConfig } from '../../context/llm/local-conf
|
|
|
10
10
|
import { KtxIngestEmbeddingPortAdapter } from '../../context/llm/embedding-port.js';
|
|
11
11
|
import { createRateLimitGovernorConfig, RateLimitGovernor } from '../../context/llm/rate-limit-governor.js';
|
|
12
12
|
import { RuntimeAgentRunner } from '../../context/llm/runtime-port.js';
|
|
13
|
+
import { getKtxCliPackageInfo } from '../../cli-runtime.js';
|
|
13
14
|
import { ktxLocalStateDbPath } from '../../context/project/local-state-db.js';
|
|
14
15
|
import { PromptService } from '../../context/prompts/prompt.service.js';
|
|
15
16
|
import { SkillsRegistryService } from '../../context/skills/skills-registry.service.js';
|
|
@@ -40,6 +41,7 @@ import { WikiWriteTool } from '../../context/wiki/tools/wiki-write.tool.js';
|
|
|
40
41
|
import { CandidateDedupService } from '../../context/ingest/context-candidates/candidate-dedup.service.js';
|
|
41
42
|
import { ContextCandidateCarryforwardService } from '../../context/ingest/context-candidates/context-candidate-carryforward.service.js';
|
|
42
43
|
import { CuratorPaginationService } from '../../context/ingest/context-candidates/curator-pagination.service.js';
|
|
44
|
+
import { SqliteContentResultCache } from '../cache/sqlite-content-result-cache.js';
|
|
43
45
|
import { createEmitHistoricSqlEvidenceTool } from './adapters/historic-sql/evidence-tool.js';
|
|
44
46
|
import { ContextEvidenceIndexService } from '../../context/ingest/context-evidence/context-evidence-index.service.js';
|
|
45
47
|
import { SqliteContextEvidenceStore } from '../../context/ingest/context-evidence/sqlite-context-evidence-store.js';
|
|
@@ -378,11 +380,11 @@ function parseKnowledgeIndexPath(file) {
|
|
|
378
380
|
const segments = file.split('/');
|
|
379
381
|
if (segments.length === 2 && segments[0] === 'global') {
|
|
380
382
|
const pageKey = segments[1].replace(/\.md$/, '');
|
|
381
|
-
return /^[a-zA-Z0-
|
|
383
|
+
return /^[a-zA-Z0-9_][a-zA-Z0-9_-]*$/.test(pageKey) ? { scope: 'GLOBAL', pageKey } : null;
|
|
382
384
|
}
|
|
383
385
|
if (segments.length === 3 && segments[0] === 'user') {
|
|
384
386
|
const pageKey = segments[2].replace(/\.md$/, '');
|
|
385
|
-
return /^[a-zA-Z0-
|
|
387
|
+
return /^[a-zA-Z0-9_][a-zA-Z0-9_-]*$/.test(pageKey) ? { scope: 'USER', pageKey } : null;
|
|
386
388
|
}
|
|
387
389
|
return null;
|
|
388
390
|
}
|
|
@@ -519,6 +521,7 @@ export function createLocalBundleIngestRuntime(options) {
|
|
|
519
521
|
mkdirSync(join(options.project.projectDir, '.ktx/cache/local-ingest'), { recursive: true });
|
|
520
522
|
const store = new SqliteBundleIngestStore({ dbPath });
|
|
521
523
|
const contextStore = new SqliteContextEvidenceStore({ dbPath });
|
|
524
|
+
const contentCache = new SqliteContentResultCache({ dbPath });
|
|
522
525
|
const embeddingProvider = options.embeddingProvider ?? null;
|
|
523
526
|
if (!embeddingProvider && options.project.config.ingest.embeddings.backend !== 'none') {
|
|
524
527
|
// Embedding-dependent stages (CandidateDedup clustering, ContextEvidenceIndex
|
|
@@ -563,6 +566,7 @@ export function createLocalBundleIngestRuntime(options) {
|
|
|
563
566
|
provenance: store,
|
|
564
567
|
reports: store,
|
|
565
568
|
canonicalPins: store,
|
|
569
|
+
contentCache,
|
|
566
570
|
registry,
|
|
567
571
|
diffSetService: new DiffSetService(store),
|
|
568
572
|
sessionWorktreeService: new SessionWorktreeService({
|
|
@@ -576,6 +580,7 @@ export function createLocalBundleIngestRuntime(options) {
|
|
|
576
580
|
storage,
|
|
577
581
|
settings: {
|
|
578
582
|
memoryIngestionModel: options.project.config.llm.models.default ?? 'local-ingest-model',
|
|
583
|
+
cliVersion: getKtxCliPackageInfo().version,
|
|
579
584
|
probeRowCount: 0,
|
|
580
585
|
workUnitMaxConcurrency: options.project.config.ingest.workUnits.maxConcurrency,
|
|
581
586
|
workUnitStepBudget: options.project.config.ingest.workUnits.stepBudget,
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import type { KtxLocalProject } from '../../context/project/project.js';
|
|
2
2
|
import type { MemoryFlowEventSink } from './memory-flow/types.js';
|
|
3
3
|
import type { KtxTableRefKey } from '../scan/table-ref.js';
|
|
4
|
-
import type { IngestTrigger, SourceAdapter, WorkUnit } from './types.js';
|
|
4
|
+
import type { IngestTrigger, SourceAdapter, SourceFetchReport, WorkUnit } from './types.js';
|
|
5
5
|
type LocalIngestStatus = 'running' | 'done' | 'error';
|
|
6
6
|
interface LocalIngestDiffPaths {
|
|
7
7
|
added: string[];
|
|
@@ -34,6 +34,8 @@ export interface LocalIngestRunRecord {
|
|
|
34
34
|
workUnits: Array<Pick<WorkUnit, 'unitKey' | 'rawFiles' | 'peerFileIndex' | 'dependencyPaths'>>;
|
|
35
35
|
evictionDeletedRawPaths: string[];
|
|
36
36
|
errors: string[];
|
|
37
|
+
/** Fetch-phase outcome (e.g. objects skipped during introspection). */
|
|
38
|
+
fetch?: SourceFetchReport;
|
|
37
39
|
}
|
|
38
40
|
export type LocalIngestReport = LocalIngestRunRecord & {
|
|
39
41
|
rawContentHashes: Record<string, string>;
|
|
@@ -10,7 +10,7 @@ import { SqliteLocalIngestStore } from './sqlite-local-ingest-store.js';
|
|
|
10
10
|
const LOCAL_AUTHOR = 'ktx';
|
|
11
11
|
const LOCAL_AUTHOR_EMAIL = 'ktx@example.com';
|
|
12
12
|
function safeSegment(kind, value) {
|
|
13
|
-
if (!/^[a-zA-Z0-
|
|
13
|
+
if (!/^[a-zA-Z0-9_][a-zA-Z0-9_-]*$/.test(value)) {
|
|
14
14
|
throw new Error(`Unsafe ${kind}: ${value}`);
|
|
15
15
|
}
|
|
16
16
|
return value;
|
|
@@ -172,6 +172,7 @@ async function runLocalStageOnlyIngestInner(options) {
|
|
|
172
172
|
if (!detected) {
|
|
173
173
|
throw new Error(`Adapter "${adapter.source}" did not recognize ${sourceDir ?? 'fetched source output'}`);
|
|
174
174
|
}
|
|
175
|
+
const fetchReport = adapter.readFetchReport ? await adapter.readFetchReport(stagedDir) : null;
|
|
175
176
|
const relativeFiles = await walkFiles(stagedDir);
|
|
176
177
|
options.memoryFlow?.update({ sourceDir });
|
|
177
178
|
options.memoryFlow?.emit({
|
|
@@ -270,6 +271,7 @@ async function runLocalStageOnlyIngestInner(options) {
|
|
|
270
271
|
})),
|
|
271
272
|
evictionDeletedRawPaths: chunkResult.eviction?.deletedRawPaths ?? [],
|
|
272
273
|
errors: [],
|
|
274
|
+
...(fetchReport ? { fetch: fetchReport } : {}),
|
|
273
275
|
};
|
|
274
276
|
if (!options.dryRun) {
|
|
275
277
|
store.saveCompletedRun({
|
|
@@ -18,6 +18,7 @@ import type { ToolContext } from '../../context/tools/base-tool.js';
|
|
|
18
18
|
import type { ToolSession } from '../../context/tools/tool-session.js';
|
|
19
19
|
import type { KnowledgeIndexPort } from '../../context/wiki/ports.js';
|
|
20
20
|
import type { KnowledgeWikiService } from '../../context/wiki/knowledge-wiki.service.js';
|
|
21
|
+
import type { ContentResultCache } from '../cache/content-result-cache.js';
|
|
21
22
|
import type { CanonicalPin } from './canonical-pins.js';
|
|
22
23
|
import type { IngestTraceLevel } from './ingest-trace.js';
|
|
23
24
|
import type { IngestReportSnapshot } from './reports.js';
|
|
@@ -108,6 +109,7 @@ export interface IngestSessionWorktreePort {
|
|
|
108
109
|
}
|
|
109
110
|
interface IngestSettingsPort {
|
|
110
111
|
memoryIngestionModel: string;
|
|
112
|
+
cliVersion: string;
|
|
111
113
|
probeRowCount: number;
|
|
112
114
|
workUnitMaxConcurrency?: number;
|
|
113
115
|
workUnitStepBudget?: number;
|
|
@@ -295,6 +297,7 @@ export interface IngestBundleRunnerDeps {
|
|
|
295
297
|
provenance: IngestProvenancePort;
|
|
296
298
|
reports: IngestReportsPort;
|
|
297
299
|
canonicalPins: IngestCanonicalPinsPort;
|
|
300
|
+
contentCache: ContentResultCache;
|
|
298
301
|
registry: SourceAdapterRegistryPort;
|
|
299
302
|
diffSetService: DiffSetComputerPort;
|
|
300
303
|
sessionWorktreeService: IngestSessionWorktreePort;
|
|
@@ -140,6 +140,17 @@ const finalizationOutcomeSchema = z.object({
|
|
|
140
140
|
actions: z.array(ingestActionSchema).default([]),
|
|
141
141
|
provenanceExclusions: z.array(finalizationProvenanceExclusionSchema).default([]),
|
|
142
142
|
});
|
|
143
|
+
const finalGatePrunedReferenceSchema = z.object({
|
|
144
|
+
kind: z.enum(['join', 'wiki_ref', 'wiki_sl_ref', 'wiki_body_ref']),
|
|
145
|
+
artifact: z.string(),
|
|
146
|
+
removedRef: z.string(),
|
|
147
|
+
absentTarget: z.string(),
|
|
148
|
+
});
|
|
149
|
+
const finalGateDroppedSourceSchema = z.object({
|
|
150
|
+
connectionId: z.string(),
|
|
151
|
+
sourceName: z.string(),
|
|
152
|
+
reason: z.string(),
|
|
153
|
+
});
|
|
143
154
|
const ingestReportSnapshotSchema = z
|
|
144
155
|
.object({
|
|
145
156
|
id: z.string().min(1),
|
|
@@ -169,9 +180,6 @@ const ingestReportSnapshotSchema = z
|
|
|
169
180
|
resolverAttempts: z.number().int().min(0).default(0),
|
|
170
181
|
resolverRepairs: z.number().int().min(0).default(0),
|
|
171
182
|
resolverFailures: z.number().int().min(0).default(0),
|
|
172
|
-
gateRepairAttempts: z.number().int().min(0).default(0),
|
|
173
|
-
gateRepairs: z.number().int().min(0).default(0),
|
|
174
|
-
gateRepairFailures: z.number().int().min(0).default(0),
|
|
175
183
|
})
|
|
176
184
|
.optional(),
|
|
177
185
|
workUnits: z.array(z.object({
|
|
@@ -198,6 +206,8 @@ const ingestReportSnapshotSchema = z
|
|
|
198
206
|
provenanceRows: z.array(provenanceDetailSchema).default([]),
|
|
199
207
|
toolTranscripts: z.array(toolTranscriptSummarySchema).default([]),
|
|
200
208
|
finalization: finalizationOutcomeSchema.optional(),
|
|
209
|
+
finalGatePrunedReferences: z.array(finalGatePrunedReferenceSchema).default([]),
|
|
210
|
+
finalGateDroppedSources: z.array(finalGateDroppedSourceSchema).default([]),
|
|
201
211
|
memoryFlow: memoryFlowReplayInputSchema.optional(),
|
|
202
212
|
})
|
|
203
213
|
.passthrough(),
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import type { MemoryAction } from '../../context/memory/types.js';
|
|
2
2
|
import type { TouchedSlSource } from '../../context/tools/touched-sl-sources.js';
|
|
3
3
|
import type { MemoryFlowReplayInput } from './memory-flow/types.js';
|
|
4
|
+
import type { FinalGateDroppedSource, FinalGatePrunedReference } from './final-gate-prune.js';
|
|
4
5
|
import type { IngestProvenanceInsert } from './ports.js';
|
|
5
6
|
import type { ArtifactResolutionRecord, ConflictResolvedRecord, EvictionAppliedRecord, StageIndex, UnmappedFallbackRecord } from './stages/stage-index.types.js';
|
|
6
7
|
import type { WikiSlRefRepair } from './wiki-sl-ref-repair.js';
|
|
@@ -79,9 +80,6 @@ export interface IngestReportBody {
|
|
|
79
80
|
resolverAttempts?: number;
|
|
80
81
|
resolverRepairs?: number;
|
|
81
82
|
resolverFailures?: number;
|
|
82
|
-
gateRepairAttempts?: number;
|
|
83
|
-
gateRepairs?: number;
|
|
84
|
-
gateRepairFailures?: number;
|
|
85
83
|
};
|
|
86
84
|
workUnits: IngestReportWorkUnit[];
|
|
87
85
|
failedWorkUnits: string[];
|
|
@@ -98,6 +96,8 @@ export interface IngestReportBody {
|
|
|
98
96
|
provenanceRows: IngestReportProvenanceDetail[];
|
|
99
97
|
toolTranscripts: IngestReportToolTranscriptSummary[];
|
|
100
98
|
finalization?: IngestReportFinalizationOutcome;
|
|
99
|
+
finalGatePrunedReferences?: FinalGatePrunedReference[];
|
|
100
|
+
finalGateDroppedSources?: FinalGateDroppedSource[];
|
|
101
101
|
wikiSlRefRepairs?: WikiSlRefRepair[];
|
|
102
102
|
wikiSlRefRepairWarnings?: string[];
|
|
103
103
|
memoryFlow?: MemoryFlowReplayInput;
|
|
@@ -12,7 +12,9 @@ export function ingestReportOutcome(report) {
|
|
|
12
12
|
if (report.body.status === 'failed') {
|
|
13
13
|
return 'error';
|
|
14
14
|
}
|
|
15
|
-
|
|
15
|
+
const hasPruneOrDrop = (report.body.finalGatePrunedReferences?.length ?? 0) > 0 ||
|
|
16
|
+
(report.body.finalGateDroppedSources?.length ?? 0) > 0;
|
|
17
|
+
if (report.body.failedWorkUnits.length === 0 && !hasPruneOrDrop) {
|
|
16
18
|
return 'done';
|
|
17
19
|
}
|
|
18
20
|
const { wikiCount, slCount } = savedMemoryCountsForReport(report);
|
|
@@ -13,6 +13,7 @@ export declare function buildWuSystemPrompt(params: {
|
|
|
13
13
|
skillsPrompt: string;
|
|
14
14
|
syncId: string;
|
|
15
15
|
sourceKey: string;
|
|
16
|
+
connectionId?: string;
|
|
16
17
|
canonicalPins?: CanonicalPin[];
|
|
17
18
|
}): string;
|
|
18
19
|
export declare function buildWuUserPrompt(input: BuildWuPromptInput): string;
|
|
@@ -6,12 +6,13 @@ import { createReadRawSpanTool } from '../tools/read-raw-span.tool.js';
|
|
|
6
6
|
import { createVerificationLedgerState, VERIFICATION_LEDGER_PROMPT, withVerificationLedger, } from '../tools/verification-ledger.tool.js';
|
|
7
7
|
const PEER_FILE_INDEX_PROMPT_LIMIT = 100;
|
|
8
8
|
export function buildWuSystemPrompt(params) {
|
|
9
|
+
const connectionLine = params.connectionId ? `\nconnectionId: ${params.connectionId}` : '';
|
|
9
10
|
const parts = [
|
|
10
11
|
params.baseFraming.trimEnd(),
|
|
11
12
|
VERIFICATION_LEDGER_PROMPT,
|
|
12
13
|
params.skillsPrompt.trimEnd(),
|
|
13
14
|
buildCanonicalPinsPromptBlock(params.canonicalPins ?? []),
|
|
14
|
-
`\n<context>\nsyncId: ${params.syncId}\nsource: ${params.sourceKey}\n</context>`,
|
|
15
|
+
`\n<context>\nsyncId: ${params.syncId}\nsource: ${params.sourceKey}${connectionLine}\n</context>`,
|
|
15
16
|
];
|
|
16
17
|
return parts.filter(Boolean).join('\n');
|
|
17
18
|
}
|
|
@@ -3,13 +3,13 @@ import type { AgentRunnerPort, KtxRuntimeToolSet, RunLoopMetrics } from '../../.
|
|
|
3
3
|
import type { CaptureSession, MemoryAction } from '../../../context/memory/types.js';
|
|
4
4
|
import { type TouchedSlSource } from '../../../context/tools/touched-sl-sources.js';
|
|
5
5
|
import { type WuValidationResult } from './validate-wu-sources.js';
|
|
6
|
+
import type { IngestWorkUnitCachedArtifactFile } from '../work-unit-cache.js';
|
|
6
7
|
import type { WorkUnit } from '../types.js';
|
|
7
8
|
export interface WorkUnitExecutionDeps {
|
|
8
9
|
sessionWorktreeGit: {
|
|
9
10
|
revParseHead(): Promise<string | null>;
|
|
10
11
|
};
|
|
11
12
|
agentRunner: AgentRunnerPort;
|
|
12
|
-
validateWikiRefs?: (actions: MemoryAction[]) => Promise<string[]>;
|
|
13
13
|
validateTouchedSources: (touched: TouchedSlSource[]) => Promise<WuValidationResult>;
|
|
14
14
|
resetHardTo: (targetSha: string) => Promise<void>;
|
|
15
15
|
buildSystemPrompt: (wu: WorkUnit) => string;
|
|
@@ -37,6 +37,7 @@ export interface WorkUnitOutcome {
|
|
|
37
37
|
slDisallowedReason?: 'lookml_connection_mismatch';
|
|
38
38
|
patchPath?: string;
|
|
39
39
|
patchTouchedPaths?: string[];
|
|
40
|
+
artifactFiles?: IngestWorkUnitCachedArtifactFile[];
|
|
40
41
|
childWorktreePath?: string;
|
|
41
42
|
/** Timing and token metrics for the work-unit agent loop, used for ingest profiling. */
|
|
42
43
|
metrics?: RunLoopMetrics;
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { isAbortError } from '../../core/abort.js';
|
|
2
2
|
import { listTouchedSlSources } from '../../../context/tools/touched-sl-sources.js';
|
|
3
|
-
import { formatInvalidWuSources } from './validate-wu-sources.js';
|
|
3
|
+
import { formatInvalidWuSources, hasBlockingWuSourceIssue, } from './validate-wu-sources.js';
|
|
4
4
|
const MAX_WORK_UNIT_PROMPT_CHARS = 240_000;
|
|
5
5
|
export async function executeWorkUnit(deps, wu) {
|
|
6
6
|
const preSha = (await deps.sessionWorktreeGit.revParseHead()) ?? '';
|
|
@@ -87,18 +87,12 @@ export async function executeWorkUnit(deps, wu) {
|
|
|
87
87
|
if (toolFailureCount > 0) {
|
|
88
88
|
return failWithReset(`${toolFailureCount} tool call(s) failed during WorkUnit ${wu.unitKey}`);
|
|
89
89
|
}
|
|
90
|
-
const danglingWikiRefs = (await deps.validateWikiRefs?.(deps.sessionActions)) ?? [];
|
|
91
|
-
if (danglingWikiRefs.length > 0) {
|
|
92
|
-
return failWithReset(`wiki references target missing page(s): ${danglingWikiRefs.join(', ')}`);
|
|
93
|
-
}
|
|
94
90
|
const touched = listTouchedSlSources(deps.captureSession.touchedSlSources);
|
|
95
91
|
if (touched.length > 0) {
|
|
96
92
|
const validation = await deps.validateTouchedSources(touched);
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
// memory-agent pattern — NOT the bundle-ingest pattern.
|
|
101
|
-
return failWithReset(`sl_validate failed for: ${formatInvalidWuSources(validation.invalidSources)}`);
|
|
93
|
+
const blockingInvalidSources = validation.invalidSources.filter(hasBlockingWuSourceIssue);
|
|
94
|
+
if (blockingInvalidSources.length > 0) {
|
|
95
|
+
return failWithReset(`sl_validate failed for: ${formatInvalidWuSources(blockingInvalidSources)}`);
|
|
102
96
|
}
|
|
103
97
|
}
|
|
104
98
|
return {
|
|
@@ -5,12 +5,24 @@ export interface InvalidWuSource {
|
|
|
5
5
|
/** `${connectionId}:${sourceName}` */
|
|
6
6
|
source: string;
|
|
7
7
|
errors: string[];
|
|
8
|
+
issues?: WuValidationIssue[];
|
|
8
9
|
}
|
|
9
10
|
export interface WuValidationResult {
|
|
10
11
|
validSources: string[];
|
|
11
12
|
invalidSources: InvalidWuSource[];
|
|
12
13
|
}
|
|
14
|
+
type WuValidationIssue = {
|
|
15
|
+
kind: 'source_validation';
|
|
16
|
+
message: string;
|
|
17
|
+
} | {
|
|
18
|
+
kind: 'missing_join_target';
|
|
19
|
+
targetSourceName: string;
|
|
20
|
+
caseMismatch: string | null;
|
|
21
|
+
message: string;
|
|
22
|
+
};
|
|
13
23
|
export declare function formatInvalidWuSources(invalid: InvalidWuSource[]): string;
|
|
24
|
+
export declare function hasBlockingWuSourceIssue(source: InvalidWuSource): boolean;
|
|
14
25
|
export declare function validateWuTouchedSources(deps: SlValidationDeps & {
|
|
15
26
|
slValidator: SlValidatorPort<SlValidationDeps>;
|
|
16
27
|
}, touched: TouchedSlSource[]): Promise<WuValidationResult>;
|
|
28
|
+
export {};
|
|
@@ -2,6 +2,14 @@ import { findMissingJoinTargets, formatMissingJoinTarget } from '../../../contex
|
|
|
2
2
|
export function formatInvalidWuSources(invalid) {
|
|
3
3
|
return invalid.map((entry) => `${entry.source} (${entry.errors.join('; ')})`).join(', ');
|
|
4
4
|
}
|
|
5
|
+
export function hasBlockingWuSourceIssue(source) {
|
|
6
|
+
const issues = source.issues ??
|
|
7
|
+
source.errors.map((message) => ({
|
|
8
|
+
kind: 'source_validation',
|
|
9
|
+
message,
|
|
10
|
+
}));
|
|
11
|
+
return issues.some((issue) => issue.kind === 'source_validation');
|
|
12
|
+
}
|
|
5
13
|
function uniqueTouchedSources(sources) {
|
|
6
14
|
const seen = new Set();
|
|
7
15
|
const unique = [];
|
|
@@ -61,8 +69,8 @@ function expandWithExistingJoinNeighbors(touched, sourcesByConnection) {
|
|
|
61
69
|
* are out of scope — they must not block unrelated work. Resolution is the
|
|
62
70
|
* Python engine's: exact source-name match within the connection.
|
|
63
71
|
*/
|
|
64
|
-
function
|
|
65
|
-
const
|
|
72
|
+
function findJoinTargetIssues(touched, sourcesByConnection) {
|
|
73
|
+
const issuesBySource = new Map();
|
|
66
74
|
const touchedByConnection = new Map();
|
|
67
75
|
for (const source of touched) {
|
|
68
76
|
const bucket = touchedByConnection.get(source.connectionId) ?? new Set();
|
|
@@ -85,11 +93,16 @@ function findJoinTargetErrors(touched, sourcesByConnection) {
|
|
|
85
93
|
continue;
|
|
86
94
|
}
|
|
87
95
|
const key = `${connectionId}:${source.name}`;
|
|
88
|
-
const
|
|
89
|
-
|
|
96
|
+
const issues = missing.map((entry) => ({
|
|
97
|
+
kind: 'missing_join_target',
|
|
98
|
+
targetSourceName: entry.to,
|
|
99
|
+
caseMismatch: entry.caseMismatch,
|
|
100
|
+
message: formatMissingJoinTarget(entry),
|
|
101
|
+
}));
|
|
102
|
+
issuesBySource.set(key, [...(issuesBySource.get(key) ?? []), ...issues]);
|
|
90
103
|
}
|
|
91
104
|
}
|
|
92
|
-
return
|
|
105
|
+
return issuesBySource;
|
|
93
106
|
}
|
|
94
107
|
export async function validateWuTouchedSources(deps, touched) {
|
|
95
108
|
if (touched.length === 0) {
|
|
@@ -101,18 +114,20 @@ export async function validateWuTouchedSources(deps, touched) {
|
|
|
101
114
|
sourcesByConnection.set(connectionId, sources);
|
|
102
115
|
}
|
|
103
116
|
const expanded = expandWithExistingJoinNeighbors(touched, sourcesByConnection);
|
|
104
|
-
const
|
|
117
|
+
const joinTargetIssues = findJoinTargetIssues(touched, sourcesByConnection);
|
|
105
118
|
const valid = [];
|
|
106
119
|
const invalid = [];
|
|
107
120
|
for (const source of expanded) {
|
|
108
121
|
const key = `${source.connectionId}:${source.sourceName}`;
|
|
109
122
|
const result = await deps.slValidator.validateSingleSource(deps, source.connectionId, source.sourceName);
|
|
110
|
-
const
|
|
123
|
+
const sourceIssues = result.errors.map((message) => ({ kind: 'source_validation', message }));
|
|
124
|
+
const issues = [...sourceIssues, ...(joinTargetIssues.get(key) ?? [])];
|
|
125
|
+
const errors = issues.map((issue) => issue.message);
|
|
111
126
|
if (errors.length === 0) {
|
|
112
127
|
valid.push(key);
|
|
113
128
|
}
|
|
114
129
|
else {
|
|
115
|
-
invalid.push({ source: key, errors });
|
|
130
|
+
invalid.push({ source: key, errors, issues });
|
|
116
131
|
}
|
|
117
132
|
}
|
|
118
133
|
return { validSources: valid, invalidSources: invalid };
|
|
@@ -1,22 +1,27 @@
|
|
|
1
1
|
import { readFile, stat } from 'node:fs/promises';
|
|
2
|
-
import { join, normalize, resolve } from 'node:path';
|
|
2
|
+
import { isAbsolute, join, normalize, relative, resolve } from 'node:path';
|
|
3
3
|
import { tool } from 'ai';
|
|
4
4
|
import { z } from 'zod';
|
|
5
5
|
const MAX_READ_RAW_FILE_BYTES = 120_000;
|
|
6
|
+
function normalizeRawPath(path) {
|
|
7
|
+
return normalize(path).replace(/^[/\\]+/, '').replace(/\\/g, '/');
|
|
8
|
+
}
|
|
6
9
|
export function createReadRawFileTool(deps) {
|
|
7
10
|
const stagedRoot = resolve(deps.stagedDir);
|
|
11
|
+
const allowedPaths = new Set([...deps.allowedPaths].map(normalizeRawPath));
|
|
8
12
|
return tool({
|
|
9
13
|
description: "Read the full text content of a raw source file inside this WorkUnit. `path` must be relative to the staged bundle root (no leading slash, no `..`) and must appear in the WorkUnit's rawFiles or dependencyPaths list.",
|
|
10
14
|
inputSchema: z.object({
|
|
11
15
|
path: z.string().describe('Path relative to the staged bundle root. Example: "views/customers/customer.lkml".'),
|
|
12
16
|
}),
|
|
13
17
|
execute: async ({ path }) => {
|
|
14
|
-
const normalized =
|
|
15
|
-
if (normalized.startsWith('..') || !
|
|
16
|
-
return `Error: path "${path}" is not accessible from this WorkUnit. Allowed paths: ${[...
|
|
18
|
+
const normalized = normalizeRawPath(path);
|
|
19
|
+
if (normalized.startsWith('..') || !allowedPaths.has(normalized)) {
|
|
20
|
+
return `Error: path "${path}" is not accessible from this WorkUnit. Allowed paths: ${[...allowedPaths].sort().join(', ')}`;
|
|
17
21
|
}
|
|
18
22
|
const absolute = resolve(join(stagedRoot, normalized));
|
|
19
|
-
|
|
23
|
+
const stagedRelative = relative(stagedRoot, absolute);
|
|
24
|
+
if (stagedRelative.startsWith('..') || isAbsolute(stagedRelative)) {
|
|
20
25
|
return `Error: path "${path}" is not accessible from this WorkUnit.`;
|
|
21
26
|
}
|
|
22
27
|
try {
|
|
@@ -1,9 +1,13 @@
|
|
|
1
1
|
import { readFile } from 'node:fs/promises';
|
|
2
|
-
import { join, normalize, resolve } from 'node:path';
|
|
2
|
+
import { isAbsolute, join, normalize, relative, resolve } from 'node:path';
|
|
3
3
|
import { tool } from 'ai';
|
|
4
4
|
import { z } from 'zod';
|
|
5
|
+
function normalizeRawPath(path) {
|
|
6
|
+
return normalize(path).replace(/^[/\\]+/, '').replace(/\\/g, '/');
|
|
7
|
+
}
|
|
5
8
|
export function createReadRawSpanTool(deps) {
|
|
6
9
|
const stagedRoot = resolve(deps.stagedDir);
|
|
10
|
+
const allowedPaths = new Set([...deps.allowedPaths].map(normalizeRawPath));
|
|
7
11
|
return tool({
|
|
8
12
|
description: 'Read a 1-based inclusive line range from a raw source file. Use this to resolve a provenance pointer like `file.lkml#L15-28` without loading the whole file into context.',
|
|
9
13
|
inputSchema: z.object({
|
|
@@ -15,12 +19,13 @@ export function createReadRawSpanTool(deps) {
|
|
|
15
19
|
if (startLine > endLine) {
|
|
16
20
|
return `Error: startLine must be <= endLine (got startLine=${startLine}, endLine=${endLine})`;
|
|
17
21
|
}
|
|
18
|
-
const normalized =
|
|
19
|
-
if (normalized.startsWith('..') || !
|
|
20
|
-
return `Error: path "${path}" is not accessible from this context. Allowed paths: ${[...
|
|
22
|
+
const normalized = normalizeRawPath(path);
|
|
23
|
+
if (normalized.startsWith('..') || !allowedPaths.has(normalized)) {
|
|
24
|
+
return `Error: path "${path}" is not accessible from this context. Allowed paths: ${[...allowedPaths].sort().join(', ')}`;
|
|
21
25
|
}
|
|
22
26
|
const absolute = resolve(join(stagedRoot, normalized));
|
|
23
|
-
|
|
27
|
+
const stagedRelative = relative(stagedRoot, absolute);
|
|
28
|
+
if (stagedRelative.startsWith('..') || isAbsolute(stagedRelative)) {
|
|
24
29
|
return `Error: path "${path}" is not accessible from this context.`;
|
|
25
30
|
}
|
|
26
31
|
try {
|
|
@@ -2,7 +2,7 @@ import { z } from 'zod';
|
|
|
2
2
|
import { BaseTool } from '../../../../context/tools/base-tool.js';
|
|
3
3
|
const discoverDataInputSchema = z.object({
|
|
4
4
|
query: z.string().optional(),
|
|
5
|
-
connectionId: z.string().regex(/^[a-zA-Z0-
|
|
5
|
+
connectionId: z.string().regex(/^[a-zA-Z0-9_][a-zA-Z0-9_-]*$/).optional(),
|
|
6
6
|
limit: z.number().int().positive().max(50).optional().default(10),
|
|
7
7
|
sourceName: z.string().optional(),
|
|
8
8
|
}).strict();
|
|
@@ -10,7 +10,7 @@ const targetSchema = z.union([
|
|
|
10
10
|
}),
|
|
11
11
|
]);
|
|
12
12
|
const entityDetailsInputSchema = z.object({
|
|
13
|
-
connectionId: z.string().regex(/^[a-zA-Z0-
|
|
13
|
+
connectionId: z.string().regex(/^[a-zA-Z0-9_][a-zA-Z0-9_-]*$/),
|
|
14
14
|
targets: z.array(targetSchema).min(1).max(50),
|
|
15
15
|
}).strict();
|
|
16
16
|
function allowedConnectionNames(context) {
|