@kaelio/ktx 0.13.1 → 0.15.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/assets/python/{kaelio_ktx-0.13.1-py3-none-any.whl → kaelio_ktx-0.15.0-py3-none-any.whl} +0 -0
- package/assets/python/manifest.json +4 -4
- package/dist/.tsbuildinfo +1 -1
- package/dist/cli-program.js +1 -1
- package/dist/commands/ingest-commands.d.ts +9 -0
- package/dist/commands/ingest-commands.js +37 -1
- package/dist/commands/knowledge-commands.js +3 -0
- package/dist/commands/setup-commands.js +15 -1
- package/dist/connection-drivers.d.ts +2 -0
- package/dist/connection-drivers.js +7 -3
- package/dist/connection.d.ts +3 -0
- package/dist/connection.js +27 -0
- package/dist/connectors/bigquery/connector.d.ts +17 -6
- package/dist/connectors/bigquery/connector.js +152 -76
- package/dist/connectors/bigquery/dialect.d.ts +2 -2
- package/dist/connectors/clickhouse/connector.d.ts +1 -0
- package/dist/connectors/clickhouse/connector.js +45 -16
- package/dist/connectors/clickhouse/dialect.d.ts +2 -2
- package/dist/connectors/mongodb/connector.d.ts +69 -0
- package/dist/connectors/mongodb/connector.js +307 -0
- package/dist/connectors/mongodb/dialect.d.ts +17 -0
- package/dist/connectors/mongodb/dialect.js +50 -0
- package/dist/connectors/mongodb/live-database-introspection.d.ts +10 -0
- package/dist/connectors/mongodb/live-database-introspection.js +24 -0
- package/dist/connectors/mongodb/schema-inference.d.ts +20 -0
- package/dist/connectors/mongodb/schema-inference.js +110 -0
- package/dist/connectors/mysql/connector.d.ts +1 -0
- package/dist/connectors/mysql/connector.js +18 -2
- package/dist/connectors/mysql/dialect.d.ts +2 -2
- package/dist/connectors/postgres/connector.d.ts +1 -0
- package/dist/connectors/postgres/connector.js +20 -3
- package/dist/connectors/postgres/dialect.d.ts +2 -2
- package/dist/connectors/snowflake/connector.d.ts +1 -0
- package/dist/connectors/snowflake/connector.js +27 -3
- package/dist/connectors/snowflake/dialect.d.ts +2 -2
- package/dist/connectors/sqlite/connector.d.ts +11 -1
- package/dist/connectors/sqlite/connector.js +120 -17
- package/dist/connectors/sqlite/dialect.d.ts +2 -2
- package/dist/connectors/sqlite/read-query-child.d.ts +1 -0
- package/dist/connectors/sqlite/read-query-child.js +23 -0
- package/dist/connectors/sqlserver/connector.d.ts +2 -0
- package/dist/connectors/sqlserver/connector.js +20 -3
- package/dist/connectors/sqlserver/dialect.d.ts +2 -2
- package/dist/context/cache/content-result-cache.d.ts +36 -0
- package/dist/context/cache/content-result-cache.js +14 -0
- package/dist/context/cache/sqlite-content-result-cache.d.ts +18 -0
- package/dist/context/cache/sqlite-content-result-cache.js +223 -0
- package/dist/context/connections/bigquery-identifiers.d.ts +1 -0
- package/dist/context/connections/bigquery-identifiers.js +7 -0
- package/dist/context/connections/configured-connections.d.ts +9 -0
- package/dist/context/connections/configured-connections.js +18 -0
- package/dist/context/connections/dialects.d.ts +30 -4
- package/dist/context/connections/dialects.js +38 -11
- package/dist/context/connections/drivers.js +21 -0
- package/dist/context/connections/gdrive-config.d.ts +19 -0
- package/dist/context/connections/gdrive-config.js +57 -0
- package/dist/context/connections/query-deadline.d.ts +31 -0
- package/dist/context/connections/query-deadline.js +37 -0
- package/dist/context/ingest/adapters/gdrive/chunk.d.ts +3 -0
- package/dist/context/ingest/adapters/gdrive/chunk.js +74 -0
- package/dist/context/ingest/adapters/gdrive/detect.d.ts +1 -0
- package/dist/context/ingest/adapters/gdrive/detect.js +20 -0
- package/dist/context/ingest/adapters/gdrive/fetch.d.ts +6 -0
- package/dist/context/ingest/adapters/gdrive/fetch.js +95 -0
- package/dist/context/ingest/adapters/gdrive/gdrive-client.d.ts +30 -0
- package/dist/context/ingest/adapters/gdrive/gdrive-client.js +132 -0
- package/dist/context/ingest/adapters/gdrive/gdrive.adapter.d.ts +11 -0
- package/dist/context/ingest/adapters/gdrive/gdrive.adapter.js +27 -0
- package/dist/context/ingest/adapters/gdrive/normalize.d.ts +2 -0
- package/dist/context/ingest/adapters/gdrive/normalize.js +256 -0
- package/dist/context/ingest/adapters/gdrive/types.d.ts +153 -0
- package/dist/context/ingest/adapters/gdrive/types.js +36 -0
- package/dist/context/ingest/adapters/live-database/daemon-introspection.js +24 -0
- package/dist/context/ingest/adapters/live-database/fetch-report.d.ts +8 -0
- package/dist/context/ingest/adapters/live-database/fetch-report.js +37 -0
- package/dist/context/ingest/adapters/live-database/live-database.adapter.d.ts +2 -1
- package/dist/context/ingest/adapters/live-database/live-database.adapter.js +9 -2
- package/dist/context/ingest/adapters/live-database/manifest.d.ts +2 -0
- package/dist/context/ingest/adapters/live-database/manifest.js +8 -4
- package/dist/context/ingest/adapters/live-database/scan-outcome.d.ts +17 -0
- package/dist/context/ingest/adapters/live-database/scan-outcome.js +40 -0
- package/dist/context/ingest/adapters/live-database/stage.js +5 -5
- package/dist/context/ingest/adapters/metabase/types.d.ts +1 -1
- package/dist/context/ingest/adapters/metabase/types.js +1 -1
- package/dist/context/ingest/adapters/sigma/chunk.d.ts +6 -0
- package/dist/context/ingest/adapters/sigma/chunk.js +119 -0
- package/dist/context/ingest/adapters/sigma/client-port.d.ts +45 -0
- package/dist/context/ingest/adapters/sigma/client-port.js +1 -0
- package/dist/context/ingest/adapters/sigma/client.d.ts +33 -0
- package/dist/context/ingest/adapters/sigma/client.js +176 -0
- package/dist/context/ingest/adapters/sigma/detect.d.ts +1 -0
- package/dist/context/ingest/adapters/sigma/detect.js +23 -0
- package/dist/context/ingest/adapters/sigma/fetch.d.ts +14 -0
- package/dist/context/ingest/adapters/sigma/fetch.js +191 -0
- package/dist/context/ingest/adapters/sigma/local-sigma.adapter.d.ts +17 -0
- package/dist/context/ingest/adapters/sigma/local-sigma.adapter.js +41 -0
- package/dist/context/ingest/adapters/sigma/project.d.ts +8 -0
- package/dist/context/ingest/adapters/sigma/project.js +186 -0
- package/dist/context/ingest/adapters/sigma/sigma.adapter.d.ts +18 -0
- package/dist/context/ingest/adapters/sigma/sigma.adapter.js +43 -0
- package/dist/context/ingest/adapters/sigma/types.d.ts +80 -0
- package/dist/context/ingest/adapters/sigma/types.js +82 -0
- package/dist/context/ingest/artifact-gates.d.ts +32 -1
- package/dist/context/ingest/artifact-gates.js +85 -18
- package/dist/context/ingest/final-gate-prune.d.ts +35 -0
- package/dist/context/ingest/final-gate-prune.js +229 -0
- package/dist/context/ingest/ingest-bundle.runner.d.ts +3 -0
- package/dist/context/ingest/ingest-bundle.runner.js +459 -240
- package/dist/context/ingest/isolated-diff/patch-integrator.d.ts +0 -11
- package/dist/context/ingest/isolated-diff/patch-integrator.js +0 -44
- package/dist/context/ingest/isolated-diff/work-unit-executor.d.ts +1 -0
- package/dist/context/ingest/isolated-diff/work-unit-executor.js +13 -4
- package/dist/context/ingest/local-adapters.d.ts +2 -1
- package/dist/context/ingest/local-adapters.js +28 -0
- package/dist/context/ingest/local-bundle-runtime.js +7 -2
- package/dist/context/ingest/local-ingest.js +1 -1
- package/dist/context/ingest/local-stage-ingest.d.ts +3 -1
- package/dist/context/ingest/local-stage-ingest.js +3 -1
- package/dist/context/ingest/ports.d.ts +3 -0
- package/dist/context/ingest/report-snapshot.js +13 -3
- package/dist/context/ingest/reports.d.ts +3 -3
- package/dist/context/ingest/reports.js +3 -1
- package/dist/context/ingest/stages/build-wu-context.d.ts +1 -0
- package/dist/context/ingest/stages/build-wu-context.js +2 -1
- package/dist/context/ingest/stages/stage-3-work-units.d.ts +2 -1
- package/dist/context/ingest/stages/stage-3-work-units.js +4 -10
- package/dist/context/ingest/stages/validate-wu-sources.d.ts +12 -0
- package/dist/context/ingest/stages/validate-wu-sources.js +23 -8
- package/dist/context/ingest/tools/read-raw-file.tool.js +10 -5
- package/dist/context/ingest/tools/read-raw-span.tool.js +10 -5
- package/dist/context/ingest/tools/warehouse-verification/discover-data.tool.js +1 -1
- package/dist/context/ingest/tools/warehouse-verification/entity-details.tool.js +1 -1
- package/dist/context/ingest/tools/warehouse-verification/sql-execution.tool.js +1 -1
- package/dist/context/ingest/types.d.ts +3 -0
- package/dist/context/ingest/wiki-body-refs.d.ts +28 -0
- package/dist/context/ingest/wiki-body-refs.js +37 -8
- package/dist/context/ingest/wiki-sl-ref-repair.d.ts +1 -0
- package/dist/context/ingest/wiki-sl-ref-repair.js +4 -0
- package/dist/context/ingest/work-unit-cache.d.ts +67 -0
- package/dist/context/ingest/work-unit-cache.js +230 -0
- package/dist/context/llm/ai-sdk-runtime.d.ts +1 -0
- package/dist/context/llm/ai-sdk-runtime.js +5 -0
- package/dist/context/llm/claude-code-runtime.d.ts +4 -1
- package/dist/context/llm/claude-code-runtime.js +38 -8
- package/dist/context/llm/codex-runtime.d.ts +4 -1
- package/dist/context/llm/codex-runtime.js +48 -35
- package/dist/context/llm/runtime-port.d.ts +26 -0
- package/dist/context/llm/subprocess-generate-object-child.d.ts +1 -0
- package/dist/context/llm/subprocess-generate-object-child.js +34 -0
- package/dist/context/llm/subprocess-generate-object.d.ts +42 -0
- package/dist/context/llm/subprocess-generate-object.js +122 -0
- package/dist/context/mcp/context-tools.d.ts +2 -0
- package/dist/context/mcp/context-tools.js +111 -17
- package/dist/context/mcp/local-project-ports.d.ts +7 -0
- package/dist/context/mcp/local-project-ports.js +29 -0
- package/dist/context/mcp/logger.d.ts +24 -0
- package/dist/context/mcp/logger.js +49 -0
- package/dist/context/mcp/server.js +2 -0
- package/dist/context/mcp/types.d.ts +17 -0
- package/dist/context/memory/memory-agent.service.js +1 -1
- package/dist/context/project/config.d.ts +72 -0
- package/dist/context/project/config.js +5 -0
- package/dist/context/project/driver-schemas.d.ts +35 -0
- package/dist/context/project/driver-schemas.js +89 -1
- package/dist/context/project/project.js +1 -1
- package/dist/context/project/setup-config.js +1 -0
- package/dist/context/scan/description-generation.d.ts +4 -0
- package/dist/context/scan/description-generation.js +100 -13
- package/dist/context/scan/enabled-tables.d.ts +5 -0
- package/dist/context/scan/enabled-tables.js +13 -1
- package/dist/context/scan/enrichment-state.d.ts +50 -7
- package/dist/context/scan/enrichment-state.js +30 -13
- package/dist/context/scan/local-enrichment-artifacts.d.ts +41 -0
- package/dist/context/scan/local-enrichment-artifacts.js +155 -32
- package/dist/context/scan/local-enrichment.d.ts +39 -4
- package/dist/context/scan/local-enrichment.js +345 -92
- package/dist/context/scan/local-scan.d.ts +4 -1
- package/dist/context/scan/local-scan.js +54 -13
- package/dist/context/scan/local-structural-artifacts.d.ts +3 -1
- package/dist/context/scan/local-structural-artifacts.js +5 -0
- package/dist/context/scan/object-introspection.d.ts +21 -0
- package/dist/context/scan/object-introspection.js +35 -0
- package/dist/context/scan/relationship-benchmarks.js +5 -3
- package/dist/context/scan/relationship-composite-candidates.d.ts +6 -3
- package/dist/context/scan/relationship-composite-candidates.js +13 -1
- package/dist/context/scan/relationship-detection-budget.d.ts +35 -0
- package/dist/context/scan/relationship-detection-budget.js +53 -0
- package/dist/context/scan/relationship-diagnostics.d.ts +5 -0
- package/dist/context/scan/relationship-diagnostics.js +2 -0
- package/dist/context/scan/relationship-discovery.d.ts +9 -3
- package/dist/context/scan/relationship-discovery.js +27 -4
- package/dist/context/scan/relationship-llm-proposal.js +36 -27
- package/dist/context/scan/relationship-profiling.d.ts +7 -3
- package/dist/context/scan/relationship-profiling.js +53 -41
- package/dist/context/scan/relationship-validation.d.ts +6 -4
- package/dist/context/scan/relationship-validation.js +46 -32
- package/dist/context/scan/sqlite-local-enrichment-state-store.d.ts +8 -1
- package/dist/context/scan/sqlite-local-enrichment-state-store.js +71 -159
- package/dist/context/scan/types.d.ts +5 -3
- package/dist/context/scan/types.js +2 -0
- package/dist/context/sl/local-query.js +5 -0
- package/dist/context/sl/pglite-sl-search-prototype.js +1 -1
- package/dist/context/sl/semantic-layer.service.js +1 -1
- package/dist/context/sl/source-files.d.ts +5 -0
- package/dist/context/sl/source-files.js +17 -11
- package/dist/context/sl/tools/connection-id-schema.js +1 -1
- package/dist/context/sql-analysis/dialect-notes.d.ts +9 -0
- package/dist/context/sql-analysis/dialect-notes.js +40 -0
- package/dist/context/sql-analysis/dialects/bigquery.md +13 -0
- package/dist/context/sql-analysis/dialects/clickhouse.md +9 -0
- package/dist/context/sql-analysis/dialects/mysql.md +9 -0
- package/dist/context/sql-analysis/dialects/postgres.md +10 -0
- package/dist/context/sql-analysis/dialects/snowflake.md +10 -0
- package/dist/context/sql-analysis/dialects/sqlite.md +11 -0
- package/dist/context/sql-analysis/dialects/tsql.md +10 -0
- package/dist/context/wiki/keys.js +1 -1
- package/dist/context/wiki/knowledge-wiki.service.d.ts +4 -4
- package/dist/context/wiki/knowledge-wiki.service.js +2 -1
- package/dist/context/wiki/local-knowledge.d.ts +13 -0
- package/dist/context/wiki/local-knowledge.js +50 -6
- package/dist/context/wiki/sqlite-knowledge-index.d.ts +2 -0
- package/dist/context/wiki/sqlite-knowledge-index.js +21 -5
- package/dist/context/wiki/tools/wiki-write.tool.d.ts +2 -0
- package/dist/context/wiki/tools/wiki-write.tool.js +27 -0
- package/dist/context/wiki/types.d.ts +6 -0
- package/dist/context-build-view.d.ts +3 -0
- package/dist/context-build-view.js +1 -0
- package/dist/knowledge.d.ts +2 -0
- package/dist/knowledge.js +12 -1
- package/dist/local-adapters.js +11 -0
- package/dist/mcp-http-server.js +15 -1
- package/dist/mcp-server-factory.d.ts +2 -0
- package/dist/mcp-server-factory.js +15 -1
- package/dist/mcp-stdio-server.js +10 -2
- package/dist/notion-page-picker.js +1 -1
- package/dist/prompts/memory_agent_external_ingest.md +2 -0
- package/dist/public-ingest.d.ts +3 -1
- package/dist/public-ingest.js +4 -0
- package/dist/scan.d.ts +3 -1
- package/dist/scan.js +7 -0
- package/dist/setup-databases.d.ts +1 -1
- package/dist/setup-databases.js +43 -1
- package/dist/setup-sources.d.ts +6 -1
- package/dist/setup-sources.js +208 -48
- package/dist/setup.d.ts +3 -0
- package/dist/setup.js +6 -1
- package/dist/skills/analytics/SKILL.md +200 -2
- package/dist/skills/gdrive_synthesize/SKILL.md +97 -0
- package/dist/skills/sigma_ingest/SKILL.md +189 -0
- package/dist/skills/wiki_capture/SKILL.md +24 -0
- package/dist/sql.js +4 -0
- package/dist/status-project.d.ts +4 -0
- package/dist/status-project.js +54 -2
- package/dist/telemetry/events.d.ts +2 -2
- package/dist/text-ingest.d.ts +4 -0
- package/dist/text-ingest.js +58 -29
- package/dist/verbatim-ingest.d.ts +46 -0
- package/dist/verbatim-ingest.js +193 -0
- package/package.json +5 -1
- package/dist/context/ingest/final-gate-repair.d.ts +0 -28
- package/dist/context/ingest/final-gate-repair.js +0 -91
|
@@ -1,9 +1,13 @@
|
|
|
1
1
|
import { readFile } from 'node:fs/promises';
|
|
2
|
-
import { join, normalize, resolve } from 'node:path';
|
|
2
|
+
import { isAbsolute, join, normalize, relative, resolve } from 'node:path';
|
|
3
3
|
import { tool } from 'ai';
|
|
4
4
|
import { z } from 'zod';
|
|
5
|
+
function normalizeRawPath(path) {
|
|
6
|
+
return normalize(path).replace(/^[/\\]+/, '').replace(/\\/g, '/');
|
|
7
|
+
}
|
|
5
8
|
export function createReadRawSpanTool(deps) {
|
|
6
9
|
const stagedRoot = resolve(deps.stagedDir);
|
|
10
|
+
const allowedPaths = new Set([...deps.allowedPaths].map(normalizeRawPath));
|
|
7
11
|
return tool({
|
|
8
12
|
description: 'Read a 1-based inclusive line range from a raw source file. Use this to resolve a provenance pointer like `file.lkml#L15-28` without loading the whole file into context.',
|
|
9
13
|
inputSchema: z.object({
|
|
@@ -15,12 +19,13 @@ export function createReadRawSpanTool(deps) {
|
|
|
15
19
|
if (startLine > endLine) {
|
|
16
20
|
return `Error: startLine must be <= endLine (got startLine=${startLine}, endLine=${endLine})`;
|
|
17
21
|
}
|
|
18
|
-
const normalized =
|
|
19
|
-
if (normalized.startsWith('..') || !
|
|
20
|
-
return `Error: path "${path}" is not accessible from this context. Allowed paths: ${[...
|
|
22
|
+
const normalized = normalizeRawPath(path);
|
|
23
|
+
if (normalized.startsWith('..') || !allowedPaths.has(normalized)) {
|
|
24
|
+
return `Error: path "${path}" is not accessible from this context. Allowed paths: ${[...allowedPaths].sort().join(', ')}`;
|
|
21
25
|
}
|
|
22
26
|
const absolute = resolve(join(stagedRoot, normalized));
|
|
23
|
-
|
|
27
|
+
const stagedRelative = relative(stagedRoot, absolute);
|
|
28
|
+
if (stagedRelative.startsWith('..') || isAbsolute(stagedRelative)) {
|
|
24
29
|
return `Error: path "${path}" is not accessible from this context.`;
|
|
25
30
|
}
|
|
26
31
|
try {
|
|
@@ -2,7 +2,7 @@ import { z } from 'zod';
|
|
|
2
2
|
import { BaseTool } from '../../../../context/tools/base-tool.js';
|
|
3
3
|
const discoverDataInputSchema = z.object({
|
|
4
4
|
query: z.string().optional(),
|
|
5
|
-
connectionId: z.string().regex(/^[a-zA-Z0-
|
|
5
|
+
connectionId: z.string().regex(/^[a-zA-Z0-9_][a-zA-Z0-9_-]*$/).optional(),
|
|
6
6
|
limit: z.number().int().positive().max(50).optional().default(10),
|
|
7
7
|
sourceName: z.string().optional(),
|
|
8
8
|
}).strict();
|
|
@@ -10,7 +10,7 @@ const targetSchema = z.union([
|
|
|
10
10
|
}),
|
|
11
11
|
]);
|
|
12
12
|
const entityDetailsInputSchema = z.object({
|
|
13
|
-
connectionId: z.string().regex(/^[a-zA-Z0-
|
|
13
|
+
connectionId: z.string().regex(/^[a-zA-Z0-9_][a-zA-Z0-9_-]*$/),
|
|
14
14
|
targets: z.array(targetSchema).min(1).max(50),
|
|
15
15
|
}).strict();
|
|
16
16
|
function allowedConnectionNames(context) {
|
|
@@ -3,7 +3,7 @@ import { assertReadOnlySql, limitSqlForExecution } from '../../../../context/con
|
|
|
3
3
|
import { sqlAnalysisDialectForDriver } from '../../../../context/sql-analysis/dialect.js';
|
|
4
4
|
import { BaseTool } from '../../../../context/tools/base-tool.js';
|
|
5
5
|
const sqlExecutionInputSchema = z.object({
|
|
6
|
-
connectionId: z.string().regex(/^[a-zA-Z0-
|
|
6
|
+
connectionId: z.string().regex(/^[a-zA-Z0-9_][a-zA-Z0-9_-]*$/),
|
|
7
7
|
sql: z.string().min(1),
|
|
8
8
|
rowLimit: z.number().int().positive().max(1000).optional().default(100),
|
|
9
9
|
}).strict();
|
|
@@ -6,6 +6,7 @@ import type { KtxTableRefKey } from '../scan/table-ref.js';
|
|
|
6
6
|
import type { MemoryFlowEventSink } from './memory-flow/types.js';
|
|
7
7
|
import type { StageIndex } from './stages/stage-index.types.js';
|
|
8
8
|
import type { WorkUnitOutcome } from './stages/stage-3-work-units.js';
|
|
9
|
+
import type { FinalGateDroppedSource, FinalGatePrunedReference } from './final-gate-prune.js';
|
|
9
10
|
export type IngestTrigger = 'upload' | 'scheduled_pull' | 'manual_resync' | 'manual_override';
|
|
10
11
|
export interface DiffSet {
|
|
11
12
|
added: string[];
|
|
@@ -190,6 +191,8 @@ export interface IngestBundleResult {
|
|
|
190
191
|
failedWorkUnits: string[];
|
|
191
192
|
artifactsWritten: number;
|
|
192
193
|
commitSha: string | null;
|
|
194
|
+
finalGatePrunedReferences?: FinalGatePrunedReference[];
|
|
195
|
+
finalGateDroppedSources?: FinalGateDroppedSource[];
|
|
193
196
|
}
|
|
194
197
|
export interface IngestJobPhase {
|
|
195
198
|
updateProgress(progress: number, message?: string): Promise<void>;
|
|
@@ -5,14 +5,17 @@ export type WikiBodyRef = {
|
|
|
5
5
|
connectionId: string | null;
|
|
6
6
|
sourceName: string;
|
|
7
7
|
entityName: string;
|
|
8
|
+
rawToken: string;
|
|
8
9
|
} | {
|
|
9
10
|
kind: 'sl_source';
|
|
10
11
|
connectionId: string | null;
|
|
11
12
|
sourceName: string;
|
|
13
|
+
rawToken: string;
|
|
12
14
|
} | {
|
|
13
15
|
kind: 'table';
|
|
14
16
|
connectionId: string | null;
|
|
15
17
|
tableRef: string;
|
|
18
|
+
rawToken: string;
|
|
16
19
|
};
|
|
17
20
|
export interface WikiBodyRefValidationInput {
|
|
18
21
|
pageKey: string;
|
|
@@ -21,6 +24,31 @@ export interface WikiBodyRefValidationInput {
|
|
|
21
24
|
loadSources(connectionId: string): Promise<SemanticLayerSource[]>;
|
|
22
25
|
tableExists(connectionId: string, tableRef: string): Promise<boolean>;
|
|
23
26
|
}
|
|
27
|
+
export type WikiBodyRefIssue = {
|
|
28
|
+
kind: 'missing_wiki_body_sl_entity';
|
|
29
|
+
pageKey: string;
|
|
30
|
+
rawToken: string;
|
|
31
|
+
connectionId?: string;
|
|
32
|
+
sourceName: string;
|
|
33
|
+
entityName: string;
|
|
34
|
+
message: string;
|
|
35
|
+
} | {
|
|
36
|
+
kind: 'missing_wiki_body_sl_source';
|
|
37
|
+
pageKey: string;
|
|
38
|
+
rawToken: string;
|
|
39
|
+
connectionId?: string;
|
|
40
|
+
sourceName: string;
|
|
41
|
+
message: string;
|
|
42
|
+
} | {
|
|
43
|
+
kind: 'missing_wiki_body_table';
|
|
44
|
+
pageKey: string;
|
|
45
|
+
rawToken: string;
|
|
46
|
+
connectionId?: string;
|
|
47
|
+
tableRef: string;
|
|
48
|
+
message: string;
|
|
49
|
+
};
|
|
24
50
|
/** @internal */
|
|
25
51
|
export declare function parseWikiBodyRefs(body: string): WikiBodyRef[];
|
|
52
|
+
export declare function findInvalidWikiBodyRefIssues(input: WikiBodyRefValidationInput): Promise<WikiBodyRefIssue[]>;
|
|
53
|
+
/** @internal */
|
|
26
54
|
export declare function findInvalidWikiBodyRefs(input: WikiBodyRefValidationInput): Promise<string[]>;
|
|
@@ -36,14 +36,14 @@ export function parseWikiBodyRefs(body) {
|
|
|
36
36
|
if (scoped.body.startsWith('source:')) {
|
|
37
37
|
const sourceName = scoped.body.slice('source:'.length).trim();
|
|
38
38
|
if (sourceName) {
|
|
39
|
-
refs.push({ kind: 'sl_source', connectionId: scoped.connectionId, sourceName });
|
|
39
|
+
refs.push({ kind: 'sl_source', connectionId: scoped.connectionId, sourceName, rawToken: token });
|
|
40
40
|
}
|
|
41
41
|
continue;
|
|
42
42
|
}
|
|
43
43
|
if (scoped.body.startsWith('table:')) {
|
|
44
44
|
const tableRef = scoped.body.slice('table:'.length).trim();
|
|
45
45
|
if (tableRef) {
|
|
46
|
-
refs.push({ kind: 'table', connectionId: scoped.connectionId, tableRef });
|
|
46
|
+
refs.push({ kind: 'table', connectionId: scoped.connectionId, tableRef, rawToken: token });
|
|
47
47
|
}
|
|
48
48
|
continue;
|
|
49
49
|
}
|
|
@@ -54,6 +54,7 @@ export function parseWikiBodyRefs(body) {
|
|
|
54
54
|
connectionId: scoped.connectionId,
|
|
55
55
|
sourceName: parts[0],
|
|
56
56
|
entityName: parts[1],
|
|
57
|
+
rawToken: token,
|
|
57
58
|
});
|
|
58
59
|
}
|
|
59
60
|
}
|
|
@@ -67,8 +68,8 @@ function entityNames(source) {
|
|
|
67
68
|
...(source.segments ?? []).map((segment) => segment.name),
|
|
68
69
|
]);
|
|
69
70
|
}
|
|
70
|
-
export async function
|
|
71
|
-
const
|
|
71
|
+
export async function findInvalidWikiBodyRefIssues(input) {
|
|
72
|
+
const issues = [];
|
|
72
73
|
const sourceCache = new Map();
|
|
73
74
|
const loadSources = async (connectionId) => {
|
|
74
75
|
const cached = sourceCache.get(connectionId);
|
|
@@ -93,20 +94,48 @@ export async function findInvalidWikiBodyRefs(input) {
|
|
|
93
94
|
if (ref.kind === 'table') {
|
|
94
95
|
const found = await Promise.all(connectionIds.map((connectionId) => input.tableExists(connectionId, ref.tableRef)));
|
|
95
96
|
if (!found.some(Boolean)) {
|
|
96
|
-
|
|
97
|
+
const target = `${ref.connectionId ? `${ref.connectionId}/` : ''}${ref.tableRef}`;
|
|
98
|
+
issues.push({
|
|
99
|
+
kind: 'missing_wiki_body_table',
|
|
100
|
+
pageKey: input.pageKey,
|
|
101
|
+
rawToken: ref.rawToken,
|
|
102
|
+
...(ref.connectionId ? { connectionId: ref.connectionId } : {}),
|
|
103
|
+
tableRef: ref.tableRef,
|
|
104
|
+
message: `${input.pageKey}: unknown raw table ${target}`,
|
|
105
|
+
});
|
|
97
106
|
}
|
|
98
107
|
continue;
|
|
99
108
|
}
|
|
100
109
|
const found = await findSource(connectionIds, ref.sourceName);
|
|
101
110
|
if (!found) {
|
|
102
111
|
if (ref.kind === 'sl_source') {
|
|
103
|
-
|
|
112
|
+
const target = `${ref.connectionId ? `${ref.connectionId}/` : ''}${ref.sourceName}`;
|
|
113
|
+
issues.push({
|
|
114
|
+
kind: 'missing_wiki_body_sl_source',
|
|
115
|
+
pageKey: input.pageKey,
|
|
116
|
+
rawToken: ref.rawToken,
|
|
117
|
+
...(ref.connectionId ? { connectionId: ref.connectionId } : {}),
|
|
118
|
+
sourceName: ref.sourceName,
|
|
119
|
+
message: `${input.pageKey}: unknown semantic-layer source ${target}`,
|
|
120
|
+
});
|
|
104
121
|
}
|
|
105
122
|
continue;
|
|
106
123
|
}
|
|
107
124
|
if (ref.kind === 'sl_entity' && !entityNames(found.source).has(ref.entityName)) {
|
|
108
|
-
|
|
125
|
+
issues.push({
|
|
126
|
+
kind: 'missing_wiki_body_sl_entity',
|
|
127
|
+
pageKey: input.pageKey,
|
|
128
|
+
rawToken: ref.rawToken,
|
|
129
|
+
...(ref.connectionId ? { connectionId: ref.connectionId } : {}),
|
|
130
|
+
sourceName: ref.sourceName,
|
|
131
|
+
entityName: ref.entityName,
|
|
132
|
+
message: `${input.pageKey}: unknown semantic-layer entity ${ref.sourceName}.${ref.entityName}`,
|
|
133
|
+
});
|
|
109
134
|
}
|
|
110
135
|
}
|
|
111
|
-
return
|
|
136
|
+
return issues;
|
|
137
|
+
}
|
|
138
|
+
/** @internal */
|
|
139
|
+
export async function findInvalidWikiBodyRefs(input) {
|
|
140
|
+
return (await findInvalidWikiBodyRefIssues(input)).map((issue) => issue.message);
|
|
112
141
|
}
|
|
@@ -57,11 +57,15 @@ export async function repairWikiSlRefs(input) {
|
|
|
57
57
|
}
|
|
58
58
|
const listed = await listFiles('wiki', true);
|
|
59
59
|
const repairs = [];
|
|
60
|
+
const deferredGlobalPageKeys = new Set(input.deferGlobalPageKeys ?? []);
|
|
60
61
|
for (const file of listed.files.sort()) {
|
|
61
62
|
const parsedPath = parseKnowledgeFilePath(file);
|
|
62
63
|
if (!parsedPath) {
|
|
63
64
|
continue;
|
|
64
65
|
}
|
|
66
|
+
if (parsedPath.scope === 'GLOBAL' && deferredGlobalPageKeys.has(parsedPath.pageKey)) {
|
|
67
|
+
continue;
|
|
68
|
+
}
|
|
65
69
|
const page = await input.wikiService.readPage(parsedPath.scope, parsedPath.scopeId, parsedPath.pageKey);
|
|
66
70
|
const refs = uniqueStringArray(page?.frontmatter.sl_refs);
|
|
67
71
|
if (!page || refs.length === 0) {
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
import type { KtxModelRole } from '../../llm/types.js';
|
|
2
|
+
import type { GitService } from '../core/git.service.js';
|
|
3
|
+
import type { MemoryAction } from '../memory/types.js';
|
|
4
|
+
import type { TouchedSlSource } from '../tools/touched-sl-sources.js';
|
|
5
|
+
import type { IngestTraceWriter } from './ingest-trace.js';
|
|
6
|
+
import type { IngestSessionWorktreePort } from './ports.js';
|
|
7
|
+
import type { WorkUnit } from './types.js';
|
|
8
|
+
export declare const INGEST_WORK_UNIT_CACHE_NAMESPACE = "ingest:work-unit";
|
|
9
|
+
export interface IngestWorkUnitCachedArtifactFile {
|
|
10
|
+
path: string;
|
|
11
|
+
beforeBase64: string | null;
|
|
12
|
+
afterBase64: string | null;
|
|
13
|
+
}
|
|
14
|
+
export interface IngestWorkUnitCachePayload {
|
|
15
|
+
schemaVersion: 2;
|
|
16
|
+
unitKey: string;
|
|
17
|
+
patchTouchedPaths: string[];
|
|
18
|
+
artifactFiles: IngestWorkUnitCachedArtifactFile[];
|
|
19
|
+
actions: MemoryAction[];
|
|
20
|
+
touchedSlSources: TouchedSlSource[];
|
|
21
|
+
slDisallowed?: boolean;
|
|
22
|
+
slDisallowedReason?: 'lookml_connection_mismatch';
|
|
23
|
+
}
|
|
24
|
+
export interface ComputeIngestWorkUnitInputHashInput {
|
|
25
|
+
stagedDir: string;
|
|
26
|
+
connectionId: string;
|
|
27
|
+
sourceKey: string;
|
|
28
|
+
unit: WorkUnit;
|
|
29
|
+
cliVersion: string;
|
|
30
|
+
promptFingerprint: string;
|
|
31
|
+
modelRole: KtxModelRole;
|
|
32
|
+
}
|
|
33
|
+
export declare function computeIngestWorkUnitInputHash(input: ComputeIngestWorkUnitInputHashInput): Promise<string>;
|
|
34
|
+
/** @internal */
|
|
35
|
+
export declare function isPruneShapedCachedReplayBase(path: string, currentContent: string, outputContent: string): boolean;
|
|
36
|
+
export declare function captureIngestWorkUnitCachedArtifactFiles(input: {
|
|
37
|
+
git: GitService;
|
|
38
|
+
workdir: string;
|
|
39
|
+
baseSha: string;
|
|
40
|
+
patchTouchedPaths: string[];
|
|
41
|
+
}): Promise<IngestWorkUnitCachedArtifactFile[]>;
|
|
42
|
+
export declare function materializeCachedWorkUnitReplayPatch(input: {
|
|
43
|
+
sessionWorktreeService: IngestSessionWorktreePort;
|
|
44
|
+
baseSha: string;
|
|
45
|
+
jobId: string;
|
|
46
|
+
unitKey: string;
|
|
47
|
+
patchPath: string;
|
|
48
|
+
artifactFiles: IngestWorkUnitCachedArtifactFile[];
|
|
49
|
+
author: {
|
|
50
|
+
name: string;
|
|
51
|
+
email: string;
|
|
52
|
+
};
|
|
53
|
+
trace: IngestTraceWriter;
|
|
54
|
+
}): Promise<'materialized' | 'unsafe_drift'>;
|
|
55
|
+
export declare function ingestWorkUnitCacheScopeKey(input: {
|
|
56
|
+
connectionId: string;
|
|
57
|
+
sourceKey: string;
|
|
58
|
+
}): string;
|
|
59
|
+
export declare function computeIngestWorkUnitPromptFingerprint(input: {
|
|
60
|
+
cliVersion: string;
|
|
61
|
+
baseFraming: string;
|
|
62
|
+
skillsPrompt: string;
|
|
63
|
+
canonicalPins: unknown[];
|
|
64
|
+
sourceKey: string;
|
|
65
|
+
connectionId: string;
|
|
66
|
+
skillNames: string[];
|
|
67
|
+
}): string;
|
|
@@ -0,0 +1,230 @@
|
|
|
1
|
+
import { mkdir, readFile, rm, writeFile } from 'node:fs/promises';
|
|
2
|
+
import { dirname, join } from 'node:path';
|
|
3
|
+
import { isDeepStrictEqual } from 'node:util';
|
|
4
|
+
import YAML from 'yaml';
|
|
5
|
+
import { stableContentHash } from '../cache/content-result-cache.js';
|
|
6
|
+
export const INGEST_WORK_UNIT_CACHE_NAMESPACE = 'ingest:work-unit';
|
|
7
|
+
async function fileDigest(stagedDir, path) {
|
|
8
|
+
try {
|
|
9
|
+
const bytes = await readFile(join(stagedDir, path));
|
|
10
|
+
return { path, status: 'present', hash: stableContentHash(bytes.toString('base64')) };
|
|
11
|
+
}
|
|
12
|
+
catch (error) {
|
|
13
|
+
if (error && typeof error === 'object' && 'code' in error && error.code === 'ENOENT') {
|
|
14
|
+
return { path, status: 'missing', hash: null };
|
|
15
|
+
}
|
|
16
|
+
throw error;
|
|
17
|
+
}
|
|
18
|
+
}
|
|
19
|
+
export async function computeIngestWorkUnitInputHash(input) {
|
|
20
|
+
const rawFiles = [...input.unit.rawFiles].sort();
|
|
21
|
+
const dependencyPaths = [...input.unit.dependencyPaths].sort();
|
|
22
|
+
const [raw, dependencies] = await Promise.all([
|
|
23
|
+
Promise.all(rawFiles.map((path) => fileDigest(input.stagedDir, path))),
|
|
24
|
+
Promise.all(dependencyPaths.map((path) => fileDigest(input.stagedDir, path))),
|
|
25
|
+
]);
|
|
26
|
+
return stableContentHash({
|
|
27
|
+
schemaVersion: 2,
|
|
28
|
+
connectionId: input.connectionId,
|
|
29
|
+
sourceKey: input.sourceKey,
|
|
30
|
+
unitKey: input.unit.unitKey,
|
|
31
|
+
rawFiles: raw,
|
|
32
|
+
dependencyPaths: dependencies,
|
|
33
|
+
slDisallowed: input.unit.slDisallowed === true,
|
|
34
|
+
slDisallowedReason: input.unit.slDisallowedReason ?? null,
|
|
35
|
+
cliVersion: input.cliVersion,
|
|
36
|
+
promptFingerprint: input.promptFingerprint,
|
|
37
|
+
modelRole: input.modelRole,
|
|
38
|
+
});
|
|
39
|
+
}
|
|
40
|
+
async function readFileBase64(workdir, path) {
|
|
41
|
+
try {
|
|
42
|
+
return (await readFile(join(workdir, path))).toString('base64');
|
|
43
|
+
}
|
|
44
|
+
catch (error) {
|
|
45
|
+
if (error && typeof error === 'object' && 'code' in error && error.code === 'ENOENT') {
|
|
46
|
+
return null;
|
|
47
|
+
}
|
|
48
|
+
throw error;
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
async function readGitFileBase64(git, path, commitSha) {
|
|
52
|
+
try {
|
|
53
|
+
return Buffer.from(await git.getFileAtCommit(path, commitSha), 'utf-8').toString('base64');
|
|
54
|
+
}
|
|
55
|
+
catch {
|
|
56
|
+
return null;
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
function decodeBase64(value) {
|
|
60
|
+
return value === null ? null : Buffer.from(value, 'base64').toString('utf-8');
|
|
61
|
+
}
|
|
62
|
+
function parseYamlObject(content) {
|
|
63
|
+
const parsed = YAML.parse(content);
|
|
64
|
+
return parsed && typeof parsed === 'object' && !Array.isArray(parsed) ? parsed : null;
|
|
65
|
+
}
|
|
66
|
+
function isSubsequenceOfDeepValues(current, output) {
|
|
67
|
+
let outputIndex = 0;
|
|
68
|
+
for (const item of current) {
|
|
69
|
+
while (outputIndex < output.length && !isDeepStrictEqual(item, output[outputIndex])) {
|
|
70
|
+
outputIndex += 1;
|
|
71
|
+
}
|
|
72
|
+
if (outputIndex >= output.length) {
|
|
73
|
+
return false;
|
|
74
|
+
}
|
|
75
|
+
outputIndex += 1;
|
|
76
|
+
}
|
|
77
|
+
return true;
|
|
78
|
+
}
|
|
79
|
+
function isSemanticLayerPruneShape(current, output) {
|
|
80
|
+
const currentYaml = parseYamlObject(current);
|
|
81
|
+
const outputYaml = parseYamlObject(output);
|
|
82
|
+
if (!currentYaml || !outputYaml) {
|
|
83
|
+
return false;
|
|
84
|
+
}
|
|
85
|
+
const currentJoins = Array.isArray(currentYaml.joins) ? currentYaml.joins : [];
|
|
86
|
+
const outputJoins = Array.isArray(outputYaml.joins) ? outputYaml.joins : [];
|
|
87
|
+
if (currentJoins.length >= outputJoins.length) {
|
|
88
|
+
return false;
|
|
89
|
+
}
|
|
90
|
+
if (!isSubsequenceOfDeepValues(currentJoins, outputJoins)) {
|
|
91
|
+
return false;
|
|
92
|
+
}
|
|
93
|
+
const normalizedOutput = { ...outputYaml, joins: currentJoins };
|
|
94
|
+
return isDeepStrictEqual(currentYaml, normalizedOutput);
|
|
95
|
+
}
|
|
96
|
+
function parseWikiPage(raw) {
|
|
97
|
+
const match = raw.match(/^---\n([\s\S]*?)\n---\n?([\s\S]*)$/);
|
|
98
|
+
if (!match) {
|
|
99
|
+
return null;
|
|
100
|
+
}
|
|
101
|
+
const frontmatter = YAML.parse(match[1] ?? '');
|
|
102
|
+
return { frontmatter, content: (match[2] ?? '').trim() };
|
|
103
|
+
}
|
|
104
|
+
function withoutRemovedWikiTokens(output, current) {
|
|
105
|
+
let projected = output;
|
|
106
|
+
for (const match of output.matchAll(/\[\[\s*([^|\]\n]+)(?:\|[^\]\n]+)?\s*\]\]/g)) {
|
|
107
|
+
const token = match[0] ?? '';
|
|
108
|
+
if (token && !current.includes(token)) {
|
|
109
|
+
projected = projected.replaceAll(token, '').replace(/[ \t]+([.,;:!?])/g, '$1');
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
for (const match of output.matchAll(/`([^`\n]+)`/g)) {
|
|
113
|
+
const token = match[0] ?? '';
|
|
114
|
+
if (token && !current.includes(token)) {
|
|
115
|
+
projected = projected.replaceAll(token, '').replace(/[ \t]+([.,;:!?])/g, '$1');
|
|
116
|
+
}
|
|
117
|
+
}
|
|
118
|
+
projected = projected.replace(/,\s*,/g, ',').replace(/[ \t]+([.,;:!?])/g, '$1');
|
|
119
|
+
return projected.trim();
|
|
120
|
+
}
|
|
121
|
+
function isWikiPruneShape(current, output) {
|
|
122
|
+
const currentPage = parseWikiPage(current);
|
|
123
|
+
const outputPage = parseWikiPage(output);
|
|
124
|
+
if (!currentPage || !outputPage) {
|
|
125
|
+
return false;
|
|
126
|
+
}
|
|
127
|
+
const currentRefs = Array.isArray(currentPage.frontmatter.refs) ? currentPage.frontmatter.refs : [];
|
|
128
|
+
const outputRefs = Array.isArray(outputPage.frontmatter.refs) ? outputPage.frontmatter.refs : [];
|
|
129
|
+
const currentSlRefs = Array.isArray(currentPage.frontmatter.sl_refs) ? currentPage.frontmatter.sl_refs : [];
|
|
130
|
+
const outputSlRefs = Array.isArray(outputPage.frontmatter.sl_refs) ? outputPage.frontmatter.sl_refs : [];
|
|
131
|
+
if (currentRefs.length > outputRefs.length || currentSlRefs.length > outputSlRefs.length) {
|
|
132
|
+
return false;
|
|
133
|
+
}
|
|
134
|
+
if (!isSubsequenceOfDeepValues(currentRefs, outputRefs) || !isSubsequenceOfDeepValues(currentSlRefs, outputSlRefs)) {
|
|
135
|
+
return false;
|
|
136
|
+
}
|
|
137
|
+
const normalizedOutputFrontmatter = {
|
|
138
|
+
...outputPage.frontmatter,
|
|
139
|
+
refs: currentRefs,
|
|
140
|
+
sl_refs: currentSlRefs,
|
|
141
|
+
};
|
|
142
|
+
if (!isDeepStrictEqual(currentPage.frontmatter, normalizedOutputFrontmatter)) {
|
|
143
|
+
return false;
|
|
144
|
+
}
|
|
145
|
+
return withoutRemovedWikiTokens(outputPage.content, currentPage.content) === currentPage.content.trim();
|
|
146
|
+
}
|
|
147
|
+
/** @internal */
|
|
148
|
+
export function isPruneShapedCachedReplayBase(path, currentContent, outputContent) {
|
|
149
|
+
if (path.startsWith('semantic-layer/') && path.endsWith('.yaml')) {
|
|
150
|
+
return isSemanticLayerPruneShape(currentContent, outputContent);
|
|
151
|
+
}
|
|
152
|
+
if (path.startsWith('wiki/') && path.endsWith('.md')) {
|
|
153
|
+
return isWikiPruneShape(currentContent, outputContent);
|
|
154
|
+
}
|
|
155
|
+
return false;
|
|
156
|
+
}
|
|
157
|
+
export async function captureIngestWorkUnitCachedArtifactFiles(input) {
|
|
158
|
+
const paths = [...new Set(input.patchTouchedPaths)].sort();
|
|
159
|
+
return Promise.all(paths.map(async (path) => ({
|
|
160
|
+
path,
|
|
161
|
+
beforeBase64: await readGitFileBase64(input.git, path, input.baseSha),
|
|
162
|
+
afterBase64: await readFileBase64(input.workdir, path),
|
|
163
|
+
})));
|
|
164
|
+
}
|
|
165
|
+
async function writeCachedFile(workdir, file) {
|
|
166
|
+
const target = join(workdir, file.path);
|
|
167
|
+
if (file.afterBase64 === null) {
|
|
168
|
+
await rm(target, { force: true });
|
|
169
|
+
return;
|
|
170
|
+
}
|
|
171
|
+
await mkdir(dirname(target), { recursive: true });
|
|
172
|
+
await writeFile(target, Buffer.from(file.afterBase64, 'base64'));
|
|
173
|
+
}
|
|
174
|
+
function cacheFileCanReplayFromCurrentBase(file, currentBase64) {
|
|
175
|
+
if (currentBase64 === file.beforeBase64 || currentBase64 === file.afterBase64) {
|
|
176
|
+
return true;
|
|
177
|
+
}
|
|
178
|
+
const current = decodeBase64(currentBase64);
|
|
179
|
+
const output = decodeBase64(file.afterBase64);
|
|
180
|
+
if (current === null || output === null) {
|
|
181
|
+
return false;
|
|
182
|
+
}
|
|
183
|
+
return isPruneShapedCachedReplayBase(file.path, current, output);
|
|
184
|
+
}
|
|
185
|
+
export async function materializeCachedWorkUnitReplayPatch(input) {
|
|
186
|
+
const replay = await input.sessionWorktreeService.create(`${input.jobId}-${input.unitKey}-cache-replay`, input.baseSha);
|
|
187
|
+
let cleanup = 'crash';
|
|
188
|
+
try {
|
|
189
|
+
for (const file of input.artifactFiles) {
|
|
190
|
+
const currentBase64 = await readFileBase64(replay.workdir, file.path);
|
|
191
|
+
if (!cacheFileCanReplayFromCurrentBase(file, currentBase64)) {
|
|
192
|
+
cleanup = 'success';
|
|
193
|
+
return 'unsafe_drift';
|
|
194
|
+
}
|
|
195
|
+
}
|
|
196
|
+
for (const file of input.artifactFiles) {
|
|
197
|
+
await writeCachedFile(replay.workdir, file);
|
|
198
|
+
}
|
|
199
|
+
const changedPaths = await replay.git.changedPaths();
|
|
200
|
+
if (changedPaths.length > 0) {
|
|
201
|
+
await replay.git.commitFiles(changedPaths, `ingest: materialize cached WorkUnit ${input.unitKey}`, input.author.name, input.author.email);
|
|
202
|
+
}
|
|
203
|
+
await replay.git.writeBinaryNoRenamePatch(input.baseSha, 'HEAD', input.patchPath);
|
|
204
|
+
await input.trace.event('debug', 'work_unit', 'work_unit_cache_patch_materialized', {
|
|
205
|
+
unitKey: input.unitKey,
|
|
206
|
+
patchPath: input.patchPath,
|
|
207
|
+
touchedPaths: changedPaths,
|
|
208
|
+
});
|
|
209
|
+
cleanup = 'success';
|
|
210
|
+
return 'materialized';
|
|
211
|
+
}
|
|
212
|
+
finally {
|
|
213
|
+
await input.sessionWorktreeService.cleanup(replay, cleanup);
|
|
214
|
+
}
|
|
215
|
+
}
|
|
216
|
+
export function ingestWorkUnitCacheScopeKey(input) {
|
|
217
|
+
return `${input.connectionId}:${input.sourceKey}`;
|
|
218
|
+
}
|
|
219
|
+
export function computeIngestWorkUnitPromptFingerprint(input) {
|
|
220
|
+
return stableContentHash({
|
|
221
|
+
schemaVersion: 1,
|
|
222
|
+
cliVersion: input.cliVersion,
|
|
223
|
+
baseFraming: input.baseFraming,
|
|
224
|
+
skillsPrompt: input.skillsPrompt,
|
|
225
|
+
canonicalPins: input.canonicalPins,
|
|
226
|
+
sourceKey: input.sourceKey,
|
|
227
|
+
connectionId: input.connectionId,
|
|
228
|
+
skillNames: [...input.skillNames].sort(),
|
|
229
|
+
});
|
|
230
|
+
}
|
|
@@ -19,6 +19,7 @@ export declare class AiSdkKtxLlmRuntime implements KtxLlmRuntimePort {
|
|
|
19
19
|
private readonly deps;
|
|
20
20
|
private readonly logger;
|
|
21
21
|
constructor(deps: AiSdkKtxLlmRuntimeDeps);
|
|
22
|
+
subprocessForkSpec(): null;
|
|
22
23
|
private generateTextWithRateLimitRetry;
|
|
23
24
|
generateText(input: KtxGenerateTextInput): Promise<string>;
|
|
24
25
|
generateObject<TOutput, TSchema extends z.ZodType<TOutput>>(input: KtxGenerateObjectInput<TOutput, TSchema>): Promise<TOutput>;
|
|
@@ -126,6 +126,11 @@ export class AiSdkKtxLlmRuntime {
|
|
|
126
126
|
this.deps = deps;
|
|
127
127
|
this.logger = deps.logger ?? noopLogger;
|
|
128
128
|
}
|
|
129
|
+
// HTTP backend: abortSignal cancels the underlying fetch natively, so there is
|
|
130
|
+
// no SDK-owned child to tree-kill.
|
|
131
|
+
subprocessForkSpec() {
|
|
132
|
+
return null;
|
|
133
|
+
}
|
|
129
134
|
async generateTextWithRateLimitRetry(provider, abortSignal, run) {
|
|
130
135
|
// maxRetryAttempts() returns 1 when no governor is present or pacing is
|
|
131
136
|
// disabled, so a 429 throws immediately instead of hammering the provider
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { query as defaultQuery, type SDKMessage, type SDKResultMessage } from '@anthropic-ai/claude-agent-sdk';
|
|
2
2
|
import { z } from 'zod';
|
|
3
3
|
import type { RateLimitGovernor } from './rate-limit-governor.js';
|
|
4
|
-
import type { KtxGenerateObjectInput, KtxGenerateTextInput, KtxLlmRuntimePort, RunLoopParams, RunLoopResult, RunLoopStopReason } from './runtime-port.js';
|
|
4
|
+
import type { KtxGenerateObjectInput, KtxGenerateStructuredJsonInput, KtxGenerateTextInput, KtxLlmRuntimePort, RunLoopParams, RunLoopResult, RunLoopStopReason, SubprocessRuntimeForkSpec } from './runtime-port.js';
|
|
5
5
|
type QueryResult = AsyncIterable<SDKMessage> & {
|
|
6
6
|
interrupt?: () => void | Promise<void>;
|
|
7
7
|
};
|
|
@@ -22,7 +22,10 @@ export declare class ClaudeCodeKtxLlmRuntime implements KtxLlmRuntimePort {
|
|
|
22
22
|
private readonly runQuery;
|
|
23
23
|
constructor(deps: ClaudeCodeKtxLlmRuntimeDeps);
|
|
24
24
|
generateText(input: KtxGenerateTextInput): Promise<string>;
|
|
25
|
+
private runStructuredQuery;
|
|
25
26
|
generateObject<TOutput, TSchema extends z.ZodType<TOutput>>(input: KtxGenerateObjectInput<TOutput, TSchema>): Promise<TOutput>;
|
|
27
|
+
generateStructuredJson(input: KtxGenerateStructuredJsonInput): Promise<unknown>;
|
|
28
|
+
subprocessForkSpec(): SubprocessRuntimeForkSpec;
|
|
26
29
|
runAgentLoop(params: RunLoopParams): Promise<RunLoopResult>;
|
|
27
30
|
}
|
|
28
31
|
export declare function runClaudeCodeAuthProbe(input: {
|
|
@@ -298,7 +298,9 @@ export class ClaudeCodeKtxLlmRuntime {
|
|
|
298
298
|
}
|
|
299
299
|
return result.result;
|
|
300
300
|
}
|
|
301
|
-
|
|
301
|
+
// Structured generation has no tools, so generateObject and
|
|
302
|
+
// generateStructuredJson (the kill-boundary child path) share this one query.
|
|
303
|
+
async runStructuredQuery(input) {
|
|
302
304
|
const options = {
|
|
303
305
|
...baseOptions({
|
|
304
306
|
projectDir: this.deps.projectDir,
|
|
@@ -310,19 +312,27 @@ export class ClaudeCodeKtxLlmRuntime {
|
|
|
310
312
|
// 5 leaves headroom without enabling unbounded loops; the json_schema
|
|
311
313
|
// constraint still forces the final answer to be the schema.
|
|
312
314
|
maxTurns: 5,
|
|
313
|
-
tools: input.tools,
|
|
314
315
|
}),
|
|
315
|
-
outputFormat: { type: 'json_schema', schema:
|
|
316
|
+
outputFormat: { type: 'json_schema', schema: input.jsonSchema },
|
|
316
317
|
};
|
|
317
|
-
|
|
318
|
-
const result = await collectResultWithRateLimitRetry({
|
|
318
|
+
return collectResultWithRateLimitRetry({
|
|
319
319
|
query: this.runQuery,
|
|
320
320
|
prompt: [input.system, input.prompt].filter(Boolean).join('\n\n'),
|
|
321
321
|
options,
|
|
322
|
-
allowedToolIds: new Set([
|
|
323
|
-
expectedMcpServerNames:
|
|
322
|
+
allowedToolIds: new Set([STRUCTURED_OUTPUT_TOOL_NAME]),
|
|
323
|
+
expectedMcpServerNames: new Set(),
|
|
324
324
|
rateLimitGovernor: this.deps.rateLimitGovernor,
|
|
325
|
-
abortSignal: input.abortSignal,
|
|
325
|
+
...(input.abortSignal ? { abortSignal: input.abortSignal } : {}),
|
|
326
|
+
});
|
|
327
|
+
}
|
|
328
|
+
async generateObject(input) {
|
|
329
|
+
const startedAt = Date.now();
|
|
330
|
+
const result = await this.runStructuredQuery({
|
|
331
|
+
role: input.role,
|
|
332
|
+
prompt: input.prompt,
|
|
333
|
+
...(input.system !== undefined ? { system: input.system } : {}),
|
|
334
|
+
jsonSchema: jsonSchema(input.schema),
|
|
335
|
+
...(input.abortSignal ? { abortSignal: input.abortSignal } : {}),
|
|
326
336
|
});
|
|
327
337
|
input.onMetrics?.({ totalMs: Date.now() - startedAt, usage: claudeTokenUsage(result) });
|
|
328
338
|
const error = resultError(result);
|
|
@@ -334,6 +344,26 @@ export class ClaudeCodeKtxLlmRuntime {
|
|
|
334
344
|
}
|
|
335
345
|
return input.schema.parse(result.structured_output);
|
|
336
346
|
}
|
|
347
|
+
async generateStructuredJson(input) {
|
|
348
|
+
const result = await this.runStructuredQuery({
|
|
349
|
+
role: input.role,
|
|
350
|
+
prompt: input.prompt,
|
|
351
|
+
...(input.system !== undefined ? { system: input.system } : {}),
|
|
352
|
+
jsonSchema: input.jsonSchema,
|
|
353
|
+
...(input.abortSignal ? { abortSignal: input.abortSignal } : {}),
|
|
354
|
+
});
|
|
355
|
+
const error = resultError(result);
|
|
356
|
+
if (error) {
|
|
357
|
+
throw error;
|
|
358
|
+
}
|
|
359
|
+
if (result.subtype !== 'success') {
|
|
360
|
+
throw new Error(`Claude Code query failed (${result.subtype})`);
|
|
361
|
+
}
|
|
362
|
+
return result.structured_output;
|
|
363
|
+
}
|
|
364
|
+
subprocessForkSpec() {
|
|
365
|
+
return { backend: 'claude-code', projectDir: this.deps.projectDir, modelSlots: this.deps.modelSlots };
|
|
366
|
+
}
|
|
337
367
|
async runAgentLoop(params) {
|
|
338
368
|
const startedAt = Date.now();
|
|
339
369
|
try {
|