@kaelio/ktx 0.13.0 → 0.14.0

package/dist/verbatim-ingest.js

@@ -0,0 +1,193 @@
+import { basename, extname, join } from 'node:path';
+import YAML from 'yaml';
+import { z } from 'zod';
+import { noopLogger } from './context/core/config.js';
+import { assertConfiguredConnectionId } from './context/connections/configured-connections.js';
+import { KtxIngestEmbeddingPortAdapter } from './context/llm/embedding-port.js';
+import { createLocalKtxEmbeddingProviderFromConfig, createLocalKtxLlmRuntimeFromConfig } from './context/llm/local-config.js';
+import { KnowledgeWikiService } from './context/wiki/knowledge-wiki.service.js';
+import { suggestFlatWikiKey } from './context/wiki/keys.js';
+import { SqliteKnowledgeIndex } from './context/wiki/sqlite-knowledge-index.js';
+const LOCAL_AUTHOR = 'ktx';
+const LOCAL_AUTHOR_EMAIL = 'ktx@example.com';
+/** Only the prefix sent to the LLM for metadata is clipped; the stored body is never clipped. */
+const METADATA_CLIP_LENGTH = 48_000;
+const VERBATIM_METADATA_SYSTEM_PROMPT = [
+    'You generate search metadata for an authoritative document that ktx stores verbatim.',
+    'You never rewrite, summarize into, or alter the document body — you only describe it.',
+    'Return a concise one- or two-sentence summary, a few topical tags, and any semantic-layer',
+    'source names the document is clearly about. Use empty arrays when none apply.',
+].join(' ');
+const verbatimMetadataSchema = z.object({
+    summary: z.string().min(1).describe('A one- or two-sentence description of what the document defines or specifies.'),
+    tags: z.array(z.string()).default([]).describe('Short topical keywords that aid lexical and semantic recall.'),
+    sl_refs: z
+        .array(z.string())
+        .default([])
+        .describe('Semantic-layer source names the document is clearly about, if any are evident.'),
+});
+const DEGRADED_SUMMARY_MAX_LENGTH = 200;
+const FRONTMATTER_PATTERN = /^---\n([\s\S]*?)\n---\n?([\s\S]*)$/;
+const HEADING_PATTERN = /^#{1,6}\s+(.+?)\s*#*\s*$/;
+function isUsageMode(value) {
+    return value === 'always' || value === 'auto' || value === 'never';
+}
+function nonEmptyString(value) {
+    return typeof value === 'string' && value.trim().length > 0 ? value : undefined;
+}
+function stringArray(value) {
+    return Array.isArray(value) ? value.filter((item) => typeof item === 'string') : [];
+}
+/** `connections` accepts a single id or a list in YAML; normalize either to a string list. */
+function stringList(value) {
+    if (typeof value === 'string') {
+        return value.trim().length > 0 ? [value] : [];
+    }
+    return stringArray(value);
+}
+function leadingHeadingText(body) {
+    const firstLine = body.trimStart().split('\n', 1)[0] ?? '';
+    const match = firstLine.match(HEADING_PATTERN);
+    return match ? match[1].trim() : null;
+}
+/** @internal */
+export function splitInputDocument(raw) {
+    const match = raw.match(FRONTMATTER_PATTERN);
+    if (!match) {
+        return { frontmatter: {}, body: raw.trim() };
+    }
+    const parsed = YAML.parse(match[1]);
+    const frontmatter = parsed !== null && typeof parsed === 'object' && !Array.isArray(parsed)
+        ? parsed
+        : {};
+    return { frontmatter, body: match[2].trim() };
+}
+/** @internal */
+export function deriveVerbatimPageKey(origin, body) {
+    if (origin.kind === 'file' && origin.path) {
+        return suggestFlatWikiKey(basename(origin.path, extname(origin.path)));
+    }
+    const heading = leadingHeadingText(body);
+    if (!heading) {
+        throw new Error('Verbatim inline text needs a leading Markdown heading to derive a stable page key. Add a "# Heading" line, or pass the content as --file <path>.');
+    }
+    return suggestFlatWikiKey(heading);
+}
+/** @internal */
+export function deriveDegradedSummary(body) {
+    const heading = leadingHeadingText(body);
+    if (heading) {
+        return heading;
+    }
+    const text = body.trim();
+    const sentence = text.match(/^([\s\S]*?[.!?])(\s|$)/);
+    const summary = sentence ? sentence[1].trim() : text;
+    if (summary.length <= DEGRADED_SUMMARY_MAX_LENGTH) {
+        return summary;
+    }
+    return `${summary.slice(0, DEGRADED_SUMMARY_MAX_LENGTH).trimEnd()}…`;
+}
+/** @internal */
+export function buildVerbatimFrontmatter(input) {
+    const { inputFrontmatter } = input;
+    const inputConnections = stringList(inputFrontmatter.connections);
+    const flagConnections = input.connectionId ? [input.connectionId] : [];
+    if (inputConnections.length > 0 &&
+        flagConnections.length > 0 &&
+        !connectionSetsEqual(inputConnections, flagConnections)) {
+        throw new Error(`Connection scope conflict: frontmatter declares connections [${inputConnections.join(', ')}] but --connection-id is "${input.connectionId}". Remove one so the intent is unambiguous.`);
+    }
+    const connections = inputConnections.length > 0 ? inputConnections : flagConnections;
+    const summary = nonEmptyString(inputFrontmatter.summary) ?? input.summary;
+    const usageMode = isUsageMode(inputFrontmatter.usage_mode) ? inputFrontmatter.usage_mode : 'auto';
+    const tags = inputFrontmatter.tags !== undefined ? stringArray(inputFrontmatter.tags) : input.tags;
+    const slRefs = inputFrontmatter.sl_refs !== undefined ? stringArray(inputFrontmatter.sl_refs) : input.slRefs;
+    const passthrough = Object.fromEntries(Object.entries(inputFrontmatter).filter(([key]) => !['summary', 'usage_mode', 'tags', 'sl_refs', 'connections'].includes(key)));
+    return {
+        ...passthrough,
+        summary,
+        usage_mode: usageMode,
+        ...(tags.length > 0 ? { tags } : {}),
+        ...(slRefs.length > 0 ? { sl_refs: slRefs } : {}),
+        ...(connections.length > 0 ? { connections } : {}),
+    };
+}
+function connectionSetsEqual(left, right) {
+    if (left.length !== right.length) {
+        return false;
+    }
+    const rightSet = new Set(right);
+    return left.every((id) => rightSet.has(id));
+}
+class LocalVerbatimIngestor {
+    deps;
+    constructor(deps) {
+        this.deps = deps;
+    }
+    async ingest(item) {
+        if (item.connectionId) {
+            assertConfiguredConnectionId(this.deps.configuredConnections, item.connectionId);
+        }
+        const { frontmatter: inputFrontmatter, body } = splitInputDocument(item.content);
+        const pageKey = deriveVerbatimPageKey(item.origin, body);
+        const generated = await this.resolveMetadata(inputFrontmatter, body);
+        const frontmatter = buildVerbatimFrontmatter({
+            inputFrontmatter,
+            summary: generated.summary,
+            tags: generated.tags,
+            slRefs: generated.slRefs,
+            ...(item.connectionId ? { connectionId: item.connectionId } : {}),
+        });
+        const connections = Array.isArray(frontmatter.connections) ? frontmatter.connections : [];
+        const existing = await this.deps.wikiService.readPage('GLOBAL', null, pageKey);
+        if (existing) {
+            if (existing.content === body) {
+                return { pageKey, outcome: 'unchanged', connections, commitHash: null };
+            }
+            throw new Error(`A different page already exists at key "${pageKey}". Re-run with a distinct document name or key, ` +
+                'or remove the existing page first — verbatim ingest never overwrites a conflicting page.');
+        }
+        const writeResult = await this.deps.wikiService.writePageAndSync('GLOBAL', null, pageKey, frontmatter, body, this.deps.author, this.deps.authorEmail, `Ingest verbatim document: ${pageKey}`);
+        return { pageKey, outcome: 'written', connections, commitHash: writeResult.commitHash ?? null };
+    }
+    /**
+     * Generated metadata is only used to gap-fill absent frontmatter fields, so the LLM is
+     * skipped entirely when summary, tags, and sl_refs are all explicit. A configured backend
+     * that fails surfaces the error (the item fails); degraded derivation is reserved for
+     * `backend: none`, never used as a silent fallback that would poison the idempotency check.
+     */
+    async resolveMetadata(inputFrontmatter, body) {
+        const needsGeneration = nonEmptyString(inputFrontmatter.summary) === undefined ||
+            inputFrontmatter.tags === undefined ||
+            inputFrontmatter.sl_refs === undefined;
+        if (this.deps.llmRuntime && needsGeneration) {
+            const clipped = body.length > METADATA_CLIP_LENGTH ? body.slice(0, METADATA_CLIP_LENGTH) : body;
+            const generated = await this.deps.llmRuntime.generateObject({
+                role: 'triage',
+                system: VERBATIM_METADATA_SYSTEM_PROMPT,
+                prompt: clipped,
+                schema: verbatimMetadataSchema,
+            });
+            return { summary: generated.summary, tags: generated.tags, slRefs: generated.sl_refs };
+        }
+        return { summary: deriveDegradedSummary(body), tags: [], slRefs: [] };
+    }
+}
+export function createLocalProjectVerbatimIngestor(project, deps = {}) {
+    const llmRuntime = deps.llmRuntime !== undefined
+        ? deps.llmRuntime
+        : createLocalKtxLlmRuntimeFromConfig(project.config.llm, { projectDir: project.projectDir });
+    const embeddingProvider = deps.embeddingProvider !== undefined
+        ? deps.embeddingProvider
+        : createLocalKtxEmbeddingProviderFromConfig(project.config.ingest.embeddings, { projectDir: project.projectDir });
+    const embeddingPort = embeddingProvider ? new KtxIngestEmbeddingPortAdapter(embeddingProvider) : null;
+    const knowledgeIndex = new SqliteKnowledgeIndex({ dbPath: join(project.projectDir, '.ktx', 'db.sqlite') });
+    const wikiService = new KnowledgeWikiService(project.fileStore, embeddingPort, knowledgeIndex, project.git, noopLogger);
+    return new LocalVerbatimIngestor({
+        wikiService,
+        llmRuntime,
+        configuredConnections: project.config.connections,
+        author: LOCAL_AUTHOR,
+        authorEmail: LOCAL_AUTHOR_EMAIL,
+    });
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@kaelio/ktx",
-  "version": "0.13.0",
+  "version": "0.14.0",
   "description": "Standalone ktx context layer for data agents",
   "author": {
     "name": "Kaelio",
@@ -42,6 +42,7 @@
     "@commander-js/extra-typings": "14.0.0",
     "@duckdb/node-api": "1.5.3-r.3",
     "@google-cloud/bigquery": "^8.3.1",
+    "google-auth-library": "10.6.2",
     "@looker/sdk": "^26.8.0",
     "@looker/sdk-node": "^26.8.0",
     "@looker/sdk-rtl": "^21.6.5",
@@ -56,11 +57,14 @@
     "ink": "^7.0.3",
     "lookml-parser": "7.1.0",
     "minimatch": "^10.2.5",
+    "mongodb": "^6.12.0",
     "mssql": "^12.5.4",
     "mysql2": "^3.22.3",
     "openai": "^6.38.0",
     "p-limit": "^7.3.0",
     "pg": "^8.21.0",
+    "pino": "^10.3.1",
+    "pino-pretty": "^13.1.3",
     "posthog-node": "^5.34.9",
     "react": "^19.2.6",
     "semver": "^7.8.1",

package/dist/context/ingest/final-gate-repair.d.ts

@@ -1,28 +0,0 @@
-import type { AgentRunnerPort } from '../../context/llm/runtime-port.js';
-import type { ConstrainedRepairResult, RepairVerification } from './constrained-repair.js';
-import type { IngestTraceWriter } from './ingest-trace.js';
-type FinalGateRepairKind = 'patch_semantic_gate' | 'final_artifact_gate';
-export type FinalGateRepairResult = ConstrainedRepairResult;
-export interface RepairFinalGateFailureInput {
-    agentRunner: AgentRunnerPort;
-    workdir: string;
-    gateError: string;
-    allowedPaths: string[];
-    trace: IngestTraceWriter;
-    repairKind: FinalGateRepairKind;
-    /**
-     * Re-runs the failed gate against the current worktree. The repair counts
-     * as successful only when this passes — editing files is not the success
-     * signal.
-     */
-    verify(changedPaths: string[]): Promise<RepairVerification>;
-    maxAttempts?: number;
-    stepBudget?: number;
-    abortSignal?: AbortSignal;
-}
-export declare function finalGateRepairPaths(input: {
-    changedWikiPageKeys: string[];
-    touchedSlSourcePaths: string[];
-}): string[];
-export declare function repairFinalGateFailure(input: RepairFinalGateFailureInput): Promise<FinalGateRepairResult>;
-export {};

package/dist/context/ingest/final-gate-repair.js

@@ -1,91 +0,0 @@
-import { z } from 'zod';
-import { runConstrainedRepairLoop } from './constrained-repair.js';
-function buildGateRepairSystemPrompt() {
-    return `<role>
-You repair one ktx isolated-diff artifact gate failure inside the integration worktree.
-</role>
-
-<rules>
-- Use read_gate_error first.
-- Read only files exposed by read_repair_file.
-- Edit only paths exposed by write_repair_file.
-- Prefer the smallest text edit that makes the gate pass.
-- Preserve accepted work-unit, reconciliation, and deterministic projection content.
-- Do not invent warehouse facts, business definitions, or semantic-layer entities.
-- If the gate error requires choosing between conflicting facts without evidence, stop without editing.
-</rules>`;
-}
-function buildGateRepairUserPrompt(input) {
-    const previousFailureBlock = input.previousFailure
-        ? `\nPrevious attempt did not pass the gate:\n${input.previousFailure}\n`
-        : '';
-    return `Repair isolated-diff artifact gates.
-
-Repair kind: ${input.repairKind}
-Attempt: ${input.attempt} of ${input.maxAttempts}
-
-Allowed files:
-${input.allowedPaths.map((path) => `- ${path}`).join('\n')}
-
-Gate error:
-${input.gateError}
-${previousFailureBlock}
-Use read_gate_error first. Then inspect only the allowed files, write the
-minimal repaired content, and stop.`;
-}
-function buildReadGateErrorTool(gateError) {
-    return {
-        read_gate_error: {
-            name: 'read_gate_error',
-            description: 'Read the artifact gate failure that must be repaired.',
-            inputSchema: z.object({}),
-            execute: async () => ({
-                markdown: gateError,
-                structured: { gateError },
-            }),
-        },
-    };
-}
-export function finalGateRepairPaths(input) {
-    return [
-        ...new Set([
-            ...input.touchedSlSourcePaths,
-            ...input.changedWikiPageKeys.map((pageKey) => `wiki/global/${pageKey}.md`),
-        ]),
-    ].sort();
-}
-export async function repairFinalGateFailure(input) {
-    return runConstrainedRepairLoop({
-        agentRunner: input.agentRunner,
-        workdir: input.workdir,
-        allowedPaths: input.allowedPaths,
-        trace: input.trace,
-        tracePhase: 'gate_repair',
-        traceEventName: 'gate_repair',
-        traceData: {
-            repairKind: input.repairKind,
-            gateError: input.gateError,
-        },
-        systemPrompt: buildGateRepairSystemPrompt(),
-        buildUserPrompt: ({ attempt, maxAttempts, previousFailure }) => buildGateRepairUserPrompt({
-            gateError: input.gateError,
-            allowedPaths: [...input.allowedPaths].sort(),
-            repairKind: input.repairKind,
-            attempt,
-            maxAttempts,
-            previousFailure,
-        }),
-        buildExtraTools: () => buildReadGateErrorTool(input.gateError),
-        verify: input.verify,
-        noChangeFailureReason: 'gate repair completed without editing an allowed path',
-        telemetryTags: {
-            operationName: 'ingest-isolated-diff-gate-repair',
-            source: input.trace.context.sourceKey,
-            jobId: input.trace.context.jobId,
-            repairKind: input.repairKind,
-        },
-        maxAttempts: input.maxAttempts,
-        stepBudget: input.stepBudget ?? 16,
-        abortSignal: input.abortSignal,
-    });
-}