@kaelio/ktx 0.13.1 → 0.14.0

package/dist/cli-program.js

@@ -68,7 +68,7 @@ export function parseBooleanStringOption(value) {
     throw new InvalidArgumentError('must be true or false');
 }
 export function parseSafeConnectionIdOption(value) {
-    if (!/^[a-zA-Z0-9][a-zA-Z0-9_-]*$/.test(value)) {
+    if (!/^[a-zA-Z0-9_][a-zA-Z0-9_-]*$/.test(value)) {
         throw new InvalidArgumentError(`Unsafe connection id: ${value}`);
     }
     return value;

package/dist/commands/ingest-commands.d.ts

@@ -1,7 +1,16 @@
 import { type Command } from '@commander-js/extra-typings';
 import { type KtxCliCommandContext } from '../cli-program.js';
+import type { KtxScanEnrichmentStage } from '../context/scan/types.js';
 import type { KtxCliDeps, KtxCliIo } from '../index.js';
 import type { KtxTextIngestArgs } from '../text-ingest.js';
+/**
+ * Parses `--stages` into an ordered, de-duplicated subset of the canonical
+ * enrichment-stage registry. An unknown or empty name is a hard parse error so
+ * a typo never silently degrades to "run everything."
+ *
+ * @internal
+ */
+export declare function parseEnrichmentStagesOption(value: string): KtxScanEnrichmentStage[];
 interface IngestCommandOptions {
     runTextIngest: (args: KtxTextIngestArgs, io: KtxCliIo, deps: KtxCliDeps) => Promise<number>;
 }

package/dist/commands/ingest-commands.js

@@ -1,9 +1,35 @@
-import { Option } from '@commander-js/extra-typings';
+import { InvalidArgumentError, Option } from '@commander-js/extra-typings';
 import { collectOption, parsePositiveIntegerOption, resolveCommandProjectDir, } from '../cli-program.js';
+import { KTX_SCAN_ENRICHMENT_STAGES } from '../context/scan/enrichment-state.js';
 import { runtimeInstallPolicyFromFlags } from '../managed-python-command.js';
 import { profileMark } from '../startup-profile.js';
 import { resolveConnectionSelection } from './connection-selection.js';
 profileMark('module:commands/ingest-commands');
+/**
+ * Parses `--stages` into an ordered, de-duplicated subset of the canonical
+ * enrichment-stage registry. An unknown or empty name is a hard parse error so
+ * a typo never silently degrades to "run everything."
+ *
+ * @internal
+ */
+export function parseEnrichmentStagesOption(value) {
+    const names = value
+        .split(',')
+        .map((name) => name.trim())
+        .filter((name) => name.length > 0);
+    if (names.length === 0) {
+        throw new InvalidArgumentError(`must be a non-empty comma-separated list of stages (${KTX_SCAN_ENRICHMENT_STAGES.join(', ')})`);
+    }
+    const valid = new Set(KTX_SCAN_ENRICHMENT_STAGES);
+    const selected = new Set();
+    for (const name of names) {
+        if (!valid.has(name)) {
+            throw new InvalidArgumentError(`unknown stage "${name}"; valid stages are ${KTX_SCAN_ENRICHMENT_STAGES.join(', ')}`);
+        }
+        selected.add(name);
+    }
+    return KTX_SCAN_ENRICHMENT_STAGES.filter((stage) => selected.has(stage));
+}
 export function registerIngestCommands(program, context, commandOptions) {
     const ingest = program
         .command('ingest')
@@ -14,8 +40,10 @@ export function registerIngestCommands(program, context, commandOptions) {
         .addOption(new Option('--query-history', 'Include database query-history usage patterns').conflicts('noQueryHistory'))
         .addOption(new Option('--no-query-history', 'Skip database query-history usage patterns'))
         .option('--query-history-window-days <days>', 'Query-history lookback window for this run', parsePositiveIntegerOption)
+        .option('--stages <stages>', 'Comma-separated enrichment stages to (re)run (descriptions,embeddings,relationships); omit to run all', parseEnrichmentStagesOption)
         .option('--text <content>', 'Capture inline text into ktx memory; repeatable', collectOption, [])
         .option('--file <path>', 'Capture a text file into ktx memory; use - for stdin; repeatable', collectOption, [])
+        .option('--verbatim', 'Store each --text/--file document body unchanged as a GLOBAL wiki page; the LLM derives only metadata', false)
         .option('--connection-id <connectionId>', 'ktx connection id to tag captured text/file notes')
         .option('--user-id <id>', 'Memory user id for text/file capture attribution', 'local-cli')
         .option('--fail-fast', 'Stop after the first failed text/file item', false)
@@ -27,6 +55,12 @@ export function registerIngestCommands(program, context, commandOptions) {
     ingest.action(async (connectionId, options, command) => {
         const projectDir = resolveCommandProjectDir(command);
         const hasTextCapture = options.text.length > 0 || options.file.length > 0;
+        if (options.verbatim === true && !hasTextCapture) {
+            command.error('error: --verbatim requires --text or --file');
+        }
+        if (options.stages !== undefined && hasTextCapture) {
+            command.error('error: --stages applies to database ingest only; it cannot be combined with --text or --file');
+        }
         if (hasTextCapture) {
             if (connectionId !== undefined) {
                 command.error('error: --text/--file does not accept a positional connection id; use --connection-id <id> to tag captured notes');
@@ -42,6 +76,7 @@ export function registerIngestCommands(program, context, commandOptions) {
                 userId: options.userId,
                 json: options.json === true,
                 failFast: options.failFast === true,
+                ...(options.verbatim === true ? { verbatim: true } : {}),
             }, context.io, context.deps));
             return;
         }
@@ -57,6 +92,7 @@ export function registerIngestCommands(program, context, commandOptions) {
             inputMode: options.input === false ? 'disabled' : 'auto',
             queryHistory,
             ...(options.queryHistoryWindowDays !== undefined ? { queryHistoryWindowDays: options.queryHistoryWindowDays } : {}),
+            ...(options.stages ? { stages: options.stages } : {}),
             cliVersion: context.packageInfo.version,
             runtimeInstallPolicy: runtimeInstallPolicyFromFlags(options),
         };

package/dist/commands/knowledge-commands.js

@@ -17,6 +17,7 @@ export function registerWikiCommands(program, context) {
         .usage('[options] [query...]')
         .argument('[query...]', 'Search query; omit to list all pages')
         .option('--user-id <id>', 'Local user id', 'local')
+        .option('-c, --connection <id>', 'Scope results to one connection (unscoped pages plus pages tagged with it)')
         .option('--limit <number>', 'Maximum search results (search mode only)', parsePositiveIntegerOption)
         .addOption(new Option('--output <mode>', 'Output mode: pretty (default in TTY), plain (TSV), or json').choices([
         'pretty',
@@ -32,6 +33,7 @@ export function registerWikiCommands(program, context) {
                 command: 'list',
                 projectDir: resolveCommandProjectDir(command),
                 userId: options.userId,
+                ...(options.connection !== undefined ? { connectionId: options.connection } : {}),
                 output: options.output,
                 json: options.json,
                 cliVersion: context.packageInfo.version,
@@ -43,6 +45,7 @@ export function registerWikiCommands(program, context) {
             projectDir: resolveCommandProjectDir(command),
             query: query.join(' '),
             userId: options.userId,
+            ...(options.connection !== undefined ? { connectionId: options.connection } : {}),
             output: options.output,
             json: options.json,
             ...(isDebugEnabled(command) ? { debug: true } : {}),

package/dist/commands/setup-commands.js

@@ -42,7 +42,8 @@ function sourceType(value) {
         value === 'metabase' ||
         value === 'looker' ||
         value === 'lookml' ||
-        value === 'notion') {
+        value === 'notion' ||
+        value === 'gdrive') {
         return value;
     }
     throw new InvalidArgumentError(`invalid choice '${value}'`);
@@ -121,6 +122,9 @@ function shouldShowSetupEntryMenu(options, command) {
         'sourceTarget',
         'metabaseDatabaseId',
         'notionCrawlMode',
+        'gdriveServiceAccountKeyRef',
+        'gdriveFolderId',
+        'gdriveRecursive',
         'skipSources',
     ].some((optionName) => optionWasSpecified(command, optionName));
 }
@@ -213,6 +217,10 @@ export function registerSetupCommands(program, context) {
         .argParser((value, previous) => [...previous, value])
         .default([])
         .hideHelp())
+        .addOption(new Option('--gdrive-service-account-key-ref <ref>', 'file: reference to a Google service account JSON key')
+        .hideHelp())
+        .addOption(new Option('--gdrive-folder-id <id>', 'Google Drive folder id to ingest').hideHelp())
+        .addOption(new Option('--gdrive-recursive', 'Recursively traverse Google Drive subfolders').hideHelp().default(false))
         .addOption(new Option('--skip-sources', 'Mark optional source setup complete with no sources').hideHelp().default(false))
         .showHelpAfterError();
     setup.hook('preAction', (_thisCommand, actionCommand) => {
@@ -349,6 +357,11 @@ export function registerSetupCommands(program, context) {
             ...(options.metabaseDatabaseId !== undefined ? { metabaseDatabaseId: options.metabaseDatabaseId } : {}),
             ...(options.notionCrawlMode ? { notionCrawlMode: options.notionCrawlMode } : {}),
             ...(options.notionRootPageId.length > 0 ? { notionRootPageIds: options.notionRootPageId } : {}),
+            ...(options.gdriveServiceAccountKeyRef
+                ? { gdriveServiceAccountKeyRef: options.gdriveServiceAccountKeyRef }
+                : {}),
+            ...(options.gdriveFolderId ? { gdriveFolderId: options.gdriveFolderId } : {}),
+            ...(options.gdriveRecursive ? { gdriveRecursive: true } : {}),
             runInitialSourceIngest: false,
             skipSources: options.skipSources === true,
             showEntryMenu: shouldShowSetupEntryMenu(options, command),

package/dist/connection-drivers.d.ts

@@ -1,3 +1,5 @@
 import type { KtxProjectConnectionConfig } from './context/project/config.js';
+/** @internal Canonical SQL-warehouse driver ids; the dialect-notes coverage test derives its required coverage from this set. */
+export declare const KTX_DATABASE_DRIVER_IDS: readonly ["sqlite", "postgres", "mysql", "clickhouse", "sqlserver", "bigquery", "snowflake"];
 export declare function normalizeConnectionDriver(connection: KtxProjectConnectionConfig): string;
 export declare function isDatabaseDriver(driver: string): boolean;

package/dist/connection-drivers.js

@@ -1,4 +1,5 @@
-const KTX_DATABASE_DRIVER_IDS = new Set([
+/** @internal Canonical SQL-warehouse driver ids; the dialect-notes coverage test derives its required coverage from this set. */
+export const KTX_DATABASE_DRIVER_IDS = [
     'sqlite',
     'postgres',
     'mysql',
@@ -6,12 +7,15 @@ const KTX_DATABASE_DRIVER_IDS = new Set([
     'sqlserver',
     'bigquery',
     'snowflake',
-]);
+];
+// mongodb is a database driver but has no SQL dialect, so it sits outside the
+// dialect-notes coverage set above.
+const databaseDriverIds = new Set([...KTX_DATABASE_DRIVER_IDS, 'mongodb']);
 export function normalizeConnectionDriver(connection) {
     return String(connection.driver ?? '')
         .trim()
         .toLowerCase();
 }
 export function isDatabaseDriver(driver) {
-    return KTX_DATABASE_DRIVER_IDS.has(driver.trim().toLowerCase());
+    return databaseDriverIds.has(driver.trim().toLowerCase());
 }

package/dist/connection.d.ts

@@ -1,6 +1,7 @@
 import type { LookerClient } from './context/ingest/adapters/looker/client.js';
 import type { MetabaseRuntimeClient } from './context/ingest/adapters/metabase/client-port.js';
 import { NotionClient } from './context/ingest/adapters/notion/notion-client.js';
+import { createGoogleDocsClients } from './context/ingest/adapters/gdrive/gdrive-client.js';
 import { testRepoConnection } from './context/ingest/repo-fetch.js';
 import { type KtxLocalProject } from './context/project/project.js';
 import type { KtxCliIo } from './index.js';
@@ -19,12 +20,14 @@ export type KtxConnectionArgs = {
 type MetabaseTestPort = Pick<MetabaseRuntimeClient, 'testConnection' | 'getDatabases' | 'cleanup'>;
 type LookerTestPort = Pick<LookerClient, 'testConnection'>;
 type NotionTestPort = Pick<NotionClient, 'retrieveBotUser'>;
+type GdriveTestPort = Pick<ReturnType<typeof createGoogleDocsClients>['drive'], 'listFiles' | 'getFile'>;
 type TestRepoConnection = typeof testRepoConnection;
 export interface KtxConnectionDeps {
     createScanConnector?: typeof createKtxCliScanConnector;
     createMetabaseClient?: (project: KtxLocalProject, connectionId: string) => Promise<MetabaseTestPort>;
     createLookerClient?: (project: KtxLocalProject, connectionId: string) => Promise<LookerTestPort>;
     createNotionClient?: (project: KtxLocalProject, connectionId: string) => Promise<NotionTestPort>;
+    createGdriveClient?: (project: KtxLocalProject, connectionId: string) => Promise<GdriveTestPort>;
     testRepoConnection?: TestRepoConnection;
 }
 export declare function runKtxConnection(args: KtxConnectionArgs, io?: KtxCliIo, deps?: KtxConnectionDeps): Promise<number>;

package/dist/connection.js

@@ -1,8 +1,11 @@
 import { DEFAULT_METABASE_CLIENT_CONFIG, DefaultMetabaseConnectionClientFactory } from './context/ingest/adapters/metabase/client.js';
 import { DefaultLookerConnectionClientFactory } from './context/ingest/adapters/looker/factory.js';
 import { NotionClient } from './context/ingest/adapters/notion/notion-client.js';
+import { parseGdriveConnectionConfig, resolveGdriveServiceAccountKey } from './context/connections/gdrive-config.js';
 import { createLocalLookerCredentialResolver } from './context/ingest/adapters/looker/local-looker.adapter.js';
 import { metabaseRuntimeConfigFromLocalConnection } from './context/ingest/adapters/metabase/local-metabase.adapter.js';
+import { createGoogleDocsClients, verifyGdriveFolderAndCountDocs } from './context/ingest/adapters/gdrive/gdrive-client.js';
+import { gdriveServiceAccountKeySchema } from './context/ingest/adapters/gdrive/types.js';
 import { testRepoConnection } from './context/ingest/repo-fetch.js';
 import { federatedConnectionListing } from './context/connections/federation.js';
 import { getDriverRegistration } from './context/connections/drivers.js';
@@ -28,6 +31,7 @@ const SUPPORTED_TEST_DRIVERS = [
     'metabase',
     'looker',
     'notion',
+    'gdrive',
     'dbt',
     'metricflow',
     'lookml',
@@ -120,6 +124,25 @@ async function testNotionConnection(project, connectionId, createClient) {
     const bot = await client.retrieveBotUser();
     return { bot: describeNotionBot(bot) };
 }
+async function createDefaultGdriveClient(project, connectionId) {
+    const connection = project.config.connections[connectionId];
+    if (!connection) {
+        throw new Error(`Connection "${connectionId}" is not configured in ktx.yaml`);
+    }
+    const parsed = parseGdriveConnectionConfig(connection);
+    const keyText = await resolveGdriveServiceAccountKey(parsed.service_account_key_ref);
+    const key = gdriveServiceAccountKeySchema.parse(JSON.parse(keyText));
+    return createGoogleDocsClients(key).drive;
+}
+async function testGdriveConnection(project, connectionId, createClient) {
+    const connection = project.config.connections[connectionId];
+    if (!connection) {
+        throw new Error(`Connection "${connectionId}" is not configured in ktx.yaml`);
+    }
+    const parsed = parseGdriveConnectionConfig(connection);
+    const client = await createClient(project, connectionId);
+    return { docs: await verifyGdriveFolderAndCountDocs(client, parsed.folder_id) };
+}
 function extractGitConnectionFields(project, connectionId, driver) {
     const connection = project.config.connections[connectionId];
     if (!connection) {
@@ -164,6 +187,10 @@ async function testConnectionByDriver(project, connectionId, deps) {
         const result = await testNotionConnection(project, connectionId, deps.createNotionClient ?? createDefaultNotionClient);
         return { driver, detailKey: 'Bot', detailValue: result.bot };
     }
+    if (driver === 'gdrive') {
+        const result = await testGdriveConnection(project, connectionId, deps.createGdriveClient ?? createDefaultGdriveClient);
+        return { driver, detailKey: 'Docs', detailValue: String(result.docs) };
+    }
     if (driver === 'dbt' || driver === 'metricflow' || driver === 'lookml') {
         const result = await testGitRepoConnection(project, connectionId, driver, deps.testRepoConnection ?? testRepoConnection);
         return { driver, detailKey: 'Repo', detailValue: result.repoUrl };

package/dist/connectors/bigquery/connector.d.ts

@@ -7,13 +7,23 @@ export interface KtxBigQueryConnectionConfig {
     credentials_json?: string;
     location?: string;
     max_bytes_billed?: number | string;
-    job_timeout_ms?: number;
+    query_timeout_ms?: number;
     [key: string]: unknown;
 }
+/**
+ * A dataset to introspect, paired with the project that hosts it. `project`
+ * defaults to the billing project (`credentials.project_id`) when an entry has
+ * no `project.` prefix; a fully-qualified `project.dataset` entry resolves to
+ * its own host project. Jobs always bill in `credentials.project_id`.
+ */
+export interface BigQueryDatasetRef {
+    project: string;
+    dataset: string;
+}
 export interface KtxBigQueryResolvedConnectionConfig {
     projectId: string;
     credentials: Record<string, unknown>;
-    datasetIds: string[];
+    datasetIds: BigQueryDatasetRef[];
     location?: string;
 }
 export interface KtxBigQueryReadOnlyQueryInput extends KtxReadOnlyQueryInput {
@@ -72,7 +82,7 @@ export interface KtxBigQueryClient {
     }): Promise<[Array<{
         id?: string;
     }>, ...unknown[]]>;
-    dataset(datasetId: string): KtxBigQueryDataset;
+    dataset(datasetId: string, projectId: string): KtxBigQueryDataset;
     createQueryJob(input: {
         query: string;
         location?: string;
@@ -94,7 +104,6 @@ export interface KtxBigQueryScanConnectorOptions {
     env?: NodeJS.ProcessEnv;
     now?: () => Date;
     maxBytesBilled?: number | string;
-    queryTimeoutMs?: number;
 }
 /** @internal */
 export declare function prepareBigQueryReadOnlyQuery(sql: string, params?: Record<string, unknown>): {
@@ -117,7 +126,7 @@ export declare class KtxBigQueryScanConnector implements KtxScanConnector {
     private readonly clientFactory;
     private readonly now;
     private readonly maxBytesBilled?;
-    private readonly queryTimeoutMs?;
+    private readonly deadlineMs;
     private readonly dialect;
     private client;
     constructor(options: KtxBigQueryScanConnectorOptions);
@@ -130,14 +139,16 @@ export declare class KtxBigQueryScanConnector implements KtxScanConnector {
     columnStats(_input: KtxColumnStatsInput, _ctx: KtxScanContext): Promise<KtxColumnStatsResult | null>;
     executeReadOnly(input: KtxBigQueryReadOnlyQueryInput, _ctx: KtxScanContext): Promise<KtxQueryResult>;
     getColumnDistinctValues(table: KtxTableRef, columnName: string, options: KtxBigQueryColumnDistinctValuesOptions): Promise<KtxBigQueryColumnDistinctValuesResult | null>;
-    getTableRowCount(tableName: string, datasetId?: string): Promise<number>;
+    getTableRowCount(tableName: string, ref?: BigQueryDatasetRef | undefined): Promise<number>;
     qTableName(table: Pick<KtxTableRef, 'name'> & Partial<Pick<KtxTableRef, 'catalog' | 'db'>>): string;
     quoteIdentifier(identifier: string): string;
     listSchemas(): Promise<string[]>;
     listTables(datasetIds?: string[]): Promise<KtxTableListEntry[]>;
+    private listTablesInProject;
     cleanup(): Promise<void>;
     private getClient;
     private requireDatasetIdsForScan;
+    private qualifiedDatasetLabel;
     private query;
     private queryRaw;
     private singleNumber;