@kaelio/ktx 0.0.0 → 0.1.0-rc.0

package/node_modules/@ktx/context/dist/ingest/adapters/looker/local-runtime-store.d.ts

@@ -0,0 +1,50 @@
+import type { LookerWarehouseConnectionInfo } from './client.js';
+import type { LookerConnectionMapping } from './mapping.js';
+import type { LookerRuntimeCursors } from './types.js';
+export type LocalLookerMappingSource = 'ktx.yaml' | 'cli' | 'refresh';
+interface LocalLookerRuntimeStoreOptions {
+    dbPath: string;
+    now?: () => Date;
+}
+export interface LocalLookerConnectionMappingListRow extends LookerConnectionMapping {
+    source: LocalLookerMappingSource;
+}
+export interface UpsertLocalLookerConnectionMappingInput {
+    lookerConnectionId: string;
+    lookerConnectionName: string;
+    ktxConnectionId: string | null;
+    source: LocalLookerMappingSource;
+}
+interface ApplyLocalLookerYamlBootstrapInput {
+    lookerConnectionId: string;
+    mappings: Array<{
+        lookerConnectionName: string;
+        ktxConnectionId: string | null;
+    }>;
+}
+export interface RefreshLocalLookerDiscoveredConnectionsInput {
+    lookerConnectionId: string;
+    discovered: LookerWarehouseConnectionInfo[];
+}
+export interface ClearLocalLookerMappingsInput {
+    lookerConnectionId: string;
+    lookerConnectionName?: string;
+}
+export interface LookerSourceStateReader {
+    readMappings(lookerConnectionId: string): Promise<LookerConnectionMapping[]>;
+    readCursors(lookerConnectionId: string): Promise<LookerRuntimeCursors>;
+}
+export declare class LocalLookerRuntimeStore implements LookerSourceStateReader {
+    private readonly db;
+    private readonly now;
+    constructor(options: LocalLookerRuntimeStoreOptions);
+    applyYamlBootstrap(input: ApplyLocalLookerYamlBootstrapInput): Promise<void>;
+    readCursors(lookerConnectionId: string): Promise<LookerRuntimeCursors>;
+    setCursors(lookerConnectionId: string, cursors: LookerRuntimeCursors): Promise<void>;
+    readMappings(lookerConnectionId: string): Promise<LookerConnectionMapping[]>;
+    listConnectionMappings(lookerConnectionId: string): Promise<LocalLookerConnectionMappingListRow[]>;
+    upsertConnectionMapping(input: UpsertLocalLookerConnectionMappingInput): Promise<void>;
+    refreshDiscoveredConnections(input: RefreshLocalLookerDiscoveredConnectionsInput): Promise<void>;
+    clearConnectionMappings(input: ClearLocalLookerMappingsInput): Promise<void>;
+}
+export {};

package/node_modules/@ktx/context/dist/ingest/adapters/looker/local-runtime-store.js

@@ -0,0 +1,196 @@
+import { mkdirSync } from 'node:fs';
+import { dirname } from 'node:path';
+import Database from 'better-sqlite3';
+export class LocalLookerRuntimeStore {
+    db;
+    now;
+    constructor(options) {
+        mkdirSync(dirname(options.dbPath), { recursive: true });
+        this.db = new Database(options.dbPath);
+        this.db.pragma('journal_mode = WAL');
+        this.db.pragma('foreign_keys = ON');
+        this.now = options.now ?? (() => new Date());
+        this.db.exec(`
+      CREATE TABLE IF NOT EXISTS local_looker_runtime_config (
+        looker_connection_id TEXT PRIMARY KEY,
+        dashboards_last_synced_at TEXT,
+        looks_last_synced_at TEXT,
+        updated_at TEXT NOT NULL
+      );
+
+      CREATE TABLE IF NOT EXISTS local_looker_connection_mappings (
+        looker_connection_id TEXT NOT NULL,
+        looker_connection_name TEXT NOT NULL,
+        ktx_connection_id TEXT,
+        looker_host TEXT,
+        looker_database TEXT,
+        looker_dialect TEXT,
+        source TEXT NOT NULL,
+        updated_at TEXT NOT NULL,
+        PRIMARY KEY (looker_connection_id, looker_connection_name)
+      );
+    `);
+    }
+    async applyYamlBootstrap(input) {
+        const timestamp = this.now().toISOString();
+        const apply = this.db.transaction(() => {
+            const existing = this.db.prepare(`
+        SELECT ktx_connection_id, source
+        FROM local_looker_connection_mappings
+        WHERE looker_connection_id = ? AND looker_connection_name = ?
+      `);
+            const insert = this.db.prepare(`
+        INSERT INTO local_looker_connection_mappings (
+          looker_connection_id,
+          looker_connection_name,
+          ktx_connection_id,
+          looker_host,
+          looker_database,
+          looker_dialect,
+          source,
+          updated_at
+        )
+        VALUES (?, ?, ?, NULL, NULL, NULL, 'ktx.yaml', ?)
+      `);
+            const updateRefreshRow = this.db.prepare(`
+        UPDATE local_looker_connection_mappings
+        SET ktx_connection_id = ?,
+            source = 'ktx.yaml',
+            updated_at = ?
+        WHERE looker_connection_id = ?
+          AND looker_connection_name = ?
+          AND source = 'refresh'
+          AND ktx_connection_id IS NULL
+      `);
+            for (const mapping of input.mappings) {
+                const row = existing.get(input.lookerConnectionId, mapping.lookerConnectionName);
+                if (!row) {
+                    insert.run(input.lookerConnectionId, mapping.lookerConnectionName, mapping.ktxConnectionId, timestamp);
+                    continue;
+                }
+                if (row.source === 'refresh' && row.ktx_connection_id === null) {
+                    updateRefreshRow.run(mapping.ktxConnectionId, timestamp, input.lookerConnectionId, mapping.lookerConnectionName);
+                }
+            }
+        });
+        apply();
+    }
+    async readCursors(lookerConnectionId) {
+        const row = this.db
+            .prepare(`
+        SELECT dashboards_last_synced_at, looks_last_synced_at
+        FROM local_looker_runtime_config
+        WHERE looker_connection_id = ?
+      `)
+            .get(lookerConnectionId);
+        return {
+            dashboardsLastSyncedAt: row?.dashboards_last_synced_at ?? null,
+            looksLastSyncedAt: row?.looks_last_synced_at ?? null,
+        };
+    }
+    async setCursors(lookerConnectionId, cursors) {
+        this.db
+            .prepare(`
+        INSERT INTO local_looker_runtime_config (
+          looker_connection_id,
+          dashboards_last_synced_at,
+          looks_last_synced_at,
+          updated_at
+        )
+        VALUES (?, ?, ?, ?)
+        ON CONFLICT(looker_connection_id) DO UPDATE SET
+          dashboards_last_synced_at = excluded.dashboards_last_synced_at,
+          looks_last_synced_at = excluded.looks_last_synced_at,
+          updated_at = excluded.updated_at
+      `)
+            .run(lookerConnectionId, cursors.dashboardsLastSyncedAt, cursors.looksLastSyncedAt, this.now().toISOString());
+    }
+    async readMappings(lookerConnectionId) {
+        return (await this.listConnectionMappings(lookerConnectionId)).map(({ source: _source, ...mapping }) => mapping);
+    }
+    async listConnectionMappings(lookerConnectionId) {
+        const rows = this.db
+            .prepare(`
+        SELECT
+          looker_connection_name,
+          ktx_connection_id,
+          looker_host,
+          looker_database,
+          looker_dialect,
+          source
+        FROM local_looker_connection_mappings
+        WHERE looker_connection_id = ?
+        ORDER BY looker_connection_name
+      `)
+            .all(lookerConnectionId);
+        return rows.map((row) => ({
+            lookerConnectionName: row.looker_connection_name,
+            ktxConnectionId: row.ktx_connection_id,
+            lookerHost: row.looker_host,
+            lookerDatabase: row.looker_database,
+            lookerDialect: row.looker_dialect,
+            source: row.source,
+        }));
+    }
+    async upsertConnectionMapping(input) {
+        this.db
+            .prepare(`
+        INSERT INTO local_looker_connection_mappings (
+          looker_connection_id,
+          looker_connection_name,
+          ktx_connection_id,
+          looker_host,
+          looker_database,
+          looker_dialect,
+          source,
+          updated_at
+        )
+        VALUES (?, ?, ?, NULL, NULL, NULL, ?, ?)
+        ON CONFLICT(looker_connection_id, looker_connection_name) DO UPDATE SET
+          ktx_connection_id = excluded.ktx_connection_id,
+          source = excluded.source,
+          updated_at = excluded.updated_at
+      `)
+            .run(input.lookerConnectionId, input.lookerConnectionName, input.ktxConnectionId, input.source, this.now().toISOString());
+    }
+    async refreshDiscoveredConnections(input) {
+        const timestamp = this.now().toISOString();
+        const update = this.db.transaction(() => {
+            const upsert = this.db.prepare(`
+        INSERT INTO local_looker_connection_mappings (
+          looker_connection_id,
+          looker_connection_name,
+          ktx_connection_id,
+          looker_host,
+          looker_database,
+          looker_dialect,
+          source,
+          updated_at
+        )
+        VALUES (?, ?, NULL, ?, ?, ?, 'refresh', ?)
+        ON CONFLICT(looker_connection_id, looker_connection_name) DO UPDATE SET
+          looker_host = excluded.looker_host,
+          looker_database = excluded.looker_database,
+          looker_dialect = excluded.looker_dialect,
+          source = excluded.source,
+          updated_at = excluded.updated_at
+      `);
+            for (const connection of input.discovered) {
+                upsert.run(input.lookerConnectionId, connection.name, connection.host, connection.database, connection.dialect, timestamp);
+            }
+        });
+        update();
+    }
+    async clearConnectionMappings(input) {
+        if (input.lookerConnectionName) {
+            this.db
+                .prepare(`
+          DELETE FROM local_looker_connection_mappings
+          WHERE looker_connection_id = ? AND looker_connection_name = ?
+        `)
+                .run(input.lookerConnectionId, input.lookerConnectionName);
+            return;
+        }
+        this.db.prepare('DELETE FROM local_looker_connection_mappings WHERE looker_connection_id = ?').run(input.lookerConnectionId);
+    }
+}

package/node_modules/@ktx/context/dist/ingest/adapters/looker/local-runtime-store.test.d.ts

@@ -0,0 +1 @@
+export {};

package/node_modules/@ktx/context/dist/ingest/adapters/looker/local-runtime-store.test.js

@@ -0,0 +1,106 @@
+import { mkdtemp } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { describe, expect, it } from 'vitest';
+import { LocalLookerRuntimeStore } from './local-runtime-store.js';
+describe('LocalLookerRuntimeStore', () => {
+    async function store() {
+        const dir = await mkdtemp(join(tmpdir(), 'ktx-looker-store-'));
+        return new LocalLookerRuntimeStore({
+            dbPath: join(dir, 'db.sqlite'),
+            now: () => new Date('2026-05-05T12:00:00.000Z'),
+        });
+    }
+    it('stores cursors and connection mappings', async () => {
+        const local = await store();
+        await local.setCursors('prod-looker', {
+            dashboardsLastSyncedAt: '2026-05-01T00:00:00.000Z',
+            looksLastSyncedAt: null,
+        });
+        await local.upsertConnectionMapping({
+            lookerConnectionId: 'prod-looker',
+            lookerConnectionName: 'bq_reporting',
+            ktxConnectionId: 'prod-warehouse',
+            source: 'cli',
+        });
+        await expect(local.readCursors('prod-looker')).resolves.toEqual({
+            dashboardsLastSyncedAt: '2026-05-01T00:00:00.000Z',
+            looksLastSyncedAt: null,
+        });
+        await expect(local.readMappings('prod-looker')).resolves.toEqual([
+            {
+                lookerConnectionName: 'bq_reporting',
+                ktxConnectionId: 'prod-warehouse',
+                lookerHost: null,
+                lookerDatabase: null,
+                lookerDialect: null,
+            },
+        ]);
+    });
+    it('refreshes discovered metadata without dropping local targets', async () => {
+        const local = await store();
+        await local.upsertConnectionMapping({
+            lookerConnectionId: 'prod-looker',
+            lookerConnectionName: 'bq_reporting',
+            ktxConnectionId: 'prod-warehouse',
+            source: 'cli',
+        });
+        await local.refreshDiscoveredConnections({
+            lookerConnectionId: 'prod-looker',
+            discovered: [
+                {
+                    name: 'bq_reporting',
+                    host: 'bigquery.googleapis.com',
+                    database: 'analytics',
+                    schema: null,
+                    dialect: 'bigquery_standard_sql',
+                },
+            ],
+        });
+        await expect(local.listConnectionMappings('prod-looker')).resolves.toEqual([
+            {
+                lookerConnectionName: 'bq_reporting',
+                ktxConnectionId: 'prod-warehouse',
+                lookerHost: 'bigquery.googleapis.com',
+                lookerDatabase: 'analytics',
+                lookerDialect: 'bigquery_standard_sql',
+                source: 'refresh',
+            },
+        ]);
+    });
+    it('applies yaml mapping intent while preserving refresh metadata and cli overrides', async () => {
+        const local = await store();
+        await local.refreshDiscoveredConnections({
+            lookerConnectionId: 'prod-looker',
+            discovered: [{ name: 'analytics', host: 'looker-db.test', database: 'warehouse', schema: null, dialect: 'postgres' }],
+        });
+        await local.upsertConnectionMapping({
+            lookerConnectionId: 'prod-looker',
+            lookerConnectionName: 'manual',
+            ktxConnectionId: 'cli-warehouse',
+            source: 'cli',
+        });
+        await local.applyYamlBootstrap({
+            lookerConnectionId: 'prod-looker',
+            mappings: [
+                { lookerConnectionName: 'analytics', ktxConnectionId: 'yaml-warehouse' },
+                { lookerConnectionName: 'manual', ktxConnectionId: 'yaml-warehouse' },
+            ],
+        });
+        await expect(local.listConnectionMappings('prod-looker')).resolves.toMatchObject([
+            {
+                lookerConnectionName: 'analytics',
+                ktxConnectionId: 'yaml-warehouse',
+                lookerHost: 'looker-db.test',
+                lookerDatabase: 'warehouse',
+                lookerDialect: 'postgres',
+                source: 'ktx.yaml',
+            },
+            {
+                lookerConnectionName: 'manual',
+                ktxConnectionId: 'cli-warehouse',
+                source: 'cli',
+            },
+        ]);
+    });
+});

package/node_modules/@ktx/context/dist/ingest/adapters/looker/looker.adapter.d.ts

@@ -0,0 +1,57 @@
+import type { ChunkResult, DiffSet, FetchContext, IngestTrigger, ScopeDescriptor, SourceAdapter } from '../../types.js';
+import { type LookerClientFactory } from './fetch.js';
+interface LookerPullSucceededContext {
+    connectionId: string;
+    sourceKey: string;
+    syncId: string;
+    trigger: IngestTrigger;
+    completedAt: Date;
+    stagedDir: string;
+}
+export interface LookerSourceAdapterDeps {
+    clientFactory: LookerClientFactory;
+    now?: () => Date;
+    onPullSucceeded?: (ctx: LookerPullSucceededContext) => Promise<void>;
+}
+export declare class LookerSourceAdapter implements SourceAdapter {
+    private readonly deps;
+    readonly source = "looker";
+    readonly skillNames: string[];
+    readonly evidenceIndexing: "documents";
+    readonly triageSupported = true;
+    constructor(deps: LookerSourceAdapterDeps);
+    detect(stagedDir: string): Promise<boolean>;
+    fetch(pullConfig: unknown, stagedDir: string, ctx: FetchContext): Promise<void>;
+    chunk(stagedDir: string, diffSet?: DiffSet): Promise<ChunkResult>;
+    readFetchReport(stagedDir: string): Promise<{
+        status: "success" | "partial";
+        retryRecommended: boolean;
+        skipped: {
+            rawPath: string;
+            entityType: "dashboard" | "look" | "explore" | "signals" | "lookml_models" | "looker_connection_mapping";
+            entityId: string | null;
+            severity: "error" | "warning";
+            statusCode: number | null;
+            message: string;
+            retryRecommended: boolean;
+            kind?: "lookml_connection_mismatch" | "unmapped_looker_connection" | "unparseable_sql_table_name" | "looker_template_unresolved" | "derived_table_not_supported" | undefined;
+            details?: Record<string, unknown> | undefined;
+        }[];
+        warnings: {
+            rawPath: string;
+            entityType: "dashboard" | "look" | "explore" | "signals" | "lookml_models" | "looker_connection_mapping";
+            entityId: string | null;
+            severity: "error" | "warning";
+            statusCode: number | null;
+            message: string;
+            retryRecommended: boolean;
+            kind?: "lookml_connection_mismatch" | "unmapped_looker_connection" | "unparseable_sql_table_name" | "looker_template_unresolved" | "derived_table_not_supported" | undefined;
+            details?: Record<string, unknown> | undefined;
+        }[];
+    } | null>;
+    listTargetConnectionIds(stagedDir: string): Promise<string[]>;
+    getTriageSignals(stagedDir: string, externalId: string): Promise<import("../../types.js").TriageSignals>;
+    describeScope(stagedDir: string): Promise<ScopeDescriptor>;
+    onPullSucceeded(ctx: LookerPullSucceededContext): Promise<void>;
+}
+export {};

package/node_modules/@ktx/context/dist/ingest/adapters/looker/looker.adapter.js

@@ -0,0 +1,47 @@
+import { chunkLookerStagedDir } from './chunk.js';
+import { detectLookerStagedDir } from './detect.js';
+import { getLookerTriageSignals } from './evidence-documents.js';
+import { fetchLookerRuntimeBundle } from './fetch.js';
+import { readLookerFetchReport } from './fetch-report.js';
+import { describeLookerScope } from './scope.js';
+import { listLookerTargetConnectionIds } from './target-connections.js';
+export class LookerSourceAdapter {
+    deps;
+    source = 'looker';
+    skillNames = ['looker_ingest'];
+    evidenceIndexing = 'documents';
+    triageSupported = true;
+    constructor(deps) {
+        this.deps = deps;
+    }
+    detect(stagedDir) {
+        return detectLookerStagedDir(stagedDir);
+    }
+    fetch(pullConfig, stagedDir, ctx) {
+        return fetchLookerRuntimeBundle({
+            pullConfig,
+            stagedDir,
+            ctx,
+            clientFactory: this.deps.clientFactory,
+            now: this.deps.now,
+        });
+    }
+    chunk(stagedDir, diffSet) {
+        return chunkLookerStagedDir(stagedDir, diffSet);
+    }
+    readFetchReport(stagedDir) {
+        return readLookerFetchReport(stagedDir);
+    }
+    listTargetConnectionIds(stagedDir) {
+        return listLookerTargetConnectionIds(stagedDir);
+    }
+    getTriageSignals(stagedDir, externalId) {
+        return getLookerTriageSignals(stagedDir, externalId);
+    }
+    describeScope(stagedDir) {
+        return describeLookerScope(stagedDir);
+    }
+    async onPullSucceeded(ctx) {
+        await this.deps.onPullSucceeded?.(ctx);
+    }
+}

package/node_modules/@ktx/context/dist/ingest/adapters/looker/looker.adapter.test.d.ts

@@ -0,0 +1 @@
+export {};

package/node_modules/@ktx/context/dist/ingest/adapters/looker/looker.adapter.test.js

@@ -0,0 +1,99 @@
+import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+import { LookerSourceAdapter } from './looker.adapter.js';
+const connectionId = '11111111-1111-4111-8111-111111111111';
+function makeClient() {
+    return {
+        listDashboards: vi.fn().mockResolvedValue([]),
+        getDashboard: vi.fn(),
+        listLooks: vi.fn().mockResolvedValue([]),
+        getLook: vi.fn(),
+        listFolders: vi.fn().mockResolvedValue({ folders: [] }),
+        listUsers: vi.fn().mockResolvedValue([]),
+        listGroups: vi.fn().mockResolvedValue([]),
+        listLookmlModels: vi.fn().mockResolvedValue({
+            models: [{ name: 'b2b', label: 'B2B', explores: [{ name: 'sales_pipeline', label: 'Sales Pipeline' }] }],
+        }),
+        getExplore: vi.fn().mockResolvedValue({
+            modelName: 'b2b',
+            exploreName: 'sales_pipeline',
+            label: 'Sales Pipeline',
+            description: null,
+            fields: { dimensions: [], measures: [] },
+            joins: [],
+        }),
+    };
+}
+describe('LookerSourceAdapter', () => {
+    let stagedDir;
+    beforeEach(async () => {
+        stagedDir = await mkdtemp(join(tmpdir(), 'looker-adapter-'));
+    });
+    afterEach(async () => {
+        await rm(stagedDir, { recursive: true, force: true });
+    });
+    it('exposes source="looker" and skillNames=["looker_ingest"]', () => {
+        const adapter = new LookerSourceAdapter({ clientFactory: { createClient: () => makeClient() } });
+        expect(adapter.source).toBe('looker');
+        expect(adapter.skillNames).toEqual(['looker_ingest']);
+    });
+    it('enables context evidence indexing and delegates triage signals', async () => {
+        const adapter = new LookerSourceAdapter({ clientFactory: { createClient: () => makeClient() } });
+        expect(adapter.evidenceIndexing).toBe('documents');
+        expect(adapter.triageSupported).toBe(true);
+        await expect(adapter.getTriageSignals?.(stagedDir, 'looker:dashboard:10')).resolves.toMatchObject({
+            objectType: 'looker_dashboard',
+        });
+    });
+    it('fetches, detects, and chunks a runtime bundle through the composed adapter', async () => {
+        const adapter = new LookerSourceAdapter({
+            clientFactory: { createClient: vi.fn().mockResolvedValue(makeClient()) },
+            now: () => new Date('2026-04-30T12:30:00.000Z'),
+        });
+        await mkdir(stagedDir, { recursive: true });
+        await adapter.fetch({ lookerConnectionId: connectionId }, stagedDir, { connectionId, sourceKey: 'looker' });
+        expect(await adapter.detect(stagedDir)).toBe(true);
+        expect(await readFile(join(stagedDir, 'explores/b2b/sales_pipeline.json'), 'utf-8')).toContain('sales_pipeline');
+        const result = await adapter.chunk(stagedDir);
+        expect(result.workUnits.map((wu) => wu.unitKey)).toEqual(['looker-explore-b2b-sales_pipeline']);
+    });
+    it('passes pull success notifications to the server callback', async () => {
+        const onPullSucceeded = vi.fn().mockResolvedValue(undefined);
+        const adapter = new LookerSourceAdapter({
+            clientFactory: { createClient: () => makeClient() },
+            onPullSucceeded,
+        });
+        const completedAt = new Date('2026-04-30T12:00:00.000Z');
+        await adapter.onPullSucceeded({
+            connectionId,
+            sourceKey: 'looker',
+            syncId: 'sync-1',
+            trigger: 'scheduled_pull',
+            completedAt,
+            stagedDir: '/tmp/staged',
+        });
+        expect(onPullSucceeded).toHaveBeenCalledWith({
+            connectionId,
+            sourceKey: 'looker',
+            syncId: 'sync-1',
+            trigger: 'scheduled_pull',
+            completedAt,
+            stagedDir: '/tmp/staged',
+        });
+    });
+    it('describes incremental fetch scope from the staged scope file', async () => {
+        await mkdir(join(stagedDir, 'dashboards'), { recursive: true });
+        await writeFile(join(stagedDir, 'looker-scope.json'), JSON.stringify({
+            mode: 'incremental',
+            knownCurrentRawPaths: ['dashboards/10.json', 'dashboards/11.json'],
+            fetchedRawPaths: ['dashboards/11.json'],
+        }, null, 2));
+        const adapter = new LookerSourceAdapter({ clientFactory: { createClient: () => makeClient() } });
+        const scope = await adapter.describeScope(stagedDir);
+        expect(scope.isPathInScope('dashboards/10.json')).toBe(false);
+        expect(scope.isPathInScope('dashboards/11.json')).toBe(true);
+        expect(scope.isPathInScope('dashboards/12.json')).toBe(true);
+    });
+});