@kaelio/ktx 0.0.0 → 0.1.0-rc.0

package/node_modules/@ktx/context/dist/ingest/adapters/dbt-extraction-golden-parity.test.js

@@ -0,0 +1,392 @@
+import { describe, expect, it } from 'vitest';
+import { matchDbtTables } from './dbt-descriptions/match-tables.js';
+import { mergeSemanticModelTables } from './dbt-descriptions/merge-semantic-model-tables.js';
+import { parseDbtSchemaFiles } from './dbt-descriptions/parse-schema.js';
+import { toDescriptionUpdates } from './dbt-descriptions/to-description-updates.js';
+import { toRelationshipUpdates } from './dbt-descriptions/to-relationship-updates.js';
+import { parseMetricflowFiles } from './metricflow/deep-parse.js';
+import { mapCrossModelMetricToSource, mapSemanticModelToSource } from './metricflow/semantic-models.js';
+const DBT_SYSTEM_EMAIL = ['system@kae', 'lio.dev'].join('');
+const metricflowYaml = `
+semantic_models:
+  - name: orders_semantic
+    description: MetricFlow order facts
+    model: ref('fct_orders')
+    defaults:
+      agg_time_dimension: ordered_at
+    entities:
+      - name: customer
+        type: foreign
+        expr: customer_id
+        description: Customer relationship
+    dimensions:
+      - name: status
+        type: categorical
+        expr: status
+        description: Order status
+      - name: ordered_at
+        type: time
+        expr: ordered_at
+    measures:
+      - name: total_revenue
+        agg: sum
+        expr: amount
+        description: Revenue
+  - name: customers_semantic
+    description: Customer dimension
+    model: ref('dim_customers')
+    entities:
+      - name: customer
+        type: primary
+        expr: id
+    dimensions:
+      - name: country
+        type: categorical
+        expr: country
+        description: Customer country
+    measures:
+      - name: customer_count
+        agg: count
+        expr: id
+        description: Customer count
+metrics:
+  - name: total_revenue
+    type: simple
+    type_params:
+      measure: total_revenue
+  - name: customer_count
+    type: simple
+    type_params:
+      measure: customer_count
+  - name: revenue_per_customer
+    description: Revenue per customer
+    type: derived
+    type_params:
+      expr: total_revenue / NULLIF(customer_count, 0)
+      metrics:
+        - name: total_revenue
+          alias: total_revenue
+        - name: customer_count
+          alias: customer_count
+`;
+const schemaYaml = `
+version: 2
+sources:
+  - name: raw
+    database: warehouse
+    schema: landing
+    tables:
+      - name: customers
+        identifier: dim_customers
+        description: Raw customer dimension
+        columns:
+          - name: id
+            description: Customer primary key
+          - name: country
+            description: Country name
+models:
+  - name: "{{ var('orders_model', 'fct_orders') }}"
+    schema: "{{ var('mart_schema', 'analytics') }}"
+    description: Modeled orders
+    columns:
+      - name: customer_id
+        description: Linked customer id
+        tests:
+          - relationships:
+              to: ref('dim_customers')
+              field: id
+      - name: status
+        description: Order status
+      - name: amount
+        description: Gross amount
+`;
+const hostTables = [
+    {
+        id: 'orders-table',
+        name: 'fct_orders',
+        catalog: 'warehouse',
+        db: 'analytics',
+        columns: [
+            { id: 'orders-customer-id', name: 'customer_id' },
+            { id: 'orders-status', name: 'status' },
+            { id: 'orders-amount', name: 'amount' },
+            { id: 'orders-ordered-at', name: 'ordered_at' },
+        ],
+    },
+    {
+        id: 'customers-table',
+        name: 'dim_customers',
+        catalog: 'warehouse',
+        db: 'landing',
+        columns: [
+            { id: 'customers-id', name: 'id' },
+            { id: 'customers-country', name: 'country' },
+        ],
+    },
+];
+describe('dbt extraction golden parity fixture', () => {
+    it('freezes the relocated MetricFlow and dbt-description contract together', () => {
+        const metricflow = parseMetricflowFiles([{ path: 'semantic_models/orders.yml', content: metricflowYaml }]);
+        expect(metricflow).toEqual({
+            semanticModels: [
+                {
+                    name: 'orders_semantic',
+                    description: 'MetricFlow order facts',
+                    modelRef: 'fct_orders',
+                    dimensions: [
+                        {
+                            name: 'status',
+                            column: 'status',
+                            type: 'string',
+                            label: 'Status',
+                            description: 'Order status',
+                        },
+                        {
+                            name: 'ordered_at',
+                            column: 'ordered_at',
+                            type: 'time',
+                            label: 'Ordered At',
+                            description: undefined,
+                        },
+                    ],
+                    measures: [
+                        {
+                            type: 'simple',
+                            name: 'total_revenue',
+                            column: 'amount',
+                            aggregation: 'sum',
+                            label: 'Total Revenue',
+                            description: 'Revenue',
+                        },
+                    ],
+                    entities: [{ name: 'customer', type: 'foreign', expr: 'customer_id', description: 'Customer relationship' }],
+                    defaultTimeDimension: 'ordered_at',
+                },
+                {
+                    name: 'customers_semantic',
+                    description: 'Customer dimension',
+                    modelRef: 'dim_customers',
+                    dimensions: [
+                        {
+                            name: 'country',
+                            column: 'country',
+                            type: 'string',
+                            label: 'Country',
+                            description: 'Customer country',
+                        },
+                    ],
+                    measures: [
+                        {
+                            type: 'simple',
+                            name: 'customer_count',
+                            column: 'id',
+                            aggregation: 'count',
+                            label: 'Customer Count',
+                            description: 'Customer count',
+                        },
+                    ],
+                    entities: [{ name: 'customer', type: 'primary', expr: 'id' }],
+                    defaultTimeDimension: null,
+                },
+            ],
+            crossModelMetrics: [
+                {
+                    name: 'revenue_per_customer',
+                    label: null,
+                    description: 'Revenue per customer',
+                    type: 'derived',
+                    expr: 'total_revenue / NULLIF(customer_count, 0)',
+                    dependsOn: [
+                        { metricName: 'orders_semantic', alias: 'total_revenue' },
+                        { metricName: 'customers_semantic', alias: 'customer_count' },
+                    ],
+                    filter: null,
+                },
+            ],
+            relationships: [
+                {
+                    fromTable: 'fct_orders',
+                    fromColumn: 'customer_id',
+                    toTable: 'dim_customers',
+                    toColumn: 'id',
+                    description: 'Customer relationship',
+                },
+            ],
+            warnings: [],
+        });
+        expect(mapSemanticModelToSource(metricflow.semanticModels[0], 'analytics.fct_orders')).toEqual({
+            name: 'fct-orders',
+            table: 'analytics.fct_orders',
+            grain: ['status', 'ordered_at'],
+            columns: [
+                { name: 'status', type: 'string', description: 'Order status' },
+                { name: 'ordered_at', type: 'time' },
+            ],
+            measures: [
+                {
+                    name: 'total_revenue',
+                    expr: 'sum(amount)',
+                    description: 'Revenue',
+                },
+            ],
+            joins: [],
+            descriptions: { dbt: 'MetricFlow order facts' },
+        });
+        expect(mapCrossModelMetricToSource(metricflow.crossModelMetrics[0])).toEqual({
+            name: 'revenue-per-customer',
+            sql: 'total_revenue / NULLIF(customer_count, 0)',
+            descriptions: { dbt: 'Revenue per customer' },
+            grain: [],
+            columns: [],
+            measures: [
+                {
+                    name: 'revenue_per_customer',
+                    expr: 'total_revenue / NULLIF(customer_count, 0)',
+                    description: 'Revenue per customer',
+                },
+            ],
+            joins: [],
+        });
+        const schema = parseDbtSchemaFiles([{ path: 'models/schema.yml', content: schemaYaml }], new Map([
+            ['orders_model', 'fct_orders'],
+            ['mart_schema', 'analytics'],
+        ]));
+        const merged = mergeSemanticModelTables(schema, metricflow.semanticModels);
+        expect(merged).toEqual({
+            projectName: null,
+            dbtVersion: null,
+            tables: [
+                {
+                    name: 'dim_customers',
+                    description: 'Raw customer dimension',
+                    database: 'warehouse',
+                    schema: 'landing',
+                    columns: [
+                        { name: 'id', description: 'Customer primary key', dataType: null },
+                        { name: 'country', description: 'Country name', dataType: null },
+                    ],
+                    resourceType: 'source',
+                },
+                {
+                    name: 'fct_orders',
+                    description: 'Modeled orders',
+                    database: null,
+                    schema: 'analytics',
+                    columns: [
+                        {
+                            name: 'customer_id',
+                            description: 'Linked customer id',
+                            dataType: null,
+                            dataTests: [
+                                {
+                                    name: 'relationships',
+                                    package: 'dbt',
+                                    kwargs: { to: "ref('dim_customers')", field: 'id' },
+                                },
+                            ],
+                        },
+                        { name: 'status', description: 'Order status', dataType: null },
+                        { name: 'amount', description: 'Gross amount', dataType: null },
+                    ],
+                    resourceType: 'model',
+                },
+            ],
+            relationships: [
+                {
+                    fromTable: 'fct_orders',
+                    fromColumn: 'customer_id',
+                    toTable: 'dim_customers',
+                    toColumn: 'id',
+                    fromSchema: 'analytics',
+                },
+            ],
+        });
+        expect(matchDbtTables(merged.tables, hostTables, 'analytics')).toEqual([
+            {
+                dbtTable: 'dim_customers',
+                dbtSchema: 'landing',
+                dbtDatabase: 'warehouse',
+                hostTableId: 'customers-table',
+                hostTableName: 'dim_customers',
+                matched: true,
+                tableDescriptionAction: 'import',
+                tableDescriptionFound: true,
+                columnsToImport: 2,
+                columnsMatched: 2,
+                columnsTotal: 2,
+                columnDescriptionsFound: 2,
+            },
+            {
+                dbtTable: 'fct_orders',
+                dbtSchema: 'analytics',
+                dbtDatabase: null,
+                hostTableId: 'orders-table',
+                hostTableName: 'fct_orders',
+                matched: true,
+                tableDescriptionAction: 'import',
+                tableDescriptionFound: true,
+                columnsToImport: 3,
+                columnsMatched: 3,
+                columnsTotal: 3,
+                columnDescriptionsFound: 3,
+            },
+        ]);
+        expect(toDescriptionUpdates({
+            connectionId: 'warehouse-1',
+            parseResult: merged,
+            hostTables,
+            targetSchema: 'analytics',
+        })).toEqual({
+            dbt: [
+                {
+                    connectionId: 'warehouse-1',
+                    table: { catalog: 'warehouse', db: 'landing', name: 'dim_customers' },
+                    source: 'dbt',
+                    tableDescription: 'Raw customer dimension',
+                    columnDescriptions: {
+                        id: 'Customer primary key',
+                        country: 'Country name',
+                    },
+                },
+                {
+                    connectionId: 'warehouse-1',
+                    table: { catalog: 'warehouse', db: 'analytics', name: 'fct_orders' },
+                    source: 'dbt',
+                    tableDescription: 'Modeled orders',
+                    columnDescriptions: {
+                        customer_id: 'Linked customer id',
+                        status: 'Order status',
+                        amount: 'Gross amount',
+                    },
+                },
+            ],
+            aiInvalidations: [
+                {
+                    connectionId: 'warehouse-1',
+                    table: { catalog: 'warehouse', db: 'landing', name: 'dim_customers' },
+                    source: 'ai',
+                },
+                {
+                    connectionId: 'warehouse-1',
+                    table: { catalog: 'warehouse', db: 'analytics', name: 'fct_orders' },
+                    source: 'ai',
+                },
+            ],
+        });
+        expect(toRelationshipUpdates({ connectionId: 'warehouse-1', parseResult: merged, hostTables })).toEqual({
+            joins: [
+                {
+                    connectionId: 'warehouse-1',
+                    fromTable: 'fct_orders',
+                    fromColumns: ['customer_id'],
+                    toTable: 'dim_customers',
+                    toColumns: ['id'],
+                    relationship: 'many_to_one',
+                    author: 'dbt',
+                    authorEmail: DBT_SYSTEM_EMAIL,
+                },
+            ],
+            skippedNoMatch: 0,
+        });
+    });
+});

package/node_modules/@ktx/context/dist/ingest/adapters/fake/fake.adapter.d.ts

@@ -0,0 +1,7 @@
+import type { ChunkResult, DiffSet, SourceAdapter } from '../../types.js';
+export declare class FakeSourceAdapter implements SourceAdapter {
+    readonly source = "fake";
+    readonly skillNames: string[];
+    detect(): Promise<boolean>;
+    chunk(stagedDir: string, diffSet?: DiffSet): Promise<ChunkResult>;
+}

package/node_modules/@ktx/context/dist/ingest/adapters/fake/fake.adapter.js

@@ -0,0 +1,42 @@
+import { readdir } from 'node:fs/promises';
+import { join, relative } from 'node:path';
+export class FakeSourceAdapter {
+    source = 'fake';
+    skillNames = [];
+    detect() {
+        return Promise.resolve(true);
+    }
+    async chunk(stagedDir, diffSet) {
+        const subDirs = (await readdir(stagedDir, { withFileTypes: true }))
+            .filter((e) => e.isDirectory())
+            .map((e) => e.name)
+            .sort();
+        const workUnits = [];
+        for (const subDir of subDirs) {
+            const entries = await readdir(join(stagedDir, subDir), { withFileTypes: true, recursive: true });
+            const rawFiles = entries
+                .filter((e) => e.isFile())
+                .map((e) => relative(stagedDir, join(e.parentPath, e.name)))
+                .sort();
+            if (rawFiles.length === 0) {
+                continue;
+            }
+            if (diffSet) {
+                const touched = new Set([...diffSet.added, ...diffSet.modified]);
+                const anyTouched = rawFiles.some((p) => touched.has(p));
+                if (!anyTouched) {
+                    continue;
+                }
+            }
+            workUnits.push({
+                unitKey: `fake-${subDir}`,
+                displayLabel: subDir,
+                rawFiles,
+                peerFileIndex: [],
+                dependencyPaths: [],
+            });
+        }
+        const eviction = diffSet && diffSet.deleted.length > 0 ? { deletedRawPaths: [...diffSet.deleted] } : undefined;
+        return { workUnits, eviction };
+    }
+}

package/node_modules/@ktx/context/dist/ingest/adapters/historic-sql/bigquery-query-history-reader.d.ts

@@ -0,0 +1,11 @@
+import type { HistoricSqlQueryHistoryReader, HistoricSqlRawQueryRow, HistoricSqlTimeWindow } from './types.js';
+export interface BigQueryHistoricSqlQueryHistoryReaderOptions {
+    projectId: string;
+    region: string;
+}
+export declare class BigQueryHistoricSqlQueryHistoryReader implements HistoricSqlQueryHistoryReader {
+    private readonly viewPath;
+    constructor(options: BigQueryHistoricSqlQueryHistoryReaderOptions);
+    probe(client: unknown): Promise<void>;
+    fetch(client: unknown, window: HistoricSqlTimeWindow, cursor?: string | null): AsyncIterable<HistoricSqlRawQueryRow>;
+}

package/node_modules/@ktx/context/dist/ingest/adapters/historic-sql/bigquery-query-history-reader.js

@@ -0,0 +1,175 @@
+import { HistoricSqlGrantsMissingError } from './errors.js';
+const BIGQUERY_GRANTS_REMEDIATION = 'Grant roles/bigquery.resourceViewer on the BigQuery project, or grant a custom role containing bigquery.jobs.listAll.';
+function queryClient(client) {
+    if (client &&
+        typeof client === 'object' &&
+        'executeQuery' in client &&
+        typeof client.executeQuery === 'function') {
+        return client;
+    }
+    throw new Error('Historic SQL BigQuery reader requires a query client with executeQuery(query)');
+}
+function grantsError(cause) {
+    const message = cause instanceof Error
+        ? cause.message
+        : typeof cause === 'string'
+            ? cause
+            : 'BigQuery principal cannot query INFORMATION_SCHEMA.JOBS_BY_PROJECT.';
+    return new HistoricSqlGrantsMissingError({
+        dialect: 'bigquery',
+        message: `Missing BigQuery audit grants for historic-SQL ingest: ${message}`,
+        remediation: BIGQUERY_GRANTS_REMEDIATION,
+        cause,
+    });
+}
+function normalizeProjectId(value) {
+    if (!/^[A-Za-z0-9_-]+$/.test(value)) {
+        throw new Error(`Invalid BigQuery project id for historic-SQL ingest: ${value}`);
+    }
+    return value;
+}
+function normalizeRegion(value) {
+    const region = value.trim().toLowerCase().replace(/^region-/, '');
+    if (!/^[a-z0-9-]+$/.test(region)) {
+        throw new Error(`Invalid BigQuery region for historic-SQL ingest: ${value}`);
+    }
+    return region;
+}
+function timestampExpression(value) {
+    const date = value instanceof Date ? value : new Date(value);
+    if (Number.isNaN(date.getTime())) {
+        throw new Error(`Invalid BigQuery query-history timestamp: ${String(value)}`);
+    }
+    return `TIMESTAMP('${date.toISOString().replace(/'/g, "\\'")}')`;
+}
+function indexByHeader(headers) {
+    const out = new Map();
+    headers.forEach((header, index) => {
+        out.set(header.toUpperCase(), index);
+    });
+    return out;
+}
+function value(row, indexes, name) {
+    const index = indexes.get(name.toUpperCase());
+    return index === undefined ? null : row[index];
+}
+function nullableString(raw) {
+    if (raw === null || raw === undefined) {
+        return null;
+    }
+    const text = String(raw);
+    return text.length > 0 ? text : null;
+}
+function requiredString(raw, field) {
+    const text = nullableString(raw);
+    if (!text) {
+        throw new Error(`BigQuery JOBS_BY_PROJECT row is missing ${field}`);
+    }
+    return text;
+}
+function nullableNumber(raw) {
+    if (raw === null || raw === undefined || raw === '') {
+        return null;
+    }
+    const number = typeof raw === 'number' ? raw : Number(raw);
+    if (!Number.isFinite(number)) {
+        return null;
+    }
+    return Math.max(0, number);
+}
+function isoTimestamp(raw, field) {
+    if (raw instanceof Date) {
+        return raw.toISOString();
+    }
+    const text = requiredString(raw, field);
+    const date = new Date(text);
+    if (Number.isNaN(date.getTime())) {
+        throw new Error(`BigQuery JOBS_BY_PROJECT row has invalid ${field}: ${text}`);
+    }
+    return date.toISOString();
+}
+function nullableIsoTimestamp(raw) {
+    if (raw === null || raw === undefined || raw === '') {
+        return null;
+    }
+    return isoTimestamp(raw, 'end_time');
+}
+function executionSucceeded(state, errorReason, errorMessage) {
+    if (errorReason || errorMessage) {
+        return false;
+    }
+    return state === null || state.toUpperCase() === 'DONE';
+}
+function combinedErrorMessage(errorReason, errorMessage) {
+    if (errorReason && errorMessage) {
+        return `${errorReason}: ${errorMessage}`;
+    }
+    return errorMessage ?? errorReason;
+}
+function mapRow(row, indexes) {
+    const errorReason = nullableString(value(row, indexes, 'error_reason'));
+    const errorMessage = nullableString(value(row, indexes, 'error_message'));
+    return {
+        id: requiredString(value(row, indexes, 'job_id'), 'job_id'),
+        sql: requiredString(value(row, indexes, 'query'), 'query'),
+        user: nullableString(value(row, indexes, 'user_email')),
+        startedAt: isoTimestamp(value(row, indexes, 'creation_time'), 'creation_time'),
+        endedAt: nullableIsoTimestamp(value(row, indexes, 'end_time')),
+        runtimeMs: nullableNumber(value(row, indexes, 'runtime_ms')),
+        success: executionSucceeded(nullableString(value(row, indexes, 'state')), errorReason, errorMessage),
+        errorMessage: combinedErrorMessage(errorReason, errorMessage),
+    };
+}
+export class BigQueryHistoricSqlQueryHistoryReader {
+    viewPath;
+    constructor(options) {
+        const projectId = normalizeProjectId(options.projectId);
+        const region = normalizeRegion(options.region);
+        this.viewPath = `\`${projectId}.region-${region}.INFORMATION_SCHEMA.JOBS_BY_PROJECT\``;
+    }
+    async probe(client) {
+        let result;
+        try {
+            result = await queryClient(client).executeQuery(`SELECT 1 FROM ${this.viewPath} LIMIT 1`);
+        }
+        catch (error) {
+            throw grantsError(error);
+        }
+        if (result.error) {
+            throw grantsError(result.error);
+        }
+    }
+    async *fetch(client, window, cursor) {
+        const start = timestampExpression(cursor ?? window.start);
+        const end = timestampExpression(window.end);
+        const sql = `
+SELECT
+  job_id,
+  query,
+  user_email,
+  creation_time,
+  end_time,
+  TIMESTAMP_DIFF(end_time, creation_time, MILLISECOND) AS runtime_ms,
+  total_slot_ms,
+  total_bytes_processed,
+  state,
+  error_result.reason AS error_reason,
+  error_result.message AS error_message,
+  statement_type
+FROM ${this.viewPath}
+WHERE creation_time >= ${start}
+  AND creation_time < ${end}
+  AND job_type = 'QUERY'
+  AND query IS NOT NULL
+  AND (statement_type IS NULL OR statement_type != 'SCRIPT')
+ORDER BY creation_time ASC, job_id ASC`.trim();
+        const result = await queryClient(client).executeQuery(sql);
+        if (result.error) {
+            throw grantsError(result.error);
+        }
+        const indexes = indexByHeader(result.headers);
+        for (const row of result.rows) {
+            yield mapRow(row, indexes);
+        }
+    }
+}

package/node_modules/@ktx/context/dist/ingest/adapters/historic-sql/bigquery-query-history-reader.test.d.ts

@@ -0,0 +1 @@
+export {};