@dotdo/postgres 0.1.0

package/dist/lineage/integration.d.ts

@@ -0,0 +1,134 @@
+/**
+ * Lineage Integration with WAL and Iceberg
+ * Task: postgres-zce - Data lineage tracking
+ *
+ * Integrates lineage tracking with WAL capture and Iceberg storage,
+ * enabling automatic lineage collection and persistence.
+ */
+import type { WALManager } from '../worker/wal';
+import type { IcebergTableMetadata } from '../iceberg/types';
+import type { LineageTrackerConfig, IcebergLineageMetadata } from './types';
+import { LineageTracker } from './tracker';
+/**
+ * Lineage integration configuration
+ */
+export interface LineageIntegrationConfig {
+    /** Lineage tracker configuration */
+    lineageConfig?: LineageTrackerConfig;
+    /** Enable automatic lineage tracking from WAL */
+    autoTrackWAL?: boolean;
+    /** Enable automatic persistence to Iceberg */
+    autoSyncIceberg?: boolean;
+    /** Sync interval in milliseconds (default: 60000 - 1 minute) */
+    syncIntervalMs?: number;
+}
+/**
+ * Lineage Integration Manager
+ *
+ * Automatically tracks lineage from WAL entries and syncs to Iceberg metadata.
+ */
+export declare class LineageIntegration {
+    private tracker;
+    private config;
+    /** @internal Reserved for future WAL manager reference */
+    private _walManager;
+    private syncInterval;
+    private unsubscribeWAL;
+    constructor(config?: LineageIntegrationConfig);
+    /**
+     * Get the lineage tracker
+     */
+    getTracker(): LineageTracker;
+    /**
+     * Attach to a WAL manager for automatic tracking
+     */
+    attachWALManager(walManager: WALManager): void;
+    /**
+     * Detach from the current WAL manager
+     */
+    detachWALManager(): void;
+    /**
+     * Start automatic sync to Iceberg
+     */
+    startAutoSync(): void;
+    /**
+     * Stop automatic sync to Iceberg
+     */
+    stopAutoSync(): void;
+    /**
+     * Manually sync lineage to Iceberg metadata
+     *
+     * This updates the Iceberg table properties with lineage information.
+     * Note: This requires access to the Iceberg catalog to update metadata.
+     */
+    syncToIceberg(): Promise<void>;
+    /**
+     * Store lineage metadata
+     *
+     * This is a placeholder - actual implementation would store in:
+     * 1. Iceberg table properties (preferred)
+     * 2. Separate lineage catalog table
+     * 3. DO storage
+     */
+    private storeLineageMetadata;
+    /**
+     * Load lineage metadata from Iceberg
+     */
+    loadFromIceberg(schema: string, table: string): Promise<void>;
+    /**
+     * Update Iceberg table metadata with lineage
+     */
+    updateIcebergMetadata(metadata: IcebergTableMetadata, schema: string, table: string): IcebergTableMetadata;
+    /**
+     * Extract lineage from Iceberg table metadata
+     */
+    extractFromIcebergMetadata(metadata: IcebergTableMetadata, schema: string, table: string): void;
+    /**
+     * Get lineage statistics
+     */
+    getStats(): import("./types").LineageStats;
+    /**
+     * Shutdown the integration
+     */
+    shutdown(): void;
+}
+/**
+ * Create a lineage integration instance
+ */
+export declare function createLineageIntegration(config?: LineageIntegrationConfig): LineageIntegration;
+/**
+ * Storage interface for lineage catalog
+ */
+export interface LineageCatalogStorage {
+    /** Store lineage metadata for a table */
+    storeLineage(schema: string, table: string, metadata: IcebergLineageMetadata): Promise<void>;
+    /** Load lineage metadata for a table */
+    loadLineage(schema: string, table: string): Promise<IcebergLineageMetadata | null>;
+    /** List all tables with lineage metadata */
+    listTrackedTables(): Promise<{
+        schema: string;
+        table: string;
+    }[]>;
+    /** Delete lineage metadata for a table */
+    deleteLineage(schema: string, table: string): Promise<void>;
+}
+/**
+ * Durable Object Storage implementation for lineage catalog
+ */
+export declare class DOLineageCatalog implements LineageCatalogStorage {
+    private state;
+    private prefix;
+    constructor(state: DurableObjectState, prefix?: string);
+    storeLineage(schema: string, table: string, metadata: IcebergLineageMetadata): Promise<void>;
+    loadLineage(schema: string, table: string): Promise<IcebergLineageMetadata | null>;
+    listTrackedTables(): Promise<{
+        schema: string;
+        table: string;
+    }[]>;
+    deleteLineage(schema: string, table: string): Promise<void>;
+}
+/**
+ * Create a DO lineage catalog
+ */
+export declare function createDOLineageCatalog(state: DurableObjectState): DOLineageCatalog;
+//# sourceMappingURL=integration.d.ts.map

package/dist/lineage/integration.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"integration.d.ts","sourceRoot":"","sources":["../../src/lineage/integration.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,KAAK,EAAE,UAAU,EAAsB,MAAM,eAAe,CAAA;AACnE,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,kBAAkB,CAAA;AAC5D,OAAO,KAAK,EAAE,oBAAoB,EAAE,sBAAsB,EAAE,MAAM,SAAS,CAAA;AAC3E,OAAO,EAAE,cAAc,EAAE,MAAM,WAAW,CAAA;AAE1C;;GAEG;AACH,MAAM,WAAW,wBAAwB;IACvC,oCAAoC;IACpC,aAAa,CAAC,EAAE,oBAAoB,CAAA;IACpC,iDAAiD;IACjD,YAAY,CAAC,EAAE,OAAO,CAAA;IACtB,8CAA8C;IAC9C,eAAe,CAAC,EAAE,OAAO,CAAA;IACzB,gEAAgE;IAChE,cAAc,CAAC,EAAE,MAAM,CAAA;CACxB;AAYD;;;;GAIG;AACH,qBAAa,kBAAkB;IAC7B,OAAO,CAAC,OAAO,CAAgB;IAC/B,OAAO,CAAC,MAAM,CAAoC;IAClD,0DAA0D;IAC1D,OAAO,CAAC,WAAW,CAA0B;IAC7C,OAAO,CAAC,YAAY,CAAsB;IAC1C,OAAO,CAAC,cAAc,CAA4B;gBAEtC,MAAM,GAAE,wBAA6B;IAMjD;;OAEG;IACH,UAAU,IAAI,cAAc;IAI5B;;OAEG;IACH,gBAAgB,CAAC,UAAU,EAAE,UAAU,GAAG,IAAI;IA6B9C;;OAEG;IACH,gBAAgB,IAAI,IAAI;IAQxB;;OAEG;IACH,aAAa,IAAI,IAAI;IAiBrB;;OAEG;IACH,YAAY,IAAI,IAAI;IAOpB;;;;;OAKG;IACG,aAAa,IAAI,OAAO,CAAC,IAAI,CAAC;IAoBpC;;;;;;;OAOG;YACW,oBAAoB;IAelC;;OAEG;IACG,eAAe,CAAC,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAMnE;;OAEG;IACH,qBAAqB,CAAC,QAAQ,EAAE,oBAAoB,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,oBAAoB;IA0B1G;;OAEG;IACH,0BAA0B,CAAC,QAAQ,EAAE,oBAAoB,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,IAAI;IAiB/F;;OAEG;IACH,QAAQ;IAIR;;OAEG;IACH,QAAQ,IAAI,IAAI;CAIjB;AAED;;GAEG;AACH,wBAAgB,wBAAwB,CAAC,MAAM,CAAC,EAAE,wBAAwB,GAAG,kBAAkB,CAE9F;AAED;;GAEG;AACH,MAAM,WAAW,qBAAqB;IACpC,yCAAyC;IACzC,YAAY,CAAC,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,sBAAsB,GAAG,OAAO,CAAC,IAAI,CAAC,CAAA;IAE5F,wCAAwC;IACxC,WAAW,CAAC,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,sBAAsB,GAAG,IAAI,CAAC,CAAA;IAElF,4CAA4C;IAC5C,iBAAiB,IAAI,OAAO,CAAC;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC,CAAA;IAEjE,0CAA0C;IAC1C,aAAa,CAAC,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAA;CAC5D;AAED;;GAEG;AACH,qBAAa,gBAAiB,YAAW,qBAAqB;IAC5D,OAAO,CAAC,KAAK,CAAoB;IACjC,OAAO,CAAC,MAAM,CAAQ;gBAEV,KAAK,EAAE,kBAAkB,EAAE,MAAM,SAAa;IAKpD,YAAY,CAAC,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,sBAAsB,GAAG,OAAO,CAAC,IAAI,CAAC;IAK5F,WAAW,CAAC,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,sBAAsB,GAAG,IAAI,CAAC;IAKlF,iBAAiB,IAAI,OAAO,CAAC;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC;IAiBjE,aAAa,CAAC,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;CAIlE;AAED;;GAEG;AACH,wBAAgB,sBAAsB,CAAC,KAAK,EAAE,kBAAkB,GAAG,gBAAgB,CAElF"}

package/dist/lineage/integration.js

@@ -0,0 +1,258 @@
+/**
+ * Lineage Integration with WAL and Iceberg
+ * Task: postgres-zce - Data lineage tracking
+ *
+ * Integrates lineage tracking with WAL capture and Iceberg storage,
+ * enabling automatic lineage collection and persistence.
+ */
+import { LineageTracker } from './tracker';
+/**
+ * Default configuration
+ */
+const DEFAULT_INTEGRATION_CONFIG = {
+    lineageConfig: {},
+    autoTrackWAL: true,
+    autoSyncIceberg: true,
+    syncIntervalMs: 60000,
+};
+/**
+ * Lineage Integration Manager
+ *
+ * Automatically tracks lineage from WAL entries and syncs to Iceberg metadata.
+ */
+export class LineageIntegration {
+    tracker;
+    config;
+    /** @internal Reserved for future WAL manager reference */
+    _walManager = null;
+    syncInterval = null;
+    unsubscribeWAL = null;
+    constructor(config = {}) {
+        this.config = { ...DEFAULT_INTEGRATION_CONFIG, ...config };
+        this.tracker = new LineageTracker(this.config.lineageConfig);
+        void this._walManager; // Reserved for future WAL manager reference
+    }
+    /**
+     * Get the lineage tracker
+     */
+    getTracker() {
+        return this.tracker;
+    }
+    /**
+     * Attach to a WAL manager for automatic tracking
+     */
+    attachWALManager(walManager) {
+        // Detach from previous manager if any
+        this.detachWALManager();
+        this._walManager = walManager;
+        if (this.config.autoTrackWAL) {
+            // Subscribe to WAL entries
+            this.unsubscribeWAL = walManager.onEntry((entry) => {
+                // Convert WALEntry to SerializedWALEntry
+                const serializedEntry = {
+                    lsn: entry.lsn.toString(),
+                    operation: entry.operation,
+                    schema: entry.schema,
+                    table: entry.table,
+                    primaryKey: entry.primaryKey,
+                    newRow: entry.newRow,
+                    oldRow: entry.oldRow,
+                    changedColumns: entry.changedColumns,
+                    transactionId: entry.transactionId,
+                    timestamp: entry.timestamp,
+                    sql: entry.sql,
+                };
+                this.tracker.trackWALEntry(serializedEntry);
+            });
+        }
+    }
+    /**
+     * Detach from the current WAL manager
+     */
+    detachWALManager() {
+        if (this.unsubscribeWAL) {
+            this.unsubscribeWAL();
+            this.unsubscribeWAL = null;
+        }
+        this._walManager = null;
+    }
+    /**
+     * Start automatic sync to Iceberg
+     */
+    startAutoSync() {
+        if (this.syncInterval !== null) {
+            return; // Already running
+        }
+        if (!this.config.autoSyncIceberg) {
+            return;
+        }
+        // Use setInterval in a way that's compatible with both Node and browser
+        this.syncInterval = setInterval(() => {
+            void this.syncToIceberg().catch((error) => {
+                console.error('Lineage auto-sync error:', error);
+            });
+        }, this.config.syncIntervalMs);
+    }
+    /**
+     * Stop automatic sync to Iceberg
+     */
+    stopAutoSync() {
+        if (this.syncInterval !== null) {
+            clearInterval(this.syncInterval);
+            this.syncInterval = null;
+        }
+    }
+    /**
+     * Manually sync lineage to Iceberg metadata
+     *
+     * This updates the Iceberg table properties with lineage information.
+     * Note: This requires access to the Iceberg catalog to update metadata.
+     */
+    async syncToIceberg() {
+        // Get all tracked tables
+        const stats = this.tracker.getStats();
+        if (stats.totalNodes === 0) {
+            return; // Nothing to sync
+        }
+        // Export metadata for each table
+        const graph = this.tracker.getGraph();
+        for (const [_nodeId, node] of graph.nodes) {
+            if (node.type !== 'table')
+                continue;
+            // Export lineage metadata
+            const metadata = this.tracker.exportIcebergMetadata(node.schema, node.name);
+            // Store in a lineage catalog (implementation would depend on storage backend)
+            await this.storeLineageMetadata(node.schema, node.name, metadata);
+        }
+    }
+    /**
+     * Store lineage metadata
+     *
+     * This is a placeholder - actual implementation would store in:
+     * 1. Iceberg table properties (preferred)
+     * 2. Separate lineage catalog table
+     * 3. DO storage
+     */
+    async storeLineageMetadata(schema, table, metadata) {
+        // For now, just log - actual implementation would update Iceberg metadata
+        console.log(`Storing lineage for ${schema}.${table}:`, metadata);
+        // In a real implementation, this would:
+        // 1. Load the Iceberg table metadata
+        // 2. Update the properties section
+        // 3. Write back the metadata
+        // 4. Or store in a separate lineage catalog table
+    }
+    /**
+     * Load lineage metadata from Iceberg
+     */
+    async loadFromIceberg(schema, table) {
+        // Placeholder - would load from Iceberg table properties
+        // For now, we'll just skip
+        console.log(`Loading lineage for ${schema}.${table}`);
+    }
+    /**
+     * Update Iceberg table metadata with lineage
+     */
+    updateIcebergMetadata(metadata, schema, table) {
+        const lineageMetadata = this.tracker.exportIcebergMetadata(schema, table);
+        // Build lineage properties, only including defined values
+        const lineageProperties = {
+            'lineage.version': lineageMetadata['lineage.version'],
+            'lineage.sources': lineageMetadata['lineage.sources'],
+            'lineage.operations': lineageMetadata['lineage.operations'],
+            'lineage.updated-at': lineageMetadata['lineage.updated-at'],
+        };
+        // Only add graph property if it's defined
+        if (lineageMetadata['lineage.graph'] !== undefined) {
+            lineageProperties['lineage.graph'] = lineageMetadata['lineage.graph'];
+        }
+        // Merge lineage metadata into Iceberg properties
+        return {
+            ...metadata,
+            properties: {
+                ...metadata.properties,
+                ...lineageProperties,
+            },
+        };
+    }
+    /**
+     * Extract lineage from Iceberg table metadata
+     */
+    extractFromIcebergMetadata(metadata, schema, table) {
+        const properties = metadata.properties || {};
+        // Check if lineage metadata exists
+        if (properties['lineage.version']) {
+            const lineageMetadata = {
+                'lineage.version': properties['lineage.version'],
+                'lineage.sources': properties['lineage.sources'] || '',
+                'lineage.operations': properties['lineage.operations'] || '',
+                'lineage.updated-at': properties['lineage.updated-at'] || '',
+                'lineage.graph': properties['lineage.graph'],
+            };
+            this.tracker.importIcebergMetadata(schema, table, lineageMetadata);
+        }
+    }
+    /**
+     * Get lineage statistics
+     */
+    getStats() {
+        return this.tracker.getStats();
+    }
+    /**
+     * Shutdown the integration
+     */
+    shutdown() {
+        this.stopAutoSync();
+        this.detachWALManager();
+    }
+}
+/**
+ * Create a lineage integration instance
+ */
+export function createLineageIntegration(config) {
+    return new LineageIntegration(config);
+}
+/**
+ * Durable Object Storage implementation for lineage catalog
+ */
+export class DOLineageCatalog {
+    state;
+    prefix;
+    constructor(state, prefix = 'lineage:') {
+        this.state = state;
+        this.prefix = prefix;
+    }
+    async storeLineage(schema, table, metadata) {
+        const key = `${this.prefix}${schema}.${table}`;
+        await this.state.storage.put(key, metadata);
+    }
+    async loadLineage(schema, table) {
+        const key = `${this.prefix}${schema}.${table}`;
+        return (await this.state.storage.get(key)) || null;
+    }
+    async listTrackedTables() {
+        const keys = await this.state.storage.list({
+            prefix: this.prefix,
+        });
+        const tables = [];
+        for (const [key] of keys) {
+            const tableId = key.slice(this.prefix.length);
+            const [schema, table] = tableId.split('.');
+            if (schema && table) {
+                tables.push({ schema, table });
+            }
+        }
+        return tables;
+    }
+    async deleteLineage(schema, table) {
+        const key = `${this.prefix}${schema}.${table}`;
+        await this.state.storage.delete(key);
+    }
+}
+/**
+ * Create a DO lineage catalog
+ */
+export function createDOLineageCatalog(state) {
+    return new DOLineageCatalog(state);
+}
+//# sourceMappingURL=integration.js.map

package/dist/lineage/integration.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"integration.js","sourceRoot":"","sources":["../../src/lineage/integration.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAKH,OAAO,EAAE,cAAc,EAAE,MAAM,WAAW,CAAA;AAgB1C;;GAEG;AACH,MAAM,0BAA0B,GAAuC;IACrE,aAAa,EAAE,EAAE;IACjB,YAAY,EAAE,IAAI;IAClB,eAAe,EAAE,IAAI;IACrB,cAAc,EAAE,KAAK;CACtB,CAAA;AAED;;;;GAIG;AACH,MAAM,OAAO,kBAAkB;IACrB,OAAO,CAAgB;IACvB,MAAM,CAAoC;IAClD,0DAA0D;IAClD,WAAW,GAAsB,IAAI,CAAA;IACrC,YAAY,GAAkB,IAAI,CAAA;IAClC,cAAc,GAAwB,IAAI,CAAA;IAElD,YAAY,SAAmC,EAAE;QAC/C,IAAI,CAAC,MAAM,GAAG,EAAE,GAAG,0BAA0B,EAAE,GAAG,MAAM,EAAE,CAAA;QAC1D,IAAI,CAAC,OAAO,GAAG,IAAI,cAAc,CAAC,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,CAAA;QAC5D,KAAK,IAAI,CAAC,WAAW,CAAA,CAAC,4CAA4C;IACpE,CAAC;IAED;;OAEG;IACH,UAAU;QACR,OAAO,IAAI,CAAC,OAAO,CAAA;IACrB,CAAC;IAED;;OAEG;IACH,gBAAgB,CAAC,UAAsB;QACrC,sCAAsC;QACtC,IAAI,CAAC,gBAAgB,EAAE,CAAA;QAEvB,IAAI,CAAC,WAAW,GAAG,UAAU,CAAA;QAE7B,IAAI,IAAI,CAAC,MAAM,CAAC,YAAY,EAAE,CAAC;YAC7B,2BAA2B;YAC3B,IAAI,CAAC,cAAc,GAAG,UAAU,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,EAAE;gBACjD,yCAAyC;gBACzC,MAAM,eAAe,GAAuB;oBAC1C,GAAG,EAAE,KAAK,CAAC,GAAG,CAAC,QAAQ,EAAE;oBACzB,SAAS,EAAE,KAAK,CAAC,SAAS;oBAC1B,MAAM,EAAE,KAAK,CAAC,MAAM;oBACpB,KAAK,EAAE,KAAK,CAAC,KAAK;oBAClB,UAAU,EAAE,KAAK,CAAC,UAAU;oBAC5B,MAAM,EAAE,KAAK,CAAC,MAAM;oBACpB,MAAM,EAAE,KAAK,CAAC,MAAM;oBACpB,cAAc,EAAE,KAAK,CAAC,cAAc;oBACpC,aAAa,EAAE,KAAK,CAAC,aAAa;oBAClC,SAAS,EAAE,KAAK,CAAC,SAAS;oBAC1B,GAAG,EAAE,KAAK,CAAC,GAAG;iBACf,CAAA;gBAED,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC,eAAe,CAAC,CAAA;YAC7C,CAAC,CAAC,CAAA;QACJ,CAAC;IACH,CAAC;IAED;;OAEG;IACH,gBAAgB;QACd,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;YACxB,IAAI,CAAC,cAAc,EAAE,CAAA;YACrB,IAAI,CAAC,cAAc,GAAG,IAAI,CAAA;QAC5B,CAAC;QACD,IAAI,CAAC,WAAW,GAAG,IAAI,CAAA;IACzB,CAAC;IAED;;OAEG;IACH,aAAa;QACX,IAAI,IAAI,CAAC,YAAY,KAAK,IAAI,EAAE,CAAC;YAC/B,OAAM,CAAC,kBAAkB;QAC3B,CAAC;QAED,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,eAAe,EAAE,CAAC;YACjC,OAAM;QACR,CAAC;QAED,wEAAwE;QACxE,IAAI,CAAC,YAAY,GAAG,WAAW,CAAC,GAAG,EAAE;YACnC,KAAK,IAAI,CAAC,aAAa,EAAE,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE;gBACxC,OAAO,CAAC,KAAK,CAAC,0BAA0B,EAAE,KAAK,CAAC,CAAA;YAClD,CAAC,CAAC,CAAA;QACJ,CAAC,EAAE,IAAI,CAAC,MAAM,CAAC,cAAc,CAAsB,CAAA;IACrD,CAAC;IAED;;OAEG;IACH,YAAY;QACV,IAAI,IAAI,CAAC,YAAY,KAAK,IAAI,EAAE,CAAC;YAC/B,aAAa,CAAC,IAAI,CAAC,YAAY,CAAC,CAAA;YAChC,IAAI,CAAC,YAAY,GAAG,IAAI,CAAA;QAC1B,CAAC;IACH,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,aAAa;QACjB,yBAAyB;QACzB,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAA;QACrC,IAAI,KAAK,CAAC,UAAU,KAAK,CAAC,EAAE,CAAC;YAC3B,OAAM,CAAC,kBAAkB;QAC3B,CAAC;QAED,iCAAiC;QACjC,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAA;QACrC,KAAK,MAAM,CAAC,OAAO,EAAE,IAAI,CAAC,IAAI,KAAK,CAAC,KAAK,EAAE,CAAC;YAC1C,IAAI,IAAI,CAAC,IAAI,KAAK,OAAO;gBAAE,SAAQ;YAEnC,0BAA0B;YAC1B,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,qBAAqB,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,IAAI,CAAC,CAAA;YAE3E,8EAA8E;YAC9E,MAAM,IAAI,CAAC,oBAAoB,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAA;QACnE,CAAC;IACH,CAAC;IAED;;;;;;;OAOG;IACK,KAAK,CAAC,oBAAoB,CAChC,MAAc,EACd,KAAa,EACb,QAAgC;QAEhC,0EAA0E;QAC1E,OAAO,CAAC,GAAG,CAAC,uBAAuB,MAAM,IAAI,KAAK,GAAG,EAAE,QAAQ,CAAC,CAAA;QAEhE,wCAAwC;QACxC,qCAAqC;QACrC,mCAAmC;QACnC,6BAA6B;QAC7B,kDAAkD;IACpD,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,eAAe,CAAC,MAAc,EAAE,KAAa;QACjD,yDAAyD;QACzD,2BAA2B;QAC3B,OAAO,CAAC,GAAG,CAAC,uBAAuB,MAAM,IAAI,KAAK,EAAE,CAAC,CAAA;IACvD,CAAC;IAED;;OAEG;IACH,qBAAqB,CAAC,QAA8B,EAAE,MAAc,EAAE,KAAa;QACjF,MAAM,eAAe,GAAG,IAAI,CAAC,OAAO,CAAC,qBAAqB,CAAC,MAAM,EAAE,KAAK,CAAC,CAAA;QAEzE,0DAA0D;QAC1D,MAAM,iBAAiB,GAA2B;YAChD,iBAAiB,EAAE,eAAe,CAAC,iBAAiB,CAAC;YACrD,iBAAiB,EAAE,eAAe,CAAC,iBAAiB,CAAC;YACrD,oBAAoB,EAAE,eAAe,CAAC,oBAAoB,CAAC;YAC3D,oBAAoB,EAAE,eAAe,CAAC,oBAAoB,CAAC;SAC5D,CAAA;QAED,0CAA0C;QAC1C,IAAI,eAAe,CAAC,eAAe,CAAC,KAAK,SAAS,EAAE,CAAC;YACnD,iBAAiB,CAAC,eAAe,CAAC,GAAG,eAAe,CAAC,eAAe,CAAC,CAAA;QACvE,CAAC;QAED,iDAAiD;QACjD,OAAO;YACL,GAAG,QAAQ;YACX,UAAU,EAAE;gBACV,GAAG,QAAQ,CAAC,UAAU;gBACtB,GAAG,iBAAiB;aACrB;SACF,CAAA;IACH,CAAC;IAED;;OAEG;IACH,0BAA0B,CAAC,QAA8B,EAAE,MAAc,EAAE,KAAa;QACtF,MAAM,UAAU,GAAG,QAAQ,CAAC,UAAU,IAAI,EAAE,CAAA;QAE5C,mCAAmC;QACnC,IAAI,UAAU,CAAC,iBAAiB,CAAC,EAAE,CAAC;YAClC,MAAM,eAAe,GAA2B;gBAC9C,iBAAiB,EAAE,UAAU,CAAC,iBAAiB,CAAC;gBAChD,iBAAiB,EAAE,UAAU,CAAC,iBAAiB,CAAC,IAAI,EAAE;gBACtD,oBAAoB,EAAE,UAAU,CAAC,oBAAoB,CAAC,IAAI,EAAE;gBAC5D,oBAAoB,EAAE,UAAU,CAAC,oBAAoB,CAAC,IAAI,EAAE;gBAC5D,eAAe,EAAE,UAAU,CAAC,eAAe,CAAC;aAC7C,CAAA;YAED,IAAI,CAAC,OAAO,CAAC,qBAAqB,CAAC,MAAM,EAAE,KAAK,EAAE,eAAe,CAAC,CAAA;QACpE,CAAC;IACH,CAAC;IAED;;OAEG;IACH,QAAQ;QACN,OAAO,IAAI,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAA;IAChC,CAAC;IAED;;OAEG;IACH,QAAQ;QACN,IAAI,CAAC,YAAY,EAAE,CAAA;QACnB,IAAI,CAAC,gBAAgB,EAAE,CAAA;IACzB,CAAC;CACF;AAED;;GAEG;AACH,MAAM,UAAU,wBAAwB,CAAC,MAAiC;IACxE,OAAO,IAAI,kBAAkB,CAAC,MAAM,CAAC,CAAA;AACvC,CAAC;AAmBD;;GAEG;AACH,MAAM,OAAO,gBAAgB;IACnB,KAAK,CAAoB;IACzB,MAAM,CAAQ;IAEtB,YAAY,KAAyB,EAAE,MAAM,GAAG,UAAU;QACxD,IAAI,CAAC,KAAK,GAAG,KAAK,CAAA;QAClB,IAAI,CAAC,MAAM,GAAG,MAAM,CAAA;IACtB,CAAC;IAED,KAAK,CAAC,YAAY,CAAC,MAAc,EAAE,KAAa,EAAE,QAAgC;QAChF,MAAM,GAAG,GAAG,GAAG,IAAI,CAAC,MAAM,GAAG,MAAM,IAAI,KAAK,EAAE,CAAA;QAC9C,MAAM,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAA;IAC7C,CAAC;IAED,KAAK,CAAC,WAAW,CAAC,MAAc,EAAE,KAAa;QAC7C,MAAM,GAAG,GAAG,GAAG,IAAI,CAAC,MAAM,GAAG,MAAM,IAAI,KAAK,EAAE,CAAA;QAC9C,OAAO,CAAC,MAAM,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAyB,GAAG,CAAC,CAAC,IAAI,IAAI,CAAA;IAC5E,CAAC;IAED,KAAK,CAAC,iBAAiB;QACrB,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAyB;YACjE,MAAM,EAAE,IAAI,CAAC,MAAM;SACpB,CAAC,CAAA;QAEF,MAAM,MAAM,GAAwC,EAAE,CAAA;QACtD,KAAK,MAAM,CAAC,GAAG,CAAC,IAAI,IAAI,EAAE,CAAC;YACzB,MAAM,OAAO,GAAG,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAA;YAC7C,MAAM,CAAC,MAAM,EAAE,KAAK,CAAC,GAAG,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,CAAA;YAC1C,IAAI,MAAM,IAAI,KAAK,EAAE,CAAC;gBACpB,MAAM,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,CAAA;YAChC,CAAC;QACH,CAAC;QAED,OAAO,MAAM,CAAA;IACf,CAAC;IAED,KAAK,CAAC,aAAa,CAAC,MAAc,EAAE,KAAa;QAC/C,MAAM,GAAG,GAAG,GAAG,IAAI,CAAC,MAAM,GAAG,MAAM,IAAI,KAAK,EAAE,CAAA;QAC9C,MAAM,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,CAAA;IACtC,CAAC;CACF;AAED;;GAEG;AACH,MAAM,UAAU,sBAAsB,CAAC,KAAyB;IAC9D,OAAO,IAAI,gBAAgB,CAAC,KAAK,CAAC,CAAA;AACpC,CAAC"}

package/dist/lineage/tracker.d.ts

@@ -0,0 +1,189 @@
+/**
+ * Data Lineage Tracker
+ * Task: postgres-zce - Data lineage tracking
+ *
+ * Tracks data origin and transformations across tables, integrating with WAL capture
+ * and Iceberg storage for complete lineage history.
+ */
+import type { SerializedWALEntry } from '../worker/wal';
+import type { LineageNode, LineageEdge, LineageGraph, LineageTrackerConfig, LineageStats, LineageQueryResult, LineageParseResult, LineageVisualizationGraph, IcebergLineageMetadata, LineageOperationType } from './types';
+/**
+ * Parse SQL to extract lineage information
+ */
+export declare function parseLineage(sql: string): LineageParseResult | null;
+/**
+ * Data Lineage Tracker
+ *
+ * Tracks data flow between tables, records transformations,
+ * and provides lineage queries for any row.
+ */
+export declare class LineageTracker {
+    private config;
+    private graph;
+    private rowLineageMap;
+    constructor(config?: LineageTrackerConfig);
+    /**
+     * Get current configuration
+     */
+    getConfig(): Required<LineageTrackerConfig>;
+    /**
+     * Update configuration
+     */
+    setConfig(config: Partial<LineageTrackerConfig>): void;
+    /**
+     * Check if lineage tracking is enabled
+     */
+    isEnabled(): boolean;
+    /**
+     * Enable or disable lineage tracking
+     */
+    setEnabled(enabled: boolean): void;
+    /**
+     * Track a WAL entry for lineage
+     */
+    trackWALEntry(entry: SerializedWALEntry): void;
+    /**
+     * Track lineage from parsed SQL
+     */
+    private trackLineage;
+    /**
+     * Track row-level lineage
+     */
+    private trackRowLineage;
+    /**
+     * Get row key for lineage map
+     */
+    private getRowKey;
+    /**
+     * Ensure a node exists in the graph
+     */
+    private ensureNode;
+    /**
+     * Add an edge to the graph
+     */
+    private addEdge;
+    /**
+     * Get lineage for a specific table
+     */
+    getTableLineage(schema: string, table: string): {
+        upstreamTables: string[];
+        downstreamTables: string[];
+        edges: LineageEdge[];
+    };
+    /**
+     * Query lineage for a specific row
+     */
+    queryRowLineage(schema: string, table: string, primaryKey: Record<string, unknown>): LineageQueryResult | null;
+    /**
+     * Visualize lineage graph for a table
+     */
+    visualizeLineage(schema: string, table: string, maxDepth?: number): LineageVisualizationGraph;
+    /**
+     * Get statistics about tracked lineage
+     */
+    getStats(): LineageStats;
+    /**
+     * Calculate the depth of a node in the graph
+     */
+    private calculateNodeDepth;
+    /**
+     * Get the last update timestamp
+     */
+    private getLastUpdateTimestamp;
+    /**
+     * Export lineage metadata for Iceberg table properties
+     */
+    exportIcebergMetadata(schema: string, table: string): IcebergLineageMetadata;
+    /**
+     * Import lineage metadata from Iceberg table properties
+     */
+    importIcebergMetadata(schema: string, table: string, metadata: IcebergLineageMetadata): void;
+    /**
+     * Query all upstream tables (data sources) for a given table
+     * Traverses the graph recursively to find all tables that feed into this one
+     */
+    queryUpstreamLineage(schema: string, table: string, maxDepth?: number): {
+        tables: {
+            schema: string;
+            table: string;
+            depth: number;
+        }[];
+        edges: LineageEdge[];
+        totalDepth: number;
+    };
+    /**
+     * Query all downstream tables (data consumers) for a given table
+     * Traverses the graph recursively to find all tables that are derived from this one
+     */
+    queryDownstreamLineage(schema: string, table: string, maxDepth?: number): {
+        tables: {
+            schema: string;
+            table: string;
+            depth: number;
+        }[];
+        edges: LineageEdge[];
+        totalDepth: number;
+    };
+    /**
+     * Find the shortest path between two tables in the lineage graph
+     */
+    findLineagePath(sourceSchema: string, sourceTable: string, targetSchema: string, targetTable: string): {
+        path: string[];
+        edges: LineageEdge[];
+    } | null;
+    /**
+     * Get impact analysis: all tables that would be affected by changes to a source table
+     */
+    getImpactAnalysis(schema: string, table: string): {
+        directlyAffected: {
+            schema: string;
+            table: string;
+        }[];
+        transitivelyAffected: {
+            schema: string;
+            table: string;
+            hops: number;
+        }[];
+        totalAffected: number;
+    };
+    /**
+     * Get data provenance: full lineage history for understanding data origin
+     */
+    getDataProvenance(schema: string, table: string): {
+        rootSources: {
+            schema: string;
+            table: string;
+        }[];
+        transformationChain: {
+            table: string;
+            operation: LineageOperationType;
+            timestamp: number;
+        }[];
+        depth: number;
+    };
+    /**
+     * List all tracked tables
+     */
+    listTrackedTables(): {
+        schema: string;
+        table: string;
+        type: LineageNode['type'];
+    }[];
+    /**
+     * Get node by ID
+     */
+    getNode(schema: string, table: string): LineageNode | null;
+    /**
+     * Clear all lineage data
+     */
+    clear(): void;
+    /**
+     * Get the full lineage graph
+     */
+    getGraph(): LineageGraph;
+}
+/**
+ * Create a lineage tracker with default configuration
+ */
+export declare function createLineageTracker(config?: LineageTrackerConfig): LineageTracker;
+//# sourceMappingURL=tracker.d.ts.map

package/dist/lineage/tracker.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"tracker.d.ts","sourceRoot":"","sources":["../../src/lineage/tracker.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,eAAe,CAAA;AACvD,OAAO,KAAK,EACV,WAAW,EACX,WAAW,EACX,YAAY,EACZ,oBAAoB,EACpB,YAAY,EACZ,kBAAkB,EAElB,kBAAkB,EAClB,yBAAyB,EAOzB,sBAAsB,EACtB,oBAAoB,EACrB,MAAM,SAAS,CAAA;AAgChB;;GAEG;AACH,wBAAgB,YAAY,CAAC,GAAG,EAAE,MAAM,GAAG,kBAAkB,GAAG,IAAI,CAqKnE;AA4hBD;;;;;GAKG;AACH,qBAAa,cAAc;IACzB,OAAO,CAAC,MAAM,CAAgC;IAC9C,OAAO,CAAC,KAAK,CAAc;IAC3B,OAAO,CAAC,aAAa,CAAuC;gBAEhD,MAAM,GAAE,oBAAyB;IAU7C;;OAEG;IACH,SAAS,IAAI,QAAQ,CAAC,oBAAoB,CAAC;IAI3C;;OAEG;IACH,SAAS,CAAC,MAAM,EAAE,OAAO,CAAC,oBAAoB,CAAC,GAAG,IAAI;IAItD;;OAEG;IACH,SAAS,IAAI,OAAO;IAIpB;;OAEG;IACH,UAAU,CAAC,OAAO,EAAE,OAAO,GAAG,IAAI;IAIlC;;OAEG;IACH,aAAa,CAAC,KAAK,EAAE,kBAAkB,GAAG,IAAI;IA4B9C;;OAEG;IACH,OAAO,CAAC,YAAY;IA4BpB;;OAEG;IACH,OAAO,CAAC,eAAe;IAqCvB;;OAEG;IACH,OAAO,CAAC,SAAS;IAIjB;;OAEG;IACH,OAAO,CAAC,UAAU;IAoBlB;;OAEG;IACH,OAAO,CAAC,OAAO;IAiBf;;OAEG;IACH,eAAe,CAAC,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG;QAC9C,cAAc,EAAE,MAAM,EAAE,CAAA;QACxB,gBAAgB,EAAE,MAAM,EAAE,CAAA;QAC1B,KAAK,EAAE,WAAW,EAAE,CAAA;KACrB;IAmCD;;OAEG;IACH,eAAe,CACb,MAAM,EAAE,MAAM,EACd,KAAK,EAAE,MAAM,EACb,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAClC,kBAAkB,GAAG,IAAI;IAuC5B;;OAEG;IACH,gBAAgB,CAAC,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,GAAG,yBAAyB;IA+I7F;;OAEG;IACH,QAAQ,IAAI,YAAY;IA8BxB;;OAEG;IACH,OAAO,CAAC,kBAAkB;IAyB1B;;OAEG;IACH,OAAO,CAAC,sBAAsB;IAY9B;;OAEG;IACH,qBAAqB,CAAC,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,sBAAsB;IAoB5E;;OAEG;IACH,qBAAqB,CAAC,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,sBAAsB,GAAG,IAAI;IA2B5F;;;OAGG;IACH,oBAAoB,CAAC,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,GAAG;QACtE,MAAM,EAAE;YAAE,MAAM,EAAE,MAAM,CAAC;YAAC,KAAK,EAAE,MAAM,CAAC;YAAC,KAAK,EAAE,MAAM,CAAA;SAAE,EAAE,CAAA;QAC1D,KAAK,EAAE,WAAW,EAAE,CAAA;QACpB,UAAU,EAAE,MAAM,CAAA;KACnB;IAqDD;;;OAGG;IACH,sBAAsB,CAAC,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,GAAG;QACxE,MAAM,EAAE;YAAE,MAAM,EAAE,MAAM,CAAC;YAAC,KAAK,EAAE,MAAM,CAAC;YAAC,KAAK,EAAE,MAAM,CAAA;SAAE,EAAE,CAAA;QAC1D,KAAK,EAAE,WAAW,EAAE,CAAA;QACpB,UAAU,EAAE,MAAM,CAAA;KACnB;IA+CD;;OAEG;IACH,eAAe,CACb,YAAY,EAAE,MAAM,EACpB,WAAW,EAAE,MAAM,EACnB,YAAY,EAAE,MAAM,EACpB,WAAW,EAAE,MAAM,GAClB;QAAE,IAAI,EAAE,MAAM,EAAE,CAAC;QAAC,KAAK,EAAE,WAAW,EAAE,CAAA;KAAE,GAAG,IAAI;IAkDlD;;OAEG;IACH,iBAAiB,CAAC,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG;QAChD,gBAAgB,EAAE;YAAE,MAAM,EAAE,MAAM,CAAC;YAAC,KAAK,EAAE,MAAM,CAAA;SAAE,EAAE,CAAA;QACrD,oBAAoB,EAAE;YAAE,MAAM,EAAE,MAAM,CAAC;YAAC,KAAK,EAAE,MAAM,CAAC;YAAC,IAAI,EAAE,MAAM,CAAA;SAAE,EAAE,CAAA;QACvE,aAAa,EAAE,MAAM,CAAA;KACtB;IAkBD;;OAEG;IACH,iBAAiB,CAAC,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG;QAChD,WAAW,EAAE;YAAE,MAAM,EAAE,MAAM,CAAC;YAAC,KAAK,EAAE,MAAM,CAAA;SAAE,EAAE,CAAA;QAChD,mBAAmB,EAAE;YAAE,KAAK,EAAE,MAAM,CAAC;YAAC,SAAS,EAAE,oBAAoB,CAAC;YAAC,SAAS,EAAE,MAAM,CAAA;SAAE,EAAE,CAAA;QAC5F,KAAK,EAAE,MAAM,CAAA;KACd;IA6BD;;OAEG;IACH,iBAAiB,IAAI;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,WAAW,CAAC,MAAM,CAAC,CAAA;KAAE,EAAE;IAcnF;;OAEG;IACH,OAAO,CAAC,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,WAAW,GAAG,IAAI;IAK1D;;OAEG;IACH,KAAK,IAAI,IAAI;IAUb;;OAEG;IACH,QAAQ,IAAI,YAAY;CAGzB;AAED;;GAEG;AACH,wBAAgB,oBAAoB,CAAC,MAAM,CAAC,EAAE,oBAAoB,GAAG,cAAc,CAElF"}