@dotdo/postgres 0.1.0

package/src/lineage/tracker.ts

@@ -0,0 +1,1618 @@
+/**
+ * Data Lineage Tracker
+ * Task: postgres-zce - Data lineage tracking
+ *
+ * Tracks data origin and transformations across tables, integrating with WAL capture
+ * and Iceberg storage for complete lineage history.
+ */
+
+import type { SerializedWALEntry } from '../worker/wal'
+import type {
+  LineageNode,
+  LineageEdge,
+  LineageGraph,
+  LineageTrackerConfig,
+  LineageStats,
+  LineageQueryResult,
+  RowLineage,
+  LineageParseResult,
+  LineageVisualizationGraph,
+  LineageVisualizationNode,
+  LineageVisualizationEdge,
+  LineageJoinType,
+  LineageAggregateFunction,
+  JoinDependency,
+  AggregateDependency,
+  IcebergLineageMetadata,
+  LineageOperationType,
+} from './types'
+import { parseWriteOperation } from '../worker/wal'
+
+/**
+ * Default lineage tracker configuration
+ */
+const DEFAULT_CONFIG: Required<LineageTrackerConfig> = {
+  enabled: true,
+  trackColumns: true,
+  trackRows: false,
+  maxDepth: 10,
+  includeTables: [],
+  excludeTables: [],
+  operations: [],
+  storeInIceberg: true,
+  useSeparateCatalog: false,
+}
+
+/**
+ * Generate a unique node ID
+ */
+function generateNodeId(schema: string, table: string): string {
+  return `${schema}.${table}`
+}
+
+/**
+ * Generate a unique edge ID
+ */
+function generateEdgeId(sourceId: string, targetId: string, timestamp: number): string {
+  return `${sourceId}->${targetId}@${timestamp}`
+}
+
+/**
+ * Parse SQL to extract lineage information
+ */
+export function parseLineage(sql: string): LineageParseResult | null {
+  // Normalize SQL
+  const normalized = sql.trim().replace(/\s+/g, ' ')
+  const upperSql = normalized.toUpperCase()
+
+  // Parse CTEs first (WITH clause) - extract source tables from CTEs
+  const ctes = extractCTEs(normalized)
+  const cteSourceTables = ctes.flatMap((cte) => cte.sourceTables)
+  const cteNames = new Set(ctes.map((cte) => cte.name))
+
+  // For CTEs, we need to get the SQL after the CTE definitions for main operation detection
+  const mainSql = stripCTEs(normalized)
+  const mainUpperSql = mainSql.toUpperCase()
+
+  // Parse INSERT INTO ... SELECT FROM
+  if (upperSql.includes('INSERT INTO') && upperSql.includes('SELECT')) {
+    const result = parseInsertSelect(normalized)
+    if (result) {
+      // Add CTE source tables
+      result.sourceTables = [...result.sourceTables, ...cteSourceTables]
+      // Add aggregations if present
+      result.aggregations = extractAggregations(normalized)
+    }
+    return result
+  }
+
+  // Parse UPDATE ... FROM
+  if (upperSql.includes('UPDATE') && upperSql.includes('FROM')) {
+    const result = parseUpdateFrom(normalized)
+    if (result) {
+      result.sourceTables = [...result.sourceTables, ...cteSourceTables]
+    }
+    return result
+  }
+
+  // Parse CREATE TABLE AS SELECT (CTAS)
+  if (upperSql.includes('CREATE TABLE') && upperSql.includes('AS') && upperSql.includes('SELECT')) {
+    return parseCreateTableAs(normalized)
+  }
+
+  // Parse DELETE with subqueries - must check before generic subquery handler
+  if (mainUpperSql.startsWith('DELETE')) {
+    const writeOp = parseWriteOperation(sql)
+    if (writeOp && writeOp.operation === 'DELETE') {
+      // Extract source tables from subqueries
+      const subqueries = extractSubqueries(normalized)
+      const subquerySourceTables = subqueries.flatMap((sq) => sq.sourceTables)
+      return {
+        operation: 'DELETE',
+        targetTable: {
+          schema: writeOp.schema,
+          table: writeOp.table,
+        },
+        sourceTables: [
+          {
+            schema: writeOp.schema,
+            table: writeOp.table,
+          },
+          ...subquerySourceTables,
+          ...cteSourceTables,
+        ],
+      }
+    }
+  }
+
+  // Parse SELECT with JOINs
+  if (upperSql.includes('SELECT') && upperSql.includes('JOIN')) {
+    const result = parseSelectJoin(normalized)
+    if (result) {
+      result.sourceTables = [...result.sourceTables, ...cteSourceTables]
+      result.aggregations = extractAggregations(normalized)
+    }
+    return result
+  }
+
+  // Parse SELECT with aggregations (GROUP BY)
+  if (upperSql.includes('SELECT') && (upperSql.includes('GROUP BY') || hasAggregateFunction(upperSql))) {
+    const result = parseSelectAggregate(normalized)
+    if (result) {
+      result.sourceTables = [...result.sourceTables, ...cteSourceTables]
+    }
+    return result
+  }
+
+  // Parse SELECT with subqueries - check if main statement (not in CTE) starts with SELECT
+  // For CTEs followed by SELECT, we should return SELECT not SUBQUERY
+  if (upperSql.includes('SELECT') && normalized.includes('(SELECT')) {
+    // If we have CTEs and main SQL starts with SELECT (e.g., WITH ... SELECT), preserve SELECT operation
+    // But only if the (SELECT is part of a CTE definition, not a subquery in WHERE/FROM
+    if (ctes.length > 0 && mainUpperSql.startsWith('SELECT') && !mainSql.includes('(SELECT')) {
+      const sourceTables = extractSourceTables(mainSql)
+      // Filter out CTE names from source tables
+      const filteredSourceTables = sourceTables.filter((t) => !cteNames.has(t.table.toLowerCase()))
+      return {
+        operation: 'SELECT',
+        sourceTables: [...filteredSourceTables, ...cteSourceTables],
+      }
+    }
+    const result = parseSelectSubquery(normalized)
+    if (result) {
+      result.sourceTables = [...result.sourceTables, ...cteSourceTables]
+    }
+    return result
+  }
+
+  // Parse simple SELECT (including CTEs followed by SELECT)
+  if (mainUpperSql.startsWith('SELECT') && mainUpperSql.includes('FROM')) {
+    const sourceTables = extractSourceTables(mainSql)
+    // Filter out CTE names from source tables
+    const filteredSourceTables = sourceTables.filter((t) => !cteNames.has(t.table.toLowerCase()))
+    if (filteredSourceTables.length > 0 || cteSourceTables.length > 0) {
+      return {
+        operation: 'SELECT',
+        sourceTables: [...filteredSourceTables, ...cteSourceTables],
+      }
+    }
+  }
+
+  // Parse simple INSERT - use local parsing for embedded quotes support
+  const simpleInsertResult = parseSimpleInsert(normalized)
+  if (simpleInsertResult) {
+    return {
+      ...simpleInsertResult,
+      sourceTables: [...simpleInsertResult.sourceTables, ...cteSourceTables],
+    }
+  }
+
+  // Parse simple UPDATE
+  const writeOp = parseWriteOperation(sql)
+  if (writeOp && writeOp.operation === 'UPDATE') {
+    return {
+      operation: 'UPDATE',
+      targetTable: {
+        schema: writeOp.schema,
+        table: writeOp.table,
+      },
+      sourceTables: [
+        {
+          schema: writeOp.schema,
+          table: writeOp.table,
+        },
+        ...cteSourceTables,
+      ],
+    }
+  }
+
+  // Parse simple DELETE
+  if (writeOp && writeOp.operation === 'DELETE') {
+    return {
+      operation: 'DELETE',
+      targetTable: {
+        schema: writeOp.schema,
+        table: writeOp.table,
+      },
+      sourceTables: [
+        {
+          schema: writeOp.schema,
+          table: writeOp.table,
+        },
+        ...cteSourceTables,
+      ],
+    }
+  }
+
+  return null
+}
+
+/**
+ * Strip CTE definitions from SQL to get the main statement
+ */
+function stripCTEs(sql: string): string {
+  // Match WITH clause and all CTE definitions
+  const ctePattern = /^WITH\s+(?:RECURSIVE\s+)?(?:[a-z_][a-z0-9_]*\s+AS\s*\([^)]+\)\s*,?\s*)+/i
+  return sql.replace(ctePattern, '').trim()
+}
+
+/**
+ * Parse simple INSERT statement with embedded quote support
+ */
+function parseSimpleInsert(sql: string): LineageParseResult | null {
+  // Match INSERT INTO with support for:
+  // - [catalog.][schema.]table (3-part or 2-part names)
+  // - "Table""Name" (embedded quotes)
+  const insertMatch = sql.match(/^INSERT\s+INTO\s+(?:([a-z_][a-z0-9_]*)\s*\.\s*)?(?:([a-z_][a-z0-9_]*)\s*\.\s*)?([a-z_][a-z0-9_]*|"(?:[^"]|"")+")(?:\s|$|\()/i)
+  if (!insertMatch) return null
+
+  // Handle 3-part names: catalog.schema.table vs 2-part: schema.table vs 1-part: table
+  let targetSchema: string
+  let targetTable: string
+  if (insertMatch[2]) {
+    // 3-part name: insertMatch[1] is catalog, [2] is schema, [3] is table
+    // Or 2-part: [1] is schema, [2] is extra (shouldn't happen with this regex)
+    targetSchema = insertMatch[2].toLowerCase()
+    targetTable = unquoteIdentifier(insertMatch[3]!)
+  } else if (insertMatch[1]) {
+    targetSchema = insertMatch[1].toLowerCase()
+    targetTable = unquoteIdentifier(insertMatch[3]!)
+  } else {
+    targetSchema = 'public'
+    targetTable = unquoteIdentifier(insertMatch[3]!)
+  }
+
+  return {
+    operation: 'INSERT',
+    targetTable: {
+      schema: targetSchema,
+      table: targetTable,
+    },
+    sourceTables: [],
+  }
+}
+
+/**
+ * Check if SQL contains aggregate functions
+ */
+function hasAggregateFunction(upperSql: string): boolean {
+  const aggregateFunctions = ['COUNT(', 'SUM(', 'AVG(', 'MIN(', 'MAX(', 'ARRAY_AGG(', 'STRING_AGG(', 'JSON_AGG(', 'JSONB_AGG(']
+  return aggregateFunctions.some((fn) => upperSql.includes(fn))
+}
+
+/**
+ * Parse INSERT INTO ... SELECT FROM statement
+ */
+function parseInsertSelect(sql: string): LineageParseResult | null {
+  const insertMatch = sql.match(/INSERT\s+INTO\s+(?:([a-z_][a-z0-9_]*)\s*\.\s*)?([a-z_][a-z0-9_]*|"[^"]+")/i)
+  if (!insertMatch) return null
+
+  const targetSchema = insertMatch[1]?.toLowerCase() ?? 'public'
+  const targetTable = unquoteIdentifier(insertMatch[2]!)
+
+  // Extract source tables from SELECT clause
+  const sourceTables = extractSourceTables(sql)
+
+  return {
+    operation: 'INSERT',
+    targetTable: {
+      schema: targetSchema,
+      table: targetTable,
+    },
+    sourceTables,
+  }
+}
+
+/**
+ * Parse UPDATE ... FROM statement
+ */
+function parseUpdateFrom(sql: string): LineageParseResult | null {
+  const updateMatch = sql.match(/UPDATE\s+(?:([a-z_][a-z0-9_]*)\s*\.\s*)?([a-z_][a-z0-9_]*|"[^"]+")/i)
+  if (!updateMatch) return null
+
+  const targetSchema = updateMatch[1]?.toLowerCase() ?? 'public'
+  const targetTable = unquoteIdentifier(updateMatch[2]!)
+
+  // Extract source tables from FROM clause
+  const sourceTables = extractSourceTables(sql)
+
+  return {
+    operation: 'UPDATE',
+    targetTable: {
+      schema: targetSchema,
+      table: targetTable,
+    },
+    sourceTables,
+  }
+}
+
+/**
+ * Parse SELECT with JOIN
+ */
+function parseSelectJoin(sql: string): LineageParseResult | null {
+  // Extract source tables
+  const sourceTables = extractSourceTables(sql)
+
+  // Extract JOIN dependencies
+  const joins = extractJoins(sql)
+
+  // Extract WHERE filter
+  const whereFilter = extractWhereFilter(sql)
+
+  return {
+    operation: 'SELECT',
+    sourceTables,
+    joins,
+    whereFilter,
+  }
+}
+
+/**
+ * Parse CREATE TABLE AS SELECT (CTAS)
+ */
+function parseCreateTableAs(sql: string): LineageParseResult | null {
+  // Match CREATE TABLE [schema.]table AS SELECT
+  const createMatch = sql.match(/CREATE\s+TABLE\s+(?:IF\s+NOT\s+EXISTS\s+)?(?:([a-z_][a-z0-9_]*)\s*\.\s*)?([a-z_][a-z0-9_]*|"[^"]+")\s+AS\s+SELECT/i)
+  if (!createMatch) return null
+
+  const targetSchema = createMatch[1]?.toLowerCase() ?? 'public'
+  const targetTable = unquoteIdentifier(createMatch[2]!)
+
+  // Extract source tables from the SELECT part
+  const sourceTables = extractSourceTables(sql)
+
+  // Extract joins if present
+  const joins = extractJoins(sql)
+
+  // Extract aggregations if present
+  const aggregations = extractAggregations(sql)
+
+  return {
+    operation: 'INSERT', // CTAS is effectively an INSERT
+    targetTable: {
+      schema: targetSchema,
+      table: targetTable,
+    },
+    sourceTables,
+    joins: joins.length > 0 ? joins : undefined,
+    aggregations: aggregations.length > 0 ? aggregations : undefined,
+  }
+}
+
+/**
+ * Parse SELECT with aggregations
+ */
+function parseSelectAggregate(sql: string): LineageParseResult | null {
+  // Extract source tables
+  const sourceTables = extractSourceTables(sql)
+
+  // Extract aggregations
+  const aggregations = extractAggregations(sql)
+
+  // Extract WHERE filter
+  const whereFilter = extractWhereFilter(sql)
+
+  return {
+    operation: 'AGGREGATE',
+    sourceTables,
+    aggregations: aggregations.length > 0 ? aggregations : undefined,
+    whereFilter,
+  }
+}
+
+/**
+ * Parse SELECT with subqueries
+ */
+function parseSelectSubquery(sql: string): LineageParseResult | null {
+  // Extract main source tables
+  const sourceTables = extractSourceTables(sql)
+
+  // Extract subqueries and their source tables
+  const subqueries = extractSubqueries(sql)
+  const subquerySourceTables = subqueries.flatMap((sq) => sq.sourceTables)
+
+  // Combine all source tables (deduplicated)
+  const allSourceTables = [...sourceTables]
+  const seen = new Set(sourceTables.map((t) => `${t.schema}.${t.table}`))
+
+  for (const st of subquerySourceTables) {
+    const key = `${st.schema}.${st.table}`
+    if (!seen.has(key)) {
+      seen.add(key)
+      allSourceTables.push(st)
+    }
+  }
+
+  return {
+    operation: 'SUBQUERY',
+    sourceTables: allSourceTables,
+  }
+}
+
+/**
+ * Extract WHERE filter from SQL
+ */
+function extractWhereFilter(sql: string): string | undefined {
+  const whereMatch = sql.match(/WHERE\s+([^;]+?)(?:\s+GROUP|\s+ORDER|\s+LIMIT|\s+HAVING|\s*$|\s*;)/i)
+  if (whereMatch) {
+    return whereMatch[1]!.trim()
+  }
+  return undefined
+}
+
+/**
+ * Extract source tables from SQL
+ */
+function extractSourceTables(sql: string): { schema: string; table: string; alias?: string }[] {
+  const tables: { schema: string; table: string; alias?: string }[] = []
+  const seen = new Set<string>()
+
+  // Match FROM clause - capture everything until WHERE, GROUP, ORDER, LIMIT, JOIN, UNION, INTERSECT, EXCEPT, or end
+  // Use a non-greedy match and explicitly handle the end case
+  // Updated regex to allow dots in table names (for schema.table)
+  const fromMatch = sql.match(/FROM\s+((?:[a-z_][a-z0-9_.]*(?:\s+(?:AS\s+)?[a-z_][a-z0-9_]*)?(?:\s*,\s*)?)+)(?:\s+WHERE|\s+GROUP|\s+ORDER|\s+LIMIT|\s+(?:INNER|LEFT|RIGHT|FULL|CROSS)?\s*JOIN|\s+UNION|\s+INTERSECT|\s+EXCEPT|\s*$|\s*;)/i)
+  if (fromMatch) {
+    const fromClause = fromMatch[1]
+    if (!fromClause) return tables
+    // Match schema.table or just table, with optional alias
+    const tablePattern = /(?:([a-z_][a-z0-9_]*)\s*\.\s*)?([a-z_][a-z0-9_]*|"[^"]+")(?:\s+(?:AS\s+)?([a-z_][a-z0-9_]*))?/gi
+    const tableMatches = fromClause.matchAll(tablePattern)
+
+    for (const match of tableMatches) {
+      const schema = match[1]?.toLowerCase() ?? 'public'
+      const table = unquoteIdentifier(match[2]!)
+      const alias = match[3]?.toLowerCase()
+      const key = `${schema}.${table}`
+
+      if (!seen.has(key)) {
+        seen.add(key)
+        tables.push({ schema, table, ...(alias !== undefined && { alias }) })
+      }
+    }
+  }
+
+  // Match JOIN clauses - extract all JOINed tables
+  const joinPattern = /(?:INNER|LEFT|RIGHT|FULL|CROSS)?\s*JOIN\s+(?:([a-z_][a-z0-9_]*)\s*\.\s*)?([a-z_][a-z0-9_]*|"[^"]+")(?:\s+(?:AS\s+)?([a-z_][a-z0-9_]*))?/gi
+  const joinMatches = sql.matchAll(joinPattern)
+
+  for (const match of joinMatches) {
+    const schema = match[1]?.toLowerCase() ?? 'public'
+    const table = unquoteIdentifier(match[2]!)
+    const alias = match[3]?.toLowerCase()
+    const key = `${schema}.${table}`
+
+    if (!seen.has(key)) {
+      seen.add(key)
+      tables.push({ schema, table, ...(alias !== undefined && { alias }) })
+    }
+  }
+
+  // Match UNION/INTERSECT/EXCEPT source tables - find nested SELECTs
+  const unionPattern = /(?:UNION|INTERSECT|EXCEPT)(?:\s+ALL)?\s+SELECT\s+.*?\s+FROM\s+(?:([a-z_][a-z0-9_]*)\s*\.\s*)?([a-z_][a-z0-9_]*|"[^"]+")/gi
+  const unionMatches = sql.matchAll(unionPattern)
+
+  for (const match of unionMatches) {
+    const schema = match[1]?.toLowerCase() ?? 'public'
+    const table = unquoteIdentifier(match[2]!)
+    const key = `${schema}.${table}`
+
+    if (!seen.has(key)) {
+      seen.add(key)
+      tables.push({ schema, table })
+    }
+  }
+
+  return tables
+}
+
+/**
+ * Extract JOIN dependencies from SQL with proper left table tracking
+ */
+function extractJoins(sql: string): JoinDependency[] {
+  const joins: JoinDependency[] = []
+  const tables = extractSourceTables(sql)
+
+  // Extract the primary table from the FROM clause (first table)
+  let currentLeftTable = tables[0] ?? { schema: 'public', table: 'unknown' }
+
+  // Enhanced JOIN pattern with ON clause capture
+  const joinPattern = /(INNER|LEFT|RIGHT|FULL|CROSS)?\s*JOIN\s+(?:([a-z_][a-z0-9_]*)\s*\.\s*)?([a-z_][a-z0-9_]*|"[^"]+")(?:\s+(?:AS\s+)?([a-z_][a-z0-9_]*))?\s+ON\s+([^;]+?)(?=\s+(?:INNER|LEFT|RIGHT|FULL|CROSS)?\s*JOIN|\s+WHERE|\s+GROUP|\s+ORDER|\s+LIMIT|\s*$|\s*;)/gi
+
+  const matches = sql.matchAll(joinPattern)
+  for (const match of matches) {
+    const joinType = (match[1]?.toUpperCase() || 'INNER') as LineageJoinType
+    const schema = match[2]?.toLowerCase() ?? 'public'
+    const table = unquoteIdentifier(match[3]!)
+    const alias = match[4]?.toLowerCase()
+    const onClause = match[5]!
+
+    // Parse join columns from ON clause
+    const joinColumns = extractJoinColumns(onClause, currentLeftTable, { schema, table, ...(alias !== undefined && { alias }) })
+
+    joins.push({
+      joinType,
+      leftTable: { schema: currentLeftTable.schema, table: currentLeftTable.table },
+      rightTable: { schema, table },
+      joinColumns,
+      timestamp: Date.now(),
+    })
+
+    // Update current left table for chained joins
+    currentLeftTable = { schema, table, ...(alias !== undefined && { alias }) }
+  }
+
+  // If no ON clause detected, try simpler pattern
+  if (joins.length === 0) {
+    const simpleJoinPattern = /(INNER|LEFT|RIGHT|FULL|CROSS)?\s*JOIN\s+(?:([a-z_][a-z0-9_]*)\s*\.\s*)?([a-z_][a-z0-9_]*|"[^"]+")/gi
+    const simpleMatches = sql.matchAll(simpleJoinPattern)
+
+    for (const match of simpleMatches) {
+      const joinType = (match[1]?.toUpperCase() || 'INNER') as LineageJoinType
+      const schema = match[2]?.toLowerCase() ?? 'public'
+      const table = unquoteIdentifier(match[3]!)
+
+      joins.push({
+        joinType,
+        leftTable: { schema: currentLeftTable.schema, table: currentLeftTable.table },
+        rightTable: { schema, table },
+        joinColumns: [],
+        timestamp: Date.now(),
+      })
+
+      currentLeftTable = { schema, table }
+    }
+  }
+
+  return joins
+}
+
+/**
+ * Extract join columns from ON clause
+ */
+function extractJoinColumns(
+  onClause: string,
+  leftTable: { schema: string; table: string; alias?: string },
+  rightTable: { schema: string; table: string; alias?: string }
+): { left: string; right: string }[] {
+  const columns: { left: string; right: string }[] = []
+
+  // Pattern to match column comparisons: table.column = table.column
+  const comparisonPattern = /([a-z_][a-z0-9_]*(?:\.[a-z_][a-z0-9_]*)?)\s*=\s*([a-z_][a-z0-9_]*(?:\.[a-z_][a-z0-9_]*)?)/gi
+  const matches = onClause.matchAll(comparisonPattern)
+
+  const leftNames = [leftTable.table, leftTable.alias].filter(Boolean) as string[]
+  const rightNames = [rightTable.table, rightTable.alias].filter(Boolean) as string[]
+
+  for (const match of matches) {
+    const col1 = match[1]!.toLowerCase()
+    const col2 = match[2]!.toLowerCase()
+
+    // Determine which column belongs to which table
+    const col1Parts = col1.split('.')
+    const col2Parts = col2.split('.')
+
+    let leftCol: string | undefined
+    let rightCol: string | undefined
+
+    // Check if col1 belongs to left or right table
+    if (col1Parts.length === 2) {
+      if (leftNames.includes(col1Parts[0]!)) {
+        leftCol = col1Parts[1]!
+      } else if (rightNames.includes(col1Parts[0]!)) {
+        rightCol = col1Parts[1]!
+      }
+    }
+
+    if (col2Parts.length === 2) {
+      if (leftNames.includes(col2Parts[0]!)) {
+        leftCol = leftCol || col2Parts[1]!
+      } else if (rightNames.includes(col2Parts[0]!)) {
+        rightCol = rightCol || col2Parts[1]!
+      }
+    }
+
+    // If both columns identified, add to result
+    if (leftCol && rightCol) {
+      columns.push({ left: leftCol, right: rightCol })
+    } else if (col1Parts.length === 1 && col2Parts.length === 1) {
+      // Unqualified columns - assume same column name on both sides
+      columns.push({ left: col1Parts[0]!, right: col2Parts[0]! })
+    }
+  }
+
+  return columns
+}
+
+/**
+ * Extract aggregations from SQL
+ */
+function extractAggregations(sql: string): AggregateDependency[] {
+  const aggregations: AggregateDependency[] = []
+  const upperSql = sql.toUpperCase()
+
+  // Check if this is an aggregation query
+  const aggregateFunctions = ['COUNT', 'SUM', 'AVG', 'MIN', 'MAX', 'ARRAY_AGG', 'STRING_AGG', 'JSON_AGG', 'JSONB_AGG']
+  const hasAggregation = aggregateFunctions.some((fn) => upperSql.includes(fn + '('))
+
+  if (!hasAggregation) {
+    return aggregations
+  }
+
+  // Extract source tables
+  const sourceTables = extractSourceTables(sql)
+  if (sourceTables.length === 0) {
+    return aggregations
+  }
+
+  // Extract aggregate columns
+  const aggregatedColumns: { column: string; function: LineageAggregateFunction; alias?: string }[] = []
+
+  for (const fn of aggregateFunctions) {
+    // Match aggregation function calls: FUNC(column) AS alias or FUNC(column)
+    const fnPattern = new RegExp(
+      `${fn}\\s*\\(\\s*(?:DISTINCT\\s+)?([a-z_][a-z0-9_]*(?:\\.[a-z_][a-z0-9_]*)?|\\*)\\s*\\)(?:\\s+AS\\s+([a-z_][a-z0-9_]*))?`,
+      'gi'
+    )
+    const matches = sql.matchAll(fnPattern)
+
+    for (const match of matches) {
+      const column = match[1]!.toLowerCase()
+      const alias = match[2]?.toLowerCase()
+
+      aggregatedColumns.push({
+        column,
+        function: fn as LineageAggregateFunction,
+        ...(alias !== undefined && { alias }),
+      })
+    }
+  }
+
+  // Extract GROUP BY columns
+  const groupByColumns: string[] = []
+  const groupByMatch = sql.match(/GROUP\s+BY\s+([^;]+?)(?:\s+HAVING|\s+ORDER|\s+LIMIT|\s*$|\s*;)/i)
+  if (groupByMatch) {
+    const groupByClause = groupByMatch[1]!
+    // Split by comma and extract column names
+    const cols = groupByClause.split(',').map((c) => c.trim().toLowerCase())
+    groupByColumns.push(...cols)
+  }
+
+  // Extract HAVING filter
+  let havingFilter: string | undefined
+  const havingMatch = sql.match(/HAVING\s+([^;]+?)(?:\s+ORDER|\s+LIMIT|\s*$|\s*;)/i)
+  if (havingMatch) {
+    havingFilter = havingMatch[1]!.trim()
+  }
+
+  // Create aggregation dependency for each source table
+  for (const sourceTable of sourceTables) {
+    aggregations.push({
+      sourceTable: {
+        schema: sourceTable.schema,
+        table: sourceTable.table,
+      },
+      aggregatedColumns,
+      groupByColumns,
+      havingFilter,
+      timestamp: Date.now(),
+    })
+  }
+
+  return aggregations
+}
+
+/**
+ * Extract CTEs (Common Table Expressions) from SQL
+ */
+function extractCTEs(sql: string): { name: string; sourceTables: { schema: string; table: string }[] }[] {
+  const ctes: { name: string; sourceTables: { schema: string; table: string }[] }[] = []
+
+  // Match WITH clause CTEs
+  const ctePattern = /WITH\s+(?:RECURSIVE\s+)?([a-z_][a-z0-9_]*)\s+AS\s*\(\s*(SELECT[^)]+)\)/gi
+  const matches = sql.matchAll(ctePattern)
+
+  for (const match of matches) {
+    const cteName = match[1]!.toLowerCase()
+    const cteQuery = match[2]!
+
+    // Extract source tables from CTE query
+    const sourceTables = extractSourceTables(cteQuery)
+
+    ctes.push({
+      name: cteName,
+      sourceTables,
+    })
+  }
+
+  return ctes
+}
+
+/**
+ * Extract subqueries from SQL
+ */
+function extractSubqueries(sql: string): { sourceTables: { schema: string; table: string }[]; alias?: string }[] {
+  const subqueries: { sourceTables: { schema: string; table: string }[]; alias?: string }[] = []
+
+  // Match subqueries in FROM clause: (SELECT ...) AS alias
+  const subqueryPattern = /\(\s*(SELECT[^)]+)\s*\)\s*(?:AS\s+)?([a-z_][a-z0-9_]*)?/gi
+  const matches = sql.matchAll(subqueryPattern)
+
+  for (const match of matches) {
+    const subquery = match[1]!
+    const alias = match[2]?.toLowerCase()
+
+    // Extract source tables from subquery
+    const sourceTables = extractSourceTables(subquery)
+
+    if (sourceTables.length > 0) {
+      subqueries.push({
+        sourceTables,
+        ...(alias !== undefined && { alias }),
+      })
+    }
+  }
+
+  return subqueries
+}
+
+/**
+ * Remove quotes from an identifier
+ */
+function unquoteIdentifier(identifier: string): string {
+  if (identifier.startsWith('"') && identifier.endsWith('"')) {
+    return identifier.slice(1, -1).replace(/""/g, '"')
+  }
+  return identifier.toLowerCase()
+}
+
+/**
+ * Data Lineage Tracker
+ *
+ * Tracks data flow between tables, records transformations,
+ * and provides lineage queries for any row.
+ */
+export class LineageTracker {
+  private config: Required<LineageTrackerConfig>
+  private graph: LineageGraph
+  private rowLineageMap: Map<string, RowLineage[]> = new Map()
+
+  constructor(config: LineageTrackerConfig = {}) {
+    this.config = { ...DEFAULT_CONFIG, ...config }
+    this.graph = {
+      nodes: new Map(),
+      edges: new Map(),
+      sourceIndex: new Map(),
+      targetIndex: new Map(),
+    }
+  }
+
+  /**
+   * Get current configuration
+   */
+  getConfig(): Required<LineageTrackerConfig> {
+    return { ...this.config }
+  }
+
+  /**
+   * Update configuration
+   */
+  setConfig(config: Partial<LineageTrackerConfig>): void {
+    this.config = { ...this.config, ...config }
+  }
+
+  /**
+   * Check if lineage tracking is enabled
+   */
+  isEnabled(): boolean {
+    return this.config.enabled
+  }
+
+  /**
+   * Enable or disable lineage tracking
+   */
+  setEnabled(enabled: boolean): void {
+    this.config.enabled = enabled
+  }
+
+  /**
+   * Track a WAL entry for lineage
+   */
+  trackWALEntry(entry: SerializedWALEntry): void {
+    if (!this.config.enabled) return
+
+    // Check table filters
+    if (this.config.includeTables.length > 0 && !this.config.includeTables.includes(entry.table)) {
+      return
+    }
+    if (this.config.excludeTables.includes(entry.table)) {
+      return
+    }
+
+    // Ensure target node exists
+    this.ensureNode(entry.schema, entry.table, 'table')
+
+    // Parse SQL for lineage if available
+    if (entry.sql) {
+      const lineage = parseLineage(entry.sql)
+      if (lineage) {
+        this.trackLineage(lineage, entry)
+      }
+    }
+
+    // Track row-level lineage if enabled
+    if (this.config.trackRows && entry.primaryKey) {
+      this.trackRowLineage(entry)
+    }
+  }
+
+  /**
+   * Track lineage from parsed SQL
+   */
+  private trackLineage(lineage: LineageParseResult, entry: SerializedWALEntry): void {
+    if (!lineage.targetTable) return
+
+    const targetNodeId = generateNodeId(lineage.targetTable.schema, lineage.targetTable.table)
+
+    // Track source tables
+    for (const source of lineage.sourceTables) {
+      const sourceNodeId = generateNodeId(source.schema, source.table)
+
+      // Ensure source node exists
+      this.ensureNode(source.schema, source.table, 'table')
+
+      // Create edge
+      this.addEdge({
+        id: generateEdgeId(sourceNodeId, targetNodeId, Date.now()),
+        sourceId: sourceNodeId,
+        targetId: targetNodeId,
+        operation: lineage.operation,
+        timestamp: entry.timestamp,
+        lsn: entry.lsn,
+        transactionId: entry.transactionId,
+        sql: entry.sql,
+        joinDependency: lineage.joins?.[0],
+        aggregateDependency: lineage.aggregations?.[0],
+      })
+    }
+  }
+
+  /**
+   * Track row-level lineage
+   */
+  private trackRowLineage(entry: SerializedWALEntry): void {
+    if (!entry.primaryKey) return
+
+    const rowKey = this.getRowKey(entry.schema, entry.table, entry.primaryKey)
+
+    const rowLineage: RowLineage = {
+      targetTable: {
+        schema: entry.schema,
+        table: entry.table,
+      },
+      targetPrimaryKey: entry.primaryKey,
+      sourceRows: [],
+      operation: entry.operation as LineageOperationType,
+      timestamp: entry.timestamp,
+      lsn: entry.lsn,
+      transactionId: entry.transactionId,
+    }
+
+    // For UPDATE operations, track the old row as source
+    if (entry.operation === 'UPDATE' && entry.oldRow) {
+      rowLineage.sourceRows.push({
+        table: {
+          schema: entry.schema,
+          table: entry.table,
+        },
+        primaryKey: entry.primaryKey,
+        contributionType: 'direct',
+      })
+    }
+
+    // Store row lineage
+    if (!this.rowLineageMap.has(rowKey)) {
+      this.rowLineageMap.set(rowKey, [])
+    }
+    this.rowLineageMap.get(rowKey)!.push(rowLineage)
+  }
+
+  /**
+   * Get row key for lineage map
+   */
+  private getRowKey(schema: string, table: string, primaryKey: Record<string, unknown>): string {
+    return `${schema}.${table}:${JSON.stringify(primaryKey)}`
+  }
+
+  /**
+   * Ensure a node exists in the graph
+   */
+  private ensureNode(schema: string, table: string, type: LineageNode['type']): void {
+    const nodeId = generateNodeId(schema, table)
+
+    if (!this.graph.nodes.has(nodeId)) {
+      const now = Date.now()
+      this.graph.nodes.set(nodeId, {
+        id: nodeId,
+        type,
+        schema,
+        name: table,
+        createdAt: now,
+        updatedAt: now,
+      })
+    } else {
+      // Update timestamp
+      const node = this.graph.nodes.get(nodeId)!
+      node.updatedAt = Date.now()
+    }
+  }
+
+  /**
+   * Add an edge to the graph
+   */
+  private addEdge(edge: LineageEdge): void {
+    // Add edge
+    this.graph.edges.set(edge.id, edge)
+
+    // Update source index
+    if (!this.graph.sourceIndex.has(edge.sourceId)) {
+      this.graph.sourceIndex.set(edge.sourceId, new Set())
+    }
+    this.graph.sourceIndex.get(edge.sourceId)!.add(edge.id)
+
+    // Update target index
+    if (!this.graph.targetIndex.has(edge.targetId)) {
+      this.graph.targetIndex.set(edge.targetId, new Set())
+    }
+    this.graph.targetIndex.get(edge.targetId)!.add(edge.id)
+  }
+
+  /**
+   * Get lineage for a specific table
+   */
+  getTableLineage(schema: string, table: string): {
+    upstreamTables: string[]
+    downstreamTables: string[]
+    edges: LineageEdge[]
+  } {
+    const nodeId = generateNodeId(schema, table)
+
+    // Get upstream (sources)
+    const upstreamEdgeIds = this.graph.targetIndex.get(nodeId) || new Set()
+    const upstreamTables = new Set<string>()
+    const edges: LineageEdge[] = []
+
+    for (const edgeId of upstreamEdgeIds) {
+      const edge = this.graph.edges.get(edgeId)
+      if (edge) {
+        upstreamTables.add(edge.sourceId)
+        edges.push(edge)
+      }
+    }
+
+    // Get downstream (targets)
+    const downstreamEdgeIds = this.graph.sourceIndex.get(nodeId) || new Set()
+    const downstreamTables = new Set<string>()
+
+    for (const edgeId of downstreamEdgeIds) {
+      const edge = this.graph.edges.get(edgeId)
+      if (edge) {
+        downstreamTables.add(edge.targetId)
+        edges.push(edge)
+      }
+    }
+
+    return {
+      upstreamTables: Array.from(upstreamTables),
+      downstreamTables: Array.from(downstreamTables),
+      edges,
+    }
+  }
+
+  /**
+   * Query lineage for a specific row
+   */
+  queryRowLineage(
+    schema: string,
+    table: string,
+    primaryKey: Record<string, unknown>
+  ): LineageQueryResult | null {
+    if (!this.config.trackRows) {
+      throw new Error('Row-level lineage tracking is not enabled')
+    }
+
+    const rowKey = this.getRowKey(schema, table, primaryKey)
+    const lineagePath = this.rowLineageMap.get(rowKey)
+
+    if (!lineagePath) {
+      return null
+    }
+
+    // Calculate depth and source tables
+    const depth = lineagePath.length
+    const sourceTables = new Set<string>()
+
+    for (const lineage of lineagePath) {
+      for (const source of lineage.sourceRows) {
+        sourceTables.add(`${source.table.schema}.${source.table.table}`)
+      }
+    }
+
+    return {
+      row: {
+        schema,
+        table,
+        primaryKey,
+      },
+      lineagePath,
+      depth,
+      sourceTables: Array.from(sourceTables)
+        .map((id) => {
+          const [schema, table] = id.split('.')
+          return schema && table ? { schema, table } : null
+        })
+        .filter((t): t is { schema: string; table: string } => t !== null),
+    }
+  }
+
+  /**
+   * Visualize lineage graph for a table
+   */
+  visualizeLineage(schema: string, table: string, maxDepth?: number): LineageVisualizationGraph {
+    // Use provided maxDepth, or fallback to config. When maxDepth is explicitly 0, use 0.
+    const depth = maxDepth !== undefined ? maxDepth : this.config.maxDepth
+    const nodeId = generateNodeId(schema, table)
+
+    const visitedNodes = new Set<string>()
+    const visitedEdges = new Set<string>()
+    const nodes: LineageVisualizationNode[] = []
+    const edges: LineageVisualizationEdge[] = []
+    const rootNodes = new Set<string>()
+    const leafNodes = new Set<string>()
+
+    // Get the starting node
+    const startingNode = this.graph.nodes.get(nodeId)
+    if (!startingNode) {
+      return {
+        nodes: [],
+        edges: [],
+        maxDepth: 0,
+        rootNodes: [],
+        leafNodes: [],
+      }
+    }
+
+    // Add starting node
+    nodes.push({
+      id: startingNode.id,
+      label: startingNode.name,
+      type: startingNode.type,
+      schema: startingNode.schema,
+      table: startingNode.name,
+      level: 0,
+    })
+
+    // If maxDepth is 0, only return the starting node
+    if (depth === 0) {
+      // Check if starting node is root or leaf
+      const hasUpstream = (this.graph.targetIndex.get(nodeId)?.size || 0) > 0
+      const hasDownstream = (this.graph.sourceIndex.get(nodeId)?.size || 0) > 0
+      if (!hasUpstream) rootNodes.add(nodeId)
+      if (!hasDownstream) leafNodes.add(nodeId)
+
+      return {
+        nodes,
+        edges: [],
+        maxDepth: 0,
+        rootNodes: Array.from(rootNodes),
+        leafNodes: Array.from(leafNodes),
+      }
+    }
+
+    // BFS traversal
+    const queue: { nodeId: string; level: number; direction: 'up' | 'down' }[] = [
+      { nodeId, level: 0, direction: 'up' },
+      { nodeId, level: 0, direction: 'down' },
+    ]
+
+    while (queue.length > 0) {
+      const { nodeId: currentId, level, direction } = queue.shift()!
+
+      if (level >= depth) continue
+      if (visitedNodes.has(`${currentId}:${direction}`)) continue
+
+      visitedNodes.add(`${currentId}:${direction}`)
+
+      const node = this.graph.nodes.get(currentId)
+      if (!node) continue
+
+      // Add node to visualization (skip if already added as starting node)
+      if (!nodes.find((n) => n.id === currentId)) {
+        nodes.push({
+          id: node.id,
+          label: node.name,
+          type: node.type,
+          schema: node.schema,
+          table: node.name,
+          level,
+        })
+      }
+
+      // Traverse edges
+      if (direction === 'up') {
+        // Upstream (sources)
+        const edgeIds = this.graph.targetIndex.get(currentId) || new Set()
+
+        if (edgeIds.size === 0) {
+          rootNodes.add(currentId)
+        }
+
+        for (const edgeId of edgeIds) {
+          if (visitedEdges.has(edgeId)) continue
+          visitedEdges.add(edgeId)
+
+          const edge = this.graph.edges.get(edgeId)
+          if (!edge) continue
+
+          edges.push({
+            id: edge.id,
+            source: edge.sourceId,
+            target: edge.targetId,
+            operation: edge.operation,
+            label: edge.operation,
+          })
+
+          queue.push({ nodeId: edge.sourceId, level: level + 1, direction: 'up' })
+        }
+      } else {
+        // Downstream (targets)
+        const edgeIds = this.graph.sourceIndex.get(currentId) || new Set()
+
+        if (edgeIds.size === 0) {
+          leafNodes.add(currentId)
+        }
+
+        for (const edgeId of edgeIds) {
+          if (visitedEdges.has(edgeId)) continue
+          visitedEdges.add(edgeId)
+
+          const edge = this.graph.edges.get(edgeId)
+          if (!edge) continue
+
+          edges.push({
+            id: edge.id,
+            source: edge.sourceId,
+            target: edge.targetId,
+            operation: edge.operation,
+            label: edge.operation,
+          })
+
+          queue.push({ nodeId: edge.targetId, level: level + 1, direction: 'down' })
+        }
+      }
+    }
+
+    return {
+      nodes,
+      edges,
+      maxDepth: nodes.length > 0 ? Math.max(...nodes.map((n) => n.level)) : 0,
+      rootNodes: Array.from(rootNodes),
+      leafNodes: Array.from(leafNodes),
+    }
+  }
+
+  /**
+   * Get statistics about tracked lineage
+   */
+  getStats(): LineageStats {
+    let totalDepth = 0
+    let depthCount = 0
+
+    // Calculate average depth by sampling nodes
+    for (const [nodeId] of this.graph.nodes) {
+      const depth = this.calculateNodeDepth(nodeId)
+      totalDepth += depth
+      depthCount++
+    }
+
+    // Estimate memory usage
+    const graphSize =
+      JSON.stringify(Array.from(this.graph.nodes.values())).length +
+      JSON.stringify(Array.from(this.graph.edges.values())).length
+    const rowLineageSize = JSON.stringify(Array.from(this.rowLineageMap.values())).length
+
+    return {
+      totalNodes: this.graph.nodes.size,
+      totalEdges: this.graph.edges.size,
+      trackedTables: this.graph.nodes.size,
+      transformationCount: Array.from(this.graph.edges.values()).filter(
+        (e) => e.operation === 'JOIN' || e.operation === 'AGGREGATE'
+      ).length,
+      avgGraphDepth: depthCount > 0 ? totalDepth / depthCount : 0,
+      lastUpdateAt: this.getLastUpdateTimestamp(),
+      memoryBytes: graphSize + rowLineageSize,
+    }
+  }
+
+  /**
+   * Calculate the depth of a node in the graph
+   */
+  private calculateNodeDepth(nodeId: string): number {
+    let maxDepth = 0
+    const visited = new Set<string>()
+    const queue: { id: string; depth: number }[] = [{ id: nodeId, depth: 0 }]
+
+    while (queue.length > 0) {
+      const { id, depth } = queue.shift()!
+
+      if (visited.has(id)) continue
+      visited.add(id)
+
+      maxDepth = Math.max(maxDepth, depth)
+
+      const edgeIds = this.graph.targetIndex.get(id) || new Set()
+      for (const edgeId of edgeIds) {
+        const edge = this.graph.edges.get(edgeId)
+        if (edge) {
+          queue.push({ id: edge.sourceId, depth: depth + 1 })
+        }
+      }
+    }
+
+    return maxDepth
+  }
+
+  /**
+   * Get the last update timestamp
+   */
+  private getLastUpdateTimestamp(): number | null {
+    let lastUpdate: number | null = null
+
+    for (const [, node] of this.graph.nodes) {
+      if (lastUpdate === null || node.updatedAt > lastUpdate) {
+        lastUpdate = node.updatedAt
+      }
+    }
+
+    return lastUpdate
+  }
+
+  /**
+   * Export lineage metadata for Iceberg table properties
+   */
+  exportIcebergMetadata(schema: string, table: string): IcebergLineageMetadata {
+    const lineage = this.getTableLineage(schema, table)
+
+    return {
+      'lineage.version': '1.0',
+      'lineage.sources': lineage.upstreamTables.join(','),
+      'lineage.operations': lineage.edges.map((e) => e.operation).join(','),
+      'lineage.updated-at': new Date().toISOString(),
+      'lineage.graph': JSON.stringify({
+        upstreamTables: lineage.upstreamTables,
+        downstreamTables: lineage.downstreamTables,
+        edges: lineage.edges.map((e) => ({
+          source: e.sourceId,
+          target: e.targetId,
+          operation: e.operation,
+        })),
+      }),
+    }
+  }
+
+  /**
+   * Import lineage metadata from Iceberg table properties
+   */
+  importIcebergMetadata(schema: string, table: string, metadata: IcebergLineageMetadata): void {
+    if (!metadata['lineage.graph']) return
+
+    try {
+      const graph = JSON.parse(metadata['lineage.graph'])
+
+      // Ensure the target node exists
+      this.ensureNode(schema, table, 'table')
+
+      // Import edges
+      for (const edge of graph.edges || []) {
+        const [sourceSchema, sourceTable] = edge.source.split('.')
+        this.ensureNode(sourceSchema, sourceTable, 'table')
+
+        this.addEdge({
+          id: generateEdgeId(edge.source, edge.target, Date.now()),
+          sourceId: edge.source,
+          targetId: edge.target,
+          operation: edge.operation,
+          timestamp: new Date(metadata['lineage.updated-at']).getTime(),
+        })
+      }
+    } catch (error) {
+      console.error('Failed to import lineage metadata:', error)
+    }
+  }
+
+  /**
+   * Query all upstream tables (data sources) for a given table
+   * Traverses the graph recursively to find all tables that feed into this one
+   */
+  queryUpstreamLineage(schema: string, table: string, maxDepth?: number): {
+    tables: { schema: string; table: string; depth: number }[]
+    edges: LineageEdge[]
+    totalDepth: number
+  } {
+    // Use provided maxDepth, or fallback to config. When maxDepth is explicitly 0, use 0.
+    const depth = maxDepth !== undefined ? maxDepth : this.config.maxDepth
+    const nodeId = generateNodeId(schema, table)
+
+    const tables: { schema: string; table: string; depth: number }[] = []
+    const edges: LineageEdge[] = []
+    const visited = new Set<string>()
+    const visitedEdges = new Set<string>()
+
+    // If maxDepth is 0, return empty results (no traversal)
+    if (depth === 0) {
+      return { tables, edges, totalDepth: 0 }
+    }
+
+    const queue: { id: string; depth: number }[] = [{ id: nodeId, depth: 0 }]
+    let totalDepth = 0
+
+    while (queue.length > 0) {
+      const { id, depth: currentDepth } = queue.shift()!
+
+      if (visited.has(id) || currentDepth >= depth) continue
+      visited.add(id)
+
+      // Get upstream edges (edges where this node is the target)
+      const upstreamEdgeIds = this.graph.targetIndex.get(id) || new Set()
+
+      for (const edgeId of upstreamEdgeIds) {
+        if (visitedEdges.has(edgeId)) continue
+        visitedEdges.add(edgeId)
+
+        const edge = this.graph.edges.get(edgeId)
+        if (!edge) continue
+
+        edges.push(edge)
+
+        // Add source node to results
+        const sourceNode = this.graph.nodes.get(edge.sourceId)
+        if (sourceNode && !visited.has(edge.sourceId)) {
+          tables.push({
+            schema: sourceNode.schema,
+            table: sourceNode.name,
+            depth: currentDepth + 1,
+          })
+          totalDepth = Math.max(totalDepth, currentDepth + 1)
+          queue.push({ id: edge.sourceId, depth: currentDepth + 1 })
+        }
+      }
+    }
+
+    return { tables, edges, totalDepth }
+  }
+
+  /**
+   * Query all downstream tables (data consumers) for a given table
+   * Traverses the graph recursively to find all tables that are derived from this one
+   */
+  queryDownstreamLineage(schema: string, table: string, maxDepth?: number): {
+    tables: { schema: string; table: string; depth: number }[]
+    edges: LineageEdge[]
+    totalDepth: number
+  } {
+    const depth = maxDepth || this.config.maxDepth
+    const nodeId = generateNodeId(schema, table)
+
+    const tables: { schema: string; table: string; depth: number }[] = []
+    const edges: LineageEdge[] = []
+    const visited = new Set<string>()
+    const visitedEdges = new Set<string>()
+
+    const queue: { id: string; depth: number }[] = [{ id: nodeId, depth: 0 }]
+    let totalDepth = 0
+
+    while (queue.length > 0) {
+      const { id, depth: currentDepth } = queue.shift()!
+
+      if (visited.has(id) || currentDepth > depth) continue
+      visited.add(id)
+
+      // Get downstream edges (edges where this node is the source)
+      const downstreamEdgeIds = this.graph.sourceIndex.get(id) || new Set()
+
+      for (const edgeId of downstreamEdgeIds) {
+        if (visitedEdges.has(edgeId)) continue
+        visitedEdges.add(edgeId)
+
+        const edge = this.graph.edges.get(edgeId)
+        if (!edge) continue
+
+        edges.push(edge)
+
+        // Add target node to results
+        const targetNode = this.graph.nodes.get(edge.targetId)
+        if (targetNode && !visited.has(edge.targetId)) {
+          tables.push({
+            schema: targetNode.schema,
+            table: targetNode.name,
+            depth: currentDepth + 1,
+          })
+          totalDepth = Math.max(totalDepth, currentDepth + 1)
+          queue.push({ id: edge.targetId, depth: currentDepth + 1 })
+        }
+      }
+    }
+
+    return { tables, edges, totalDepth }
+  }
+
+  /**
+   * Find the shortest path between two tables in the lineage graph
+   */
+  findLineagePath(
+    sourceSchema: string,
+    sourceTable: string,
+    targetSchema: string,
+    targetTable: string
+  ): { path: string[]; edges: LineageEdge[] } | null {
+    const sourceId = generateNodeId(sourceSchema, sourceTable)
+    const targetId = generateNodeId(targetSchema, targetTable)
+
+    if (!this.graph.nodes.has(sourceId) || !this.graph.nodes.has(targetId)) {
+      return null
+    }
+
+    // BFS to find shortest path
+    const visited = new Set<string>()
+    const parent = new Map<string, { nodeId: string; edge: LineageEdge }>()
+    const queue: string[] = [sourceId]
+
+    visited.add(sourceId)
+
+    while (queue.length > 0) {
+      const current = queue.shift()!
+
+      if (current === targetId) {
+        // Reconstruct path
+        const path: string[] = []
+        const edges: LineageEdge[] = []
+        let node = targetId
+
+        while (node !== sourceId) {
+          path.unshift(node)
+          const parentInfo = parent.get(node)!
+          edges.unshift(parentInfo.edge)
+          node = parentInfo.nodeId
+        }
+        path.unshift(sourceId)
+
+        return { path, edges }
+      }
+
+      // Explore downstream edges
+      const downstreamEdgeIds = this.graph.sourceIndex.get(current) || new Set()
+      for (const edgeId of downstreamEdgeIds) {
+        const edge = this.graph.edges.get(edgeId)
+        if (edge && !visited.has(edge.targetId)) {
+          visited.add(edge.targetId)
+          parent.set(edge.targetId, { nodeId: current, edge })
+          queue.push(edge.targetId)
+        }
+      }
+    }
+
+    return null // No path found
+  }
+
+  /**
+   * Get impact analysis: all tables that would be affected by changes to a source table
+   */
+  getImpactAnalysis(schema: string, table: string): {
+    directlyAffected: { schema: string; table: string }[]
+    transitivelyAffected: { schema: string; table: string; hops: number }[]
+    totalAffected: number
+  } {
+    const downstream = this.queryDownstreamLineage(schema, table)
+
+    const directlyAffected = downstream.tables
+      .filter((t) => t.depth === 1)
+      .map((t) => ({ schema: t.schema, table: t.table }))
+
+    const transitivelyAffected = downstream.tables
+      .filter((t) => t.depth > 1)
+      .map((t) => ({ schema: t.schema, table: t.table, hops: t.depth }))
+
+    return {
+      directlyAffected,
+      transitivelyAffected,
+      totalAffected: downstream.tables.length,
+    }
+  }
+
+  /**
+   * Get data provenance: full lineage history for understanding data origin
+   */
+  getDataProvenance(schema: string, table: string): {
+    rootSources: { schema: string; table: string }[]
+    transformationChain: { table: string; operation: LineageOperationType; timestamp: number }[]
+    depth: number
+  } {
+    const upstream = this.queryUpstreamLineage(schema, table)
+
+    // Find root sources (tables with no upstream dependencies)
+    const rootSources: { schema: string; table: string }[] = []
+    for (const t of upstream.tables) {
+      const nodeId = generateNodeId(t.schema, t.table)
+      const hasUpstream = (this.graph.targetIndex.get(nodeId)?.size || 0) > 0
+      if (!hasUpstream) {
+        rootSources.push({ schema: t.schema, table: t.table })
+      }
+    }
+
+    // Build transformation chain
+    const transformationChain = upstream.edges
+      .sort((a, b) => a.timestamp - b.timestamp)
+      .map((e) => ({
+        table: e.targetId,
+        operation: e.operation,
+        timestamp: e.timestamp,
+      }))
+
+    return {
+      rootSources,
+      transformationChain,
+      depth: upstream.totalDepth,
+    }
+  }
+
+  /**
+   * List all tracked tables
+   */
+  listTrackedTables(): { schema: string; table: string; type: LineageNode['type'] }[] {
+    const tables: { schema: string; table: string; type: LineageNode['type'] }[] = []
+
+    for (const [, node] of this.graph.nodes) {
+      tables.push({
+        schema: node.schema,
+        table: node.name,
+        type: node.type,
+      })
+    }
+
+    return tables
+  }
+
+  /**
+   * Get node by ID
+   */
+  getNode(schema: string, table: string): LineageNode | null {
+    const nodeId = generateNodeId(schema, table)
+    return this.graph.nodes.get(nodeId) || null
+  }
+
+  /**
+   * Clear all lineage data
+   */
+  clear(): void {
+    this.graph = {
+      nodes: new Map(),
+      edges: new Map(),
+      sourceIndex: new Map(),
+      targetIndex: new Map(),
+    }
+    this.rowLineageMap.clear()
+  }
+
+  /**
+   * Get the full lineage graph
+   */
+  getGraph(): LineageGraph {
+    return this.graph
+  }
+}
+
+/**
+ * Create a lineage tracker with default configuration
+ */
+export function createLineageTracker(config?: LineageTrackerConfig): LineageTracker {
+  return new LineageTracker(config)
+}