@sixb/ducklake 0.1.8 → 0.2.0

package/src/internal/sql-lineage/rejection.ts

@@ -0,0 +1,16 @@
+/**
+ * A transform whose lineage cannot be derived exactly. The message names the construct and becomes
+ * the `reason` of an unsupported analysis; it never escapes the analysis as an error.
+ */
+export class SqlLineageRejected extends Error {
+  override readonly name = "SqlLineageRejected"
+}
+
+export function reject(reason: string): never {
+  throw new SqlLineageRejected(reason)
+}
+
+/** DuckDB serialized something the analysis does not recognize, most likely after an upgrade. */
+export function rejectPlanFormat(detail: string): never {
+  reject(`unexpected DuckDB plan format: ${detail}`)
+}

package/src/internal/sql-lineage/statement-gate.ts

@@ -0,0 +1,164 @@
+import { isJsonObject, type JsonObject, listField, objectField, stringField } from "./plan-json"
+import { reject } from "./rejection"
+import { PURE_TABLE_FUNCTIONS } from "./table-functions"
+
+/**
+ * Checks the parse tree (`json_serialize_sql`) before DuckDB binds the query. Binding already runs
+ * table functions' bind code, and table macros such as `query()` only expand during binding, so
+ * they must be refused from the parse tree. PIVOT is refused here because its output columns depend
+ * on data the empty analysis tables do not have.
+ */
+export function assertSingleSelect(statements: readonly unknown[]): void {
+  const [statement] = statements
+  if (statements.length !== 1 || statement === undefined) {
+    reject("the transform must be exactly one SELECT statement")
+  }
+
+  visit(statement, (node) => {
+    if (node.type === "PIVOT") {
+      const unpivot = listField(node, "unpivot_names").length > 0
+      reject(`${unpivot ? "UNPIVOT" : "PIVOT"} is not supported`)
+    }
+
+    const unread = readsBeyondSources(node)
+    if (unread) reject(unread)
+  })
+}
+
+/**
+ * Whether the statement reads nothing but its sources, for a transform the analysis rejected: then
+ * every column it outputs depends on at most every column of its sources. PIVOT reads only its
+ * input.
+ *
+ * Data reaches a query through its table references. Each must be one of `sourceTables`, or a
+ * common table expression an enclosing query defines; a table function must compute from its
+ * arguments alone. A name a query does not define resolves to a table of the lake.
+ *
+ * A function must carry a name DuckDB itself defines, one of `builtinFunctions`: a macro the
+ * lake's setup defines can read any table, and its call shows none.
+ */
+export function readsOnlySources(
+  statements: readonly unknown[],
+  sourceTables: ReadonlySet<string>,
+  builtinFunctions: ReadonlySet<string>
+): boolean {
+  const [statement] = statements
+  if (statements.length !== 1 || statement === undefined) return false
+
+  let only = true
+  visit(statement, (node, ctes) => {
+    if (readsBeyondSources(node)) only = false
+    else if (node.type === "BASE_TABLE" && !readsSourceOrCte(node, sourceTables, ctes)) only = false
+    else if (callsUnknownFunction(node, builtinFunctions)) only = false
+  })
+
+  return only
+}
+
+/** What makes a node read outside the transform's sources, if anything. */
+function readsBeyondSources(node: JsonObject): string | undefined {
+  if (node.type === "SHOW_REF") {
+    return "DESCRIBE, SHOW and SUMMARIZE are not supported"
+  }
+
+  if (node.type === "TABLE_FUNCTION") {
+    const name = stringField(objectField(node, "function"), "function_name")
+    if (!PURE_TABLE_FUNCTIONS.has(name)) return `table function '${name}' is not supported`
+  }
+
+  if (node.class === "PARAMETER") {
+    return "query parameters are not supported"
+  }
+
+  return undefined
+}
+
+function readsSourceOrCte(
+  table: JsonObject,
+  sourceTables: ReadonlySet<string>,
+  ctes: ReadonlySet<string>
+): boolean {
+  if (qualified(table, "schema_name") || qualified(table, "catalog_name")) return false
+
+  const name = stringField(table, "table_name")
+  return sourceTables.has(name) || ctes.has(name.toLowerCase())
+}
+
+/**
+ * DuckDB writes syntax such as `EXTRACT`, `[1, 2]` or `{a: 1}` as calls to functions of schema
+ * `main`, so only another schema or a catalog qualifies a call.
+ */
+function callsUnknownFunction(node: JsonObject, builtinFunctions: ReadonlySet<string>): boolean {
+  if (node.class !== "FUNCTION" && node.class !== "WINDOW") return false
+  if (qualified(node, "catalog") || (qualified(node, "schema") && node.schema !== "main")) {
+    return true
+  }
+
+  const name = node.function_name
+  return typeof name !== "string" || !builtinFunctions.has(name.toLowerCase())
+}
+
+function qualified(node: JsonObject, key: string): boolean {
+  const value = node[key]
+  return typeof value === "string" && value !== ""
+}
+
+/**
+ * Visit every node of a parse tree, with the names of the common table expressions in scope, which
+ * DuckDB matches without regard to case.
+ *
+ * A query's body sees all of its expressions. An expression's own query sees only those defined
+ * before it, and a recursive one sees itself in its recursive part alone: any other name resolves
+ * to a table of the lake, so `WITH raw AS (SELECT * FROM raw)` reads the lake's `raw`.
+ */
+function visit(
+  value: unknown,
+  check: (node: JsonObject, ctes: ReadonlySet<string>) => void,
+  ctes: ReadonlySet<string> = new Set()
+): void {
+  if (Array.isArray(value)) {
+    for (const item of value) visit(item, check, ctes)
+    return
+  }
+
+  if (!isJsonObject(value)) return
+
+  const entries = cteEntries(value)
+  const defined = new Set(ctes)
+  for (const { name, query } of entries) {
+    visit(query, check, new Set(defined))
+    defined.add(name)
+  }
+
+  check(value, defined)
+
+  const recursive =
+    value.type === "RECURSIVE_CTE_NODE" && typeof value.cte_name === "string"
+      ? new Set([...defined, value.cte_name.toLowerCase()])
+      : undefined
+
+  for (const [key, child] of Object.entries(value)) {
+    if (key === "cte_map" && entries.length > 0) continue
+    visit(child, check, key === "right" && recursive ? recursive : defined)
+  }
+}
+
+/**
+ * A query's common table expressions, in the order it defines them. A map in a shape this does not
+ * know gives none: its nodes are then visited as any others, with no name in scope.
+ */
+function cteEntries(
+  node: JsonObject
+): readonly { readonly name: string; readonly query: unknown }[] {
+  const map = node.cte_map
+  if (!isJsonObject(map) || !Array.isArray(map.map)) return []
+
+  const entries: { name: string; query: unknown }[] = []
+  for (const entry of map.map) {
+    if (!isJsonObject(entry) || typeof entry.key !== "string" || !isJsonObject(entry.value)) {
+      return []
+    }
+    entries.push({ name: entry.key.toLowerCase(), query: entry.value.query })
+  }
+  return entries
+}

package/src/internal/sql-lineage/table-functions.ts

@@ -0,0 +1,11 @@
+/**
+ * Table functions that compute only from their arguments. Every other table function reads outside
+ * the transform's sources (files, catalogs, settings) or runs SQL of its own, like `query()`.
+ */
+export const PURE_TABLE_FUNCTIONS: ReadonlySet<string> = new Set([
+  "range",
+  "generate_series",
+  "unnest",
+  "json_each",
+  "json_tree",
+])

package/src/internal/sql-transform-relations.ts

@@ -1,4 +1,5 @@
 import { randomUUID } from "node:crypto"
+import type { DatasetColumnDefinition, DatasetDefinition } from "@sixb/core"
 import {
   LakeStorageError,
   type SqlTransformBody,
@@ -6,7 +7,7 @@ import {
 } from "@sixb/core/lake-storage"
 import type { DuckLakeStorageOptions } from "../types"
 import { encodeDatasetTableName } from "./names"
-import { qualifiedTableName } from "./sql"
+import { qualifiedTableName, quoteIdentifier } from "./sql"
 import { parseVersionId } from "./versions"
 
 const RELATION_PLACEHOLDER_PREFIX = "__sixb_sql_transform_relation_"
@@ -15,17 +16,17 @@ const RELATION_PLACEHOLDER_PATTERN = /__sixb_sql_transform_relation_[0-9a-f]{32}
 export interface DuckLakeSqlTransformSourceRelation {
   readonly datasetId: string
   readonly versionId: string
+  /** The only columns the transform may read, in order. Every column when omitted. */
+  readonly columns?: readonly string[]
 }
 
-export interface RenderDuckLakeSqlTransformSqlInput {
-  readonly options: Pick<DuckLakeStorageOptions, "alias">
-  readonly sources: Readonly<Record<string, DuckLakeSqlTransformSourceRelation>>
-  readonly sql: SqlTransformBody<"duckdb">
-}
-
-interface DuckLakeSqlTransformRelationContext {
-  readonly relations: Readonly<Record<string, SqlTransformRelation>>
-  render(sql: string): string
+/**
+ * The SQL a transform body returned, with a placeholder where each source relation belongs. The
+ * body runs once; the same SQL then renders against the lake and against the lineage analysis.
+ */
+export interface DuckLakeSqlTransformTemplate {
+  /** Replaces each placeholder with its source's relation SQL, keyed by source name. */
+  render(relations: ReadonlyMap<string, string>): string
 }
 
 class DuckLakeSqlTransformRelation implements SqlTransformRelation {
@@ -36,42 +37,35 @@ class DuckLakeSqlTransformRelation implements SqlTransformRelation {
   }
 }
 
-export function renderDuckLakeSqlTransformSql(input: RenderDuckLakeSqlTransformSqlInput): string {
-  const context = createDuckLakeSqlTransformRelationContext(input)
-  const sql = input.sql(context.relations)
-
-  if (typeof sql !== "string") {
-    throw new LakeStorageError("[SixbDuckLake] SQL transform body must return a SQL string.")
-  }
-
-  return context.render(sql)
-}
-
-function createDuckLakeSqlTransformRelationContext(
-  input: Omit<RenderDuckLakeSqlTransformSqlInput, "sql">
-): DuckLakeSqlTransformRelationContext {
+export function createDuckLakeSqlTransformTemplate(
+  sourceNames: readonly string[],
+  body: SqlTransformBody<"duckdb">
+): DuckLakeSqlTransformTemplate {
   const token = randomUUID().replaceAll("-", "")
   const relations: Record<string, SqlTransformRelation> = Object.create(null)
-  const replacements = new Map<string, string>()
+  const sourceNamesByPlaceholder = new Map<string, string>()
 
-  let index = 0
-  for (const [sourceName, source] of Object.entries(input.sources)) {
+  sourceNames.forEach((sourceName, index) => {
     const placeholder = `${RELATION_PLACEHOLDER_PREFIX}${token}_${index}__`
     relations[sourceName] = Object.freeze(new DuckLakeSqlTransformRelation(placeholder))
-    replacements.set(placeholder, renderDuckLakeSourceRelation(input.options, source))
-    index += 1
+    sourceNamesByPlaceholder.set(placeholder, sourceName)
+  })
+
+  const sql = body(Object.freeze(relations))
+  if (typeof sql !== "string") {
+    throw new LakeStorageError("[SixbDuckLake] SQL transform body must return a SQL string.")
   }
 
   return {
-    relations: Object.freeze(relations),
-    render(sql) {
+    render(relationSql) {
       const rendered = sql.replace(RELATION_PLACEHOLDER_PATTERN, (placeholder) => {
-        const replacement = replacements.get(placeholder)
-        if (replacement === undefined) {
+        const sourceName = sourceNamesByPlaceholder.get(placeholder)
+        const relation = sourceName === undefined ? undefined : relationSql.get(sourceName)
+        if (relation === undefined) {
           throwUnresolvedPlaceholder()
         }
 
-        return replacement
+        return relation
       })
 
       if (rendered.includes(RELATION_PLACEHOLDER_PREFIX)) {
@@ -83,13 +77,69 @@ function createDuckLakeSqlTransformRelationContext(
   }
 }
 
-function renderDuckLakeSourceRelation(
+/** Each source as DuckLake time travel to its resolved version. */
+export function renderDuckLakeSourceRelations(
   options: Pick<DuckLakeStorageOptions, "alias">,
-  source: DuckLakeSqlTransformSourceRelation
-): string {
-  const snapshotId = parseVersionId(source.versionId)
-  const tableName = encodeDatasetTableName(source.datasetId)
-  return `(SELECT * FROM ${qualifiedTableName(options, tableName)} AT (VERSION => ${snapshotId}))`
+  sources: Readonly<Record<string, DuckLakeSqlTransformSourceRelation>>
+): ReadonlyMap<string, string> {
+  return new Map(
+    Object.entries(sources).map(([sourceName, source]) => {
+      const snapshotId = parseVersionId(source.versionId)
+      const table = qualifiedTableName(options, encodeDatasetTableName(source.datasetId))
+      return [
+        sourceName,
+        renderSourceRelation(`${table} AT (VERSION => ${snapshotId})`, source.columns),
+      ]
+    })
+  )
+}
+
+/**
+ * The shape of every source relation. A pick lists its columns, so the transform cannot read any
+ * other column of the source.
+ */
+export function renderSourceRelation(tableSql: string, columns?: readonly string[]): string {
+  const columnsSql =
+    columns === undefined ? "*" : columns.map((column) => quoteIdentifier(column)).join(", ")
+  return `(SELECT ${columnsSql} FROM ${tableSql})`
+}
+
+/** The columns a source exposes to a transform: its pick, in order, or every column. */
+export function pickSourceColumns(
+  sourceName: string,
+  dataset: DatasetDefinition,
+  columns?: readonly string[]
+): readonly DatasetColumnDefinition[] {
+  if (columns === undefined) {
+    return dataset.schema.columns
+  }
+
+  if (columns.length === 0) {
+    throw new LakeStorageError(
+      `[SixbDuckLake] SQL transform source '${sourceName}' must pick at least one column.`
+    )
+  }
+
+  const columnsByName = new Map(dataset.schema.columns.map((column) => [column.name, column]))
+  const picked = new Set<string>()
+
+  return columns.map((columnName) => {
+    const column = columnsByName.get(columnName)
+    if (column === undefined) {
+      throw new LakeStorageError(
+        `[SixbDuckLake] SQL transform source '${sourceName}' picks column '${columnName}', which dataset '${dataset.id}' does not have.`
+      )
+    }
+
+    if (picked.has(columnName)) {
+      throw new LakeStorageError(
+        `[SixbDuckLake] SQL transform source '${sourceName}' picks column '${columnName}' more than once.`
+      )
+    }
+
+    picked.add(columnName)
+    return column
+  })
 }
 
 function throwUnresolvedPlaceholder(): never {

package/src/internal/versions.ts

@@ -1,4 +1,4 @@
-import type { DatasetVersion } from "@sixb/core/lake-storage"
+import type { DatasetProducer, DatasetVersion } from "@sixb/core/lake-storage"
 import { LakeStorageError } from "@sixb/core/lake-storage"
 
 export interface SixbSchemaChangeMetadata {
@@ -15,6 +15,8 @@ export interface SixbCommitMetadata {
   readonly rowCount?: number
   readonly validatedPrimaryKeyColumns?: readonly string[]
   readonly schemaChange?: SixbSchemaChangeMetadata
+  /** As `encodeLineage` stored it: decoded only for the versions a read returns. */
+  readonly lineage?: unknown
 }
 
 export function parseVersionId(versionId: string): string {
@@ -57,6 +59,7 @@ export function parseCommitMetadata(value: unknown): SixbCommitMetadata | undefi
 
   const mode = parsed.sixb.mode
   const commitId = parsed.sixb.commitId
+  const producer = parseDatasetProducer(parsed.sixb.producer)
   const rowCount = parsed.sixb.rowCount
   const validatedPrimaryKeyColumns = parsed.sixb.validatedPrimaryKeyColumns
 
@@ -67,7 +70,7 @@ export function parseCommitMetadata(value: unknown): SixbCommitMetadata | undefi
     ...(mode === "snapshot" || mode === "append" || mode === "merge" || mode === "schema"
       ? { mode }
       : {}),
-    ...(isDatasetProducer(parsed.sixb.producer) ? { producer: parsed.sixb.producer } : {}),
+    ...(producer === undefined ? {} : { producer }),
     ...(isDatasetVersionRefs(parsed.sixb.inputs) ? { inputs: parsed.sixb.inputs } : {}),
     ...(typeof commitId === "string" && isRowCount(rowCount) ? { rowCount } : {}),
     ...(typeof commitId === "string" && isValidatedPrimaryKeyColumns(validatedPrimaryKeyColumns)
@@ -76,10 +79,22 @@ export function parseCommitMetadata(value: unknown): SixbCommitMetadata | undefi
     ...(isSchemaChangeMetadata(parsed.sixb.schemaChange)
       ? { schemaChange: parsed.sixb.schemaChange }
       : {}),
+    ...(parsed.sixb.lineage === undefined ? {} : { lineage: parsed.sixb.lineage }),
   }
 }
 
-export function parseInlineDataChange(
+/** Whether a DuckLake snapshot's `changes_made` names one of the tables, for any change. */
+export function changesTouchTables(changesMade: string, tableIds: readonly bigint[]): boolean {
+  const ids = new Set(tableIds.map((tableId) => tableId.toString()))
+  return changesMade.split(",").some((change) => ids.has(change.split(":")[1] ?? ""))
+}
+
+/**
+ * Whether a DuckLake snapshot changed a table's rows, and whether it removed any. DuckLake records
+ * every row change in `changes_made`, for inlined rows (`inlined_insert:<id>`) as for files
+ * (`inserted_into_table:<id>`, `deleted_from_table:<id>`); altering a table changes none.
+ */
+export function parseTableDataChange(
   changesMade: string,
   tableId: bigint
 ): { readonly hasDataChange: boolean; readonly hasDeleteChange: boolean } {
@@ -110,13 +125,25 @@ function isRecord(value: unknown): value is Readonly<Record<string, unknown>> {
   return typeof value === "object" && value !== null && !Array.isArray(value)
 }
 
-function isDatasetProducer(value: unknown): value is NonNullable<DatasetVersion["producer"]> {
-  return (
-    isRecord(value) &&
-    (value.kind === "sync" || value.kind === "pipeline" || value.kind === "ingest") &&
-    optionalString(value.id) &&
-    optionalString(value.runId)
-  )
+/**
+ * Malformed fields are dropped rather than failing the read, and never the producer's kind: a
+ * pipeline version must keep reading as one, since a pipeline version without lineage is
+ * classified with every marking.
+ */
+function parseDatasetProducer(value: unknown): DatasetProducer | undefined {
+  if (
+    !isRecord(value) ||
+    (value.kind !== "sync" && value.kind !== "pipeline" && value.kind !== "ingest")
+  ) {
+    return undefined
+  }
+
+  return {
+    kind: value.kind,
+    ...(typeof value.id === "string" ? { id: value.id } : {}),
+    ...(typeof value.runId === "string" ? { runId: value.runId } : {}),
+    ...(typeof value.stepId === "string" ? { stepId: value.stepId } : {}),
+  }
 }
 
 function isDatasetVersionRefs(value: unknown): value is NonNullable<DatasetVersion["inputs"]> {
@@ -138,10 +165,6 @@ function isSchemaChangeMetadata(value: unknown): value is SixbSchemaChangeMetada
   )
 }
 
-function optionalString(value: unknown): boolean {
-  return value === undefined || typeof value === "string"
-}
-
 function isRowCount(value: unknown): value is number {
   return typeof value === "number" && Number.isSafeInteger(value) && value >= 0
 }

package/src/types.ts

@@ -1,3 +1,5 @@
+import type { DatasetVersionRef } from "@sixb/core/lake-storage"
+
 /**
  * DuckLake catalog connection settings.
  *
@@ -119,19 +121,27 @@ export type DuckDbSecretOptions =
   | AzureSecretOptions
 
 export interface DuckLakeMaintenanceOptions {
-  /** How long to keep snapshots before they can expire. Default: "7 days". */
+  /**
+   * How long to keep snapshots before they can expire. Default: "7 days". Each dataset's latest
+   * version and the `protect` versions are kept whatever their age.
+   */
   readonly expireOlderThan?: string
   /** How long to keep deleted/orphaned files before reclaim. Default: expireOlderThan. */
   readonly deleteOlderThan?: string
   /** Report what would be expired/deleted without changing the lake. Default: false. */
   readonly dryRun?: boolean
+  /** Versions to keep whatever their age, such as those projections applied. */
+  readonly protect?: readonly DatasetVersionRef[]
 }
 
 export interface DuckLakeMaintenanceReport {
   readonly dryRun: boolean
   readonly expireOlderThan: string
   readonly deleteOlderThan: string
+  /** Snapshots expired, or that would expire on a dry run. */
   readonly snapshots: number
+  /** Snapshots older than `expireOlderThan` kept because a version they hold is still needed. */
+  readonly keptSnapshots: number
   readonly oldFiles: number
   readonly orphanedFiles: number
 }