@sixb/core 0.1.19 → 0.1.21

package/src/lake-storage/in-memory.ts

@@ -5,6 +5,7 @@ import { getDatasetRowValidationFailure } from "../datasets/validation"
 import { resolveDatasetChangeColumns } from "./changes"
 import { mergeStrictDatasetDefinition } from "./definition-updates"
 import { LakeConcurrencyError, LakeStorageError } from "./errors"
+import { lineageAtCommit } from "./lineage"
 import type {
   BeginDatasetMergeInput,
   CommitDatasetMergeInput,
@@ -42,7 +43,7 @@ import type {
   ReadDatasetChangesInput,
   ReadDatasetRowsInput,
 } from "./types"
-import { hasDatasetInputChanges } from "./version-inputs"
+import { hasDatasetProvenanceChanges } from "./version-inputs"
 
 function cloneDatasetDefinition(definition: DatasetDefinition): DatasetDefinition {
   return structuredClone(definition)
@@ -230,6 +231,7 @@ class InMemoryLakeMergeSession implements LakeMergeSession {
 }
 
 export class InMemoryLakeStorage implements LakeStorage {
+  readonly recordsLineage = true
   private readonly datasets = new Map<string, DatasetDefinition>()
   private readonly versionsByDataset = new Map<string, DatasetVersion[]>()
   private readonly rowsByVersionId = new Map<string, readonly DatasetRow[]>()
@@ -568,6 +570,12 @@ export class InMemoryLakeStorage implements LakeStorage {
 
       const visibleRows = [...nextByKey.values()]
       const versionId = `ver_${randomUUID()}`
+      const lineage = lineageAtCommit({
+        mode: "merge",
+        schema: definition.schema,
+        producer: options.merge.producer,
+        parent: latestVersion,
+      })
       const version: DatasetVersion = {
         datasetId: options.merge.dataset.id,
         versionId,
@@ -577,6 +585,7 @@ export class InMemoryLakeStorage implements LakeStorage {
         schema: definition.schema,
         producer: options.merge.producer ? structuredClone(options.merge.producer) : undefined,
         inputs: options.merge.inputs ? structuredClone(options.merge.inputs) : undefined,
+        ...(lineage && { lineage }),
         rowCount: visibleRows.length,
         sizeBytes: new TextEncoder().encode(
           visibleRows.map((row) => JSON.stringify(row)).join("\n")
@@ -623,11 +632,24 @@ export class InMemoryLakeStorage implements LakeStorage {
 
     this.assertKeyedWriteIsUnique(mode, options.rows, latestVersion, definition)
 
+    const lineage = lineageAtCommit({
+      mode,
+      schema: definition.schema,
+      producer: options.write.producer,
+      written: options.write.lineage,
+      parent: latestVersion,
+    })
+
     // Content-identical snapshots and empty appends reuse the latest version
-    // unless explicitly supplied input lineage has changed.
+    // unless explicitly supplied provenance has changed.
     if (
       latestVersion &&
-      !hasDatasetInputChanges(latestVersion.inputs, options.write.inputs) &&
+      !hasDatasetProvenanceChanges(latestVersion, {
+        ...options.write,
+        datasetId: definition.id,
+        schema: definition.schema,
+        lineage,
+      }) &&
       this.isUnchangedWrite(mode, latestVersion, options.rows, definition.schema)
     ) {
       return { ...latestVersion, outcome: "unchanged" }
@@ -652,6 +674,7 @@ export class InMemoryLakeStorage implements LakeStorage {
       schema: definition.schema,
       producer: options.write.producer ? structuredClone(options.write.producer) : undefined,
       inputs: options.write.inputs ? structuredClone(options.write.inputs) : undefined,
+      ...(lineage && { lineage }),
       rowCount: visibleRows.length,
       sizeBytes,
     }

package/src/lake-storage/index.ts

@@ -21,6 +21,8 @@ export {
 } from "./definition-updates"
 export { LakeConcurrencyError, LakeStorageError } from "./errors"
 export { InMemoryLakeStorage } from "./in-memory"
+export type { LineageAtCommitInput } from "./lineage"
+export { lineageAtCommit, resolveStepLineage } from "./lineage"
 export type {
   BeginDatasetMergeInput,
   CommitDatasetMergeInput,
@@ -41,13 +43,18 @@ export {
   reconcileDatasetSequences,
 } from "./source-ordering"
 export type {
+  AnalyzeSqlTransformInput,
+  AnalyzeSqlTransformSource,
   ExecuteSqlTransformInput,
   LakeSqlExecutor,
   LakeSqlTransformCapabilities,
   LakeStorageWithSql,
   PreviewSqlTransformInput,
   SqlDialect,
+  SqlTransformAnalysis,
   SqlTransformBody,
+  SqlTransformColumnRef,
+  SqlTransformLineage,
   SqlTransformRelation,
   SqlTransformSource,
 } from "./sql-transforms"
@@ -56,11 +63,15 @@ export type {
   CommitDatasetWriteInput,
   DatasetCatalogState,
   DatasetChanges,
+  DatasetColumnLineage,
+  DatasetColumnRef,
   DatasetLatestVersionSummary,
   DatasetProducer,
   DatasetRow,
   DatasetRowChange,
+  DatasetUnknownLineage,
   DatasetVersion,
+  DatasetVersionLineage,
   DatasetVersionMode,
   DatasetVersionRef,
   DatasetWriteCommitResult,
@@ -74,4 +85,5 @@ export type {
   ReadDatasetChangesInput,
   ReadDatasetRowsInput,
 } from "./types"
-export { hasDatasetInputChanges } from "./version-inputs"
+export type { DatasetCommitProvenance } from "./version-inputs"
+export { hasDatasetProvenanceChanges } from "./version-inputs"

package/src/lake-storage/lineage.ts

@@ -0,0 +1,385 @@
+import type { DatasetSchema } from "../datasets"
+import { LakeStorageError } from "./errors"
+import type {
+  DatasetColumnLineage,
+  DatasetColumnRef,
+  DatasetProducer,
+  DatasetUnknownLineage,
+  DatasetVersion,
+  DatasetVersionLineage,
+  DatasetVersionMode,
+} from "./types"
+
+/**
+ * Resolved column lineage: each dataset version records every dataset column its values come
+ * from, across every upstream step and every row it keeps from its parent. Classifying a version
+ * reads only its own record, so the lake may drop older versions without changing any markings.
+ *
+ * Writers resolve the lineage of the rows they add (`resolveStepLineage`), and providers merge in
+ * what the version keeps of its parent when they commit it (`lineageAtCommit`).
+ *
+ * A version that records no lineage was written before Sixb recorded it, or by a pipeline write
+ * that passed none. Such a pipeline version could hold anything; any other one holds only what its
+ * own columns declare. A version computed from, or keeping rows of, a version whose lineage is
+ * unknown records `{ unknown }`, naming where it was lost.
+ */
+
+/** The value dependencies of one column, or undefined when the lineage does not list it. */
+export function lineageColumn(
+  lineage: DatasetColumnLineage,
+  column: string
+): readonly DatasetColumnRef[] | undefined {
+  // An own-property check: a column named like an Object.prototype member is not listed.
+  return Object.hasOwn(lineage.columns, column) ? lineage.columns[column] : undefined
+}
+
+export function isUnknownLineage(lineage: DatasetVersionLineage): lineage is DatasetUnknownLineage {
+  return "unknown" in lineage
+}
+
+/** What a version records about its lineage, before or after it is committed. */
+export type LineageRecord = Pick<
+  DatasetVersion,
+  "datasetId" | "schema" | "producer" | "lineage"
+> & {
+  readonly versionId?: string
+}
+
+/**
+ * The lineage a version stands for, with the rules for versions that record none applied. A
+ * stored value that is not a lineage is unknown: reading it as absent could declassify it.
+ *
+ * Only a provider that declares `recordsLineage` stores lineage on every version it commits, so
+ * only its versions without lineage are known to predate it. Any other provider may have dropped
+ * the lineage it was given, and a version of it that records none is unknown, whatever wrote it.
+ *
+ * Reading no other version is what makes the rules approximate for versions written before
+ * Sixb recorded lineage: one written outside a pipeline (a sync, an ingestion, a schema change)
+ * on top of a pipeline version holds that version's rows, yet reads as its own columns only.
+ * Treating every such version as unknown instead would classify every older sync dataset with
+ * every marking, for good, since a merge keeps its parent's rows.
+ */
+export function versionLineage(
+  version: LineageRecord,
+  provider: { readonly recordsLineage?: boolean }
+): DatasetVersionLineage {
+  const stored: unknown = version.lineage
+  const subject =
+    version.versionId === undefined
+      ? `a version of '${version.datasetId}'`
+      : `version '${version.versionId}' of '${version.datasetId}'`
+
+  if (stored === undefined) {
+    if (provider.recordsLineage !== true) {
+      return {
+        unknown: `${subject} recorded no column lineage, and its lake provider does not declare that it records lineage`,
+      }
+    }
+
+    return version.producer?.kind === "pipeline"
+      ? { unknown: `${subject} recorded no column lineage` }
+      : ownColumns(version.schema)
+  }
+
+  return isDatasetVersionLineage(stored)
+    ? stored
+    : { unknown: `${subject} records a column lineage Sixb cannot read` }
+}
+
+/**
+ * What a provider committing a version, or a pipeline step resolving the lineage it passes to one,
+ * reads its own versions as. Whatever a provider that does not record lineage commits stays
+ * without lineage, so it is classified as unknown regardless.
+ */
+export const COMMITTING_PROVIDER = { recordsLineage: true } as const
+
+export interface LineageAtCommitInput {
+  readonly mode: DatasetVersionMode
+  /** Columns of the version being committed. */
+  readonly schema: DatasetSchema
+  readonly producer?: DatasetProducer
+  /** The lineage the writer passed, as `BeginDatasetWriteInput.lineage`. */
+  readonly written?: DatasetVersionLineage
+  /**
+   * The dataset's latest version, which an append, a merge, or a schema change keeps the rows of.
+   * A snapshot replaces it, so it is ignored for one.
+   */
+  readonly parent?: Pick<
+    DatasetVersion,
+    "datasetId" | "versionId" | "schema" | "producer" | "lineage"
+  > | null
+}
+
+/**
+ * The lineage a provider stores on a version it commits. Every provider calls it, with the latest
+ * version as `parent`, before it decides whether the commit changes anything
+ * (`hasDatasetProvenanceChanges`), and stores the result as `DatasetVersion.lineage`.
+ *
+ * - A snapshot records what was written.
+ * - An append or a merge records what was written and what its parent records, column by column.
+ * - A schema change records its parent's lineage. A column it adds holds no value yet.
+ *
+ * Without written lineage, a pipeline write records none, which classifies the version with every
+ * marking; any other write depends on its own columns only. Returns undefined when nothing is to
+ * be stored.
+ */
+export function lineageAtCommit(input: LineageAtCommitInput): DatasetVersionLineage | undefined {
+  const parent = input.mode === "snapshot" ? undefined : (input.parent ?? undefined)
+
+  if (input.mode === "schema") {
+    return parent ? keptLineage(parent, input.schema) : ownColumns(input.schema)
+  }
+
+  const written = writtenLineage(input)
+  if (!written || isUnknownLineage(written) || !parent) return written
+
+  const kept = keptLineage(parent, input.schema)
+  if (isUnknownLineage(kept)) return kept
+
+  return canonicalLineage(
+    [...written.rows, ...kept.rows],
+    input.schema.columns.flatMap(({ name }) => {
+      const writtenColumn = lineageColumn(written, name)
+      const keptColumn = lineageColumn(kept, name)
+      return writtenColumn && keptColumn ? [[name, [...writtenColumn, ...keptColumn]] as const] : []
+    })
+  )
+}
+
+/**
+ * Lineage of a pipeline step's output across every upstream hop: each column a step reads stands
+ * for itself and for everything the input version it read records for it.
+ *
+ * Every output column also depends on the rows of every input version the step read, whether or
+ * not it names a column of that input: `SELECT count(*) FROM rich`, or a constant selected from
+ * it, reveals which rows `rich` holds, and so what decided them upstream.
+ *
+ * `step` is the step's own lineage, whose refs name columns of `inputs`, the input versions the
+ * step read. The pipeline worker passes it for `run` steps; a SQL provider passes it for the SQL it
+ * executes. The result is unknown when an input's lineage is unknown, or when a column it depends
+ * on comes from a dataset none of `inputs` is a version of.
+ */
+export function resolveStepLineage(
+  step: DatasetColumnLineage,
+  inputs: readonly DatasetVersion[]
+): DatasetVersionLineage {
+  if (!isDatasetColumnLineage(step)) {
+    throw new LakeStorageError("[SixbLake] A pipeline step's column lineage is malformed.")
+  }
+
+  const inputsByDataset = new Map<string, DatasetVersion[]>()
+  for (const input of inputs) {
+    const versions = inputsByDataset.get(input.datasetId)
+    if (versions) versions.push(input)
+    else inputsByDataset.set(input.datasetId, [input])
+  }
+
+  const lineages = new Map<DatasetVersion, DatasetVersionLineage>()
+  const lineageOf = (version: DatasetVersion) => {
+    let lineage = lineages.get(version)
+    if (!lineage) {
+      lineage = versionLineage(version, COMMITTING_PROVIDER)
+      lineages.set(version, lineage)
+    }
+    return lineage
+  }
+
+  const resolved = new Map<string, readonly DatasetColumnRef[] | { readonly unknown: string }>()
+  /** The ref and everything the input versions record for it, or why that is unknown. */
+  const resolve = (ref: DatasetColumnRef) => {
+    const key = refKey(ref)
+    let result = resolved.get(key)
+    if (!result) {
+      result = resolveRef(ref, inputsByDataset.get(ref.datasetId), lineageOf)
+      resolved.set(key, result)
+    }
+    return result
+  }
+
+  const expand = (refs: readonly DatasetColumnRef[]) => {
+    const expanded: DatasetColumnRef[] = []
+    for (const ref of refs) {
+      const result = resolve(ref)
+      if ("unknown" in result) return result
+      expanded.push(...result)
+    }
+    return expanded
+  }
+
+  const rows = expand(step.rows)
+  if ("unknown" in rows) return rows
+
+  for (const input of inputs) {
+    const lineage = lineageOf(input)
+    if (isUnknownLineage(lineage)) return lineage
+    rows.push(...lineage.rows)
+  }
+
+  const columns: (readonly [string, readonly DatasetColumnRef[]])[] = []
+  for (const [column, refs] of Object.entries(step.columns)) {
+    const values = expand(refs)
+    if ("unknown" in values) return values
+    columns.push([column, values])
+  }
+
+  return canonicalLineage(rows, columns)
+}
+
+function resolveRef(
+  ref: DatasetColumnRef,
+  versions: readonly DatasetVersion[] | undefined,
+  lineageOf: (version: DatasetVersion) => DatasetVersionLineage
+): readonly DatasetColumnRef[] | { readonly unknown: string } {
+  if (!versions) {
+    return { unknown: `a pipeline step read '${ref.datasetId}' without recording which version` }
+  }
+
+  const refs = [ref]
+  for (const version of versions) {
+    const lineage = lineageOf(version)
+    if (isUnknownLineage(lineage)) return lineage
+
+    const values = lineageColumn(lineage, ref.column)
+    // A column the input version does not have holds no value; one it does not list, anything.
+    if (!values && version.schema.columns.some(({ name }) => name === ref.column)) {
+      const subject = `version '${version.versionId}' of '${version.datasetId}'`
+      return { unknown: `${subject} recorded no lineage for column '${ref.column}'` }
+    }
+
+    // The input's rows are every output column's dependencies already.
+    refs.push(...(values ?? []))
+  }
+
+  return refs
+}
+
+/**
+ * Whether two lineages classify alike: the same dependencies per column, whatever their order
+ * and repetition, or both unknown.
+ */
+export function sameLineage(left: DatasetVersionLineage, right: DatasetVersionLineage): boolean {
+  return lineageKey(left) === lineageKey(right)
+}
+
+function lineageKey(lineage: DatasetVersionLineage): string {
+  if (isUnknownLineage(lineage)) return "unknown"
+  const canonical = canonicalLineage(lineage.rows, Object.entries(lineage.columns))
+  return JSON.stringify([canonical.rows, Object.entries(canonical.columns)])
+}
+
+/** What the version keeps of its parent, restricted to the columns it has. */
+function keptLineage(parent: LineageRecord, schema: DatasetSchema): DatasetVersionLineage {
+  const lineage = versionLineage(parent, COMMITTING_PROVIDER)
+  if (isUnknownLineage(lineage)) return lineage
+
+  const parentColumns = new Set(parent.schema.columns.map(({ name }) => name))
+  return canonicalLineage(
+    lineage.rows,
+    schema.columns.flatMap(({ name }) => {
+      const values = lineageColumn(lineage, name)
+      // A column the parent does not have holds no value in the rows it keeps.
+      if (values) return [[name, values] as const]
+      return parentColumns.has(name) ? [] : [[name, []] as const]
+    })
+  )
+}
+
+function writtenLineage(input: LineageAtCommitInput): DatasetVersionLineage | undefined {
+  const { written } = input
+
+  if (written === undefined) {
+    return input.producer?.kind === "pipeline" ? undefined : ownColumns(input.schema)
+  }
+  if (!isDatasetVersionLineage(written)) {
+    throw new LakeStorageError("[SixbLake] The column lineage passed to the write is malformed.")
+  }
+  if (isUnknownLineage(written)) return { unknown: written.unknown }
+
+  return canonicalLineage(
+    written.rows,
+    input.schema.columns.flatMap(({ name }) => {
+      const values = lineageColumn(written, name)
+      return values ? [[name, values] as const] : []
+    })
+  )
+}
+
+function ownColumns(schema: DatasetSchema): DatasetColumnLineage {
+  return { rows: [], columns: Object.fromEntries(schema.columns.map(({ name }) => [name, []])) }
+}
+
+/**
+ * Refs deduplicated and sorted, columns sorted: equal lineages store and compare alike.
+ *
+ * Every column depends on `rows`, so a column lists only what it depends on beyond them. Without
+ * this, each hop would copy the rows of every input into every column again.
+ */
+function canonicalLineage(
+  rows: readonly DatasetColumnRef[],
+  columns: Iterable<readonly [string, readonly DatasetColumnRef[]]>
+): DatasetColumnLineage {
+  const canonicalRows = canonicalRefs(rows)
+  const inRows = new Set(canonicalRows.map(refKey))
+  const sortedColumns = [...columns].sort(([left], [right]) => compare(left, right))
+
+  return {
+    rows: canonicalRows,
+    columns: Object.fromEntries(
+      sortedColumns.map(([column, refs]) => [column, canonicalRefs(refs, inRows)])
+    ),
+  }
+}
+
+function canonicalRefs(
+  refs: readonly DatasetColumnRef[],
+  exclude?: ReadonlySet<string>
+): readonly DatasetColumnRef[] {
+  if (refs.length === 0) return []
+
+  const unique = new Map<string, DatasetColumnRef>()
+  for (const { datasetId, column } of refs) {
+    const key = refKey({ datasetId, column })
+    if (!exclude?.has(key)) unique.set(key, { datasetId, column })
+  }
+
+  return [...unique.entries()].sort(([left], [right]) => compare(left, right)).map(([, ref]) => ref)
+}
+
+function refKey(ref: DatasetColumnRef): string {
+  return `${ref.datasetId}\u0000${ref.column}`
+}
+
+function compare(left: string, right: string): number {
+  return left < right ? -1 : left > right ? 1 : 0
+}
+
+function isDatasetVersionLineage(value: unknown): value is DatasetVersionLineage {
+  if (typeof value !== "object" || value === null) return false
+  if ("unknown" in value) return typeof value.unknown === "string"
+  return isDatasetColumnLineage(value)
+}
+
+function isDatasetColumnLineage(value: unknown): value is DatasetColumnLineage {
+  if (typeof value !== "object" || value === null) return false
+  if (!("rows" in value) || !isRefs(value.rows)) return false
+  if (!("columns" in value)) return false
+
+  const { columns } = value
+  if (typeof columns !== "object" || columns === null || Array.isArray(columns)) return false
+  return Object.values(columns).every(isRefs)
+}
+
+function isRefs(value: unknown): value is readonly DatasetColumnRef[] {
+  return (
+    Array.isArray(value) &&
+    value.every(
+      (ref: unknown) =>
+        typeof ref === "object" &&
+        ref !== null &&
+        "datasetId" in ref &&
+        typeof ref.datasetId === "string" &&
+        "column" in ref &&
+        typeof ref.column === "string"
+    )
+  )
+}

package/src/lake-storage/sql-transforms.ts

@@ -24,7 +24,21 @@ export interface LakeSqlExecutor<TSqlDialect extends SqlDialect = SqlDialect> {
   readonly capabilities: LakeSqlTransformCapabilities
 
   preview(input: PreviewSqlTransformInput<TSqlDialect>): AsyncIterable<DatasetRow>
+  /**
+   * Records the resolved lineage of the executed SQL on the committed version, as the writer of a
+   * run step passes `BeginDatasetWriteInput.lineage`: the provider analyzes the SQL it executes
+   * (as `analyze`), maps that lineage through the source versions it read with
+   * `resolveStepLineage`, and stores what `lineageAtCommit` returns for it, which merges in what an
+   * append keeps of its parent. SQL it cannot analyze exactly but that reads only its sources
+   * depends on every column it reads, as a run step does. Otherwise it passes no lineage, and the
+   * version is classified with every marking.
+   */
   execute(input: ExecuteSqlTransformInput<TSqlDialect>): Promise<DatasetWriteCommitResult>
+  /**
+   * Column lineage of a transform, bound against the sources' column schemas without reading any
+   * data. A provider without it cannot run SQL over marked columns.
+   */
+  analyze?(input: AnalyzeSqlTransformInput<TSqlDialect>): Promise<SqlTransformAnalysis>
 }
 
 export interface LakeSqlTransformCapabilities {
@@ -34,6 +48,7 @@ export interface LakeSqlTransformCapabilities {
 }
 
 export interface ExecuteSqlTransformInput<TSqlDialect extends SqlDialect = SqlDialect> {
+  /** The source versions read. The lineage the provider records is resolved through them. */
   readonly sources: Readonly<Record<string, SqlTransformSource>>
   readonly sql: SqlTransformBody<TSqlDialect>
   readonly target: DatasetDefinition
@@ -53,6 +68,36 @@ export interface PreviewSqlTransformInput<TSqlDialect extends SqlDialect = SqlDi
 export interface SqlTransformSource {
   readonly dataset: DatasetDefinition
   readonly versionId?: string
+  /** Columns the transform may read, in order. Every column of the dataset when omitted. */
+  readonly columns?: readonly string[]
+}
+
+export interface AnalyzeSqlTransformInput<TSqlDialect extends SqlDialect = SqlDialect> {
+  readonly sources: Readonly<Record<string, AnalyzeSqlTransformSource>>
+  readonly sql: SqlTransformBody<TSqlDialect>
+}
+
+export interface AnalyzeSqlTransformSource {
+  readonly dataset: DatasetDefinition
+  /** Columns the transform may read, in order. Every column of the dataset when omitted. */
+  readonly columns?: readonly string[]
+}
+
+export type SqlTransformAnalysis =
+  | { readonly supported: true; readonly lineage: SqlTransformLineage }
+  /** The provider could not analyze this SQL exactly; `reason` names the construct. */
+  | { readonly supported: false; readonly reason: string }
+
+/** Lineage of one transform, by source name. Same semantics as `DatasetColumnLineage`. */
+export interface SqlTransformLineage {
+  readonly rows: readonly SqlTransformColumnRef[]
+  readonly columns: Readonly<Record<string, readonly SqlTransformColumnRef[]>>
+}
+
+export interface SqlTransformColumnRef {
+  /** Source name, as passed in `sources`. */
+  readonly source: string
+  readonly column: string
 }
 
 export interface SqlTransformRelation {

package/src/lake-storage/types.ts

@@ -26,6 +26,37 @@ export interface DatasetProducer {
   readonly stepId?: string
 }
 
+/** A column of a dataset, by dataset id and column name. */
+export interface DatasetColumnRef {
+  readonly datasetId: string
+  readonly column: string
+}
+
+/**
+ * Which dataset columns the values of a version, or of one transform, come from.
+ *
+ * A column depends on the columns `rows` lists and on the ones `columns` lists for it. A column
+ * missing from `columns` could depend on anything.
+ */
+export interface DatasetColumnLineage {
+  /** Columns that decide which rows exist or their order. Every column depends on them. */
+  readonly rows: readonly DatasetColumnRef[]
+  /** Columns whose values flow into each column. Lists every column. */
+  readonly columns: Readonly<Record<string, readonly DatasetColumnRef[]>>
+}
+
+/**
+ * A version whose rows come, at least in part, from a version whose lineage is unknown, such as
+ * a pipeline version written before Sixb recorded lineage. Every column could hold anything.
+ */
+export interface DatasetUnknownLineage {
+  /** Names the version or write the lineage was lost at, for diagnostics. */
+  readonly unknown: string
+}
+
+/** What a dataset version records about where everything it holds comes from. */
+export type DatasetVersionLineage = DatasetColumnLineage | DatasetUnknownLineage
+
 export interface DatasetVersion {
   readonly datasetId: string
   readonly versionId: string
@@ -35,6 +66,16 @@ export interface DatasetVersion {
   readonly schema: DatasetSchema
   readonly producer?: DatasetProducer
   readonly inputs?: readonly DatasetVersionRef[]
+  /**
+   * Every dataset column the values of this version come from, across every upstream step, and
+   * including the rows it keeps from its parent. Markings are read from it, so classifying a
+   * version reads no other version.
+   *
+   * Providers store the value `lineageAtCommit` returns, and return it as stored: Sixb validates
+   * it when it reads it. Absent on a version written before its provider recorded it
+   * (`LakeStorage.recordsLineage`), and on a pipeline version whose writer passed none.
+   */
+  readonly lineage?: DatasetVersionLineage
   readonly rowCount?: number
   readonly sizeBytes?: number
 }
@@ -76,6 +117,13 @@ export interface BeginDatasetWriteInput {
   readonly mode?: DatasetWriteMode
   readonly producer?: DatasetProducer
   readonly inputs?: readonly DatasetVersionRef[]
+  /**
+   * The resolved lineage of the rows this write adds: for a pipeline step, its own lineage mapped
+   * through its input versions with `resolveStepLineage`. The provider merges in what the version
+   * keeps of its parent at commit (`lineageAtCommit`). A pipeline write without it is classified
+   * with every marking; any other write without it depends on its own columns only.
+   */
+  readonly lineage?: DatasetVersionLineage
 }
 
 export interface ReadDatasetRowsInput {
@@ -137,7 +185,7 @@ export interface DatasetWriteCommitResult extends DatasetVersion {
    * Whether this operation created the returned version or reused the
    * unchanged latest version. When a latest version exists, providers must
    * report `unchanged` and reuse it for commits that change neither visible
-   * dataset content nor explicitly supplied input lineage: a snapshot whose
+   * dataset content nor provenance (`hasDatasetProvenanceChanges`): a snapshot whose
    * rows equal the latest visible rows
    * (order-insensitive, with equal duplicate counts), or an append of zero
    * rows. Without a latest version, an empty snapshot or append still creates
@@ -156,6 +204,16 @@ export interface LakeWriteSession {
 
 export interface LakeStorage {
   readonly standard?: LakeStandardDescriptor
+  /**
+   * Whether this provider stores `DatasetVersion.lineage` on every version it commits, as
+   * `lineageAtCommit` returns it.
+   *
+   * A version without lineage from such a provider was committed before it recorded lineage: a
+   * pipeline version could hold anything, any other version what its own columns declare. Without
+   * this declaration, every version without lineage could hold anything, so a provider that drops
+   * the lineage it is given cannot declassify the rows a merge keeps of a pipeline version.
+   */
+  readonly recordsLineage?: boolean
 
   /**
    * Read-only preflight for hosts that want to fail before serving traffic.