@sixb/core 0.1.20 → 0.1.22

package/src/testing/lake-storage-contract.ts

@@ -2,7 +2,14 @@ import { describe, expect, test } from "bun:test"
 import type { FileRef } from "../blob-storage"
 import { col, type DatasetDefinition, defineDataset } from "../datasets"
 import { LakeStorageError } from "../lake-storage/errors"
-import type { DatasetRow, LakeStorage } from "../lake-storage/types"
+import type {
+  BeginDatasetWriteInput,
+  DatasetColumnRef,
+  DatasetProducer,
+  DatasetRow,
+  DatasetWriteCommitResult,
+  LakeStorage,
+} from "../lake-storage/types"
 
 export type LakeStorageSchemaEvolutionCapability = "strict" | "addNullableColumns"
 
@@ -16,6 +23,8 @@ export interface LakeStorageContractSuiteOptions<TStorage extends LakeStorage =
    * `strict`, which allows metadata-only merges but rejects schema evolution.
    */
   readonly schemaEvolution?: LakeStorageSchemaEvolutionCapability
+  /** Open the same lake again, to check what a provider persists. */
+  readonly reopen?: (storage: TStorage) => TStorage | Promise<TStorage>
   /**
    * Version id that is syntactically valid for the provider but should not
    * exist. Defaults to a generic id accepted by providers without format
@@ -65,6 +74,30 @@ const invoiceRef: FileRef = {
   logicalPath: "invoices/invoice.txt",
 }
 
+const pipeline: DatasetProducer = { kind: "pipeline", id: "contract-pipeline", stepId: "copy" }
+const source = (column: string): DatasetColumnRef => ({ datasetId: "contract.source", column })
+
+/** A step that copies `contract.source` columns, as resolved lineage. */
+const copied = {
+  rows: [],
+  columns: { customerName: [source("name")], orderId: [source("id")] },
+}
+/** The same step, before a fix: it wrote amounts into `customerName`. */
+const fromAmount = {
+  rows: [],
+  columns: { customerName: [source("amount")], orderId: [source("id")] },
+}
+
+async function write(
+  storage: LakeStorage,
+  input: Pick<BeginDatasetWriteInput, "producer" | "lineage" | "mode">,
+  rows: readonly DatasetRow[]
+): Promise<DatasetWriteCommitResult> {
+  const session = await storage.beginWrite({ dataset: writeDataset, ...input })
+  await session.writeRows(rows)
+  return session.commit()
+}
+
 /**
  * Runs the shared `LakeStorage` contract against any provider.
  *
@@ -91,36 +124,27 @@ export function runLakeStorageContractSuite<TStorage extends LakeStorage>(
 
   describe(label, () => {
     describe("dataset definitions", () => {
-      test("creates, gets, lists, and reuses dataset definitions", async () => {
+      test("creates, gets, lists, and reuses plain and keyed dataset definitions", async () => {
+        // Regression guard: omitting primaryKey from mergeDatasetDefinition makes reuse lose
+        // the key and fails the keyed case.
         await withStorage(async (storage) => {
-          expect(await storage.getDataset(definitionDataset.id)).toBeNull()
+          const datasets = [keyedDefinitionDataset, definitionDataset]
+          for (const dataset of datasets) {
+            expect(await storage.getDataset(dataset.id)).toBeNull()
 
-          const created = await storage.createDataset(definitionDataset)
-          const repeated = await storage.createDataset(definitionDataset)
+            const created = await storage.createDataset(dataset)
+            const repeated = await storage.createDataset(dataset)
 
-          expect(created).toEqual(definitionDataset)
-          expect(repeated).toEqual(definitionDataset)
-          expect(await storage.getDataset(definitionDataset.id)).toEqual(definitionDataset)
+            expect(created).toEqual(dataset)
+            expect(repeated).toEqual(dataset)
+            expect(await storage.getDataset(dataset.id)).toEqual(dataset)
+            expect(await storage.listVersions(dataset.id)).toEqual([])
+          }
           expect(await storage.getDataset("missing.dataset")).toBeNull()
-          expect(await storage.listDatasets()).toEqual([definitionDataset])
-          expect(await storage.listVersions(definitionDataset.id)).toEqual([])
-        })
-      })
-
-      test("round-trips keyed definitions through create, get, list, and reuse", async () => {
-        // Regression guard: omitting primaryKey from mergeDatasetDefinition makes reuse lose
-        // the key and fails this contract case.
-        await withStorage(async (storage) => {
-          const created = await storage.createDataset(keyedDefinitionDataset)
-          const repeated = await storage.createDataset(keyedDefinitionDataset)
-
-          expect(created).toEqual(keyedDefinitionDataset)
-          expect(repeated).toEqual(keyedDefinitionDataset)
-          expect(await storage.getDataset(keyedDefinitionDataset.id)).toEqual(
-            keyedDefinitionDataset
+          const listed = await storage.listDatasets()
+          expect([...listed].sort((left, right) => left.id.localeCompare(right.id))).toEqual(
+            datasets
           )
-          expect(await storage.listDatasets()).toEqual([keyedDefinitionDataset])
-          expect(await storage.listVersions(keyedDefinitionDataset.id)).toEqual([])
         })
       })
 
@@ -168,7 +192,9 @@ export function runLakeStorageContractSuite<TStorage extends LakeStorage>(
 
           await expect(
             storage.assertDatasetDefinitionsCompatible([changedDataset])
-          ).rejects.toThrow("changing column 'orderId' type")
+          ).rejects.toThrow(
+            /Lake dataset definition check failed[\s\S]*changing column 'orderId' type/
+          )
           await expect(storage.createDataset(changedDataset)).rejects.toBeInstanceOf(
             LakeStorageError
           )
@@ -471,6 +497,7 @@ export function runLakeStorageContractSuite<TStorage extends LakeStorage>(
             rowCount: 2,
           })
 
+          // `listVersions` orders by creation time: a later millisecond keeps the order exact.
           await Bun.sleep(2)
 
           const appendWrite = await storage.beginWrite({
@@ -612,7 +639,7 @@ export function runLakeStorageContractSuite<TStorage extends LakeStorage>(
 
       test("versions changed input lineage even when output rows are identical", () =>
         withStorage(async (storage) => {
-          // Regression proof: omit hasDatasetInputChanges in the provider's no-op branch.
+          // Regression proof: omit hasDatasetProvenanceChanges in the provider's no-op branch.
           // The updated input reference is lost and the stale writer is no longer fenced.
           await storage.createDataset(writeDataset)
           await storage.createDataset(definitionDataset)
@@ -644,6 +671,68 @@ export function runLakeStorageContractSuite<TStorage extends LakeStorage>(
           expect(await collectRows(storage.readRows({ datasetId: writeDataset.id }))).toEqual(rows)
         }))
 
+      test("versions changed column lineage even when output rows are identical", () =>
+        withStorage(async (storage) => {
+          // Regression proof: compare only inputs in the provider's no-op branch. The rerun
+          // reuses the version without lineage, and its rows stay classified with every marking.
+          await storage.createDataset(writeDataset)
+          const rows = [{ orderId: "ord_1", customerName: "Ada" }]
+          const commit = async (
+            write: Pick<BeginDatasetWriteInput, "producer" | "lineage" | "mode">,
+            expectedLatestVersionId: string | null,
+            written: readonly DatasetRow[] = rows
+          ) => {
+            const session = await storage.beginWrite({ dataset: writeDataset, ...write })
+            await session.writeRows(written)
+            return session.commit({ expectedLatestVersionId })
+          }
+
+          const initial = await commit({ producer: pipeline }, null)
+          expect(initial.lineage).toBeUndefined()
+
+          const analyzed = await commit({ producer: pipeline, lineage: copied }, initial.versionId)
+          expect(analyzed).toMatchObject({ outcome: "created", lineage: copied })
+
+          // Order and repetition do not change a lineage.
+          const reordered = {
+            rows: [],
+            columns: {
+              orderId: [...copied.columns.orderId, ...copied.columns.orderId],
+              customerName: copied.columns.customerName,
+            },
+          }
+          expect(
+            await commit({ producer: pipeline, lineage: reordered }, analyzed.versionId)
+          ).toMatchObject({ outcome: "unchanged", versionId: analyzed.versionId })
+
+          // A run that recorded no lineage is a change too: reusing the analyzed version would
+          // classify its rows by a lineage the run did not record.
+          const unanalyzed = await commit({ producer: pipeline }, analyzed.versionId)
+          expect(unanalyzed).toMatchObject({ outcome: "created" })
+          expect(
+            (await storage.getVersion(writeDataset.id, unanalyzed.versionId))?.lineage
+          ).toBeUndefined()
+
+          // An empty append compares what it would store, after merging its parent's lineage.
+          const analyzedAgain = await commit(
+            { producer: pipeline, lineage: copied },
+            unanalyzed.versionId
+          )
+
+          const append = { producer: pipeline, mode: "append" } as const
+          const narrower = { rows: [], columns: { ...copied.columns, customerName: [] } }
+          expect(
+            await commit({ ...append, lineage: narrower }, analyzedAgain.versionId, [])
+          ).toMatchObject({ outcome: "unchanged", versionId: analyzedAgain.versionId })
+
+          const wider = { rows: [source("tenant")], columns: copied.columns }
+          const widened = await commit({ ...append, lineage: wider }, analyzedAgain.versionId, [])
+          expect(widened).toMatchObject({
+            outcome: "created",
+            lineage: { rows: [source("tenant")], columns: copied.columns },
+          })
+        }))
+
       test("creates an initial empty append version", async () => {
         await withStorage(async (storage) => {
           await storage.createDataset(writeDataset)
@@ -728,6 +817,7 @@ export function runLakeStorageContractSuite<TStorage extends LakeStorage>(
             versionId: seedVersion.versionId,
           })
 
+          // `listVersions` orders by creation time: a later millisecond keeps the order exact.
           await Bun.sleep(2)
 
           const changedWrite = await storage.beginWrite({ dataset: writeDataset, mode: "snapshot" })
@@ -755,8 +845,6 @@ export function runLakeStorageContractSuite<TStorage extends LakeStorage>(
           await seedWrite.writeRows([rowA, rowA, rowB])
           const seedVersion = await seedWrite.commit()
 
-          await Bun.sleep(2)
-
           // Same distinct rows and row count, but different duplicate counts:
           // this is new content, not a no-op.
           const shiftedWrite = await storage.beginWrite({ dataset: writeDataset, mode: "snapshot" })
@@ -853,6 +941,7 @@ export function runLakeStorageContractSuite<TStorage extends LakeStorage>(
           await write.abort()
           await write.abort()
 
+          await expect(write.writeRows([])).rejects.toThrow("already closed")
           expect(await storage.getLatestVersion(writeDataset.id)).toBeNull()
           expect(await storage.listVersions(writeDataset.id)).toEqual([])
           await expect(
@@ -939,6 +1028,156 @@ export function runLakeStorageContractSuite<TStorage extends LakeStorage>(
         })
       })
     })
+
+    describe("resolved column lineage", () => {
+      test("stores the lineage of every version, in a canonical form, across reopening", async () => {
+        let storage = await options.createStorage()
+        try {
+          // A provider that stores lineage says so: Sixb then reads a version without lineage as
+          // one committed before the provider recorded it, not as one whose lineage was dropped.
+          expect(storage.recordsLineage).toBe(true)
+
+          await storage.createDataset(writeDataset)
+          const synced = await write(storage, { producer: { kind: "sync", id: "contract-sync" } }, [
+            { orderId: "ord_1", customerName: "Ada" },
+          ])
+
+          // Outside pipelines, a version depends on its own columns only.
+          expect(synced.lineage).toEqual({ rows: [], columns: { customerName: [], orderId: [] } })
+
+          const repeated = {
+            rows: [source("tenant"), source("tenant")],
+            columns: {
+              orderId: [source("id")],
+              customerName: [source("name"), source("amount"), source("name")],
+              dropped: [source("id")],
+            },
+          }
+          const computed = await write(storage, { producer: pipeline, lineage: repeated }, [
+            { orderId: "ord_1", customerName: "Ada" },
+          ])
+
+          const canonical = {
+            rows: [source("tenant")],
+            columns: {
+              customerName: [source("amount"), source("name")],
+              orderId: [source("id")],
+            },
+          }
+          expect(computed.lineage).toEqual(canonical)
+
+          if (options.reopen) storage = await options.reopen(storage)
+
+          expect((await storage.getVersion(writeDataset.id, computed.versionId))?.lineage).toEqual(
+            canonical
+          )
+          expect((await storage.getLatestVersion(writeDataset.id))?.lineage).toEqual(canonical)
+
+          const listed = await storage.listVersions(writeDataset.id)
+          expect(Object.fromEntries(listed.map((v) => [v.versionId, v.lineage]))).toEqual({
+            [computed.versionId]: canonical,
+            [synced.versionId]: synced.lineage,
+          })
+        } finally {
+          await options.teardown?.(storage)
+        }
+      })
+
+      test("an append keeps its parent's lineage and a snapshot replaces it", () =>
+        withStorage(async (storage) => {
+          await storage.createDataset(writeDataset)
+          await write(storage, { producer: pipeline, lineage: fromAmount }, [
+            { orderId: "ord_1", customerName: "120.00" },
+          ])
+
+          const appended = await write(
+            storage,
+            { producer: pipeline, lineage: copied, mode: "append" },
+            [{ orderId: "ord_2", customerName: "Grace" }]
+          )
+
+          expect(appended.lineage).toEqual({
+            rows: [],
+            columns: {
+              customerName: [source("amount"), source("name")],
+              orderId: [source("id")],
+            },
+          })
+
+          const replaced = await write(storage, { producer: pipeline, lineage: copied }, [
+            { orderId: "ord_2", customerName: "Grace" },
+          ])
+          expect(replaced.lineage).toEqual(copied)
+        }))
+
+      test("a version that keeps rows of an unknown history records it as unknown", () =>
+        withStorage(async (storage) => {
+          await storage.createDataset(writeDataset)
+          const unanalyzed = await write(storage, { producer: pipeline }, [
+            { orderId: "ord_1", customerName: "Ada" },
+          ])
+          expect(unanalyzed.lineage).toBeUndefined()
+
+          const unknown = {
+            unknown: `version '${unanalyzed.versionId}' of '${writeDataset.id}' recorded no column lineage`,
+          }
+
+          const synced = await write(
+            storage,
+            { producer: { kind: "sync", id: "contract-sync" }, mode: "append" },
+            [{ orderId: "ord_2", customerName: "Grace" }]
+          )
+          expect(synced.lineage).toEqual(unknown)
+
+          // The reason keeps naming where the lineage was lost.
+          const computed = await write(
+            storage,
+            { producer: pipeline, lineage: copied, mode: "append" },
+            [{ orderId: "ord_3", customerName: "Katherine" }]
+          )
+
+          expect(computed.lineage).toEqual(unknown)
+          expect((await storage.getVersion(writeDataset.id, computed.versionId))?.lineage).toEqual(
+            unknown
+          )
+
+          const replaced = await write(storage, { producer: pipeline, lineage: copied }, [
+            { orderId: "ord_3", customerName: "Katherine" },
+          ])
+          expect(replaced.lineage).toEqual(copied)
+        }))
+
+      test("a schema change keeps its parent's lineage", () =>
+        withStorage(async (storage) => {
+          if (schemaEvolution === "strict") return
+
+          const initial = defineDataset("contract.lineage.schema_evolution", {
+            schema: [col("invoiceId", "string")],
+          })
+          const widened = defineDataset("contract.lineage.schema_evolution", {
+            schema: [col("invoiceId", "string"), col("currency", "string", { nullable: true })],
+          })
+
+          await storage.createDataset(initial)
+          const session = await storage.beginWrite({
+            dataset: initial,
+            producer: pipeline,
+            lineage: { rows: [source("tenant")], columns: { invoiceId: [source("id")] } },
+          })
+          await session.writeRows([{ invoiceId: "inv_1" }])
+          await session.commit()
+
+          await storage.createDataset(widened)
+          const [schemaVersion] = await storage.listVersions(initial.id, 1)
+          expect(schemaVersion?.mode).toBe("schema")
+
+          // Its rows hold no currency yet.
+          expect(schemaVersion?.lineage).toEqual({
+            rows: [source("tenant")],
+            columns: { currency: [], invoiceId: [source("id")] },
+          })
+        }))
+    })
   })
 }
 

package/src/testing/materializer-storage-contract.ts

@@ -862,6 +862,20 @@ export function runMaterializerStorageContractSuite<TStorage extends Storage>(
           expectedLinks: [],
         })
       ).rejects.toMatchObject({ kind: "expectation" })
+      // Same version, different commit: the object was deleted and recreated since it was read.
+      const current = await storage.objects.getByPrimaryId({
+        projectId: "materializer-storage-contract",
+        ...device,
+      })
+      if (!current) throw new Error("Expected the setup commit to create the device.")
+      await expect(
+        commitExpecting("recreated-object", {
+          expectedObjects: [
+            { ref: device, exists: true, version: current.version, lastCommitId: "stale-commit" },
+          ],
+          expectedLinks: [],
+        })
+      ).rejects.toMatchObject({ kind: "expectation" })
       await expect(
         commitExpecting("stale-link", {
           expectedObjects: [],
@@ -1443,8 +1457,10 @@ export function runMaterializerStorageContractSuite<TStorage extends Storage>(
 
         let retries = 0
         const recording = observeActionRunRecords(storage)
-        const second = contractMaterializer(recording.storage, {}, () => {
-          retries += 1
+        const second = contractMaterializer(recording.storage, {
+          onSerializationRetry: () => {
+            retries += 1
+          },
         })
           .withScope(await scopeOf(secondRun.executionId))
           .edits.commit(commitOf(secondRun, "second"))
@@ -1618,6 +1634,7 @@ export function runMaterializerStorageContractSuite<TStorage extends Storage>(
     let failNextCommit = true
     let stagedIdentities = 0
     let plannedAt: string | undefined
+    let now = new Date("2026-03-01T00:00:00.000Z")
     const observed = observePlanning(
       storage,
       async (planned, commit) => {
@@ -1631,7 +1648,7 @@ export function runMaterializerStorageContractSuite<TStorage extends Storage>(
         throw new Error("commit lost")
       }
     )
-    const materializer = contractMaterializer(observed)
+    const materializer = contractMaterializer(observed, { clock: () => now })
     try {
       const execution = await claimDevices(storage, 1)
       const values = [sourceEntry("a", "A"), sourceEntry("b", "B", "a")]
@@ -1640,7 +1657,7 @@ export function runMaterializerStorageContractSuite<TStorage extends Storage>(
       )
       expect(stagedIdentities).toBe(3)
       // The redelivery's clock has moved on; its commit keeps the time its events were planned at.
-      await new Promise((resolve) => setTimeout(resolve, 5))
+      now = new Date("2026-03-01T00:00:01.000Z")
       const result = await publishDevices(materializer, storage, 1, values, execution)
       expect(stagedIdentities).toBe(3)
       expect(result.committedAt).toBe(plannedAt!)
@@ -1984,7 +2001,7 @@ async function claim(
   const identityBase = {
     projectionId: input.projectionId,
     datasetVersion: input.datasetVersion,
-    ontologyRevision: projections.ontologyRevision,
+    ontologyRevision: resolved.ontologyRevision,
     projectionRevision: resolved.projectionRevision,
     ownershipHash: resolved.ownershipHash,
   }
@@ -2027,17 +2044,27 @@ const CONTRACT_PROJECT = "materializer-storage-contract"
 
 function contractMaterializer(
   storage: Storage,
-  batching: { readonly transactionReplanRows?: number } = {},
-  onSerializationRetry?: () => void
+  options: {
+    readonly transactionReplanRows?: number
+    readonly onSerializationRetry?: () => void
+    readonly clock?: () => Date
+  } = {}
 ): OntologyMaterializer {
+  const { transactionReplanRows, onSerializationRetry, clock } = options
   return createOntologyMaterializer({
     projectId: CONTRACT_PROJECT,
     ontology,
     projections,
     storage,
     dependencies: {
-      batching: { sourceStageRows: 1, statePageRows: 1, planChunkRows: 1, ...batching },
+      batching: {
+        sourceStageRows: 1,
+        statePageRows: 1,
+        planChunkRows: 1,
+        ...(transactionReplanRows === undefined ? {} : { transactionReplanRows }),
+      },
       ...(onSerializationRetry === undefined ? {} : { onSerializationRetry }),
+      ...(clock === undefined ? {} : { clock }),
     },
   })
 }
@@ -2107,7 +2134,6 @@ function observePlanning(
   return observe(storage, false)
 }
 
-/** The same storage, whose transactions wait before they finalize until `release` is called. */
 /** Resolves `reached` when a transaction of `storage` first records an Action run. */
 function observeActionRunRecords(storage: ContractStorage) {
   const reached = Promise.withResolvers<void>()
@@ -2128,6 +2154,7 @@ function observeActionRunRecords(storage: ContractStorage) {
             },
             recordEffects: (input) => actionRuns.recordEffects(input),
             getById: (input) => actionRuns.getById(input),
+            getByExecutionIds: (input) => actionRuns.getByExecutionIds(input),
             list: (input) => actionRuns.list(input),
           }
           return run(
@@ -2142,6 +2169,7 @@ function observeActionRunRecords(storage: ContractStorage) {
   return { storage: observed, reached: reached.promise }
 }
 
+/** The same storage, whose transactions wait before they finalize until `release` is called. */
 function pauseBeforeFinalize(storage: ContractStorage) {
   let reach!: () => void
   const reached = new Promise<void>((resolve) => {