@sixb/core 0.1.7 → 0.1.8

package/src/lake-storage/in-memory.ts

@@ -2,19 +2,26 @@ import { randomUUID } from "node:crypto"
 import type { DatasetDefinition, DatasetSchema, MergeChange } from "../datasets"
 import { getDatasetRowValidationError } from "../datasets/validation"
 import { mergeStrictDatasetDefinition } from "./definition-updates"
-import { LakeStorageError } from "./errors"
+import { LakeConcurrencyError, LakeStorageError } from "./errors"
 import type {
   BeginDatasetMergeInput,
   CommitDatasetMergeInput,
   DatasetMergeCommitResult,
   LakeMergeSession,
 } from "./merge"
+import { retryDatasetMergeCommit } from "./merge"
 import {
   cloneDatasetMergeChange,
   encodeDatasetPrimaryKey,
   getDatasetMergeChangeValidationError,
   getDatasetPrimaryKeyColumns,
 } from "./merge-validation"
+import {
+  assertUnsequencedDatasetWrite,
+  type DatasetSequenceState,
+  datasetSequenceChange,
+  reconcileDatasetSequences,
+} from "./source-ordering"
 import type {
   BeginDatasetWriteInput,
   CommitDatasetWriteInput,
@@ -27,6 +34,7 @@ import type {
   LakeWriteSession,
   ReadDatasetRowsInput,
 } from "./types"
+import { hasDatasetInputChanges } from "./version-inputs"
 
 function cloneDatasetDefinition(definition: DatasetDefinition): DatasetDefinition {
   return structuredClone(definition)
@@ -177,12 +185,16 @@ class InMemoryLakeMergeSession implements LakeMergeSession {
   async commit(input?: CommitDatasetMergeInput): Promise<DatasetMergeCommitResult> {
     this.assertOpen()
     this.closed = true
-    return this.storage.commitMerge({
-      merge: this.input,
-      baseVersionId: this.baseVersionId,
-      changes: this.changes,
-      commit: input,
-    })
+    return retryDatasetMergeCommit(this.input, input, async (rebase) =>
+      this.storage.commitMerge({
+        merge: this.input,
+        baseVersionId: rebase
+          ? ((await this.storage.getLatestVersion(this.input.dataset.id))?.versionId ?? null)
+          : this.baseVersionId,
+        changes: this.changes,
+        commit: input,
+      })
+    )
   }
 
   async abort(): Promise<void> {
@@ -201,6 +213,7 @@ export class InMemoryLakeStorage implements LakeStorage {
   private readonly datasets = new Map<string, DatasetDefinition>()
   private readonly versionsByDataset = new Map<string, DatasetVersion[]>()
   private readonly rowsByVersionId = new Map<string, readonly DatasetRow[]>()
+  private readonly sequencesByVersionId = new Map<string, Map<string, DatasetSequenceState>>()
   private readonly latestVersionIdByDataset = new Map<string, string>()
   private readonly commitLocks = new Map<string, Promise<void>>()
 
@@ -295,6 +308,7 @@ export class InMemoryLakeStorage implements LakeStorage {
       throw new LakeStorageError(`[LakeStorage] Unknown dataset '${input.dataset.id}'`)
     }
 
+    assertUnsequencedDatasetWrite(definition)
     return new InMemoryLakeWriteSession(this, {
       ...input,
       dataset: definition,
@@ -318,7 +332,7 @@ export class InMemoryLakeStorage implements LakeStorage {
       input.expectedLatestVersionId !== undefined &&
       latestVersion?.versionId !== input.expectedLatestVersionId
     ) {
-      throw new LakeStorageError(
+      throw new LakeConcurrencyError(
         `[LakeStorage] Optimistic merge start failed for dataset '${definition.id}': expected latest version '${input.expectedLatestVersionId}', found '${latestVersion?.versionId ?? "none"}'`
       )
     }
@@ -398,9 +412,10 @@ export class InMemoryLakeStorage implements LakeStorage {
       }
 
       const latestVersion = await this.getLatestVersion(options.merge.dataset.id)
+      options.commit?.signal?.throwIfAborted()
       const actualVersionId = latestVersion?.versionId ?? null
       if (actualVersionId !== options.baseVersionId) {
-        throw new LakeStorageError(
+        throw new LakeConcurrencyError(
           `[LakeStorage] Optimistic merge commit failed for dataset '${options.merge.dataset.id}': expected latest version '${options.baseVersionId ?? "none"}', found '${actualVersionId ?? "none"}'`
         )
       }
@@ -408,9 +423,21 @@ export class InMemoryLakeStorage implements LakeStorage {
       const previousRows = this.rowsByVersionId.get(latestVersion?.versionId ?? "") ?? []
       const previousByKey = this.rowsByPrimaryKey(previousRows, definition)
       const finalChanges = new Map<string, MergeChange<DatasetRow, DatasetRow>>()
-      for (const change of options.changes) {
-        const value = change.kind === "upsert" ? change.row : change.key
-        finalChanges.set(encodeDatasetPrimaryKey(definition, value), change)
+      const ordered =
+        definition.sequenceBy === undefined
+          ? undefined
+          : reconcileDatasetSequences(
+              definition,
+              this.sequencesByVersionId.get(actualVersionId ?? "") ?? new Map(),
+              options.changes.map((change) => datasetSequenceChange(definition, change))
+            )
+      if (ordered) {
+        for (const [key, index] of ordered.accepted) finalChanges.set(key, options.changes[index]!)
+      } else {
+        for (const change of options.changes) {
+          const value = change.kind === "upsert" ? change.row : change.key
+          finalChanges.set(encodeDatasetPrimaryKey(definition, value), change)
+        }
       }
 
       const nextByKey = new Map(previousByKey)
@@ -422,7 +449,13 @@ export class InMemoryLakeStorage implements LakeStorage {
         }
       }
 
-      if (this.sameKeyedRowContent(previousByKey, nextByKey, definition.schema)) {
+      const createInitialVersion = options.commit?.createInitialVersion && latestVersion === null
+      if (
+        !createInitialVersion &&
+        (ordered
+          ? ordered.accepted.size === 0
+          : this.sameKeyedRowContent(previousByKey, nextByKey, definition.schema))
+      ) {
         return {
           outcome: "unchanged",
           version: latestVersion ? cloneDatasetVersion(latestVersion) : null,
@@ -454,6 +487,7 @@ export class InMemoryLakeStorage implements LakeStorage {
         visibleRows.map((row) => cloneRow(row))
       )
       this.latestVersionIdByDataset.set(options.merge.dataset.id, versionId)
+      if (ordered) this.sequencesByVersionId.set(versionId, ordered.states)
 
       return { outcome: "created", version: cloneDatasetVersion(version) }
     })
@@ -470,13 +504,14 @@ export class InMemoryLakeStorage implements LakeStorage {
     }
 
     const mode = options.write.mode ?? "snapshot"
+    assertUnsequencedDatasetWrite(definition)
     const latestVersion = await this.getLatestVersion(options.write.dataset.id)
 
     if (options.commit?.expectedLatestVersionId !== undefined) {
-      const actual = latestVersion?.versionId
+      const actual = latestVersion?.versionId ?? null
       if (actual !== options.commit.expectedLatestVersionId) {
-        throw new LakeStorageError(
-          `[LakeStorage] Optimistic commit failed for dataset '${options.write.dataset.id}': expected latest version '${options.commit.expectedLatestVersionId}', found '${actual ?? "none"}'`
+        throw new LakeConcurrencyError(
+          `[LakeStorage] Optimistic commit failed for dataset '${options.write.dataset.id}': expected latest version '${options.commit.expectedLatestVersionId ?? "none"}', found '${actual ?? "none"}'`
         )
       }
     }
@@ -484,9 +519,10 @@ export class InMemoryLakeStorage implements LakeStorage {
     this.assertKeyedWriteIsUnique(mode, options.rows, latestVersion, definition)
 
     // Content-identical snapshots and empty appends reuse the latest version
-    // instead of creating a new one.
+    // unless explicitly supplied input lineage has changed.
     if (
       latestVersion &&
+      !hasDatasetInputChanges(latestVersion.inputs, options.write.inputs) &&
       this.isUnchangedWrite(mode, latestVersion, options.rows, definition.schema)
     ) {
       return { ...latestVersion, outcome: "unchanged" }

package/src/lake-storage/index.ts

@@ -6,6 +6,7 @@ export type {
   DatasetSchema,
   MergeChange,
 } from "../datasets"
+export { assertDatasetDefinition } from "../datasets/validation"
 export type { JsonValue } from "../json"
 export type {
   DatasetDefinitionUpdatePlan,
@@ -17,7 +18,7 @@ export {
   mergeStrictDatasetDefinition,
   planDatasetDefinitionUpdate,
 } from "./definition-updates"
-export { LakeStorageError } from "./errors"
+export { LakeConcurrencyError, LakeStorageError } from "./errors"
 export { InMemoryLakeStorage } from "./in-memory"
 export type {
   BeginDatasetMergeInput,
@@ -25,12 +26,19 @@ export type {
   DatasetMergeCommitResult,
   LakeMergeSession,
 } from "./merge"
+export { retryDatasetMergeCommit } from "./merge"
 export {
   cloneDatasetMergeChange,
   encodeDatasetPrimaryKey,
   getDatasetMergeChangeValidationError,
   getDatasetPrimaryKeyColumns,
 } from "./merge-validation"
+export type { DatasetSequenceChange, DatasetSequenceState } from "./source-ordering"
+export {
+  assertUnsequencedDatasetWrite,
+  datasetSequenceChange,
+  reconcileDatasetSequences,
+} from "./source-ordering"
 export type {
   ExecuteSqlTransformInput,
   LakeSqlExecutor,
@@ -60,3 +68,4 @@ export type {
   LakeWriteSession,
   ReadDatasetRowsInput,
 } from "./types"
+export { hasDatasetInputChanges } from "./version-inputs"

package/src/lake-storage/merge-validation.ts

@@ -1,5 +1,6 @@
 import type { DatasetDefinition, MergeChange } from "../datasets"
 import { getDatasetRowValidationError } from "../datasets"
+import { getDatasetSequenceValidationError } from "../datasets/sequence"
 import { isPlainRecord } from "../json"
 import { LakeStorageError } from "./errors"
 import type { DatasetRow } from "./types"
@@ -58,7 +59,11 @@ export function getDatasetMergeChangeValidationError(
     if (shapeError) {
       return shapeError
     }
-    return getDeleteKeyValidationError(change.key, dataset, primaryKeyColumns)
+    const keyError = getDeleteKeyValidationError(change.key, dataset, primaryKeyColumns)
+    if (keyError) return keyError
+    return dataset.sequenceBy === undefined
+      ? null
+      : getDatasetSequenceValidationError(dataset, change.sequence)
   }
 
   return `Dataset '${dataset.id}' merge change kind must be 'upsert' or 'delete'.`
@@ -70,7 +75,11 @@ export function cloneDatasetMergeChange(
 ): MergeChange<DatasetRow, DatasetRow> {
   return change.kind === "upsert"
     ? { kind: "upsert", row: structuredClone(change.row) }
-    : { kind: "delete", key: structuredClone(change.key) }
+    : {
+        kind: "delete",
+        key: structuredClone(change.key),
+        ...(change.sequence !== undefined ? { sequence: structuredClone(change.sequence) } : {}),
+      }
 }
 
 function getChangeShapeError(
@@ -79,6 +88,7 @@ function getChangeShapeError(
   payloadField: "row" | "key"
 ): string | null {
   const allowedFields = new Set(["kind", payloadField])
+  if (payloadField === "key" && dataset.sequenceBy !== undefined) allowedFields.add("sequence")
   for (const field of Object.keys(change)) {
     if (!allowedFields.has(field)) {
       return `Dataset '${dataset.id}' ${change.kind as string} change contains unknown field '${field}'.`

package/src/lake-storage/merge.ts

@@ -1,4 +1,5 @@
 import type { DatasetDefinition, MergeChange } from "../datasets"
+import { LakeConcurrencyError } from "./errors"
 import type { DatasetProducer, DatasetRow, DatasetVersion, DatasetVersionRef } from "./types"
 
 export interface BeginDatasetMergeInput {
@@ -12,6 +13,40 @@ export interface BeginDatasetMergeInput {
 
 export interface CommitDatasetMergeInput {
   readonly commitMessage?: string
+  /** Create an addressable first version even without changes; reuse an existing version on no-op. */
+  readonly createInitialVersion?: boolean
+  /** Retry sequenced merges up to three times total; ignored when an explicit version guard is set. */
+  readonly retryOnConflict?: boolean
+  /** Checked before each attempt; a durable commit is never cancelled retroactively. */
+  readonly signal?: AbortSignal
+}
+
+/** Providers retain their staging session throughout this bounded commit loop. */
+export async function retryDatasetMergeCommit(
+  merge: BeginDatasetMergeInput,
+  commit: CommitDatasetMergeInput | undefined,
+  run: (rebase: boolean) => Promise<DatasetMergeCommitResult>
+): Promise<DatasetMergeCommitResult> {
+  const attempts =
+    commit?.retryOnConflict &&
+    merge.dataset.sequenceBy !== undefined &&
+    merge.expectedLatestVersionId === undefined
+      ? 3
+      : 1
+  for (let attempt = 0; ; attempt += 1) {
+    commit?.signal?.throwIfAborted()
+    try {
+      return await run(attempt > 0)
+    } catch (error) {
+      if (!(error instanceof LakeConcurrencyError) || attempts === 1) throw error
+      if (attempt + 1 === attempts) {
+        throw new LakeConcurrencyError(
+          `[SixbLake] Dataset '${merge.dataset.id}' merge exhausted ${attempts} concurrency attempts. No changes from this operation were committed.`,
+          { cause: error }
+        )
+      }
+    }
+  }
 }
 
 /**
@@ -23,13 +58,16 @@ export type DatasetMergeCommitResult =
   | { readonly outcome: "unchanged"; readonly version: DatasetVersion | null }
 
 export interface LakeMergeSession {
-  /** Stage ordered complete-row upserts and exact primary-key deletes. */
+  /**
+   * Stage ordered complete-row upserts and exact primary-key deletes. With sequenceBy, deletes
+   * require a sequence, stale changes are ignored, and conflicting ties abort the whole merge.
+   */
   writeChanges(
     changes:
       | Iterable<MergeChange<DatasetRow, DatasetRow>>
       | AsyncIterable<MergeChange<DatasetRow, DatasetRow>>
   ): Promise<void>
-  /** Commit only if the latest version still matches the version captured when the session began. */
+  /** Commit against the captured version; opt-in retries rebase sequenced changes on a fresh head. */
   commit(input?: CommitDatasetMergeInput): Promise<DatasetMergeCommitResult>
   abort(): Promise<void>
 }

package/src/lake-storage/source-ordering.ts

@@ -0,0 +1,95 @@
+import { createHash } from "node:crypto"
+import type { DatasetDefinition, MergeChange } from "../datasets"
+import { datasetSequenceValue } from "../datasets/sequence"
+import { stableJsonStringify } from "../json"
+import { normalizeDecimalValue } from "../ontology/decimal"
+import { LakeStorageError } from "./errors"
+import { encodeDatasetPrimaryKey } from "./merge-validation"
+import type { DatasetRow } from "./types"
+
+/** Current source revision; a null content digest is a deletion tombstone. */
+export interface DatasetSequenceState {
+  readonly sequence: string
+  readonly content: string | null
+}
+
+export interface DatasetSequenceChange extends DatasetSequenceState {
+  readonly key: string
+}
+
+export function datasetSequenceChange(
+  dataset: DatasetDefinition,
+  change: MergeChange<DatasetRow, DatasetRow>
+): DatasetSequenceChange {
+  const value = change.kind === "upsert" ? change.row[dataset.sequenceBy ?? ""] : change.sequence
+  return {
+    key: encodeDatasetPrimaryKey(dataset, change.kind === "upsert" ? change.row : change.key),
+    sequence: datasetSequenceValue(dataset, value).toString(),
+    content: change.kind === "delete" ? null : rowDigest(dataset, change.row),
+  }
+}
+
+/** Process in submission order against a private copy; conflicts publish no partial state. */
+export function reconcileDatasetSequences(
+  dataset: DatasetDefinition,
+  previous: ReadonlyMap<string, DatasetSequenceState>,
+  changes: readonly DatasetSequenceChange[]
+): { states: Map<string, DatasetSequenceState>; accepted: Map<string, number> } {
+  const states = new Map(previous)
+  const accepted = new Map<string, number>()
+  for (const [index, change] of changes.entries()) {
+    const current = states.get(change.key)
+    if (current) {
+      const difference = BigInt(change.sequence) - BigInt(current.sequence)
+      if (difference < 0n) continue
+      if (difference === 0n) {
+        if (current.content !== change.content) {
+          throw new LakeStorageError(
+            `[SixbLake] Dataset '${dataset.id}' has conflicting content at source sequence '${change.sequence}' for key ${change.key}.`
+          )
+        }
+        continue
+      }
+    }
+    states.set(change.key, { sequence: change.sequence, content: change.content })
+    accepted.set(change.key, index)
+  }
+  return { states, accepted }
+}
+
+function rowDigest(dataset: DatasetDefinition, row: DatasetRow): string {
+  const content: Record<string, unknown> = Object.create(null)
+  for (const column of dataset.schema.columns) {
+    const value = row[column.name]
+    // Nullable additions and omitted/undefined/null values have identical content.
+    if (value == null) continue
+    switch (column.type) {
+      case "int64":
+        content[column.name] = BigInt(value as string | number).toString()
+        break
+      case "timestamp":
+        content[column.name] = new Date(value as string | Date).toISOString()
+        break
+      case "date":
+        content[column.name] = new Date(value as string | Date).toISOString().slice(0, 10)
+        break
+      case "decimal":
+        content[column.name] = normalizeDecimalValue(value as string)
+        break
+      default:
+        content[column.name] = value
+    }
+  }
+  return createHash("sha256")
+    .update(stableJsonStringify(JSON.parse(JSON.stringify(content))))
+    .digest("hex")
+}
+
+/** Until row reconciliation is supported, replacement writes must not erase source history. */
+export function assertUnsequencedDatasetWrite(dataset: DatasetDefinition): void {
+  if (dataset.sequenceBy !== undefined) {
+    throw new LakeStorageError(
+      `[SixbLake] Dataset '${dataset.id}' declares sequenceBy; use beginMerge with source-ordered changes.`
+    )
+  }
+}

package/src/lake-storage/sql-transforms.ts

@@ -39,7 +39,8 @@ export interface ExecuteSqlTransformInput<TSqlDialect extends SqlDialect = SqlDi
   readonly target: DatasetDefinition
   readonly mode: DatasetWriteMode
   readonly producer?: DatasetProducer
-  readonly expectedLatestVersionId?: string
+  /** Atomic target version guard; null requires that no version exists. */
+  readonly expectedLatestVersionId?: string | null
   readonly commitMessage?: string
 }
 

package/src/lake-storage/types.ts

@@ -19,7 +19,8 @@ export interface DatasetVersionRef {
 }
 
 export interface DatasetProducer {
-  readonly kind: "sync" | "pipeline"
+  readonly kind: "sync" | "pipeline" | "ingest"
+  /** For ingestion, the bound execution id; otherwise the sync/pipeline definition id. */
   readonly id?: string
   readonly runId?: string
   readonly stepId?: string
@@ -73,7 +74,8 @@ export interface ReadDatasetRowsInput {
 }
 
 export interface CommitDatasetWriteInput {
-  readonly expectedLatestVersionId?: string
+  /** Atomic version guard; null requires that no version exists, undefined leaves it unguarded. */
+  readonly expectedLatestVersionId?: string | null
   readonly commitMessage?: string
 }
 
@@ -81,11 +83,14 @@ export interface DatasetWriteCommitResult extends DatasetVersion {
   /**
    * Whether this operation created the returned version or reused the
    * unchanged latest version. When a latest version exists, providers must
-   * report `unchanged` and reuse it for commits that cannot change visible
-   * dataset content: a snapshot whose rows equal the latest visible rows
+   * report `unchanged` and reuse it for commits that change neither visible
+   * dataset content nor explicitly supplied input lineage: a snapshot whose
+   * rows equal the latest visible rows
    * (order-insensitive, with equal duplicate counts), or an append of zero
    * rows. Without a latest version, an empty snapshot or append still creates
-   * the first addressable dataset version.
+   * the first addressable dataset version. New input versions create a version
+   * even for identical rows, so a newer pipeline calculation fences concurrent
+   * work based on older inputs.
    */
   readonly outcome: "created" | "unchanged"
 }

package/src/lake-storage/version-inputs.ts

@@ -0,0 +1,14 @@
+import type { DatasetVersionRef } from "./types"
+
+/** Input lineage participates in write identity when the caller supplies it. */
+export function hasDatasetInputChanges(
+  previous: readonly DatasetVersionRef[] | undefined,
+  next: readonly DatasetVersionRef[] | undefined
+): boolean {
+  if (next === undefined) return false
+  const keys = (refs: readonly DatasetVersionRef[]) =>
+    new Set(refs.map((ref) => JSON.stringify([ref.datasetId, ref.versionId])))
+  const before = keys(previous ?? [])
+  const after = keys(next)
+  return before.size !== after.size || [...after].some((key) => !before.has(key))
+}