@sixb/core 0.1.7 → 0.1.8

package/src/bootstrap/ontology-type-manifest.ts

@@ -1,4 +1,4 @@
-import { mkdir, readdir, readFile, writeFile } from "node:fs/promises"
+import { access, mkdir, readdir, readFile, writeFile } from "node:fs/promises"
 import { dirname, extname, join, relative, resolve, sep } from "node:path"
 import { pathToFileURL } from "node:url"
 import type { OntologyDocumentInput } from "../ontology/registry"
@@ -75,11 +75,17 @@ export async function generateOntologyTypeManifest(
   const discovery = await discoverOntologyTypeManifest(projectRoot)
 
   if (discovery.moduleCount === 0) {
-    return {
-      ...discovery,
-      path: outFile,
-      written: false,
-      skipped: true,
+    // Leave projects without ontology untouched, but clear a previously generated map.
+    try {
+      await access(outFile)
+    } catch (error) {
+      if (!isNotFoundError(error)) throw error
+      return {
+        ...discovery,
+        path: outFile,
+        written: false,
+        skipped: true,
+      }
     }
   }
 

package/src/datasets/builders.ts

@@ -15,6 +15,7 @@ type DatasetColumnOptions = {
 type DefineDatasetOptions = {
   readonly schema: readonly DatasetColumnDefinition[]
   readonly primaryKey?: DatasetPrimaryKey
+  readonly sequenceBy?: string
   readonly partitionBy?: readonly string[]
   readonly description?: string
 }
@@ -23,6 +24,7 @@ type DeriveDatasetOptions<TParent extends DatasetDefinition> = {
   readonly pick?: readonly DatasetColumnNameOf<TParent>[]
   readonly add?: readonly DatasetColumnDefinition[]
   readonly primaryKey?: DatasetPrimaryKey
+  readonly sequenceBy?: string
   readonly partitionBy?: readonly string[]
   readonly description?: string
 }
@@ -58,8 +60,22 @@ type PrimaryKeyColumnNameOf<TColumns extends readonly DatasetColumnDefinition[]>
 
 type CheckedDefineDatasetOptions<TOptions extends DefineDatasetOptions> = TOptions & {
   readonly primaryKey?: DatasetPrimaryKey<PrimaryKeyColumnNameOf<TOptions["schema"]>>
+  readonly sequenceBy?: SequenceColumnNameOf<TOptions["schema"]>
 }
 
+type SequenceColumnNameOf<TColumns extends readonly DatasetColumnDefinition[]> =
+  string extends ColumnNameOf<TColumns>
+    ? string
+    : {
+        [TName in ColumnNameOf<TColumns>]: ColumnDefinitionOf<TColumns, TName>["type"] extends
+          | "timestamp"
+          | "int64"
+          ? ColumnDefinitionOf<TColumns, TName> extends { readonly nullable: true }
+            ? never
+            : TName
+          : never
+      }[ColumnNameOf<TColumns>]
+
 type PickColumns<
   TColumns extends readonly DatasetColumnDefinition[],
   TNames extends readonly ColumnNameOf<TColumns>[],
@@ -107,8 +123,9 @@ type DatasetColumnResult<
 
 type DatasetDefinitionResult<TId extends string, TOptions extends DefineDatasetOptions> = Omit<
   DatasetDefinition<TId, TOptions["schema"]>,
-  "primaryKey" | "partitionBy" | "description"
+  "primaryKey" | "sequenceBy" | "partitionBy" | "description"
 > &
+  FieldFromOptions<TOptions, "sequenceBy", string> &
   FieldFromOptions<TOptions, "primaryKey", DatasetPrimaryKey> &
   FieldFromOptions<TOptions, "partitionBy", readonly string[]> &
   FieldFromOptions<TOptions, "description", string>
@@ -119,8 +136,9 @@ type DerivedDatasetDefinitionResult<
   TOptions extends DeriveDatasetOptions<TParent> | undefined,
 > = Omit<
   DatasetDefinition<TId, DerivedColumns<TParent, TOptions>>,
-  "primaryKey" | "partitionBy" | "description"
+  "primaryKey" | "sequenceBy" | "partitionBy" | "description"
 > &
+  FieldFromOptions<TOptions, "sequenceBy", string> &
   FieldFromOptions<TOptions, "primaryKey", DatasetPrimaryKey> &
   FieldFromOptions<TOptions, "partitionBy", readonly string[]> &
   FieldFromOptions<TOptions, "description", string>
@@ -130,6 +148,7 @@ type CheckedDeriveDatasetOptions<
   TOptions extends DeriveDatasetOptions<TParent>,
 > = TOptions & {
   readonly primaryKey?: DatasetPrimaryKey<PrimaryKeyColumnNameOf<DerivedColumns<TParent, TOptions>>>
+  readonly sequenceBy?: SequenceColumnNameOf<DerivedColumns<TParent, TOptions>>
 }
 
 export function col<const TName extends string, const TType extends DatasetColumnType>(
@@ -177,6 +196,7 @@ function createDatasetDefinition(id: string, options: DefineDatasetOptions): Dat
         }
       : {}),
     ...(options.partitionBy !== undefined ? { partitionBy: [...options.partitionBy] } : {}),
+    ...(options.sequenceBy !== undefined ? { sequenceBy: options.sequenceBy } : {}),
     ...(options.description !== undefined ? { description: options.description } : {}),
   }
 
@@ -230,6 +250,7 @@ function deriveDatasetDefinition(
   return createDatasetDefinition(id, {
     schema: [...pickedColumns, ...(options.add ?? [])],
     ...(options.primaryKey !== undefined ? { primaryKey: options.primaryKey } : {}),
+    ...(options.sequenceBy !== undefined ? { sequenceBy: options.sequenceBy } : {}),
     ...(options.partitionBy !== undefined ? { partitionBy: options.partitionBy } : {}),
     ...(options.description !== undefined ? { description: options.description } : {}),
   })

package/src/datasets/changes.ts

@@ -1,13 +1,25 @@
 export type MergeChange<Row, Key> =
   | { readonly kind: "upsert"; readonly row: Row }
-  | { readonly kind: "delete"; readonly key: Key }
+  | { readonly kind: "delete"; readonly key: Key; readonly sequence?: string | number | Date }
 
 export const change = {
   upsert<const Row extends object>(row: Row): { readonly kind: "upsert"; readonly row: Row } {
     return { kind: "upsert", row }
   },
 
-  delete<const Key extends object>(key: Key): { readonly kind: "delete"; readonly key: Key } {
-    return { kind: "delete", key }
-  },
+  delete: deleteChange,
+}
+
+function deleteChange<const Key extends object>(
+  key: Key
+): { readonly kind: "delete"; readonly key: Key }
+function deleteChange<const Key extends object, const Sequence extends string | number | Date>(
+  key: Key,
+  options: { readonly sequence: Sequence }
+): { readonly kind: "delete"; readonly key: Key; readonly sequence: Sequence }
+function deleteChange<Key extends object>(
+  key: Key,
+  options?: { readonly sequence: string | number | Date }
+): { readonly kind: "delete"; readonly key: Key; readonly sequence?: string | number | Date } {
+  return { kind: "delete", key, ...(options ? { sequence: options.sequence } : {}) }
 }

package/src/datasets/execution.ts

@@ -1,24 +1,95 @@
-import { isAllowed } from "../authorization"
+import { assertPrivileged, isRuntimeAllowed } from "../authorization"
+import type { BlobStorage } from "../blob-storage"
+import type { ExecutionContext } from "../execution"
+import { resolveRuntimeAuthorizationForProject } from "../execution/authorization"
+import type { DatasetMergeCommitResult } from "../lake-storage/merge"
+import { getDatasetPrimaryKeyColumns } from "../lake-storage/merge-validation"
+import type { DatasetRow, LakeStorage } from "../lake-storage/types"
 import type { SixbRuntimeContext } from "../runtime/types"
+import type { MergeChange } from "./changes"
 import type { DatasetDefinition } from "./types"
+import { writeDataset } from "./write"
+
+export interface DatasetIngestInput {
+  readonly changes:
+    | Iterable<MergeChange<DatasetRow, DatasetRow>>
+    | AsyncIterable<MergeChange<DatasetRow, DatasetRow>>
+  readonly signal?: AbortSignal
+}
+
+/** Source commit result; downstream pipelines and projections run asynchronously. */
+export type DatasetIngestResult = DatasetMergeCommitResult & { readonly rowsRead: number }
 
 export interface DatasetsRuntime {
   list(): readonly DatasetDefinition[]
   getById(datasetId: string): DatasetDefinition | null
+  ingest(dataset: DatasetDefinition, input: DatasetIngestInput): Promise<DatasetIngestResult>
 }
 
 export function createDatasetsRuntime(
-  runtime: Pick<SixbRuntimeContext, "authorization">,
-  source: Pick<DatasetsRuntime, "list" | "getById">
+  runtime: SixbRuntimeContext,
+  execution: ExecutionContext,
+  source: Pick<DatasetsRuntime, "list" | "getById">,
+  lakeStorage: LakeStorage,
+  blobStorage: Pick<BlobStorage, "stat">
 ): DatasetsRuntime {
+  const authority = resolveRuntimeAuthorizationForProject(runtime)
   const allowed = (datasetId: string) =>
-    isAllowed(runtime.authorization, { kind: "dataset.view", datasetId })
+    isRuntimeAllowed(runtime, { kind: "dataset.view", datasetId })
 
   return {
-    list: () => source.list().filter((dataset) => allowed(dataset.id)),
+    list: () =>
+      authority.type === "denied" || authority.type === "delegated"
+        ? []
+        : source.list().filter((dataset) => allowed(dataset.id)),
     getById: (datasetId) => {
+      if (authority.type === "denied" || authority.type === "delegated") return null
       const dataset = source.getById(datasetId)
       return dataset && allowed(datasetId) ? dataset : null
     },
+    async ingest(dataset, input) {
+      assertPrivileged(runtime, "datasets.ingest")
+      const registered = source.getById(dataset.id)
+      if (!registered) throw new Error(`[Sixb] Dataset '${dataset.id}' is not registered.`)
+      if (getDatasetPrimaryKeyColumns(registered) === null) {
+        throw new Error(`[Sixb] Dataset '${dataset.id}' must define a primaryKey before ingestion.`)
+      }
+      if (typeof lakeStorage.beginMerge !== "function") {
+        throw new Error("[Sixb] Dataset ingestion requires a lake provider with merge support.")
+      }
+      const producer = { kind: "ingest" as const, id: execution.id }
+      const result = await writeDataset({
+        lakeStorage,
+        blobStorage,
+        dataset: registered,
+        mode: "merge",
+        signal: input.signal ?? new AbortController().signal,
+        producer,
+        readValues: async () =>
+          (async function* () {
+            for await (const value of input.changes) yield { value }
+          })(),
+      })
+      if (result.outcome === "created") {
+        await runtime.events.emit(
+          {
+            events: [
+              {
+                type: "dataset.version.committed",
+                payload: {
+                  datasetId: registered.id,
+                  versionId: result.version.versionId,
+                  createdAt: result.version.createdAt.toISOString(),
+                  producer,
+                },
+              },
+            ],
+            correlationId: execution.correlationId,
+          },
+          { source: "SixbDatasets" }
+        )
+      }
+      return result
+    },
   }
 }

package/src/datasets/index.ts

@@ -2,6 +2,7 @@ export { col, defineDataset } from "./builders"
 export type { MergeChange } from "./changes"
 export { change } from "./changes"
 export { DatasetValidationError } from "./errors"
+export type { DatasetIngestInput, DatasetIngestResult, DatasetsRuntime } from "./execution"
 export type {
   DatasetColumnDefinition,
   DatasetColumnDefinitionOf,

package/src/datasets/sequence.ts

@@ -0,0 +1,77 @@
+import type { DatasetDefinition } from "./types"
+
+const MIN_INT64 = -(1n << 63n)
+const MAX_INT64 = (1n << 63n) - 1n
+
+/** Normalize source ordering to an exact integer (timestamp milliseconds or signed int64). */
+export function datasetSequenceValue(dataset: DatasetDefinition, value: unknown): bigint {
+  const column = dataset.schema.columns.find((column) => column.name === dataset.sequenceBy)
+  if (column?.type === "int64") {
+    if (
+      (typeof value === "number" && Number.isSafeInteger(value)) ||
+      (typeof value === "string" && /^-?\d+$/.test(value))
+    ) {
+      const integer = BigInt(value)
+      if (integer >= MIN_INT64 && integer <= MAX_INT64) return integer
+    }
+    throw new Error(
+      `Dataset '${dataset.id}' sequence must be a signed int64; use a string outside the safe integer range.`
+    )
+  }
+  if (column?.type === "timestamp") {
+    const millis = parseDatasetTimestamp(value)
+    if (millis !== null) return BigInt(millis)
+    throw new Error(
+      `Dataset '${dataset.id}' sequence must be a valid Date or ISO timestamp with an explicit timezone and at most millisecond precision.`
+    )
+  }
+  throw new Error(
+    `Dataset '${dataset.id}' sequenceBy must reference a non-nullable timestamp or int64 column.`
+  )
+}
+
+/** Date.parse alone normalizes impossible calendar dates instead of rejecting them. */
+export function parseDatasetTimestamp(value: unknown): number | null {
+  if (value instanceof Date && Number.isFinite(value.getTime())) return value.getTime()
+  if (typeof value === "string") {
+    const match =
+      /^([+-]\d{6}|\d{4})-(\d{2})-(\d{2})T(\d{2}):(\d{2}):(\d{2})(?:\.(\d{1,3}))?(Z|[+-]\d{2}:\d{2})$/.exec(
+        value
+      )
+    if (match) {
+      const [, year, month, day, hour, minute, second, , zone] = match
+      const numericYear = Number(year)
+      const leapYear = numericYear % 4 === 0 && (numericYear % 100 !== 0 || numericYear % 400 === 0)
+      const days =
+        Number(month) === 2 ? (leapYear ? 29 : 28) : [4, 6, 9, 11].includes(Number(month)) ? 30 : 31
+      const validZone =
+        zone === "Z" || (Number(zone?.slice(1, 3)) <= 23 && Number(zone?.slice(4)) <= 59)
+      const millis = Date.parse(value)
+      if (
+        Number(month) >= 1 &&
+        Number(month) <= 12 &&
+        Number(day) >= 1 &&
+        Number(day) <= days &&
+        Number(hour) < 24 &&
+        Number(minute) < 60 &&
+        Number(second) < 60 &&
+        validZone &&
+        Number.isFinite(millis)
+      )
+        return millis
+    }
+  }
+  return null
+}
+
+export function getDatasetSequenceValidationError(
+  dataset: DatasetDefinition,
+  value: unknown
+): string | null {
+  try {
+    datasetSequenceValue(dataset, value)
+    return null
+  } catch (error) {
+    return error instanceof Error ? error.message : String(error)
+  }
+}

package/src/datasets/types.ts

@@ -36,6 +36,7 @@ export interface DatasetDefinition<
   readonly id: TId
   readonly schema: DatasetSchema<TColumns>
   readonly primaryKey?: DatasetPrimaryKey
+  readonly sequenceBy?: string
   readonly partitionBy?: readonly string[]
   readonly description?: string
 }

package/src/datasets/validation.ts

@@ -2,6 +2,7 @@ import { isFileRef } from "../blob-storage"
 import { isJsonValue, isPlainRecord } from "../json"
 import { isDecimalString } from "../ontology/decimal"
 import { DatasetValidationError } from "./errors"
+import { getDatasetSequenceValidationError, parseDatasetTimestamp } from "./sequence"
 import type {
   DatasetColumnDefinition,
   DatasetColumnType,
@@ -185,6 +186,20 @@ export function assertDatasetDefinition(
     assertDatasetPrimaryKey(definition.primaryKey, definition.schema, createError)
   }
 
+  if (definition.sequenceBy !== undefined) {
+    const column = definition.schema.columns.find((column) => column.name === definition.sequenceBy)
+    if (
+      definition.primaryKey === undefined ||
+      !column ||
+      column.nullable ||
+      !["timestamp", "int64"].includes(column.type)
+    ) {
+      throw createError(
+        "Dataset sequenceBy requires a primaryKey and a non-nullable timestamp or int64 column."
+      )
+    }
+  }
+
   if (definition.partitionBy !== undefined) {
     if (!Array.isArray(definition.partitionBy)) {
       throw createError("Dataset partitionBy must be an array of column names.")
@@ -268,6 +283,9 @@ function getColumnValidationError(
 ): string | null {
   const hasValue = Object.hasOwn(row, column.name)
   const value = row[column.name]
+  if (column.name === dataset.sequenceBy) {
+    return getDatasetSequenceValidationError(dataset, value)
+  }
 
   // Treat `undefined` the same as an omitted field so callers either send a real
   // value or use `null` explicitly on nullable columns.
@@ -285,6 +303,14 @@ function getColumnValidationError(
     return null
   }
 
+  // Every timestamp contributes to a sequenced row's millisecond-precision content digest.
+  // Reject ambiguous or lossy values before staging, even when another column orders the row.
+  if (dataset.sequenceBy !== undefined && column.type === "timestamp") {
+    return parseDatasetTimestamp(value) === null
+      ? `Dataset '${dataset.id}' column '${column.name}' must be a valid Date or ISO timestamp with an explicit timezone and at most millisecond precision.`
+      : null
+  }
+
   if (!matchesColumnType(value, column.type)) {
     return `Dataset '${dataset.id}' column '${column.name}' must match type '${column.type}'.`
   }

package/src/datasets/write.ts

@@ -0,0 +1,240 @@
+import type { BlobStorage, FileRef } from "../blob-storage"
+import { isFileRef } from "../blob-storage"
+import { isPlainRecord } from "../json"
+import type { DatasetMergeCommitResult } from "../lake-storage/merge"
+import { getDatasetMergeChangeValidationError } from "../lake-storage/merge-validation"
+import type {
+  DatasetProducer,
+  DatasetRow,
+  DatasetWriteMode,
+  LakeStorage,
+} from "../lake-storage/types"
+import type { MergeChange } from "./changes"
+import type { DatasetDefinition } from "./types"
+import { getDatasetRowValidationError } from "./validation"
+
+export interface DatasetWriteValue {
+  readonly value: unknown
+}
+
+export interface WriteDatasetInput<TValue extends DatasetWriteValue = DatasetWriteValue> {
+  readonly lakeStorage: LakeStorage
+  readonly blobStorage: Pick<BlobStorage, "stat">
+  readonly dataset: DatasetDefinition
+  readonly mode: DatasetWriteMode | "merge"
+  readonly signal: AbortSignal
+  /** Lazy so a merge's latest-version guard is checked before fetching source data. */
+  readValues(): Promise<Iterable<TValue> | AsyncIterable<TValue>>
+  readonly producer?: DatasetProducer
+  readonly expectedLatestVersionId?: string
+  readonly commitMessage?: string
+  /** Diagnostic label for the caller, including its component prefix. */
+  readonly sourceLabel?: string
+  /** Progress includes validated values even if the operation subsequently fails. */
+  onRead?(rowsRead: number): void
+  /** Attach caller-owned source provenance to a validation failure. */
+  mapValidationError?(error: unknown, value: TValue, itemIndex: number): unknown
+}
+
+export type WriteDatasetResult = DatasetMergeCommitResult & { readonly rowsRead: number }
+
+/**
+ * Validate, stage, and commit one dataset write using existing lake sessions.
+ *
+ * The caller owns fetching/cancelling its source, execution bookkeeping, and publishing the
+ * dataset-update notification for a `created` result. This operation returns immediately after
+ * the lake commit; it does not wait for downstream pipelines or projections.
+ */
+export async function writeDataset<TValue extends DatasetWriteValue>(
+  input: WriteDatasetInput<TValue>
+): Promise<WriteDatasetResult> {
+  const { lakeStorage, dataset, signal } = input
+  let abortWrite: (() => Promise<void>) | undefined
+  let rowsRead = 0
+  const onRead = () => {
+    rowsRead += 1
+    input.onRead?.(rowsRead)
+  }
+
+  throwIfAborted(signal)
+  await lakeStorage.createDataset(dataset)
+  try {
+    throwIfAborted(signal)
+    const reconcileSnapshot = dataset.sequenceBy !== undefined && input.mode === "snapshot"
+    if (input.mode === "merge" || reconcileSnapshot) {
+      const session = await lakeStorage.beginMerge({
+        dataset,
+        expectedLatestVersionId: input.expectedLatestVersionId,
+        producer: input.producer,
+      })
+      abortWrite = () => session.abort()
+      throwIfAborted(signal)
+      const values = await input.readValues()
+      throwIfAborted(signal)
+      await session.writeChanges(
+        validatedValues(
+          input,
+          values,
+          async (value, itemIndex) =>
+            reconcileSnapshot
+              ? { kind: "upsert" as const, row: await validateRow(value, input, itemIndex) }
+              : validateMergeChange(value, input, itemIndex),
+          onRead
+        )
+      )
+      throwIfAborted(signal)
+      const result = await session.commit({
+        commitMessage: input.commitMessage,
+        createInitialVersion: reconcileSnapshot,
+        retryOnConflict: dataset.sequenceBy !== undefined,
+        signal,
+      })
+      return { ...result, rowsRead }
+    }
+
+    const values = await input.readValues()
+    throwIfAborted(signal)
+    const session = await lakeStorage.beginWrite({
+      dataset,
+      mode: input.mode,
+      producer: input.producer,
+    })
+    abortWrite = () => session.abort()
+    await session.writeRows(
+      validatedValues(
+        input,
+        values,
+        (value, itemIndex) => validateRow(value, input, itemIndex),
+        onRead
+      )
+    )
+    throwIfAborted(signal)
+
+    // Preserve sync semantics: an empty append must not invent an initial dataset version.
+    if (rowsRead === 0 && input.mode === "append") {
+      const version = await lakeStorage.getLatestVersion(dataset.id)
+      await session.abort()
+      return { outcome: "unchanged", version, rowsRead }
+    }
+
+    const { outcome, ...version } = await session.commit({
+      expectedLatestVersionId: input.expectedLatestVersionId,
+      commitMessage: input.commitMessage,
+    })
+    return { outcome, version, rowsRead }
+  } catch (error) {
+    // Cleanup must not replace the source, validation, cancellation, or commit failure.
+    await abortWrite?.().catch(() => {})
+    throw error
+  }
+}
+
+async function* validatedValues<TValue extends DatasetWriteValue, TResult>(
+  input: WriteDatasetInput<TValue>,
+  values: Iterable<TValue> | AsyncIterable<TValue>,
+  validate: (value: unknown, itemIndex: number) => Promise<TResult>,
+  onRead: () => void
+): AsyncIterable<TResult> {
+  let itemIndex = 0
+  for await (const sourceValue of values) {
+    throwIfAborted(input.signal)
+    itemIndex += 1
+    let value: TResult
+    try {
+      value = await validate(sourceValue.value, itemIndex)
+    } catch (error) {
+      throw input.mapValidationError
+        ? input.mapValidationError(error, sourceValue, itemIndex)
+        : error
+    }
+    onRead()
+    yield value
+  }
+}
+
+function sourceLabel(input: Pick<WriteDatasetInput, "dataset" | "sourceLabel">): string {
+  return input.sourceLabel ?? `[Sixb] Dataset '${input.dataset.id}' writer`
+}
+
+async function validateRow(
+  value: unknown,
+  input: WriteDatasetInput,
+  itemIndex: number
+): Promise<DatasetRow> {
+  if (!isPlainRecord(value)) {
+    throw new Error(
+      `${sourceLabel(input)} returned an invalid row at item ${itemIndex}. Dataset rows must be plain objects.`
+    )
+  }
+  const validationError = getDatasetRowValidationError(value, input.dataset)
+  if (validationError) {
+    throw new Error(
+      `${sourceLabel(input)} returned an invalid row at item ${itemIndex}. ${validationError}`
+    )
+  }
+  await verifyRowFileRefs(value, input, itemIndex)
+  return value
+}
+
+async function validateMergeChange(
+  value: unknown,
+  input: WriteDatasetInput,
+  itemIndex: number
+): Promise<MergeChange<DatasetRow, DatasetRow>> {
+  const validationError = getDatasetMergeChangeValidationError(value, input.dataset)
+  if (validationError) {
+    throw new Error(
+      `${sourceLabel(input)} returned an invalid merge change at item ${itemIndex}. ${validationError}`
+    )
+  }
+  const change = value as MergeChange<DatasetRow, DatasetRow>
+  if (change.kind === "upsert") await verifyRowFileRefs(change.row, input, itemIndex)
+  return change
+}
+
+async function verifyRowFileRefs(
+  row: DatasetRow,
+  input: WriteDatasetInput,
+  itemIndex: number
+): Promise<void> {
+  for (const column of input.dataset.schema.columns) {
+    if (column.type !== "fileRef") continue
+    const value = row[column.name]
+    if (isFileRef(value)) await verifyFileRef(value, input, column.name, itemIndex)
+  }
+}
+
+async function verifyFileRef(
+  fileRef: FileRef,
+  input: WriteDatasetInput,
+  columnName: string,
+  itemIndex: number
+): Promise<void> {
+  const blobInfo = await input.blobStorage.stat(fileRef.blobId)
+  const context = `${sourceLabel(input)} returned row ${itemIndex} with dataset '${input.dataset.id}' column '${columnName}'`
+  if (!blobInfo) {
+    throw new Error(`${context} referencing unknown blob '${fileRef.blobId}'.`)
+  }
+  if (blobInfo.digest !== fileRef.digest) {
+    throw new Error(
+      `${context} referencing blob '${fileRef.blobId}' with digest '${fileRef.digest}', but blob storage has '${blobInfo.digest}'.`
+    )
+  }
+  if (blobInfo.sizeBytes !== fileRef.sizeBytes) {
+    throw new Error(
+      `${context} referencing blob '${fileRef.blobId}' with size ${fileRef.sizeBytes}, but blob storage has ${blobInfo.sizeBytes}.`
+    )
+  }
+}
+
+function throwIfAborted(signal: AbortSignal): void {
+  if (!signal.aborted) return
+  if (signal.reason instanceof Error) throw signal.reason
+  const error = new Error(
+    typeof signal.reason === "string" && signal.reason.length > 0
+      ? signal.reason
+      : "[Sixb] Dataset write was cancelled."
+  )
+  error.name = "AbortError"
+  throw error
+}