@sixb/core 0.1.6 → 0.1.8

package/src/datasets/execution.ts

@@ -1,24 +1,95 @@
-import { isAllowed } from "../authorization"
+import { assertPrivileged, isRuntimeAllowed } from "../authorization"
+import type { BlobStorage } from "../blob-storage"
+import type { ExecutionContext } from "../execution"
+import { resolveRuntimeAuthorizationForProject } from "../execution/authorization"
+import type { DatasetMergeCommitResult } from "../lake-storage/merge"
+import { getDatasetPrimaryKeyColumns } from "../lake-storage/merge-validation"
+import type { DatasetRow, LakeStorage } from "../lake-storage/types"
 import type { SixbRuntimeContext } from "../runtime/types"
+import type { MergeChange } from "./changes"
 import type { DatasetDefinition } from "./types"
+import { writeDataset } from "./write"
+
+export interface DatasetIngestInput {
+  readonly changes:
+    | Iterable<MergeChange<DatasetRow, DatasetRow>>
+    | AsyncIterable<MergeChange<DatasetRow, DatasetRow>>
+  readonly signal?: AbortSignal
+}
+
+/** Source commit result; downstream pipelines and projections run asynchronously. */
+export type DatasetIngestResult = DatasetMergeCommitResult & { readonly rowsRead: number }
 
 export interface DatasetsRuntime {
   list(): readonly DatasetDefinition[]
   getById(datasetId: string): DatasetDefinition | null
+  ingest(dataset: DatasetDefinition, input: DatasetIngestInput): Promise<DatasetIngestResult>
 }
 
 export function createDatasetsRuntime(
-  runtime: Pick<SixbRuntimeContext, "authorization">,
-  source: Pick<DatasetsRuntime, "list" | "getById">
+  runtime: SixbRuntimeContext,
+  execution: ExecutionContext,
+  source: Pick<DatasetsRuntime, "list" | "getById">,
+  lakeStorage: LakeStorage,
+  blobStorage: Pick<BlobStorage, "stat">
 ): DatasetsRuntime {
+  const authority = resolveRuntimeAuthorizationForProject(runtime)
   const allowed = (datasetId: string) =>
-    isAllowed(runtime.authorization, { kind: "dataset.view", datasetId })
+    isRuntimeAllowed(runtime, { kind: "dataset.view", datasetId })
 
   return {
-    list: () => source.list().filter((dataset) => allowed(dataset.id)),
+    list: () =>
+      authority.type === "denied" || authority.type === "delegated"
+        ? []
+        : source.list().filter((dataset) => allowed(dataset.id)),
     getById: (datasetId) => {
+      if (authority.type === "denied" || authority.type === "delegated") return null
       const dataset = source.getById(datasetId)
       return dataset && allowed(datasetId) ? dataset : null
     },
+    async ingest(dataset, input) {
+      assertPrivileged(runtime, "datasets.ingest")
+      const registered = source.getById(dataset.id)
+      if (!registered) throw new Error(`[Sixb] Dataset '${dataset.id}' is not registered.`)
+      if (getDatasetPrimaryKeyColumns(registered) === null) {
+        throw new Error(`[Sixb] Dataset '${dataset.id}' must define a primaryKey before ingestion.`)
+      }
+      if (typeof lakeStorage.beginMerge !== "function") {
+        throw new Error("[Sixb] Dataset ingestion requires a lake provider with merge support.")
+      }
+      const producer = { kind: "ingest" as const, id: execution.id }
+      const result = await writeDataset({
+        lakeStorage,
+        blobStorage,
+        dataset: registered,
+        mode: "merge",
+        signal: input.signal ?? new AbortController().signal,
+        producer,
+        readValues: async () =>
+          (async function* () {
+            for await (const value of input.changes) yield { value }
+          })(),
+      })
+      if (result.outcome === "created") {
+        await runtime.events.emit(
+          {
+            events: [
+              {
+                type: "dataset.version.committed",
+                payload: {
+                  datasetId: registered.id,
+                  versionId: result.version.versionId,
+                  createdAt: result.version.createdAt.toISOString(),
+                  producer,
+                },
+              },
+            ],
+            correlationId: execution.correlationId,
+          },
+          { source: "SixbDatasets" }
+        )
+      }
+      return result
+    },
   }
 }

package/src/datasets/index.ts

@@ -2,6 +2,7 @@ export { col, defineDataset } from "./builders"
 export type { MergeChange } from "./changes"
 export { change } from "./changes"
 export { DatasetValidationError } from "./errors"
+export type { DatasetIngestInput, DatasetIngestResult, DatasetsRuntime } from "./execution"
 export type {
   DatasetColumnDefinition,
   DatasetColumnDefinitionOf,

package/src/datasets/sequence.ts

@@ -0,0 +1,77 @@
+import type { DatasetDefinition } from "./types"
+
+const MIN_INT64 = -(1n << 63n)
+const MAX_INT64 = (1n << 63n) - 1n
+
+/** Normalize source ordering to an exact integer (timestamp milliseconds or signed int64). */
+export function datasetSequenceValue(dataset: DatasetDefinition, value: unknown): bigint {
+  const column = dataset.schema.columns.find((column) => column.name === dataset.sequenceBy)
+  if (column?.type === "int64") {
+    if (
+      (typeof value === "number" && Number.isSafeInteger(value)) ||
+      (typeof value === "string" && /^-?\d+$/.test(value))
+    ) {
+      const integer = BigInt(value)
+      if (integer >= MIN_INT64 && integer <= MAX_INT64) return integer
+    }
+    throw new Error(
+      `Dataset '${dataset.id}' sequence must be a signed int64; use a string outside the safe integer range.`
+    )
+  }
+  if (column?.type === "timestamp") {
+    const millis = parseDatasetTimestamp(value)
+    if (millis !== null) return BigInt(millis)
+    throw new Error(
+      `Dataset '${dataset.id}' sequence must be a valid Date or ISO timestamp with an explicit timezone and at most millisecond precision.`
+    )
+  }
+  throw new Error(
+    `Dataset '${dataset.id}' sequenceBy must reference a non-nullable timestamp or int64 column.`
+  )
+}
+
+/** Date.parse alone normalizes impossible calendar dates instead of rejecting them. */
+export function parseDatasetTimestamp(value: unknown): number | null {
+  if (value instanceof Date && Number.isFinite(value.getTime())) return value.getTime()
+  if (typeof value === "string") {
+    const match =
+      /^([+-]\d{6}|\d{4})-(\d{2})-(\d{2})T(\d{2}):(\d{2}):(\d{2})(?:\.(\d{1,3}))?(Z|[+-]\d{2}:\d{2})$/.exec(
+        value
+      )
+    if (match) {
+      const [, year, month, day, hour, minute, second, , zone] = match
+      const numericYear = Number(year)
+      const leapYear = numericYear % 4 === 0 && (numericYear % 100 !== 0 || numericYear % 400 === 0)
+      const days =
+        Number(month) === 2 ? (leapYear ? 29 : 28) : [4, 6, 9, 11].includes(Number(month)) ? 30 : 31
+      const validZone =
+        zone === "Z" || (Number(zone?.slice(1, 3)) <= 23 && Number(zone?.slice(4)) <= 59)
+      const millis = Date.parse(value)
+      if (
+        Number(month) >= 1 &&
+        Number(month) <= 12 &&
+        Number(day) >= 1 &&
+        Number(day) <= days &&
+        Number(hour) < 24 &&
+        Number(minute) < 60 &&
+        Number(second) < 60 &&
+        validZone &&
+        Number.isFinite(millis)
+      )
+        return millis
+    }
+  }
+  return null
+}
+
+export function getDatasetSequenceValidationError(
+  dataset: DatasetDefinition,
+  value: unknown
+): string | null {
+  try {
+    datasetSequenceValue(dataset, value)
+    return null
+  } catch (error) {
+    return error instanceof Error ? error.message : String(error)
+  }
+}

package/src/datasets/types.ts

@@ -36,6 +36,7 @@ export interface DatasetDefinition<
   readonly id: TId
   readonly schema: DatasetSchema<TColumns>
   readonly primaryKey?: DatasetPrimaryKey
+  readonly sequenceBy?: string
   readonly partitionBy?: readonly string[]
   readonly description?: string
 }

package/src/datasets/validation.ts

@@ -2,6 +2,7 @@ import { isFileRef } from "../blob-storage"
 import { isJsonValue, isPlainRecord } from "../json"
 import { isDecimalString } from "../ontology/decimal"
 import { DatasetValidationError } from "./errors"
+import { getDatasetSequenceValidationError, parseDatasetTimestamp } from "./sequence"
 import type {
   DatasetColumnDefinition,
   DatasetColumnType,
@@ -185,6 +186,20 @@ export function assertDatasetDefinition(
     assertDatasetPrimaryKey(definition.primaryKey, definition.schema, createError)
   }
 
+  if (definition.sequenceBy !== undefined) {
+    const column = definition.schema.columns.find((column) => column.name === definition.sequenceBy)
+    if (
+      definition.primaryKey === undefined ||
+      !column ||
+      column.nullable ||
+      !["timestamp", "int64"].includes(column.type)
+    ) {
+      throw createError(
+        "Dataset sequenceBy requires a primaryKey and a non-nullable timestamp or int64 column."
+      )
+    }
+  }
+
   if (definition.partitionBy !== undefined) {
     if (!Array.isArray(definition.partitionBy)) {
       throw createError("Dataset partitionBy must be an array of column names.")
@@ -268,6 +283,9 @@ function getColumnValidationError(
 ): string | null {
   const hasValue = Object.hasOwn(row, column.name)
   const value = row[column.name]
+  if (column.name === dataset.sequenceBy) {
+    return getDatasetSequenceValidationError(dataset, value)
+  }
 
   // Treat `undefined` the same as an omitted field so callers either send a real
   // value or use `null` explicitly on nullable columns.
@@ -285,6 +303,14 @@ function getColumnValidationError(
     return null
   }
 
+  // Every timestamp contributes to a sequenced row's millisecond-precision content digest.
+  // Reject ambiguous or lossy values before staging, even when another column orders the row.
+  if (dataset.sequenceBy !== undefined && column.type === "timestamp") {
+    return parseDatasetTimestamp(value) === null
+      ? `Dataset '${dataset.id}' column '${column.name}' must be a valid Date or ISO timestamp with an explicit timezone and at most millisecond precision.`
+      : null
+  }
+
   if (!matchesColumnType(value, column.type)) {
     return `Dataset '${dataset.id}' column '${column.name}' must match type '${column.type}'.`
   }

package/src/datasets/write.ts

@@ -0,0 +1,240 @@
+import type { BlobStorage, FileRef } from "../blob-storage"
+import { isFileRef } from "../blob-storage"
+import { isPlainRecord } from "../json"
+import type { DatasetMergeCommitResult } from "../lake-storage/merge"
+import { getDatasetMergeChangeValidationError } from "../lake-storage/merge-validation"
+import type {
+  DatasetProducer,
+  DatasetRow,
+  DatasetWriteMode,
+  LakeStorage,
+} from "../lake-storage/types"
+import type { MergeChange } from "./changes"
+import type { DatasetDefinition } from "./types"
+import { getDatasetRowValidationError } from "./validation"
+
+export interface DatasetWriteValue {
+  readonly value: unknown
+}
+
+export interface WriteDatasetInput<TValue extends DatasetWriteValue = DatasetWriteValue> {
+  readonly lakeStorage: LakeStorage
+  readonly blobStorage: Pick<BlobStorage, "stat">
+  readonly dataset: DatasetDefinition
+  readonly mode: DatasetWriteMode | "merge"
+  readonly signal: AbortSignal
+  /** Lazy so a merge's latest-version guard is checked before fetching source data. */
+  readValues(): Promise<Iterable<TValue> | AsyncIterable<TValue>>
+  readonly producer?: DatasetProducer
+  readonly expectedLatestVersionId?: string
+  readonly commitMessage?: string
+  /** Diagnostic label for the caller, including its component prefix. */
+  readonly sourceLabel?: string
+  /** Progress includes validated values even if the operation subsequently fails. */
+  onRead?(rowsRead: number): void
+  /** Attach caller-owned source provenance to a validation failure. */
+  mapValidationError?(error: unknown, value: TValue, itemIndex: number): unknown
+}
+
+export type WriteDatasetResult = DatasetMergeCommitResult & { readonly rowsRead: number }
+
+/**
+ * Validate, stage, and commit one dataset write using existing lake sessions.
+ *
+ * The caller owns fetching/cancelling its source, execution bookkeeping, and publishing the
+ * dataset-update notification for a `created` result. This operation returns immediately after
+ * the lake commit; it does not wait for downstream pipelines or projections.
+ */
+export async function writeDataset<TValue extends DatasetWriteValue>(
+  input: WriteDatasetInput<TValue>
+): Promise<WriteDatasetResult> {
+  const { lakeStorage, dataset, signal } = input
+  let abortWrite: (() => Promise<void>) | undefined
+  let rowsRead = 0
+  const onRead = () => {
+    rowsRead += 1
+    input.onRead?.(rowsRead)
+  }
+
+  throwIfAborted(signal)
+  await lakeStorage.createDataset(dataset)
+  try {
+    throwIfAborted(signal)
+    const reconcileSnapshot = dataset.sequenceBy !== undefined && input.mode === "snapshot"
+    if (input.mode === "merge" || reconcileSnapshot) {
+      const session = await lakeStorage.beginMerge({
+        dataset,
+        expectedLatestVersionId: input.expectedLatestVersionId,
+        producer: input.producer,
+      })
+      abortWrite = () => session.abort()
+      throwIfAborted(signal)
+      const values = await input.readValues()
+      throwIfAborted(signal)
+      await session.writeChanges(
+        validatedValues(
+          input,
+          values,
+          async (value, itemIndex) =>
+            reconcileSnapshot
+              ? { kind: "upsert" as const, row: await validateRow(value, input, itemIndex) }
+              : validateMergeChange(value, input, itemIndex),
+          onRead
+        )
+      )
+      throwIfAborted(signal)
+      const result = await session.commit({
+        commitMessage: input.commitMessage,
+        createInitialVersion: reconcileSnapshot,
+        retryOnConflict: dataset.sequenceBy !== undefined,
+        signal,
+      })
+      return { ...result, rowsRead }
+    }
+
+    const values = await input.readValues()
+    throwIfAborted(signal)
+    const session = await lakeStorage.beginWrite({
+      dataset,
+      mode: input.mode,
+      producer: input.producer,
+    })
+    abortWrite = () => session.abort()
+    await session.writeRows(
+      validatedValues(
+        input,
+        values,
+        (value, itemIndex) => validateRow(value, input, itemIndex),
+        onRead
+      )
+    )
+    throwIfAborted(signal)
+
+    // Preserve sync semantics: an empty append must not invent an initial dataset version.
+    if (rowsRead === 0 && input.mode === "append") {
+      const version = await lakeStorage.getLatestVersion(dataset.id)
+      await session.abort()
+      return { outcome: "unchanged", version, rowsRead }
+    }
+
+    const { outcome, ...version } = await session.commit({
+      expectedLatestVersionId: input.expectedLatestVersionId,
+      commitMessage: input.commitMessage,
+    })
+    return { outcome, version, rowsRead }
+  } catch (error) {
+    // Cleanup must not replace the source, validation, cancellation, or commit failure.
+    await abortWrite?.().catch(() => {})
+    throw error
+  }
+}
+
+async function* validatedValues<TValue extends DatasetWriteValue, TResult>(
+  input: WriteDatasetInput<TValue>,
+  values: Iterable<TValue> | AsyncIterable<TValue>,
+  validate: (value: unknown, itemIndex: number) => Promise<TResult>,
+  onRead: () => void
+): AsyncIterable<TResult> {
+  let itemIndex = 0
+  for await (const sourceValue of values) {
+    throwIfAborted(input.signal)
+    itemIndex += 1
+    let value: TResult
+    try {
+      value = await validate(sourceValue.value, itemIndex)
+    } catch (error) {
+      throw input.mapValidationError
+        ? input.mapValidationError(error, sourceValue, itemIndex)
+        : error
+    }
+    onRead()
+    yield value
+  }
+}
+
+function sourceLabel(input: Pick<WriteDatasetInput, "dataset" | "sourceLabel">): string {
+  return input.sourceLabel ?? `[Sixb] Dataset '${input.dataset.id}' writer`
+}
+
+async function validateRow(
+  value: unknown,
+  input: WriteDatasetInput,
+  itemIndex: number
+): Promise<DatasetRow> {
+  if (!isPlainRecord(value)) {
+    throw new Error(
+      `${sourceLabel(input)} returned an invalid row at item ${itemIndex}. Dataset rows must be plain objects.`
+    )
+  }
+  const validationError = getDatasetRowValidationError(value, input.dataset)
+  if (validationError) {
+    throw new Error(
+      `${sourceLabel(input)} returned an invalid row at item ${itemIndex}. ${validationError}`
+    )
+  }
+  await verifyRowFileRefs(value, input, itemIndex)
+  return value
+}
+
+async function validateMergeChange(
+  value: unknown,
+  input: WriteDatasetInput,
+  itemIndex: number
+): Promise<MergeChange<DatasetRow, DatasetRow>> {
+  const validationError = getDatasetMergeChangeValidationError(value, input.dataset)
+  if (validationError) {
+    throw new Error(
+      `${sourceLabel(input)} returned an invalid merge change at item ${itemIndex}. ${validationError}`
+    )
+  }
+  const change = value as MergeChange<DatasetRow, DatasetRow>
+  if (change.kind === "upsert") await verifyRowFileRefs(change.row, input, itemIndex)
+  return change
+}
+
+async function verifyRowFileRefs(
+  row: DatasetRow,
+  input: WriteDatasetInput,
+  itemIndex: number
+): Promise<void> {
+  for (const column of input.dataset.schema.columns) {
+    if (column.type !== "fileRef") continue
+    const value = row[column.name]
+    if (isFileRef(value)) await verifyFileRef(value, input, column.name, itemIndex)
+  }
+}
+
+async function verifyFileRef(
+  fileRef: FileRef,
+  input: WriteDatasetInput,
+  columnName: string,
+  itemIndex: number
+): Promise<void> {
+  const blobInfo = await input.blobStorage.stat(fileRef.blobId)
+  const context = `${sourceLabel(input)} returned row ${itemIndex} with dataset '${input.dataset.id}' column '${columnName}'`
+  if (!blobInfo) {
+    throw new Error(`${context} referencing unknown blob '${fileRef.blobId}'.`)
+  }
+  if (blobInfo.digest !== fileRef.digest) {
+    throw new Error(
+      `${context} referencing blob '${fileRef.blobId}' with digest '${fileRef.digest}', but blob storage has '${blobInfo.digest}'.`
+    )
+  }
+  if (blobInfo.sizeBytes !== fileRef.sizeBytes) {
+    throw new Error(
+      `${context} referencing blob '${fileRef.blobId}' with size ${fileRef.sizeBytes}, but blob storage has ${blobInfo.sizeBytes}.`
+    )
+  }
+}
+
+function throwIfAborted(signal: AbortSignal): void {
+  if (!signal.aborted) return
+  if (signal.reason instanceof Error) throw signal.reason
+  const error = new Error(
+    typeof signal.reason === "string" && signal.reason.length > 0
+      ? signal.reason
+      : "[Sixb] Dataset write was cancelled."
+  )
+  error.name = "AbortError"
+  throw error
+}

package/src/error-reporting/types.ts

@@ -15,8 +15,10 @@ export interface SixbRunIdentityByKind {
   }
   readonly agent: {
     readonly runId: string
-    readonly agentId: string
-  }
+  } & (
+    | { readonly threadId: string; readonly parentRunId?: never }
+    | { readonly parentRunId: string; readonly threadId?: never }
+  )
   readonly pipeline: {
     readonly runId: string
     readonly pipelineId: string

package/src/errors/catalog.ts

@@ -18,6 +18,14 @@ export const SIXB_ERROR_DEFINITIONS = {
     publicMessage: "Agent execution failed.",
     retryable: false,
   },
+  "ai.usage_limit_exceeded": {
+    publicMessage: "AI usage limit exceeded.",
+    retryable: false,
+  },
+  "ai.usage_limit_unavailable": {
+    publicMessage: "AI usage limit could not be evaluated.",
+    retryable: true,
+  },
   "connector.adapter_invalid": {
     publicMessage: "Connector adapter returned an invalid result.",
     retryable: false,

package/src/events/execution.ts

@@ -1,4 +1,5 @@
 import { assertPrivileged, canViewEvent } from "../authorization"
+import { resolveRuntimeAuthorizationForProject } from "../execution/authorization"
 import type { SixbRuntimeContext } from "../runtime/types"
 import type {
   EventsAppendInput,
@@ -21,8 +22,31 @@ export interface EventsRuntime {
 }
 
 export function createEventsRuntime(runtime: SixbRuntimeContext): EventsRuntime {
+  const authority = resolveRuntimeAuthorizationForProject(runtime)
+  const visibleEvents = (events: readonly StoredDomainEvent[]): readonly StoredDomainEvent[] => {
+    switch (authority.type) {
+      case "denied":
+      case "delegated":
+        return []
+      case "principal":
+        return events.filter((event) => canViewEvent(authority.context, event))
+      case "unrestricted":
+        return [...events]
+    }
+  }
+
   return {
-    canRead: (event) => canViewEvent(runtime.authorization, event),
+    canRead: (event) => {
+      switch (authority.type) {
+        case "denied":
+        case "delegated":
+          return false
+        case "principal":
+          return canViewEvent(authority.context, event)
+        case "unrestricted":
+          return true
+      }
+    },
     append: (input) => {
       assertPrivileged(runtime, "events.append")
       return runtime.events.append(input)
@@ -32,14 +56,40 @@ export function createEventsRuntime(runtime: SixbRuntimeContext): EventsRuntime
       return runtime.events.emit(input, options)
     },
     read: async (input) => {
+      switch (authority.type) {
+        case "denied":
+        case "delegated":
+          return []
+        case "principal":
+        case "unrestricted":
+          break
+      }
       const events = await runtime.events.read(input)
-      return events.filter((event) => canViewEvent(runtime.authorization, event))
+      return visibleEvents(events)
     },
-    latestCursor: () => runtime.events.latestCursor(),
-    subscribe: (input, handler) =>
-      runtime.events.subscribe(input, (events) => {
-        const visible = events.filter((event) => canViewEvent(runtime.authorization, event))
+    latestCursor: () => {
+      switch (authority.type) {
+        case "denied":
+        case "delegated":
+          return Promise.resolve(undefined)
+        case "principal":
+        case "unrestricted":
+          return runtime.events.latestCursor()
+      }
+    },
+    subscribe: (input, handler) => {
+      switch (authority.type) {
+        case "denied":
+        case "delegated":
+          return Promise.resolve(() => {})
+        case "principal":
+        case "unrestricted":
+          break
+      }
+      return runtime.events.subscribe(input, (events) => {
+        const visible = visibleEvents(events)
         if (visible.length > 0) handler(visible)
-      }),
+      })
+    },
   }
 }

package/src/events/types/datasets.ts

@@ -1,3 +1,4 @@
+import type { DatasetProducer } from "../../lake-storage/types"
 import type { EventEnvelope } from "../envelope"
 
 export interface DatasetVersionCommittedEvent extends EventEnvelope {
@@ -9,12 +10,7 @@ export interface DatasetVersionCommittedEvent extends EventEnvelope {
     versionId: string
     /** Canonical UTC ISO timestamp from the committed immutable dataset version. */
     createdAt: string
-    producer: {
-      kind: "sync" | "pipeline"
-      id?: string
-      runId?: string
-      stepId?: string
-    }
+    producer: DatasetProducer
   }
 }