@sixb/core 0.1.6 → 0.1.8

package/src/lake-storage/source-ordering.ts

@@ -0,0 +1,95 @@
+import { createHash } from "node:crypto"
+import type { DatasetDefinition, MergeChange } from "../datasets"
+import { datasetSequenceValue } from "../datasets/sequence"
+import { stableJsonStringify } from "../json"
+import { normalizeDecimalValue } from "../ontology/decimal"
+import { LakeStorageError } from "./errors"
+import { encodeDatasetPrimaryKey } from "./merge-validation"
+import type { DatasetRow } from "./types"
+
+/** Current source revision; a null content digest is a deletion tombstone. */
+export interface DatasetSequenceState {
+  readonly sequence: string
+  readonly content: string | null
+}
+
+export interface DatasetSequenceChange extends DatasetSequenceState {
+  readonly key: string
+}
+
+export function datasetSequenceChange(
+  dataset: DatasetDefinition,
+  change: MergeChange<DatasetRow, DatasetRow>
+): DatasetSequenceChange {
+  const value = change.kind === "upsert" ? change.row[dataset.sequenceBy ?? ""] : change.sequence
+  return {
+    key: encodeDatasetPrimaryKey(dataset, change.kind === "upsert" ? change.row : change.key),
+    sequence: datasetSequenceValue(dataset, value).toString(),
+    content: change.kind === "delete" ? null : rowDigest(dataset, change.row),
+  }
+}
+
+/** Process in submission order against a private copy; conflicts publish no partial state. */
+export function reconcileDatasetSequences(
+  dataset: DatasetDefinition,
+  previous: ReadonlyMap<string, DatasetSequenceState>,
+  changes: readonly DatasetSequenceChange[]
+): { states: Map<string, DatasetSequenceState>; accepted: Map<string, number> } {
+  const states = new Map(previous)
+  const accepted = new Map<string, number>()
+  for (const [index, change] of changes.entries()) {
+    const current = states.get(change.key)
+    if (current) {
+      const difference = BigInt(change.sequence) - BigInt(current.sequence)
+      if (difference < 0n) continue
+      if (difference === 0n) {
+        if (current.content !== change.content) {
+          throw new LakeStorageError(
+            `[SixbLake] Dataset '${dataset.id}' has conflicting content at source sequence '${change.sequence}' for key ${change.key}.`
+          )
+        }
+        continue
+      }
+    }
+    states.set(change.key, { sequence: change.sequence, content: change.content })
+    accepted.set(change.key, index)
+  }
+  return { states, accepted }
+}
+
+function rowDigest(dataset: DatasetDefinition, row: DatasetRow): string {
+  const content: Record<string, unknown> = Object.create(null)
+  for (const column of dataset.schema.columns) {
+    const value = row[column.name]
+    // Nullable additions and omitted/undefined/null values have identical content.
+    if (value == null) continue
+    switch (column.type) {
+      case "int64":
+        content[column.name] = BigInt(value as string | number).toString()
+        break
+      case "timestamp":
+        content[column.name] = new Date(value as string | Date).toISOString()
+        break
+      case "date":
+        content[column.name] = new Date(value as string | Date).toISOString().slice(0, 10)
+        break
+      case "decimal":
+        content[column.name] = normalizeDecimalValue(value as string)
+        break
+      default:
+        content[column.name] = value
+    }
+  }
+  return createHash("sha256")
+    .update(stableJsonStringify(JSON.parse(JSON.stringify(content))))
+    .digest("hex")
+}
+
+/** Until row reconciliation is supported, replacement writes must not erase source history. */
+export function assertUnsequencedDatasetWrite(dataset: DatasetDefinition): void {
+  if (dataset.sequenceBy !== undefined) {
+    throw new LakeStorageError(
+      `[SixbLake] Dataset '${dataset.id}' declares sequenceBy; use beginMerge with source-ordered changes.`
+    )
+  }
+}

package/src/lake-storage/sql-transforms.ts

@@ -39,7 +39,8 @@ export interface ExecuteSqlTransformInput<TSqlDialect extends SqlDialect = SqlDi
   readonly target: DatasetDefinition
   readonly mode: DatasetWriteMode
   readonly producer?: DatasetProducer
-  readonly expectedLatestVersionId?: string
+  /** Atomic target version guard; null requires that no version exists. */
+  readonly expectedLatestVersionId?: string | null
   readonly commitMessage?: string
 }
 

package/src/lake-storage/types.ts

@@ -19,7 +19,8 @@ export interface DatasetVersionRef {
 }
 
 export interface DatasetProducer {
-  readonly kind: "sync" | "pipeline"
+  readonly kind: "sync" | "pipeline" | "ingest"
+  /** For ingestion, the bound execution id; otherwise the sync/pipeline definition id. */
   readonly id?: string
   readonly runId?: string
   readonly stepId?: string
@@ -73,7 +74,8 @@ export interface ReadDatasetRowsInput {
 }
 
 export interface CommitDatasetWriteInput {
-  readonly expectedLatestVersionId?: string
+  /** Atomic version guard; null requires that no version exists, undefined leaves it unguarded. */
+  readonly expectedLatestVersionId?: string | null
   readonly commitMessage?: string
 }
 
@@ -81,11 +83,14 @@ export interface DatasetWriteCommitResult extends DatasetVersion {
   /**
    * Whether this operation created the returned version or reused the
    * unchanged latest version. When a latest version exists, providers must
-   * report `unchanged` and reuse it for commits that cannot change visible
-   * dataset content: a snapshot whose rows equal the latest visible rows
+   * report `unchanged` and reuse it for commits that change neither visible
+   * dataset content nor explicitly supplied input lineage: a snapshot whose
+   * rows equal the latest visible rows
    * (order-insensitive, with equal duplicate counts), or an append of zero
    * rows. Without a latest version, an empty snapshot or append still creates
-   * the first addressable dataset version.
+   * the first addressable dataset version. New input versions create a version
+   * even for identical rows, so a newer pipeline calculation fences concurrent
+   * work based on older inputs.
    */
   readonly outcome: "created" | "unchanged"
 }

package/src/lake-storage/version-inputs.ts

@@ -0,0 +1,14 @@
+import type { DatasetVersionRef } from "./types"
+
+/** Input lineage participates in write identity when the caller supplies it. */
+export function hasDatasetInputChanges(
+  previous: readonly DatasetVersionRef[] | undefined,
+  next: readonly DatasetVersionRef[] | undefined
+): boolean {
+  if (next === undefined) return false
+  const keys = (refs: readonly DatasetVersionRef[]) =>
+    new Set(refs.map((ref) => JSON.stringify([ref.datasetId, ref.versionId])))
+  const before = keys(previous ?? [])
+  const after = keys(next)
+  return before.size !== after.size || [...after].some((key) => !before.has(key))
+}

package/src/models/catalog.ts

@@ -0,0 +1,137 @@
+import { RuntimeError } from "../runtime/errors"
+import {
+  defineLanguageModel,
+  type LanguageModelDefinition,
+  type ModelDefinition,
+} from "./definitions"
+import type { LanguageModel } from "./language-model"
+
+/** Stable identity of one configured language-model binding. */
+export interface LanguageModelRef {
+  readonly provider: string
+  readonly modelId: string
+}
+
+/** The configured language models, with the project default. */
+export interface LanguageModelCatalog {
+  readonly default: LanguageModelEntry
+  list(): readonly LanguageModelEntry[]
+  getByRef(ref: LanguageModelRef): LanguageModelEntry | null
+}
+
+/**
+ * One configured language model.
+ *
+ * The identity describes the configured binding, not only a vendor model. For example,
+ * A gateway binding and a direct provider binding for the same vendor model are distinct because their
+ * `provider` values differ.
+ */
+export interface LanguageModelEntry extends LanguageModelRef {
+  readonly model: LanguageModel
+}
+
+/** Models a project allows Sixb to use, organized by technical model kind. */
+export interface ModelCatalog {
+  readonly language: LanguageModelCatalog
+}
+
+export interface ModelCatalogInput {
+  /** Ordered; the first entry is the project default. */
+  readonly language: readonly LanguageModel[]
+}
+
+/** Build the immutable project model catalog. Rejects invalid, duplicate, and empty catalogs. */
+export function createModelCatalog(input: ModelCatalogInput): ModelCatalog {
+  if (!Array.isArray(input?.language)) {
+    throw new RuntimeError("[Sixb] 'models.language' must be an array of Sixb language models.")
+  }
+
+  const entries: LanguageModelEntry[] = []
+  const byProvider = new Map<string, Map<string, LanguageModelEntry>>()
+
+  for (const [index, model] of input.language.entries()) {
+    assertLanguageModel(model, index)
+
+    let byModelId = byProvider.get(model.providerId)
+    if (byModelId === undefined) {
+      byModelId = new Map()
+      byProvider.set(model.providerId, byModelId)
+    }
+    if (byModelId.has(model.modelId)) {
+      throw new RuntimeError(
+        `[Sixb] Duplicate language model '${model.providerId}/${model.modelId}' in 'models'. Each provider and model id pair may be configured once.`
+      )
+    }
+
+    const entry = Object.freeze({
+      provider: model.providerId,
+      modelId: model.modelId,
+      model,
+    })
+    byModelId.set(model.modelId, entry)
+    entries.push(entry)
+  }
+
+  const [defaultEntry] = entries
+  if (defaultEntry === undefined) {
+    throw new RuntimeError(
+      "[Sixb] 'models.language' needs at least one model. Configure one or omit 'models' from createSixb()."
+    )
+  }
+
+  const listed = Object.freeze(entries.slice())
+  const language: LanguageModelCatalog = Object.freeze({
+    default: defaultEntry,
+    list: () => listed,
+    getByRef: (ref: LanguageModelRef) => byProvider.get(ref.provider)?.get(ref.modelId) ?? null,
+  })
+
+  return Object.freeze({ language })
+}
+
+function assertLanguageModel(model: unknown, index: number): asserts model is LanguageModel {
+  if ((typeof model !== "object" && typeof model !== "function") || model === null) {
+    throw invalidLanguageModel(index, "expected a Sixb LanguageModel instance")
+  }
+
+  const candidate = model as Record<string, unknown>
+  assertModelIdentifier(candidate.providerId, "provider", index)
+  assertModelIdentifier(candidate.modelId, "modelId", index)
+  if (typeof candidate.stream !== "function") {
+    throw invalidLanguageModel(index, "expected 'stream' to be a function")
+  }
+  try {
+    const definition = defineLanguageModel(candidate.definition as LanguageModelDefinition)
+    if (
+      definition.providerId !== candidate.providerId ||
+      definition.modelId !== candidate.modelId
+    ) {
+      throw new Error("definition identity does not match the binding")
+    }
+  } catch (error) {
+    throw invalidLanguageModel(index, error instanceof Error ? error.message : "invalid definition")
+  }
+}
+
+function assertModelIdentifier(value: unknown, field: "provider" | "modelId", index: number): void {
+  if (typeof value !== "string" || value.length === 0) {
+    throw invalidLanguageModel(index, `expected '${field}' to be a non-empty string`)
+  }
+  if (value.trim() !== value) {
+    throw invalidLanguageModel(index, `expected '${field}' not to have surrounding whitespace`)
+  }
+}
+
+function invalidLanguageModel(index: number, detail: string): RuntimeError {
+  return new RuntimeError(
+    `[Sixb] Invalid language model at 'models.language[${index}]': ${detail}.`
+  )
+}
+
+/** Provider-owned metadata lookup, independent from the project's configured bindings. */
+export interface ModelDefinitionCatalog<TDefinition extends ModelDefinition = ModelDefinition> {
+  get(modelId: string): Promise<TDefinition | undefined>
+  list(): Promise<readonly TDefinition[]>
+}
+
+export type LanguageModelDefinitionCatalog = ModelDefinitionCatalog<LanguageModelDefinition>

package/src/models/definitions.ts

@@ -0,0 +1,202 @@
+import {
+  MODEL_REASONING_EFFORTS,
+  type ModelCapabilities,
+  type ModelReasoningBudgetCapabilities,
+  type ModelReasoningCapabilities,
+} from "./language-model"
+
+export type ModelKind = "language" | "image" | "video" | "embedding"
+
+/** Common catalog identity shared by every present and future model runtime. */
+export interface ModelDefinition {
+  readonly kind: ModelKind
+  readonly providerId: string
+  readonly modelId: string
+  readonly name?: string
+  readonly description?: string
+  readonly family?: string
+  readonly tags?: readonly string[]
+  readonly releaseDate?: string
+}
+
+/** Serializable facts about one concrete model offering from one provider. */
+export interface LanguageModelDefinition extends ModelDefinition {
+  readonly kind: "language"
+  readonly knowledgeCutoff?: string
+  readonly contextWindow?: number
+  /** Maximum input tokens, distinct from a shared input/output context window. */
+  readonly maxInputTokens?: number
+  readonly maxOutputTokens?: number
+  readonly capabilities: ModelCapabilities
+}
+
+export function defineLanguageModel(definition: LanguageModelDefinition): LanguageModelDefinition {
+  if (definition.kind !== "language") {
+    throw new TypeError("[Sixb] Language model definitions must use kind 'language'.")
+  }
+  assertModelId(definition.providerId, "providerId")
+  assertModelId(definition.modelId, "modelId")
+  assertOptionalString(definition.name, "name")
+  assertOptionalString(definition.description, "description")
+  assertOptionalString(definition.family, "family")
+  assertOptionalString(definition.releaseDate, "releaseDate")
+  assertOptionalString(definition.knowledgeCutoff, "knowledgeCutoff")
+  const tags = freezeStrings(definition.tags, "tags")
+  const capabilities = defineModelCapabilities(definition.capabilities)
+  assertOptionalPositiveInteger(definition.contextWindow, "contextWindow")
+  assertOptionalPositiveInteger(definition.maxInputTokens, "maxInputTokens")
+  assertOptionalPositiveInteger(definition.maxOutputTokens, "maxOutputTokens")
+  return Object.freeze({
+    kind: "language",
+    providerId: definition.providerId,
+    modelId: definition.modelId,
+    ...(definition.name === undefined ? {} : { name: definition.name }),
+    ...(definition.description === undefined ? {} : { description: definition.description }),
+    ...(definition.family === undefined ? {} : { family: definition.family }),
+    ...(tags === undefined ? {} : { tags }),
+    ...(definition.releaseDate === undefined ? {} : { releaseDate: definition.releaseDate }),
+    ...(definition.knowledgeCutoff === undefined
+      ? {}
+      : { knowledgeCutoff: definition.knowledgeCutoff }),
+    ...(definition.contextWindow === undefined ? {} : { contextWindow: definition.contextWindow }),
+    ...(definition.maxInputTokens === undefined
+      ? {}
+      : { maxInputTokens: definition.maxInputTokens }),
+    ...(definition.maxOutputTokens === undefined
+      ? {}
+      : { maxOutputTokens: definition.maxOutputTokens }),
+    capabilities,
+  })
+}
+
+function defineModelCapabilities(capabilities: unknown): ModelCapabilities {
+  assertRecord(capabilities, "capabilities")
+  const inputMediaTypes =
+    capabilities.inputMediaTypes === "any"
+      ? "any"
+      : freezeStrings(capabilities.inputMediaTypes, "capabilities.inputMediaTypes")
+  for (const [field, value] of Object.entries(capabilities)) {
+    if (field === "inputMediaTypes" || field === "reasoning") continue
+    if (value !== undefined && typeof value !== "boolean") {
+      throw new TypeError(`[Sixb] Model capability '${field}' must be boolean.`)
+    }
+  }
+  const reasoning = freezeReasoningCapabilities(capabilities.reasoning)
+  const flags: {
+    localTools?: boolean
+    parallelToolCalls?: boolean
+    nativeStructuredOutput?: boolean
+    providerExecutedTools?: boolean
+  } = {}
+  for (const key of [
+    "localTools",
+    "parallelToolCalls",
+    "nativeStructuredOutput",
+    "providerExecutedTools",
+  ] as const) {
+    const value = capabilities[key]
+    if (typeof value === "boolean") flags[key] = value
+  }
+  return Object.freeze({
+    ...(inputMediaTypes === undefined ? {} : { inputMediaTypes }),
+    ...(reasoning === undefined ? {} : { reasoning }),
+    ...flags,
+  })
+}
+
+function freezeReasoningCapabilities(reasoning: unknown): ModelCapabilities["reasoning"] {
+  if (reasoning === undefined || reasoning === false) return reasoning
+  assertRecord(reasoning, "reasoning")
+  if (reasoning.canDisable !== undefined && typeof reasoning.canDisable !== "boolean") {
+    throw new TypeError("[Sixb] Model reasoning capability 'canDisable' must be boolean.")
+  }
+  const efforts = freezeReasoningEfforts(reasoning.efforts)
+  const budgetTokens = freezeReasoningBudgetCapabilities(reasoning.budgetTokens)
+  return Object.freeze({
+    ...(reasoning.canDisable === undefined ? {} : { canDisable: reasoning.canDisable }),
+    ...(efforts === undefined ? {} : { efforts }),
+    ...(budgetTokens === undefined ? {} : { budgetTokens }),
+  } satisfies ModelReasoningCapabilities)
+}
+
+function freezeReasoningEfforts(
+  efforts: unknown
+): readonly (typeof MODEL_REASONING_EFFORTS)[number][] | undefined {
+  if (efforts === undefined) return undefined
+  if (!Array.isArray(efforts)) {
+    throw new TypeError("[Sixb] Model reasoning capability 'efforts' must be an array.")
+  }
+  const seen = new Set<string>()
+  return Object.freeze(
+    efforts.map((value: unknown) => {
+      const effort = MODEL_REASONING_EFFORTS.find((allowed) => allowed === value)
+      if (effort === undefined) {
+        throw new TypeError(`[Sixb] Model reasoning effort '${String(value)}' is invalid.`)
+      }
+      if (seen.has(effort)) {
+        throw new TypeError(`[Sixb] Model reasoning effort '${effort}' is duplicated.`)
+      }
+      seen.add(effort)
+      return effort
+    })
+  )
+}
+
+function freezeReasoningBudgetCapabilities(
+  budget: unknown
+): ModelReasoningBudgetCapabilities | undefined {
+  if (budget === undefined) return undefined
+  assertRecord(budget, "reasoning budgetTokens")
+  assertOptionalNonnegativeInteger(budget.min, "capabilities.reasoning.budgetTokens.min")
+  assertOptionalNonnegativeInteger(budget.max, "capabilities.reasoning.budgetTokens.max")
+  if (budget.min !== undefined && budget.max !== undefined && budget.max < budget.min) {
+    throw new TypeError(
+      "[Sixb] Model reasoning token budget maximum must not be below its minimum."
+    )
+  }
+  return Object.freeze({
+    ...(budget.min === undefined ? {} : { min: budget.min }),
+    ...(budget.max === undefined ? {} : { max: budget.max }),
+  })
+}
+
+function assertModelId(value: string, field: string): void {
+  if (!value.trim()) throw new TypeError(`[Sixb] Model ${field} must not be empty.`)
+}
+
+function assertOptionalPositiveInteger(value: number | undefined, field: string): void {
+  if (value !== undefined && (!Number.isSafeInteger(value) || value <= 0)) {
+    throw new TypeError(`[Sixb] Model ${field} must be a positive safe integer.`)
+  }
+}
+
+function assertOptionalNonnegativeInteger(
+  value: unknown,
+  field: string
+): asserts value is number | undefined {
+  if (
+    value !== undefined &&
+    (typeof value !== "number" || !Number.isSafeInteger(value) || value < 0)
+  ) {
+    throw new TypeError(`[Sixb] Model ${field} must be a nonnegative safe integer.`)
+  }
+}
+
+function assertOptionalString(value: string | undefined, field: string): void {
+  if (value !== undefined && typeof value !== "string") {
+    throw new TypeError(`[Sixb] Model ${field} must be a string.`)
+  }
+}
+
+function freezeStrings(value: unknown, field: string): readonly string[] | undefined {
+  if (value === undefined) return undefined
+  if (!Array.isArray(value) || value.some((entry) => typeof entry !== "string" || !entry.trim())) {
+    throw new TypeError(`[Sixb] Model ${field} must contain nonempty strings.`)
+  }
+  return Object.freeze([...value])
+}
+
+function assertRecord(value: unknown, field: string): asserts value is Record<string, unknown> {
+  if (value === null || typeof value !== "object" || Array.isArray(value))
+    throw new TypeError(`[Sixb] Model ${field} must be an object.`)
+}

package/src/models/errors.ts

@@ -0,0 +1,83 @@
+import type { ModelFinishReason, ModelUsage } from "./events"
+import type { ModelCallCost } from "./pricing"
+
+/** Catalog transport/access failed; callers may use local metadata without masking invalid definitions. */
+export class ModelCatalogUnavailableError extends Error {
+  readonly name = "ModelCatalogUnavailableError"
+}
+
+/** A provider or provider protocol rejected a model request. */
+export class ModelProviderError extends Error {
+  readonly name = "ModelProviderError"
+
+  constructor(
+    message: string,
+    readonly providerId: string,
+    readonly modelId: string,
+    options?: ErrorOptions & {
+      readonly status?: number
+      readonly code?: string
+      readonly requestId?: string
+      readonly retryAfterMs?: number
+      readonly retryable?: boolean
+    }
+  ) {
+    super(message, options)
+    this.status = options?.status
+    this.code = options?.code
+    this.requestId = options?.requestId
+    this.retryAfterMs = options?.retryAfterMs
+    this.retryable = options?.retryable ?? false
+  }
+
+  readonly status?: number
+  readonly code?: string
+  readonly requestId?: string
+  readonly retryAfterMs?: number
+  readonly retryable: boolean
+}
+
+export class ModelStreamError extends Error {
+  readonly name = "ModelStreamError"
+}
+
+export class UnsupportedModelFeatureError extends Error {
+  readonly name = "UnsupportedModelFeatureError"
+}
+
+export class StructuredOutputError extends Error {
+  readonly name = "StructuredOutputError"
+
+  constructor(
+    message: string,
+    options?: ErrorOptions & {
+      readonly text?: string
+      readonly providerId?: string
+      readonly modelId?: string
+      readonly responseId?: string
+      readonly responseModelId?: string
+      readonly finishReason?: ModelFinishReason
+      readonly usage?: ModelUsage
+      readonly cost?: ModelCallCost
+    }
+  ) {
+    super(message, options)
+    this.text = options?.text
+    this.providerId = options?.providerId
+    this.modelId = options?.modelId
+    this.responseId = options?.responseId
+    this.responseModelId = options?.responseModelId
+    this.finishReason = options?.finishReason
+    this.usage = options?.usage
+    this.cost = options?.cost
+  }
+
+  readonly text?: string
+  readonly providerId?: string
+  readonly modelId?: string
+  readonly responseId?: string
+  readonly responseModelId?: string
+  readonly finishReason?: ModelFinishReason
+  readonly usage?: ModelUsage
+  readonly cost?: ModelCallCost
+}