@sixb/core 0.1.7 → 0.1.9

package/src/index.ts

@@ -9,6 +9,10 @@ export type {
   InferSchemaOrRef,
   Interface,
   LinkCardinality,
+  LinkPathSelection,
+  LinkPathSelectionBuilder,
+  LinkPathSelectionInput,
+  LinkPathSelectionMode,
   LinkToken,
   LinkTokenMap,
   MapSchema,
@@ -47,10 +51,10 @@ export {
   isObjectRefSchema,
   link,
   OntologyRegistry,
+  objectRef,
   prop,
   ref,
   stringEnum,
-  validateSchemaOrRefValue,
   valueTypeRef,
 } from "./ontology"
 
@@ -122,9 +126,6 @@ export {
   isObjectActionDefinition,
   optional,
   param,
-  requestAction,
-  requestActionAndWait,
-  waitForActionRun,
 } from "./actions"
 /**
  * `ctx.edits()` rejects an invalid batch with this, so it crosses a public boundary and users need
@@ -331,24 +332,26 @@ export {
 
 // ── Logging ─────────────────────────────────────────────────
 
+export type { ConsoleLoggerOptions } from "./logging/console-logger"
+export { ConsoleLogger, noopLogger, noopLoggerProvider } from "./logging/console-logger"
+export type {
+  LoggingService,
+  LogsObservabilityOptions,
+  ObservabilityOptions,
+} from "./logging/service"
 export type {
-  ConsoleLoggerOptions,
   LogContext,
   LogEntry,
   LogFields,
   Logger,
   LoggerProvider,
-  LoggingService,
   LogLevel,
   LogRecord,
   LogRunRef,
-  LogsObservabilityOptions,
-  ObservabilityOptions,
   SixbRunKind,
   StoredLogLine,
-} from "./logging"
+} from "./logging/types"
 export {
-  ConsoleLogger,
   isLevelEnabled,
   isLogLevel,
   isLogRecord,
@@ -356,10 +359,8 @@ export {
   isStoredLogLine,
   LOG_LEVELS,
   logLevelsAtOrAbove,
-  noopLogger,
-  noopLoggerProvider,
   SIXB_RUN_KINDS,
-} from "./logging"
+} from "./logging/types"
 
 // ── Predicates ─────────────────────────────────────────────
 
@@ -421,7 +422,6 @@ export type { AuthorizationContext, GrantIndex, GrantKind } from "./authorizatio
 export {
   AuthorizationError,
   canAccessApplication,
-  emptyGrantIndex,
   isAllowed,
   isApplicationAccessControlled,
   resolveAuthorizationContext,
@@ -474,6 +474,7 @@ export type {
   RunGrantTarget,
   SecurityDefinitionCatalog,
   Selection,
+  ShareGrant,
   ViewGrant,
   ViewGrantTarget,
 } from "./security"
@@ -487,6 +488,29 @@ export {
   SecurityValidationError,
 } from "./security"
 
+// ── Shares ────────────────────────────────────────────────
+
+export type {
+  DefineShareOptions,
+  IssueSharedAccessByIdInput,
+  IssueSharedAccessInput,
+  ListSharedAccessByIdInput,
+  ListSharedAccessInput,
+  ShareActionGrant,
+  ShareActionGrantBuilder,
+  ShareDefinition,
+  SharedAccessGrant,
+  SharedAccessGrantListResult,
+  SharedAccessInvitation,
+  ShareErrorReason,
+  ShareScopeGrant,
+  SharesRuntime,
+  ShareTarget,
+  ShareViewGrant,
+  ShareViewGrantBuilder,
+} from "./shares"
+export { defineShare, ShareDefinitionError, ShareError } from "./shares"
+
 // ── Storage ────────────────────────────────────────────────
 // Config contract + operator migration API + in-memory dev providers.
 // The full storage-provider contract lives at `@sixb/core/storage`.
@@ -563,12 +587,14 @@ export type {
   CommandResult,
   CreateSandboxOptions,
   ExecOptions,
+  ResumeSandboxOptions,
   RunCommandOptions,
   Sandbox,
   SandboxFactory,
   SandboxFileRecord,
   SandboxNetworkPolicy,
   SandboxNetworkTarget,
+  SandboxSessionOptions,
   SandboxStatus,
 } from "./sandboxes"
 export {
@@ -695,7 +721,7 @@ export type {
   SyncRunRequestResult,
   SyncTargetBuilder,
 } from "./syncs"
-export { defineSync, requestSyncRun } from "./syncs"
+export { defineSync } from "./syncs"
 
 // ── Schedules ───────────────────────────────────────────────
 
@@ -755,7 +781,6 @@ export {
   isPipelineDefinition,
   isPipelineStepDefinition,
   PipelineError,
-  requestPipelineRun,
 } from "./pipelines"
 
 // ── Workflows ───────────────────────────────────────────────
@@ -821,7 +846,6 @@ export {
   isInterventionDefinition,
   isStepDefinition,
   isWorkflowDefinition,
-  requestWorkflowRun,
   WorkflowDefinitionError,
   WorkflowValidationError,
 } from "./workflows"
@@ -854,7 +878,7 @@ export type {
 } from "./agents/execution"
 export type { BlobsRuntime } from "./blob-storage/execution"
 export type { ConnectorRuntime } from "./connectors/execution"
-export type { DatasetsRuntime } from "./datasets/execution"
+export type { DatasetIngestInput, DatasetIngestResult, DatasetsRuntime } from "./datasets/execution"
 export type {
   SixbActionPhaseFailedContext,
   SixbErrorContext,
@@ -983,20 +1007,10 @@ export type {
   AgentFileDataProjection,
   AgentFileDataResolverInput,
   AgentFilePart,
-  AgentInboundUiMessage,
-  AgentInboundUiMessagePart,
   AgentMessage,
   AgentMessagePart,
   AgentMessagePartType,
   AgentMessageRole,
-  AgentModelAssistantPart,
-  AgentModelFilePart,
-  AgentModelMessage,
-  AgentModelReasoningPart,
-  AgentModelTextPart,
-  AgentModelToolCallPart,
-  AgentModelToolOutput,
-  AgentModelToolResultPart,
   AgentProviderStatePart,
   AgentReasoning,
   AgentReasoningLevel,
@@ -1025,9 +1039,6 @@ export type {
   AgentToolRunContext,
   AgentToolRunInfo,
   AgentToolTextContent,
-  AgentUiMessage,
-  AgentUiMessagePart,
-  AgentUiToolPart,
   AgentUsageReference,
   InferAgentToolInput,
   InferAgentToolInputSchema,
@@ -1046,7 +1057,6 @@ export {
   MAX_AGENT_APP_STATE_ENTRY_BYTES,
   MAX_AGENT_APP_STATE_TOTAL_BYTES,
   MAX_AGENT_CONTEXT_ENTRIES,
-  requestAgentRun,
 } from "./agents"
 export type { AgentThreadRecord, ListAgentThreadsResult } from "./storage/agents"
 
@@ -1059,6 +1069,14 @@ export type {
   ModelCatalog,
   ModelCatalogInput,
 } from "./models"
+export type {
+  InferLanguageModelOutput,
+  LanguageModelGenerateInput,
+  LanguageModelGenerateResult,
+  LanguageModelOutputShape,
+  LanguageModelsRuntime,
+  ModelsRuntime,
+} from "./models/generation-types"
 
 // ── Scheduling ──────────────────────────────────────────────
 
@@ -1066,6 +1084,12 @@ export { CronValidationError } from "./schedules"
 
 // ── Projections ─────────────────────────────────────────────
 
+export type {
+  ActionDescriptor,
+  ActionDescriptorBinding,
+  ActionParamDescriptor,
+  ActionPhaseDescriptor,
+} from "./actions/descriptor"
 export type {
   ForeignKeyDescriptor,
   LinkProjectionDefinition,
@@ -1083,7 +1107,6 @@ export type {
   TelemetryProjectionDefinition,
   TelemetryProjectionPropertyMapping,
 } from "./projections"
-
 export {
   defineProjection,
   fromForeignKey,

package/src/lake-storage/definition-updates.ts

@@ -1,4 +1,5 @@
 import type { DatasetColumnDefinition, DatasetDefinition } from "../datasets"
+import { assertDatasetDefinition } from "../datasets/validation"
 import { LakeStorageError } from "./errors"
 import type { LakeStorage } from "./types"
 
@@ -36,6 +37,12 @@ export function planDatasetDefinitionUpdate(
   requested: DatasetDefinition
 ): DatasetDefinitionUpdatePlan {
   assertSameDataset(existing, requested)
+  assertDatasetDefinition(requested, (message) => new LakeStorageError(`[SixbLake] ${message}`))
+  if (existing.sequenceBy !== requested.sequenceBy) {
+    throw new LakeStorageError(
+      `[SixbLake] Dataset '${requested.id}' sequenceBy is immutable. Create a new dataset and backfill to adopt or change source ordering.`
+    )
+  }
   assertPrimaryKeyUnchanged(existing, requested)
 
   const addedColumns = getAddedColumnsAfterValidatingExistingColumns(existing, requested)
@@ -66,6 +73,7 @@ export function mergeStrictDatasetDefinition(options: {
   readonly existing?: DatasetDefinition | null
   readonly next: DatasetDefinition
 }): DatasetDefinition {
+  assertDatasetDefinition(options.next, (message) => new LakeStorageError(`[SixbLake] ${message}`))
   if (!options.existing) {
     return structuredClone(options.next)
   }
@@ -221,6 +229,7 @@ function mergeDatasetDefinition(
     id: requested.id,
     schema: { columns },
     ...(primaryKey !== undefined ? { primaryKey: structuredClone(primaryKey) } : {}),
+    ...(existing.sequenceBy !== undefined ? { sequenceBy: existing.sequenceBy } : {}),
     ...(partitionBy !== undefined ? { partitionBy: [...partitionBy] } : {}),
     ...(description !== undefined ? { description } : {}),
   }

package/src/lake-storage/errors.ts

@@ -2,5 +2,10 @@
  * Error for lake-storage invariants and invalid lake operations.
  */
 export class LakeStorageError extends Error {
-  readonly name = "LakeStorageError"
+  readonly name: string = "LakeStorageError"
+}
+
+/** A known, uncommitted version/transaction conflict; safe to retry staged changes. */
+export class LakeConcurrencyError extends LakeStorageError {
+  override readonly name = "LakeConcurrencyError"
 }

package/src/lake-storage/in-memory.ts

@@ -2,19 +2,26 @@ import { randomUUID } from "node:crypto"
 import type { DatasetDefinition, DatasetSchema, MergeChange } from "../datasets"
 import { getDatasetRowValidationError } from "../datasets/validation"
 import { mergeStrictDatasetDefinition } from "./definition-updates"
-import { LakeStorageError } from "./errors"
+import { LakeConcurrencyError, LakeStorageError } from "./errors"
 import type {
   BeginDatasetMergeInput,
   CommitDatasetMergeInput,
   DatasetMergeCommitResult,
   LakeMergeSession,
 } from "./merge"
+import { retryDatasetMergeCommit } from "./merge"
 import {
   cloneDatasetMergeChange,
   encodeDatasetPrimaryKey,
   getDatasetMergeChangeValidationError,
   getDatasetPrimaryKeyColumns,
 } from "./merge-validation"
+import {
+  assertUnsequencedDatasetWrite,
+  type DatasetSequenceState,
+  datasetSequenceChange,
+  reconcileDatasetSequences,
+} from "./source-ordering"
 import type {
   BeginDatasetWriteInput,
   CommitDatasetWriteInput,
@@ -27,6 +34,7 @@ import type {
   LakeWriteSession,
   ReadDatasetRowsInput,
 } from "./types"
+import { hasDatasetInputChanges } from "./version-inputs"
 
 function cloneDatasetDefinition(definition: DatasetDefinition): DatasetDefinition {
   return structuredClone(definition)
@@ -177,12 +185,16 @@ class InMemoryLakeMergeSession implements LakeMergeSession {
   async commit(input?: CommitDatasetMergeInput): Promise<DatasetMergeCommitResult> {
     this.assertOpen()
     this.closed = true
-    return this.storage.commitMerge({
-      merge: this.input,
-      baseVersionId: this.baseVersionId,
-      changes: this.changes,
-      commit: input,
-    })
+    return retryDatasetMergeCommit(this.input, input, async (rebase) =>
+      this.storage.commitMerge({
+        merge: this.input,
+        baseVersionId: rebase
+          ? ((await this.storage.getLatestVersion(this.input.dataset.id))?.versionId ?? null)
+          : this.baseVersionId,
+        changes: this.changes,
+        commit: input,
+      })
+    )
   }
 
   async abort(): Promise<void> {
@@ -201,6 +213,7 @@ export class InMemoryLakeStorage implements LakeStorage {
   private readonly datasets = new Map<string, DatasetDefinition>()
   private readonly versionsByDataset = new Map<string, DatasetVersion[]>()
   private readonly rowsByVersionId = new Map<string, readonly DatasetRow[]>()
+  private readonly sequencesByVersionId = new Map<string, Map<string, DatasetSequenceState>>()
   private readonly latestVersionIdByDataset = new Map<string, string>()
   private readonly commitLocks = new Map<string, Promise<void>>()
 
@@ -295,6 +308,7 @@ export class InMemoryLakeStorage implements LakeStorage {
       throw new LakeStorageError(`[LakeStorage] Unknown dataset '${input.dataset.id}'`)
     }
 
+    assertUnsequencedDatasetWrite(definition)
     return new InMemoryLakeWriteSession(this, {
       ...input,
       dataset: definition,
@@ -318,7 +332,7 @@ export class InMemoryLakeStorage implements LakeStorage {
       input.expectedLatestVersionId !== undefined &&
       latestVersion?.versionId !== input.expectedLatestVersionId
     ) {
-      throw new LakeStorageError(
+      throw new LakeConcurrencyError(
         `[LakeStorage] Optimistic merge start failed for dataset '${definition.id}': expected latest version '${input.expectedLatestVersionId}', found '${latestVersion?.versionId ?? "none"}'`
       )
     }
@@ -398,9 +412,10 @@ export class InMemoryLakeStorage implements LakeStorage {
       }
 
       const latestVersion = await this.getLatestVersion(options.merge.dataset.id)
+      options.commit?.signal?.throwIfAborted()
       const actualVersionId = latestVersion?.versionId ?? null
       if (actualVersionId !== options.baseVersionId) {
-        throw new LakeStorageError(
+        throw new LakeConcurrencyError(
           `[LakeStorage] Optimistic merge commit failed for dataset '${options.merge.dataset.id}': expected latest version '${options.baseVersionId ?? "none"}', found '${actualVersionId ?? "none"}'`
         )
       }
@@ -408,9 +423,21 @@ export class InMemoryLakeStorage implements LakeStorage {
       const previousRows = this.rowsByVersionId.get(latestVersion?.versionId ?? "") ?? []
       const previousByKey = this.rowsByPrimaryKey(previousRows, definition)
       const finalChanges = new Map<string, MergeChange<DatasetRow, DatasetRow>>()
-      for (const change of options.changes) {
-        const value = change.kind === "upsert" ? change.row : change.key
-        finalChanges.set(encodeDatasetPrimaryKey(definition, value), change)
+      const ordered =
+        definition.sequenceBy === undefined
+          ? undefined
+          : reconcileDatasetSequences(
+              definition,
+              this.sequencesByVersionId.get(actualVersionId ?? "") ?? new Map(),
+              options.changes.map((change) => datasetSequenceChange(definition, change))
+            )
+      if (ordered) {
+        for (const [key, index] of ordered.accepted) finalChanges.set(key, options.changes[index]!)
+      } else {
+        for (const change of options.changes) {
+          const value = change.kind === "upsert" ? change.row : change.key
+          finalChanges.set(encodeDatasetPrimaryKey(definition, value), change)
+        }
       }
 
       const nextByKey = new Map(previousByKey)
@@ -422,7 +449,13 @@ export class InMemoryLakeStorage implements LakeStorage {
         }
       }
 
-      if (this.sameKeyedRowContent(previousByKey, nextByKey, definition.schema)) {
+      const createInitialVersion = options.commit?.createInitialVersion && latestVersion === null
+      if (
+        !createInitialVersion &&
+        (ordered
+          ? ordered.accepted.size === 0
+          : this.sameKeyedRowContent(previousByKey, nextByKey, definition.schema))
+      ) {
         return {
           outcome: "unchanged",
           version: latestVersion ? cloneDatasetVersion(latestVersion) : null,
@@ -454,6 +487,7 @@ export class InMemoryLakeStorage implements LakeStorage {
         visibleRows.map((row) => cloneRow(row))
       )
       this.latestVersionIdByDataset.set(options.merge.dataset.id, versionId)
+      if (ordered) this.sequencesByVersionId.set(versionId, ordered.states)
 
       return { outcome: "created", version: cloneDatasetVersion(version) }
     })
@@ -470,13 +504,14 @@ export class InMemoryLakeStorage implements LakeStorage {
     }
 
     const mode = options.write.mode ?? "snapshot"
+    assertUnsequencedDatasetWrite(definition)
     const latestVersion = await this.getLatestVersion(options.write.dataset.id)
 
     if (options.commit?.expectedLatestVersionId !== undefined) {
-      const actual = latestVersion?.versionId
+      const actual = latestVersion?.versionId ?? null
       if (actual !== options.commit.expectedLatestVersionId) {
-        throw new LakeStorageError(
-          `[LakeStorage] Optimistic commit failed for dataset '${options.write.dataset.id}': expected latest version '${options.commit.expectedLatestVersionId}', found '${actual ?? "none"}'`
+        throw new LakeConcurrencyError(
+          `[LakeStorage] Optimistic commit failed for dataset '${options.write.dataset.id}': expected latest version '${options.commit.expectedLatestVersionId ?? "none"}', found '${actual ?? "none"}'`
         )
       }
     }
@@ -484,9 +519,10 @@ export class InMemoryLakeStorage implements LakeStorage {
     this.assertKeyedWriteIsUnique(mode, options.rows, latestVersion, definition)
 
     // Content-identical snapshots and empty appends reuse the latest version
-    // instead of creating a new one.
+    // unless explicitly supplied input lineage has changed.
     if (
       latestVersion &&
+      !hasDatasetInputChanges(latestVersion.inputs, options.write.inputs) &&
       this.isUnchangedWrite(mode, latestVersion, options.rows, definition.schema)
     ) {
       return { ...latestVersion, outcome: "unchanged" }

package/src/lake-storage/index.ts

@@ -6,6 +6,7 @@ export type {
   DatasetSchema,
   MergeChange,
 } from "../datasets"
+export { assertDatasetDefinition } from "../datasets/validation"
 export type { JsonValue } from "../json"
 export type {
   DatasetDefinitionUpdatePlan,
@@ -17,7 +18,7 @@ export {
   mergeStrictDatasetDefinition,
   planDatasetDefinitionUpdate,
 } from "./definition-updates"
-export { LakeStorageError } from "./errors"
+export { LakeConcurrencyError, LakeStorageError } from "./errors"
 export { InMemoryLakeStorage } from "./in-memory"
 export type {
   BeginDatasetMergeInput,
@@ -25,12 +26,19 @@ export type {
   DatasetMergeCommitResult,
   LakeMergeSession,
 } from "./merge"
+export { retryDatasetMergeCommit } from "./merge"
 export {
   cloneDatasetMergeChange,
   encodeDatasetPrimaryKey,
   getDatasetMergeChangeValidationError,
   getDatasetPrimaryKeyColumns,
 } from "./merge-validation"
+export type { DatasetSequenceChange, DatasetSequenceState } from "./source-ordering"
+export {
+  assertUnsequencedDatasetWrite,
+  datasetSequenceChange,
+  reconcileDatasetSequences,
+} from "./source-ordering"
 export type {
   ExecuteSqlTransformInput,
   LakeSqlExecutor,
@@ -60,3 +68,4 @@ export type {
   LakeWriteSession,
   ReadDatasetRowsInput,
 } from "./types"
+export { hasDatasetInputChanges } from "./version-inputs"

package/src/lake-storage/merge-validation.ts

@@ -1,5 +1,6 @@
 import type { DatasetDefinition, MergeChange } from "../datasets"
 import { getDatasetRowValidationError } from "../datasets"
+import { getDatasetSequenceValidationError } from "../datasets/sequence"
 import { isPlainRecord } from "../json"
 import { LakeStorageError } from "./errors"
 import type { DatasetRow } from "./types"
@@ -58,7 +59,11 @@ export function getDatasetMergeChangeValidationError(
     if (shapeError) {
       return shapeError
     }
-    return getDeleteKeyValidationError(change.key, dataset, primaryKeyColumns)
+    const keyError = getDeleteKeyValidationError(change.key, dataset, primaryKeyColumns)
+    if (keyError) return keyError
+    return dataset.sequenceBy === undefined
+      ? null
+      : getDatasetSequenceValidationError(dataset, change.sequence)
   }
 
   return `Dataset '${dataset.id}' merge change kind must be 'upsert' or 'delete'.`
@@ -70,7 +75,11 @@ export function cloneDatasetMergeChange(
 ): MergeChange<DatasetRow, DatasetRow> {
   return change.kind === "upsert"
     ? { kind: "upsert", row: structuredClone(change.row) }
-    : { kind: "delete", key: structuredClone(change.key) }
+    : {
+        kind: "delete",
+        key: structuredClone(change.key),
+        ...(change.sequence !== undefined ? { sequence: structuredClone(change.sequence) } : {}),
+      }
 }
 
 function getChangeShapeError(
@@ -79,6 +88,7 @@ function getChangeShapeError(
   payloadField: "row" | "key"
 ): string | null {
   const allowedFields = new Set(["kind", payloadField])
+  if (payloadField === "key" && dataset.sequenceBy !== undefined) allowedFields.add("sequence")
   for (const field of Object.keys(change)) {
     if (!allowedFields.has(field)) {
       return `Dataset '${dataset.id}' ${change.kind as string} change contains unknown field '${field}'.`

package/src/lake-storage/merge.ts

@@ -1,4 +1,5 @@
 import type { DatasetDefinition, MergeChange } from "../datasets"
+import { LakeConcurrencyError } from "./errors"
 import type { DatasetProducer, DatasetRow, DatasetVersion, DatasetVersionRef } from "./types"
 
 export interface BeginDatasetMergeInput {
@@ -12,6 +13,40 @@ export interface BeginDatasetMergeInput {
 
 export interface CommitDatasetMergeInput {
   readonly commitMessage?: string
+  /** Create an addressable first version even without changes; reuse an existing version on no-op. */
+  readonly createInitialVersion?: boolean
+  /** Retry sequenced merges up to three times total; ignored when an explicit version guard is set. */
+  readonly retryOnConflict?: boolean
+  /** Checked before each attempt; a durable commit is never cancelled retroactively. */
+  readonly signal?: AbortSignal
+}
+
+/** Providers retain their staging session throughout this bounded commit loop. */
+export async function retryDatasetMergeCommit(
+  merge: BeginDatasetMergeInput,
+  commit: CommitDatasetMergeInput | undefined,
+  run: (rebase: boolean) => Promise<DatasetMergeCommitResult>
+): Promise<DatasetMergeCommitResult> {
+  const attempts =
+    commit?.retryOnConflict &&
+    merge.dataset.sequenceBy !== undefined &&
+    merge.expectedLatestVersionId === undefined
+      ? 3
+      : 1
+  for (let attempt = 0; ; attempt += 1) {
+    commit?.signal?.throwIfAborted()
+    try {
+      return await run(attempt > 0)
+    } catch (error) {
+      if (!(error instanceof LakeConcurrencyError) || attempts === 1) throw error
+      if (attempt + 1 === attempts) {
+        throw new LakeConcurrencyError(
+          `[SixbLake] Dataset '${merge.dataset.id}' merge exhausted ${attempts} concurrency attempts. No changes from this operation were committed.`,
+          { cause: error }
+        )
+      }
+    }
+  }
 }
 
 /**
@@ -23,13 +58,16 @@ export type DatasetMergeCommitResult =
   | { readonly outcome: "unchanged"; readonly version: DatasetVersion | null }
 
 export interface LakeMergeSession {
-  /** Stage ordered complete-row upserts and exact primary-key deletes. */
+  /**
+   * Stage ordered complete-row upserts and exact primary-key deletes. With sequenceBy, deletes
+   * require a sequence, stale changes are ignored, and conflicting ties abort the whole merge.
+   */
   writeChanges(
     changes:
       | Iterable<MergeChange<DatasetRow, DatasetRow>>
       | AsyncIterable<MergeChange<DatasetRow, DatasetRow>>
   ): Promise<void>
-  /** Commit only if the latest version still matches the version captured when the session began. */
+  /** Commit against the captured version; opt-in retries rebase sequenced changes on a fresh head. */
   commit(input?: CommitDatasetMergeInput): Promise<DatasetMergeCommitResult>
   abort(): Promise<void>
 }