@sixb/core 0.1.8 → 0.1.10

package/src/index.ts

@@ -55,7 +55,6 @@ export {
   prop,
   ref,
   stringEnum,
-  validateSchemaOrRefValue,
   valueTypeRef,
 } from "./ontology"
 
@@ -127,9 +126,6 @@ export {
   isObjectActionDefinition,
   optional,
   param,
-  requestAction,
-  requestActionAndWait,
-  waitForActionRun,
 } from "./actions"
 /**
  * `ctx.edits()` rejects an invalid batch with this, so it crosses a public boundary and users need
@@ -167,9 +163,15 @@ export {
 
 // ── Broker ─────────────────────────────────────────────────
 
-export type { Broker, BrokerCursor } from "./broker"
+export type {
+  Broker,
+  BrokerCursor,
+  BrokerRetention,
+  BrokerStreamRetention,
+  InMemoryBrokerOptions,
+} from "./broker"
 export { InMemoryBroker } from "./broker"
-export type { JsonValue, ReadonlyJsonValue } from "./json"
+export type { JsonValue, ReadonlyJsonObject, ReadonlyJsonValue } from "./json"
 export {
   assertJsonValue,
   cloneJsonValue,
@@ -336,24 +338,26 @@ export {
 
 // ── Logging ─────────────────────────────────────────────────
 
+export type { ConsoleLoggerOptions } from "./logging/console-logger"
+export { ConsoleLogger, noopLogger, noopLoggerProvider } from "./logging/console-logger"
+export type {
+  LoggingService,
+  LogsObservabilityOptions,
+  ObservabilityOptions,
+} from "./logging/service"
 export type {
-  ConsoleLoggerOptions,
   LogContext,
   LogEntry,
   LogFields,
   Logger,
   LoggerProvider,
-  LoggingService,
   LogLevel,
   LogRecord,
   LogRunRef,
-  LogsObservabilityOptions,
-  ObservabilityOptions,
   SixbRunKind,
   StoredLogLine,
-} from "./logging"
+} from "./logging/types"
 export {
-  ConsoleLogger,
   isLevelEnabled,
   isLogLevel,
   isLogRecord,
@@ -361,10 +365,8 @@ export {
   isStoredLogLine,
   LOG_LEVELS,
   logLevelsAtOrAbove,
-  noopLogger,
-  noopLoggerProvider,
   SIXB_RUN_KINDS,
-} from "./logging"
+} from "./logging/types"
 
 // ── Predicates ─────────────────────────────────────────────
 
@@ -426,7 +428,6 @@ export type { AuthorizationContext, GrantIndex, GrantKind } from "./authorizatio
 export {
   AuthorizationError,
   canAccessApplication,
-  emptyGrantIndex,
   isAllowed,
   isApplicationAccessControlled,
   resolveAuthorizationContext,
@@ -592,12 +593,14 @@ export type {
   CommandResult,
   CreateSandboxOptions,
   ExecOptions,
+  ResumeSandboxOptions,
   RunCommandOptions,
   Sandbox,
   SandboxFactory,
   SandboxFileRecord,
   SandboxNetworkPolicy,
   SandboxNetworkTarget,
+  SandboxSessionOptions,
   SandboxStatus,
 } from "./sandboxes"
 export {
@@ -684,6 +687,8 @@ export type {
   WebhookBodyFormat,
   WebhookBodyParser,
   WebhookBodySchema,
+  WebhookConnection,
+  WebhookConnections,
   WebhookConnectorClient,
   WebhookDefinition,
   WebhookHandlerContext,
@@ -724,7 +729,7 @@ export type {
   SyncRunRequestResult,
   SyncTargetBuilder,
 } from "./syncs"
-export { defineSync, requestSyncRun } from "./syncs"
+export { defineSync } from "./syncs"
 
 // ── Schedules ───────────────────────────────────────────────
 
@@ -784,7 +789,6 @@ export {
   isPipelineDefinition,
   isPipelineStepDefinition,
   PipelineError,
-  requestPipelineRun,
 } from "./pipelines"
 
 // ── Workflows ───────────────────────────────────────────────
@@ -850,7 +854,6 @@ export {
   isInterventionDefinition,
   isStepDefinition,
   isWorkflowDefinition,
-  requestWorkflowRun,
   WorkflowDefinitionError,
   WorkflowValidationError,
 } from "./workflows"
@@ -1062,7 +1065,6 @@ export {
   MAX_AGENT_APP_STATE_ENTRY_BYTES,
   MAX_AGENT_APP_STATE_TOTAL_BYTES,
   MAX_AGENT_CONTEXT_ENTRIES,
-  requestAgentRun,
 } from "./agents"
 export type { AgentThreadRecord, ListAgentThreadsResult } from "./storage/agents"
 
@@ -1075,6 +1077,14 @@ export type {
   ModelCatalog,
   ModelCatalogInput,
 } from "./models"
+export type {
+  InferLanguageModelOutput,
+  LanguageModelGenerateInput,
+  LanguageModelGenerateResult,
+  LanguageModelOutputShape,
+  LanguageModelsRuntime,
+  ModelsRuntime,
+} from "./models/generation-types"
 
 // ── Scheduling ──────────────────────────────────────────────
 

package/src/lake-storage/changes.ts

@@ -0,0 +1,43 @@
+import type { DatasetColumnDefinition } from "../datasets"
+import { LakeStorageError } from "./errors"
+import type { DatasetVersion, ReadDatasetChangesInput } from "./types"
+
+/** Shared eligibility checks for providers comparing pinned row images. */
+export function resolveDatasetChangeColumns(
+  input: ReadDatasetChangesInput,
+  before: DatasetVersion | null,
+  after: DatasetVersion | null
+): readonly DatasetColumnDefinition[] | null {
+  for (const value of [input.datasetId, input.fromVersionId, input.toVersionId]) {
+    if (typeof value !== "string" || value.trim().length === 0) {
+      throw new LakeStorageError(
+        "[LakeStorage] Change reads require explicit dataset and version ids."
+      )
+    }
+  }
+  if (
+    input.keyColumns.length === 0 ||
+    new Set(input.keyColumns).size !== input.keyColumns.length ||
+    input.columns.length === 0 ||
+    new Set(input.columns).size !== input.columns.length ||
+    input.columns.some((column) => typeof column !== "string" || column.length === 0) ||
+    input.keyColumns.some((column) => !input.columns.includes(column))
+  ) {
+    throw new LakeStorageError(
+      "[LakeStorage] Change reads require distinct columns including every key column."
+    )
+  }
+  if (!before || !after) return null
+  const oldColumns = new Map(before.schema.columns.map((column) => [column.name, column]))
+  const newColumns = new Map(after.schema.columns.map((column) => [column.name, column]))
+  const columns: DatasetColumnDefinition[] = []
+  for (const name of input.columns) {
+    const previous = oldColumns.get(name)
+    const next = newColumns.get(name)
+    if (!previous || !next || previous.type !== next.type) return null
+    // Projection identities are nonblank strings. Other key types keep the full-read path.
+    if (input.keyColumns.includes(name) && next.type !== "string") return null
+    columns.push(next)
+  }
+  return columns
+}

package/src/lake-storage/in-memory.ts

@@ -1,6 +1,8 @@
 import { randomUUID } from "node:crypto"
+import { isDeepStrictEqual } from "node:util"
 import type { DatasetDefinition, DatasetSchema, MergeChange } from "../datasets"
 import { getDatasetRowValidationError } from "../datasets/validation"
+import { resolveDatasetChangeColumns } from "./changes"
 import { mergeStrictDatasetDefinition } from "./definition-updates"
 import { LakeConcurrencyError, LakeStorageError } from "./errors"
 import type {
@@ -26,12 +28,15 @@ import type {
   BeginDatasetWriteInput,
   CommitDatasetWriteInput,
   DatasetCatalogState,
+  DatasetChanges,
   DatasetRow,
+  DatasetRowChange,
   DatasetVersion,
   DatasetWriteCommitResult,
   DatasetWriteMode,
   LakeStorage,
   LakeWriteSession,
+  ReadDatasetChangesInput,
   ReadDatasetRowsInput,
 } from "./types"
 import { hasDatasetInputChanges } from "./version-inputs"
@@ -389,6 +394,74 @@ export class InMemoryLakeStorage implements LakeStorage {
     }
   }
 
+  async readChanges(input: ReadDatasetChangesInput): Promise<DatasetChanges | null> {
+    input.signal?.throwIfAborted()
+    const before = await this.getVersion(input.datasetId, input.fromVersionId)
+    const after = await this.getVersion(input.datasetId, input.toVersionId)
+    if (!resolveDatasetChangeColumns(input, before, after)) return null
+    const previous = this.rowsByVersionId.get(input.fromVersionId)!
+    const next = this.rowsByVersionId.get(input.toVersionId)!
+    const index = (rows: readonly DatasetRow[]) => {
+      const result = new Map<string, DatasetRow>()
+      for (const row of rows) {
+        input.signal?.throwIfAborted()
+        const values = input.keyColumns.map((column) => row[column])
+        if (values.some((value) => typeof value !== "string" || value.trim().length === 0))
+          return null
+        const key = JSON.stringify(values)
+        if (result.has(key)) return null
+        result.set(key, row)
+      }
+      return result
+    }
+    const oldRows = index(previous)
+    const newRows = index(next)
+    if (!oldRows || !newRows) return null
+    let pending: DatasetRowChange[] = []
+    for (const [key, row] of oldRows) {
+      const newRow = newRows.get(key)
+      if (
+        !newRow ||
+        input.columns.some(
+          (column) => !isDeepStrictEqual(row[column] ?? null, newRow[column] ?? null)
+        )
+      ) {
+        pending.push({
+          before: selectColumns(row, input.columns),
+          after: newRow ? selectColumns(newRow, input.columns) : null,
+        })
+      }
+    }
+    for (const [key, row] of newRows) {
+      if (!oldRows.has(key))
+        pending.push({ before: null, after: selectColumns(row, input.columns) })
+    }
+    let closed = false
+    const changeCount = pending.length
+    return {
+      fromRowCount: previous.length,
+      toRowCount: next.length,
+      changeCount,
+      changes: (async function* () {
+        if (closed) throw new LakeStorageError("[LakeStorage] Change reader is closed.")
+        try {
+          for (let i = 0; i < changeCount; i++) {
+            input.signal?.throwIfAborted()
+            if (closed) throw new LakeStorageError("[LakeStorage] Change reader is closed.")
+            yield structuredClone(pending[i]!)
+          }
+        } finally {
+          closed = true
+          pending = []
+        }
+      })(),
+      async close() {
+        closed = true
+        pending = []
+      },
+    }
+  }
+
   async commitWrite(options: {
     write: BeginDatasetWriteInput
     rows: readonly DatasetRow[]

package/src/lake-storage/index.ts

@@ -8,6 +8,7 @@ export type {
 } from "../datasets"
 export { assertDatasetDefinition } from "../datasets/validation"
 export type { JsonValue } from "../json"
+export { resolveDatasetChangeColumns } from "./changes"
 export type {
   DatasetDefinitionUpdatePlan,
   DatasetMetadataUpdatePlan,
@@ -54,9 +55,11 @@ export type {
   BeginDatasetWriteInput,
   CommitDatasetWriteInput,
   DatasetCatalogState,
+  DatasetChanges,
   DatasetLatestVersionSummary,
   DatasetProducer,
   DatasetRow,
+  DatasetRowChange,
   DatasetVersion,
   DatasetVersionMode,
   DatasetVersionRef,
@@ -66,6 +69,7 @@ export type {
   LakeStandardId,
   LakeStorage,
   LakeWriteSession,
+  ReadDatasetChangesInput,
   ReadDatasetRowsInput,
 } from "./types"
 export { hasDatasetInputChanges } from "./version-inputs"

package/src/lake-storage/types.ts

@@ -73,6 +73,39 @@ export interface ReadDatasetRowsInput {
   readonly offset?: number
 }
 
+export interface ReadDatasetChangesInput {
+  readonly datasetId: string
+  readonly fromVersionId: string
+  readonly toVersionId: string
+  /** Stable, unique row identity in both versions; it need not be the dataset primary key. */
+  readonly keyColumns: readonly string[]
+  /** Complete inputs of the consumer's calculation, including key columns and timestamps. */
+  readonly columns: readonly string[]
+  readonly signal?: AbortSignal
+}
+
+export interface DatasetRowChange {
+  /** Null for an insertion. Otherwise includes every requested column. */
+  readonly before: DatasetRow | null
+  /** Null for a deletion. Otherwise includes every requested column. */
+  readonly after: DatasetRow | null
+}
+
+/** A complete, pinned change set. Call close in finally, including when iteration never starts. */
+export interface DatasetChanges {
+  readonly fromRowCount: number
+  readonly toRowCount: number
+  /** Exact number of records the iterator must deliver before EOF. */
+  readonly changeCount: number
+  /**
+   * At most one record per key, with at least one non-null image. Includes every changed key;
+   * providers may conservatively include equal images (for example, reordered JSON fields).
+   * Order is unspecified. Consumers must exhaust the stream before publishing its result.
+   */
+  readonly changes: AsyncIterable<DatasetRowChange>
+  close(): Promise<void>
+}
+
 export interface CommitDatasetWriteInput {
   /** Atomic version guard; null requires that no version exists, undefined leaves it unguarded. */
   readonly expectedLatestVersionId?: string | null
@@ -138,6 +171,13 @@ export interface LakeStorage {
   /** Explicit immutable-version reads have stable physical order and stable offset semantics. */
   readRows(input: ReadDatasetRowsInput): AsyncIterable<DatasetRow>
 
+  /**
+   * Optional comparison of two immutable versions. Null means the snapshots, schemas or unique
+   * keys cannot support a complete delta: use a full read. Operational errors must propagate.
+   * Iteration releases the provider connection between bounded pages, allowing other lake work.
+   */
+  readChanges?(input: ReadDatasetChangesInput): Promise<DatasetChanges | null>
+
   /** Release external resources. Optional: a provider that owns none omits it. */
   close?(): void | Promise<void>
 }

package/src/logging/index.ts

@@ -1,25 +1,5 @@
-export {
-  ConsoleLogger,
-  type ConsoleLoggerOptions,
-  noopLogger,
-  noopLoggerProvider,
-} from "./console-logger"
-export {
-  LoggingService,
-  type LoggingServiceOptions,
-  type LogsObservabilityOptions,
-  type LogsPage,
-  type LogsReadInput,
-  type LogsSubscribeInput,
-  type LogsTailInput,
-  type ObservabilityOptions,
-  resolveLoggingService,
-} from "./service"
-export {
-  DEFAULT_LOGS_MAX_BYTES,
-  DEFAULT_LOGS_RETENTION,
-  LOGS_STREAM,
-} from "./stream"
+/** Browser-safe logging contracts and record readers. */
+export type * from "./types"
 export {
   isLevelEnabled,
   isLogLevel,
@@ -27,17 +7,6 @@ export {
   isSixbRunKind,
   isStoredLogLine,
   LOG_LEVELS,
-  type LogContext,
-  type LogEntry,
-  type LogFields,
-  type Logger,
-  type LoggerProvider,
-  type LogLevel,
-  type LogRecord,
-  type LogRunRef,
   logLevelsAtOrAbove,
-  normalizeLogError,
   SIXB_RUN_KINDS,
-  type SixbRunKind,
-  type StoredLogLine,
 } from "./types"

package/src/logging/internal.ts

@@ -0,0 +1,43 @@
+export {
+  ConsoleLogger,
+  type ConsoleLoggerOptions,
+  noopLogger,
+  noopLoggerProvider,
+} from "./console-logger"
+export {
+  LoggingService,
+  type LoggingServiceOptions,
+  type LogsObservabilityOptions,
+  type LogsPage,
+  type LogsReadInput,
+  type LogsSubscribeInput,
+  type LogsTailInput,
+  type ObservabilityOptions,
+  resolveLoggingService,
+} from "./service"
+export {
+  DEFAULT_LOGS_MAX_BYTES,
+  DEFAULT_LOGS_RETENTION,
+  LOGS_STREAM,
+} from "./stream"
+export {
+  isLevelEnabled,
+  isLogLevel,
+  isLogRecord,
+  isSixbRunKind,
+  isStoredLogLine,
+  LOG_LEVELS,
+  type LogContext,
+  type LogEntry,
+  type LogFields,
+  type Logger,
+  type LoggerProvider,
+  type LogLevel,
+  type LogRecord,
+  type LogRunRef,
+  logLevelsAtOrAbove,
+  normalizeLogError,
+  SIXB_RUN_KINDS,
+  type SixbRunKind,
+  type StoredLogLine,
+} from "./types"

package/src/logging/service.ts

@@ -213,7 +213,7 @@ export class LoggingService {
         const lines = records.map(hydrateLogRecord)
         if (lines.length === 0) return
         try {
-          handler(lines)
+          return handler(lines)
         } catch {
           // Subscriber failures never break broker delivery.
         }

package/src/materialization/index.ts

@@ -51,6 +51,8 @@ export type {
   ProjectionRunFinishInput,
   ProjectionRunTerminalDecision,
   ProjectionSourceAssertion,
+  ProjectionSourceBase,
+  ProjectionSourceDeletion,
   ProjectionSourceEntry,
   ProjectionSourceRef,
   ProjectionSourceReplacement,

package/src/materialization/model.ts

@@ -169,11 +169,24 @@ export interface ProjectionSourceEntry {
   readonly assertions: readonly ProjectionSourceAssertion[]
 }
 
+export interface ProjectionSourceDeletion {
+  readonly root: ProjectionEntityRef
+  readonly deleted: true
+}
+
+/** Exact committed source state against which a complete delta was calculated. */
+export interface ProjectionSourceBase {
+  readonly materializationId: string
+  readonly lastCommitId: string
+}
+
 export interface ProjectionSourceReplacement {
   readonly source: ProjectionSourceRef
   readonly datasetVersion: PinnedDatasetVersion
   readonly execution: ProjectionExecution
-  readonly entries: AsyncIterable<ProjectionSourceEntry>
+  /** Without a base, entries replace the whole source. With a base, unchanged roots are retained. */
+  readonly entries: AsyncIterable<ProjectionSourceEntry | ProjectionSourceDeletion>
+  readonly base?: ProjectionSourceBase
   readonly signal?: AbortSignal
 }
 

package/src/materializer/index.ts

@@ -42,6 +42,8 @@ export type {
   ProjectionProtocolIdentity,
   ProjectionRunFinishInput,
   ProjectionSourceAssertion,
+  ProjectionSourceBase,
+  ProjectionSourceDeletion,
   ProjectionSourceEntry,
   ProjectionSourceRef,
   ProjectionSourceReplacement,

package/src/materializer/materializer.ts

@@ -4,12 +4,14 @@ import type {
   OntologyEditCommit,
   ProjectionCommitResult,
   ProjectionRunFinishInput,
+  ProjectionSourceRef,
   ProjectionSourceReplacement,
   TelemetryAppend,
   TelemetryCommitResult,
 } from "../materialization/model"
 import type { OntologyRegistry } from "../ontology"
 import type { ProjectionRegistry } from "../projections/registry"
+import type { OntologySourceRecord } from "../storage/ontology"
 import {
   createMaterializerContext,
   type MaterializerContext,
@@ -34,6 +36,7 @@ export interface OntologyMaterializerContract {
     commit(input: MaterializerCommand<OntologyEditCommit>): Promise<EditCommitResult>
   }
   readonly projections: {
+    getActive(source: ProjectionSourceRef): Promise<OntologySourceRecord | null>
     replace(
       input: MaterializerCommand<ProjectionSourceReplacement>
     ): Promise<ProjectionCommitResult>
@@ -50,6 +53,7 @@ export interface BoundOntologyMaterializer {
     commit(input: OntologyEditCommit): Promise<EditCommitResult>
   }
   readonly projections: {
+    getActive(source: ProjectionSourceRef): Promise<OntologySourceRecord | null>
     replace(input: ProjectionSourceReplacement): Promise<ProjectionCommitResult>
     finishRun(input: ProjectionRunFinishInput): Promise<void>
   }
@@ -64,6 +68,11 @@ export class OntologyMaterializer implements OntologyMaterializerContract {
       commitEdits(this.context, command),
   }
   readonly projections = {
+    getActive: (source: ProjectionSourceRef) =>
+      this.context.storage.ontology.sources.getActive({
+        projectId: this.context.projectId,
+        source,
+      }),
     replace: (command: MaterializerCommand<ProjectionSourceReplacement>) =>
       replaceProjection(this.context, command),
     finishRun: (command: MaterializerCommand<ProjectionRunFinishInput>) =>
@@ -80,6 +89,7 @@ export class OntologyMaterializer implements OntologyMaterializerContract {
         commit: (input: OntologyEditCommit) => this.edits.commit({ scope, input }),
       }),
       projections: Object.freeze({
+        getActive: (source: ProjectionSourceRef) => this.projections.getActive(source),
         replace: (input: ProjectionSourceReplacement) => this.projections.replace({ scope, input }),
         finishRun: (input: ProjectionRunFinishInput) =>
           this.projections.finishRun({ scope, input }),

package/src/materializer/projections/entry-validator.ts

@@ -1,5 +1,5 @@
 import { MaterializationValidationError } from "../../materialization/errors"
-import type { ProjectionSourceEntry } from "../../materialization/model"
+import type { ProjectionEntityRef, ProjectionSourceEntry } from "../../materialization/model"
 import { linkOwnershipKey } from "../../materialization/refs"
 import type { OntologyRegistry } from "../../ontology"
 import type { ProjectionRegistry } from "../../projections/registry"
@@ -30,6 +30,24 @@ export function createProjectionEntryValidator(
   return (entry) => validateProjectionEntry(ontology, resolved, policy, entry)
 }
 
+export function validateProjectionRootRef(
+  resolved: ResolvedSourceProjection,
+  root: ProjectionEntityRef
+): void {
+  const definition = resolved.definition
+  const owned =
+    definition._tag === "ObjectProjectionDefinition"
+      ? root.kind === "object" && root.ref.objectTypeId === definition.objectTypeId
+      : root.kind === "link" &&
+        root.ref.source.objectTypeId === definition.sourceObjectTypeId &&
+        root.ref.target.objectTypeId === definition.targetObjectTypeId &&
+        root.ref.linkId === definition.linkId
+  if (!owned)
+    throw new MaterializationValidationError(
+      "Projection root is outside its owned type or link scope."
+    )
+}
+
 function createValidationPolicy(resolved: ResolvedSourceProjection): ProjectionValidationPolicy {
   const ownedProperties = new Map(
     resolved.ownership.objects.map((object) => [object.objectTypeId, new Set(object.propertyIds)])