@sixb/core 0.1.9 → 0.1.11

package/src/lake-storage/index.ts

@@ -8,6 +8,7 @@ export type {
 } from "../datasets"
 export { assertDatasetDefinition } from "../datasets/validation"
 export type { JsonValue } from "../json"
+export { resolveDatasetChangeColumns } from "./changes"
 export type {
   DatasetDefinitionUpdatePlan,
   DatasetMetadataUpdatePlan,
@@ -54,9 +55,11 @@ export type {
   BeginDatasetWriteInput,
   CommitDatasetWriteInput,
   DatasetCatalogState,
+  DatasetChanges,
   DatasetLatestVersionSummary,
   DatasetProducer,
   DatasetRow,
+  DatasetRowChange,
   DatasetVersion,
   DatasetVersionMode,
   DatasetVersionRef,
@@ -66,6 +69,7 @@ export type {
   LakeStandardId,
   LakeStorage,
   LakeWriteSession,
+  ReadDatasetChangesInput,
   ReadDatasetRowsInput,
 } from "./types"
 export { hasDatasetInputChanges } from "./version-inputs"

package/src/lake-storage/types.ts

@@ -71,6 +71,41 @@ export interface ReadDatasetRowsInput {
   readonly columns?: readonly string[]
   readonly limit?: number
   readonly offset?: number
+  /** Cancel an active read or a read waiting for provider capacity. */
+  readonly signal?: AbortSignal
+}
+
+export interface ReadDatasetChangesInput {
+  readonly datasetId: string
+  readonly fromVersionId: string
+  readonly toVersionId: string
+  /** Stable, unique row identity in both versions; it need not be the dataset primary key. */
+  readonly keyColumns: readonly string[]
+  /** Complete inputs of the consumer's calculation, including key columns and timestamps. */
+  readonly columns: readonly string[]
+  readonly signal?: AbortSignal
+}
+
+export interface DatasetRowChange {
+  /** Null for an insertion. Otherwise includes every requested column. */
+  readonly before: DatasetRow | null
+  /** Null for a deletion. Otherwise includes every requested column. */
+  readonly after: DatasetRow | null
+}
+
+/** A complete, pinned change set. Call close in finally, including when iteration never starts. */
+export interface DatasetChanges {
+  readonly fromRowCount: number
+  readonly toRowCount: number
+  /** Exact number of records the iterator must deliver before EOF. */
+  readonly changeCount: number
+  /**
+   * At most one record per key, with at least one non-null image. Includes every changed key;
+   * providers may conservatively include equal images (for example, reordered JSON fields).
+   * Order is unspecified. Consumers must exhaust the stream before publishing its result.
+   */
+  readonly changes: AsyncIterable<DatasetRowChange>
+  close(): Promise<void>
 }
 
 export interface CommitDatasetWriteInput {
@@ -138,6 +173,13 @@ export interface LakeStorage {
   /** Explicit immutable-version reads have stable physical order and stable offset semantics. */
   readRows(input: ReadDatasetRowsInput): AsyncIterable<DatasetRow>
 
+  /**
+   * Optional comparison of two immutable versions. Null means the snapshots, schemas or unique
+   * keys cannot support a complete delta: use a full read. Operational errors must propagate.
+   * Iteration releases the provider connection between bounded pages, allowing other lake work.
+   */
+  readChanges?(input: ReadDatasetChangesInput): Promise<DatasetChanges | null>
+
   /** Release external resources. Optional: a provider that owns none omits it. */
   close?(): void | Promise<void>
 }

package/src/logging/service.ts

@@ -213,7 +213,7 @@ export class LoggingService {
         const lines = records.map(hydrateLogRecord)
         if (lines.length === 0) return
         try {
-          handler(lines)
+          return handler(lines)
         } catch {
           // Subscriber failures never break broker delivery.
         }

package/src/maintenance/ontology-maintenance.ts

@@ -93,6 +93,7 @@ export class OntologyMaintenance {
 
   /** Runs one coalesced pass. Exposed for deterministic hosting and tests. */
   runNow(): Promise<void> {
+    if (this.stopping) return this.stopping
     this.pass ??= this.runPass().finally(() => {
       this.pass = null
     })
@@ -244,12 +245,15 @@ export class OntologyMaintenance {
       this.timer = null
     }
     const activePass = this.pass ?? Promise.resolve()
-    const stopping = settlesWithin(activePass, this.shutdownTimeoutMs).then((settled) => {
+    const stopping = settlesWithin(activePass, this.shutdownTimeoutMs).then(async (settled) => {
       if (!settled) {
         this.reportError(
-          new Error(`[Sixb] Ontology maintenance did not stop within ${this.shutdownTimeoutMs}ms.`)
+          new Error(
+            `[Sixb] Ontology maintenance is still stopping after ${this.shutdownTimeoutMs}ms; waiting for active storage operations.`
+          )
         )
       }
+      await activePass
     })
     this.stopping = stopping.finally(() => {
       this.stopping = null

package/src/maintenance/types.ts

@@ -7,7 +7,7 @@ export interface OntologyMaintenanceOptions {
   readonly terminalSourceRetentionMs?: number
   /** Maximum deletions attempted per cleanup domain and pass. Defaults to 1,000. */
   readonly cleanupLimit?: number
-  /** Maximum time to wait for an active pass during shutdown. Defaults to 30 seconds. */
+  /** Warn after this shutdown grace period, then wait for active operations. Defaults to 30 seconds. */
   readonly shutdownTimeoutMs?: number
 }
 

package/src/materialization/index.ts

@@ -51,6 +51,8 @@ export type {
   ProjectionRunFinishInput,
   ProjectionRunTerminalDecision,
   ProjectionSourceAssertion,
+  ProjectionSourceBase,
+  ProjectionSourceDeletion,
   ProjectionSourceEntry,
   ProjectionSourceRef,
   ProjectionSourceReplacement,

package/src/materialization/model.ts

@@ -169,11 +169,24 @@ export interface ProjectionSourceEntry {
   readonly assertions: readonly ProjectionSourceAssertion[]
 }
 
+export interface ProjectionSourceDeletion {
+  readonly root: ProjectionEntityRef
+  readonly deleted: true
+}
+
+/** Exact committed source state against which a complete delta was calculated. */
+export interface ProjectionSourceBase {
+  readonly materializationId: string
+  readonly lastCommitId: string
+}
+
 export interface ProjectionSourceReplacement {
   readonly source: ProjectionSourceRef
   readonly datasetVersion: PinnedDatasetVersion
   readonly execution: ProjectionExecution
-  readonly entries: AsyncIterable<ProjectionSourceEntry>
+  /** Without a base, entries replace the whole source. With a base, unchanged roots are retained. */
+  readonly entries: AsyncIterable<ProjectionSourceEntry | ProjectionSourceDeletion>
+  readonly base?: ProjectionSourceBase
   readonly signal?: AbortSignal
 }
 

package/src/materializer/README.md

@@ -17,7 +17,7 @@ runtime telemetry       -> telemetry.append (origin: telemetry/runtime)
 
 Committed facts are published from the transactional outbox after the commit resolves.
 `OntologyOutboxDispatcher` owns that protocol: ingresses call `notify()` for prompt, non-blocking
-in-process delivery, and a process may call `start()` to host the durable poll loop. Publication is
+in-process delivery; `OntologyMaintenance` hosts recovery catch-up. Publication is
 best effort — delivery may lag, but a committed fact is never lost.
 
 ## Common commit pipeline

package/src/materializer/index.ts

@@ -42,6 +42,8 @@ export type {
   ProjectionProtocolIdentity,
   ProjectionRunFinishInput,
   ProjectionSourceAssertion,
+  ProjectionSourceBase,
+  ProjectionSourceDeletion,
   ProjectionSourceEntry,
   ProjectionSourceRef,
   ProjectionSourceReplacement,

package/src/materializer/materializer.ts

@@ -4,12 +4,14 @@ import type {
   OntologyEditCommit,
   ProjectionCommitResult,
   ProjectionRunFinishInput,
+  ProjectionSourceRef,
   ProjectionSourceReplacement,
   TelemetryAppend,
   TelemetryCommitResult,
 } from "../materialization/model"
 import type { OntologyRegistry } from "../ontology"
 import type { ProjectionRegistry } from "../projections/registry"
+import type { OntologySourceRecord } from "../storage/ontology"
 import {
   createMaterializerContext,
   type MaterializerContext,
@@ -34,6 +36,7 @@ export interface OntologyMaterializerContract {
     commit(input: MaterializerCommand<OntologyEditCommit>): Promise<EditCommitResult>
   }
   readonly projections: {
+    getActive(source: ProjectionSourceRef): Promise<OntologySourceRecord | null>
     replace(
       input: MaterializerCommand<ProjectionSourceReplacement>
     ): Promise<ProjectionCommitResult>
@@ -50,6 +53,7 @@ export interface BoundOntologyMaterializer {
     commit(input: OntologyEditCommit): Promise<EditCommitResult>
   }
   readonly projections: {
+    getActive(source: ProjectionSourceRef): Promise<OntologySourceRecord | null>
     replace(input: ProjectionSourceReplacement): Promise<ProjectionCommitResult>
     finishRun(input: ProjectionRunFinishInput): Promise<void>
   }
@@ -64,6 +68,11 @@ export class OntologyMaterializer implements OntologyMaterializerContract {
       commitEdits(this.context, command),
   }
   readonly projections = {
+    getActive: (source: ProjectionSourceRef) =>
+      this.context.storage.ontology.sources.getActive({
+        projectId: this.context.projectId,
+        source,
+      }),
     replace: (command: MaterializerCommand<ProjectionSourceReplacement>) =>
       replaceProjection(this.context, command),
     finishRun: (command: MaterializerCommand<ProjectionRunFinishInput>) =>
@@ -80,6 +89,7 @@ export class OntologyMaterializer implements OntologyMaterializerContract {
         commit: (input: OntologyEditCommit) => this.edits.commit({ scope, input }),
       }),
       projections: Object.freeze({
+        getActive: (source: ProjectionSourceRef) => this.projections.getActive(source),
         replace: (input: ProjectionSourceReplacement) => this.projections.replace({ scope, input }),
         finishRun: (input: ProjectionRunFinishInput) =>
           this.projections.finishRun({ scope, input }),

package/src/materializer/projections/entry-validator.ts

@@ -1,5 +1,5 @@
 import { MaterializationValidationError } from "../../materialization/errors"
-import type { ProjectionSourceEntry } from "../../materialization/model"
+import type { ProjectionEntityRef, ProjectionSourceEntry } from "../../materialization/model"
 import { linkOwnershipKey } from "../../materialization/refs"
 import type { OntologyRegistry } from "../../ontology"
 import type { ProjectionRegistry } from "../../projections/registry"
@@ -30,6 +30,24 @@ export function createProjectionEntryValidator(
   return (entry) => validateProjectionEntry(ontology, resolved, policy, entry)
 }
 
+export function validateProjectionRootRef(
+  resolved: ResolvedSourceProjection,
+  root: ProjectionEntityRef
+): void {
+  const definition = resolved.definition
+  const owned =
+    definition._tag === "ObjectProjectionDefinition"
+      ? root.kind === "object" && root.ref.objectTypeId === definition.objectTypeId
+      : root.kind === "link" &&
+        root.ref.source.objectTypeId === definition.sourceObjectTypeId &&
+        root.ref.target.objectTypeId === definition.targetObjectTypeId &&
+        root.ref.linkId === definition.linkId
+  if (!owned)
+    throw new MaterializationValidationError(
+      "Projection root is outside its owned type or link scope."
+    )
+}
+
 function createValidationPolicy(resolved: ResolvedSourceProjection): ProjectionValidationPolicy {
   const ownedProperties = new Map(
     resolved.ownership.objects.map((object) => [object.objectTypeId, new Set(object.propertyIds)])

package/src/materializer/projections/replace.ts

@@ -10,6 +10,8 @@ import type {
   ProjectionCommitResult,
   ProjectionExecution,
   ProjectionMaterializationIdentity,
+  ProjectionSourceBase,
+  ProjectionSourceDeletion,
   ProjectionSourceEntry,
   ProjectionSourceRef,
   ProjectionSourceReplacement,
@@ -19,6 +21,7 @@ import type {
   OntologyCommitRecord,
   OntologyCommitWrite,
   OntologyMaterializationStorage,
+  OntologySourceRecord,
   OntologyStorage,
 } from "../../storage/ontology"
 import type { MaterializerContext, MaterializerStorage } from "../context"
@@ -60,7 +63,8 @@ interface PreparedProjectionReplacement {
   readonly source: ProjectionSourceRef
   readonly datasetVersion: PinnedDatasetVersion
   readonly execution: ProjectionExecution
-  readonly entries: AsyncIterable<ProjectionSourceEntry>
+  readonly entries: AsyncIterable<ProjectionSourceEntry | ProjectionSourceDeletion>
+  readonly base?: ProjectionSourceBase
   readonly signal?: AbortSignal
   readonly resolved: ResolvedSourceProjection
   readonly projectionKind: "object" | "link"
@@ -128,6 +132,7 @@ function prepareProjectionReplacement(
     datasetVersion,
     execution,
     entries: raw.input.entries,
+    ...(raw.input.base ? { base: Object.freeze({ ...raw.input.base }) } : {}),
     resolved,
     projectionKind,
     runIdentity,
@@ -237,6 +242,7 @@ async function prepareProjectionCandidate(
     source: command.source,
   })
   validateProjectionWatermark(active, command.datasetVersion)
+  assertDeltaBase(active, command)
   return {
     materializationId: context.materializationId(),
     createdAt: context.clock().toISOString(),
@@ -263,6 +269,7 @@ async function stageProjectionCandidate(
     ownershipHash: command.resolved.ownershipHash,
     createdAt: candidate.createdAt,
     entries: command.entries,
+    ...(command.base ? { base: command.base } : {}),
     validateEntry: createProjectionEntryValidator(context.ontology, command.resolved),
   }
   const staged = await stageCandidateEntries(context, input, command.signal)
@@ -312,6 +319,14 @@ async function executeProjectionTransaction(
   )
   if (replay) return projectionReplayResult(replay, command)
 
+  if (command.base) {
+    const active = await storage.ontology.sources.getActive({
+      projectId: context.projectId,
+      source: command.source,
+    })
+    validateProjectionWatermark(active, command.datasetVersion)
+    assertDeltaBase(active, command)
+  }
   const origin = projectionOrigin(command)
   throwIfAborted(command.signal)
   const session = await storage.ontology.materializations.begin({
@@ -478,3 +493,23 @@ function validateProjectionWatermark(
   if (!active) return
   assertPinnedDatasetWatermark(active.datasetVersion, next, "Projection replacement")
 }
+
+function assertDeltaBase(
+  active: OntologySourceRecord | null,
+  command: PreparedProjectionReplacement
+): void {
+  if (
+    command.base &&
+    (!active ||
+      active.materializationId !== command.base.materializationId ||
+      active.lastCommitId !== command.base.lastCommitId ||
+      active.projectionRevision !== command.runIdentity.projectionRevision ||
+      active.ontologyRevision !== command.runIdentity.ontologyRevision ||
+      active.ownershipHash !== command.runIdentity.ownershipHash)
+  ) {
+    throw new MaterializationConflictError(
+      "source-materialization",
+      "Projection delta no longer matches the active source or its definition."
+    )
+  }
+}

package/src/materializer/projections/source-materialization.ts

@@ -1,16 +1,19 @@
+import { MaterializationValidationError } from "../../materialization/errors"
 import type {
   PinnedDatasetVersion,
   ProjectionExecution,
+  ProjectionSourceBase,
+  ProjectionSourceDeletion,
   ProjectionSourceEntry,
   ProjectionSourceRef,
 } from "../../materialization/model"
 import { utf8JsonByteLength } from "../../materialization/refs"
-import type { StageSourceAssertion } from "../../storage/ontology"
+import type { StageSourceAssertion, StageSourceRoot } from "../../storage/ontology"
 import type { MaterializerContext } from "../context"
 import { throwIfAborted } from "../shared/abort"
 import { chunkBySize } from "../shared/chunking"
-import { normalizeProjectionSourceEntry } from "../shared/normalize"
-import type { ProjectionEntryValidator } from "./entry-validator"
+import { normalizeProjectionEntity, normalizeProjectionSourceEntry } from "../shared/normalize"
+import { type ProjectionEntryValidator, validateProjectionRootRef } from "./entry-validator"
 
 export interface StagedProjectionMaterialization {
   readonly rootCount: number
@@ -37,7 +40,8 @@ export async function stageProjectionMaterialization(
     readonly projectionRevision: string
     readonly ownershipHash: string
     readonly createdAt: string
-    readonly entries: AsyncIterable<ProjectionSourceEntry>
+    readonly entries: AsyncIterable<ProjectionSourceEntry | ProjectionSourceDeletion>
+    readonly base?: ProjectionSourceBase
     readonly validateEntry: ProjectionEntryValidator
     readonly signal?: AbortSignal
   }
@@ -54,13 +58,28 @@ export async function stageProjectionMaterialization(
     ownershipHash: input.ownershipHash,
     ontologyRevision: context.projectionRegistry.ontologyRevision,
     createdAt: input.createdAt,
+    ...(input.base ? { base: input.base } : {}),
   })
 
   let rootCount = 0
   let assertionCount = 0
-  async function* assertions(): AsyncIterable<StageSourceAssertion> {
+  async function* assertions(): AsyncIterable<StageSourceAssertion | StageSourceRoot> {
     for await (const rawEntry of input.entries) {
       throwIfAborted(input.signal)
+      if ("deleted" in rawEntry) {
+        if (!input.base || rawEntry.deleted !== true) {
+          throw new MaterializationValidationError(
+            "Explicit root deletions require a pinned source base."
+          )
+        }
+        const root = normalizeProjectionEntity(rawEntry.root)
+        validateProjectionRootRef(
+          context.projectionRegistry.resolveSource(input.source.projectionId),
+          root
+        )
+        yield { root, stagingOrdinal: rootCount++ }
+        continue
+      }
       const entry = input.validateEntry(normalizeProjectionSourceEntry(rawEntry))
       const stagingOrdinal = rootCount
       for (const assertion of entry.assertions) {
@@ -82,7 +101,8 @@ export async function stageProjectionMaterialization(
       source: input.source,
       materializationId: input.materializationId,
       execution: input.execution,
-      rows,
+      rows: rows.filter((row): row is StageSourceAssertion => "assertion" in row),
+      deletions: rows.filter((row) => !("assertion" in row)),
     })
   }
 

package/src/materializer/shared/normalize.ts

@@ -344,7 +344,7 @@ export function normalizeProjectionExecution(input: ProjectionExecution): Projec
   })
 }
 
-function normalizeProjectionEntity(entity: ProjectionEntityRef): ProjectionEntityRef {
+export function normalizeProjectionEntity(entity: ProjectionEntityRef): ProjectionEntityRef {
   return entity.kind === "object"
     ? Object.freeze({ kind: "object", ref: normalizeObjectRef(entity.ref) })
     : Object.freeze({ kind: "link", ref: normalizeLinkRef(entity.ref) })

package/src/models/definitions.ts

@@ -14,6 +14,10 @@ export interface ModelDefinition {
   readonly modelId: string
   readonly name?: string
   readonly description?: string
+  /** Model author for display; does not change the provider/model routing identity. */
+  readonly publisher?: { readonly id: string; readonly name: string }
+  /** Hosting service label, when the model is offered through an intermediary. */
+  readonly via?: string
   readonly family?: string
   readonly tags?: readonly string[]
   readonly releaseDate?: string
@@ -38,6 +42,17 @@ export function defineLanguageModel(definition: LanguageModelDefinition): Langua
   assertModelId(definition.modelId, "modelId")
   assertOptionalString(definition.name, "name")
   assertOptionalString(definition.description, "description")
+  assertOptionalString(definition.via, "via")
+  if (definition.via !== undefined && !definition.via.trim())
+    throw new TypeError("[Sixb] Model via must not be empty.")
+  const publisher = definition.publisher
+  if (publisher !== undefined) {
+    assertRecord(publisher, "publisher")
+    for (const [field, value] of Object.entries({ id: publisher.id, name: publisher.name })) {
+      if (typeof value !== "string" || !value.trim() || value !== value.trim())
+        throw new TypeError(`[Sixb] Model publisher.${field} must be a nonempty trimmed string.`)
+    }
+  }
   assertOptionalString(definition.family, "family")
   assertOptionalString(definition.releaseDate, "releaseDate")
   assertOptionalString(definition.knowledgeCutoff, "knowledgeCutoff")
@@ -52,6 +67,10 @@ export function defineLanguageModel(definition: LanguageModelDefinition): Langua
     modelId: definition.modelId,
     ...(definition.name === undefined ? {} : { name: definition.name }),
     ...(definition.description === undefined ? {} : { description: definition.description }),
+    ...(publisher === undefined
+      ? {}
+      : { publisher: Object.freeze({ id: publisher.id, name: publisher.name }) }),
+    ...(definition.via === undefined ? {} : { via: definition.via }),
     ...(definition.family === undefined ? {} : { family: definition.family }),
     ...(tags === undefined ? {} : { tags }),
     ...(definition.releaseDate === undefined ? {} : { releaseDate: definition.releaseDate }),

package/src/models/pricing.ts

@@ -335,8 +335,12 @@ function partitionInputUsage(usage: ModelUsage, rateCard: LanguageModelRateCard)
     ...(rateCard.cacheReadInput === undefined ? {} : { cacheRead }),
     ...(hasSpecificWrites
       ? {
-          cacheWrite5m: usage.cacheWrite5mInputTokens ?? 0,
-          cacheWrite1h: usage.cacheWrite1hInputTokens ?? 0,
+          ...(rateCard.cacheWriteInput5m !== undefined || rateCard.cacheWriteInput !== undefined
+            ? { cacheWrite5m: usage.cacheWrite5mInputTokens ?? 0 }
+            : {}),
+          ...(rateCard.cacheWriteInput1h !== undefined || rateCard.cacheWriteInput !== undefined
+            ? { cacheWrite1h: usage.cacheWrite1hInputTokens ?? 0 }
+            : {}),
         }
       : rateCard.cacheWriteInput === undefined
         ? {}

package/src/pipelines/run-dispatch.ts

@@ -101,7 +101,10 @@ export async function dispatchPipelineRun(
   input: DispatchPipelineRunInput
 ): Promise<PipelineRunRequestResult> {
   const runId = createPipelineRunId(input.runId)
-  const persisted = await persistPipelineRun(input, runId)
+  let persisted = await persistPipelineRun(input, runId)
+  if (!persisted.publish && input.validateExistingRun && persisted.run.status === "queued") {
+    persisted = await reusePipelineRun(input, persisted.run)
+  }
   if (!persisted.publish) return existingPipelineRunResult(persisted.run)
   return publishPipelineRun(input, persisted)
 }
@@ -155,6 +158,15 @@ async function reusePipelineRun(
 ): Promise<PersistedPipelineRun> {
   assertExistingRunMatchesRequest(existing, input)
   await input.validateExistingRun?.(existing, input.storage)
+  if (input.validateExistingRun && existing.status === "queued") {
+    const execution = await input.storage.executions.getById({
+      projectId: input.projectId,
+      id: existing.executionId,
+    })
+    if (!execution)
+      throw new PipelineRunError(`[Sixb] Execution '${existing.executionId}' was not found.`)
+    return { publish: true, run: existing, execution, queuedAt: existing.queuedAt, created: false }
+  }
   if (
     existing.status !== "failed" ||
     existing.error?.code !== "queue.enqueue_failed" ||
@@ -200,6 +212,11 @@ async function publishPipelineRun(
         },
       ],
     })
+    if (!job || job.id !== persisted.run.id) {
+      throw new PipelineRunError(
+        `[Sixb] Queue did not acknowledge Pipeline run '${persisted.run.id}'.`
+      )
+    }
     return {
       pipelineId: persisted.run.pipelineId,
       runId: persisted.run.id,
@@ -208,6 +225,9 @@ async function publishPipelineRun(
       created: persisted.created,
     }
   } catch (error) {
+    // Automatic consumers retain the batch and retry. Keep uncertain delivery queued:
+    // a lost reply does not prove that the queue rejected the job.
+    if (input.validateExistingRun) throw error
     const failedAt = new Date()
     const failure = toEnqueueFailure(error, persisted.run, failedAt)
     const failed = await requirePipelineRunStorage(input.storage).finish({