@sixb/core 0.1.10 → 0.1.12

package/src/materializer/edits/vectors.ts

@@ -0,0 +1,100 @@
+import {
+  MaterializationConflictError,
+  MaterializationValidationError,
+} from "../../materialization/errors"
+import type { OntologyEditCommit } from "../../materialization/model"
+import { objectRefKey } from "../../materialization/refs"
+import { normalizeVector, vectorConfiguration, vectorSources } from "../../objects/vectors/profile"
+import type { Storage } from "../../storage"
+import { objectBatchKey } from "../../storage/objects/keys"
+import type { MaterializationSession, OntologyVectorStorage } from "../../storage/ontology"
+import type { MaterializerContext } from "../context"
+
+export async function commitVectorWrites(
+  context: MaterializerContext,
+  storage: Storage,
+  input: OntologyEditCommit,
+  commitId: string,
+  session: MaterializationSession
+): Promise<void> {
+  if (input.mode !== "atomic" || !input.vectorWrites?.length) return
+  const vectors = storage.ontology.vectors
+  if (!vectors)
+    throw new MaterializationValidationError("Storage does not support vector profiles.")
+  const rows = await storage.objects.getByPrimaryIdBatch({
+    projectId: context.projectId,
+    items: input.vectorWrites.map((write) => write.input.ref),
+  })
+  const expectations = new Map(
+    input.expectedObjects.map((expected) => [objectRefKey(expected.ref), expected])
+  )
+  const entries: Parameters<OntologyVectorStorage["writeBatch"]>[0]["entries"][number][] = []
+  const seen = new Set<string>()
+  for (const write of input.vectorWrites) {
+    const prepared = write.input
+    const key = JSON.stringify([objectRefKey(prepared.ref), prepared.profile])
+    if (seen.has(key)) throw new MaterializationValidationError(`Duplicate vector write: ${key}`)
+    seen.add(key)
+    const profileDefinition = context.ontology.resolveObjectType(prepared.ref.objectTypeId).search
+      ?.vectors?.[prepared.profile]
+    if (
+      prepared.projectId !== context.projectId ||
+      !profileDefinition ||
+      vectorConfiguration(profileDefinition) !== prepared.configuration
+    ) {
+      throw new MaterializationConflictError(
+        "effective-state",
+        "Vector profile configuration changed; index it again."
+      )
+    }
+    const expected = expectations.get(objectRefKey(prepared.ref))
+    if (
+      !expected?.exists ||
+      !prepared.expectedObject?.exists ||
+      objectRefKey(prepared.expectedObject.ref) !== objectRefKey(prepared.ref) ||
+      expected.version !== prepared.expectedObject.version ||
+      expected.lastCommitId !== prepared.expectedObject.lastCommitId
+    ) {
+      throw new MaterializationValidationError(
+        "Vector writes must fence their prepared object revision."
+      )
+    }
+    const row = rows.get(objectBatchKey(prepared.ref.objectTypeId, prepared.ref.primaryId))
+    if (
+      !row ||
+      row.version !== expected.version ||
+      row.lastCommitId !== expected.lastCommitId ||
+      vectorSources(profileDefinition.source, row.properties).sourceFingerprint !==
+        prepared.sourceFingerprint
+    ) {
+      throw new MaterializationConflictError(
+        "effective-state",
+        "Vector sources changed; index the profile again."
+      )
+    }
+    entries.push({
+      expectedCommitId: prepared.expectedVectorCommitId,
+      value: {
+        ref: prepared.ref,
+        profile: prepared.profile,
+        configuration: prepared.configuration,
+        source: profileDefinition.source,
+        sourceFingerprint: prepared.sourceFingerprint,
+        values: normalizeVector(write.values, profileDefinition.model.definition.dimensions),
+        lastCommitId: commitId,
+      },
+    })
+  }
+
+  await vectors.writeBatch({ session, projectId: context.projectId, entries })
+  await storage.ontology.vectorIndexing?.complete({
+    session,
+    projectId: context.projectId,
+    entries: input.vectorWrites.map(({ input: prepared }) => ({
+      ref: prepared.ref,
+      profile: prepared.profile,
+      configuration: prepared.configuration,
+      sourceFingerprint: prepared.sourceFingerprint,
+    })),
+  })
+}

package/src/materializer/effective/validate.ts

@@ -1,3 +1,4 @@
+import { withFailureMessage } from "../../errors/failure-message"
 import type { JsonValue } from "../../json"
 import { MaterializationValidationError } from "../../materialization/errors"
 import type {
@@ -113,8 +114,11 @@ export function validateEffectiveObject(
         property.required &&
         properties[property.id] === undefined
       ) {
-        throw new Error(
-          `[Sixb] Missing required property '${property.id}' for object type '${objectType.id}'`
+        throw withFailureMessage(
+          new Error(
+            `[Sixb] Missing required property '${property.id}' for object type '${objectType.id}'`
+          ),
+          `Missing required property '${objectType.id}.${property.id}'.`
         )
       }
     }
@@ -175,6 +179,9 @@ function materializationValidation<T>(run: () => T): T {
     return run()
   } catch (error) {
     if (error instanceof MaterializationValidationError) throw error
-    throw new MaterializationValidationError(error instanceof Error ? error.message : String(error))
+    throw new MaterializationValidationError(
+      error instanceof Error ? error.message : String(error),
+      { cause: error }
+    )
   }
 }

package/src/materializer/effective/vector-indexing.ts

@@ -0,0 +1,84 @@
+import { randomUUID } from "node:crypto"
+import type { OntologyObjectRef } from "../../materialization/model"
+import { vectorConfiguration, vectorSources } from "../../objects/vectors/profile"
+import type { Storage } from "../../storage"
+import { objectBatchKey } from "../../storage/objects/keys"
+import type { MaterializationSession } from "../../storage/ontology"
+import type { VectorIndexingRequest } from "../../storage/ontology/vector-indexing"
+import type { MaterializerContext } from "../context"
+import type { MaterializationPlanItem } from "../execution/plan-stream"
+
+/** Source assertions are not effective values: schedule only after conflict resolution. */
+export async function scheduleVectorChanges(
+  context: Pick<MaterializerContext, "projectId" | "ontology" | "clock">,
+  storage: Storage,
+  items: readonly MaterializationPlanItem[],
+  session: MaterializationSession,
+  projection = false
+): Promise<void> {
+  const indexing = storage.ontology.vectorIndexing
+  if (!indexing) return
+  const upserts = items
+    .filter((item) => item.kind === "object-upsert")
+    .filter(
+      (item) =>
+        Object.keys(
+          context.ontology.resolveObjectType(item.value.row.ref.objectTypeId).search?.vectors ?? {}
+        ).length > 0
+    )
+  const previous = await storage.objects.getByPrimaryIdBatch({
+    projectId: context.projectId,
+    items: upserts.map((item) => item.value.row.ref),
+  })
+  const requests: VectorIndexingRequest[] = []
+  const deleted: OntologyObjectRef[] = []
+  // Groups exist only for this materialization page and are persisted in its transaction.
+  const groups = new Map<string, { id: string; count: number; bytes: number }>()
+  for (const item of items) {
+    if (item.kind === "object-delete") {
+      deleted.push(item.value.ref)
+      continue
+    }
+    if (item.kind !== "object-upsert") continue
+    const row = item.value.row
+    const profiles = context.ontology.resolveObjectType(row.ref.objectTypeId).search?.vectors ?? {}
+    const before = previous.get(objectBatchKey(row.ref.objectTypeId, row.ref.primaryId))
+    for (const [profile, definition] of Object.entries(profiles)) {
+      const { sourceFingerprint, text } = vectorSources(definition.source, row.properties)
+      if (
+        before &&
+        vectorSources(definition.source, before.properties).sourceFingerprint === sourceFingerprint
+      )
+        continue
+      const configuration = vectorConfiguration(definition)
+      const bytes = Buffer.byteLength(text)
+      const key = JSON.stringify([row.ref.objectTypeId, profile, configuration, row.lastCommitId])
+      let group = projection ? groups.get(key) : undefined
+      if (projection && (!group || group.count === 32 || group.bytes + bytes > 32_768)) {
+        group = { id: randomUUID(), count: 0, bytes: 0 }
+        groups.set(key, group)
+      }
+      if (group) {
+        group.count++
+        group.bytes += bytes
+      }
+      requests.push({
+        ...(group ? { batchId: group.id } : {}),
+        id: randomUUID(),
+        ref: row.ref,
+        profile,
+        sourceFingerprint,
+        configuration,
+        sourceCommitId: row.lastCommitId,
+      })
+    }
+  }
+  if (requests.length || deleted.length)
+    await indexing.schedule({
+      projectId: context.projectId,
+      session,
+      requests,
+      deleted,
+      availableAt: context.clock().toISOString(),
+    })
+}

package/src/materializer/effective/vectors.ts

@@ -0,0 +1,47 @@
+import { objectRefKey } from "../../materialization/refs"
+import { vectorSources } from "../../objects/vectors/profile"
+import type { MaterializationSession } from "../../storage/ontology"
+import type { OntologyVectorStorage } from "../../storage/ontology/vectors"
+import type { MaterializationPlanItem } from "../execution/plan-stream"
+
+/** Invalidate a bounded materialization page, including stored profiles no longer declared. */
+export async function invalidateVectorChanges(
+  projectId: string,
+  vectors: OntologyVectorStorage | undefined,
+  items: readonly MaterializationPlanItem[],
+  session: MaterializationSession
+): Promise<void> {
+  if (!vectors) return
+  const changes = new Map(
+    items.flatMap((item) => {
+      if (item.kind !== "object-upsert" && item.kind !== "object-delete") return []
+      const ref = item.kind === "object-upsert" ? item.value.row.ref : item.value.ref
+      return [[objectRefKey(ref), { ref, item }] as const]
+    })
+  )
+  if (!changes.size) return
+
+  const stored = await vectors.listBatch({
+    projectId,
+    refs: [...changes.values()].map(({ ref }) => ref),
+  })
+  const removals: Parameters<OntologyVectorStorage["removeBatch"]>[0]["entries"][number][] = []
+  for (const entry of stored) {
+    const change = changes.get(objectRefKey(entry.ref))
+    if (!change) continue
+    const { item } = change
+    if (
+      item.kind === "object-upsert" &&
+      entry.source.every((id) => {
+        const value = item.value.row.properties[id]
+        return value === undefined || value === null || typeof value === "string"
+      }) &&
+      vectorSources(entry.source, item.value.row.properties).sourceFingerprint ===
+        entry.sourceFingerprint
+    )
+      continue
+
+    removals.push({ ref: entry.ref, profile: entry.profile, expectedCommitId: entry.lastCommitId })
+  }
+  if (removals.length) await vectors.removeBatch({ session, projectId, entries: removals })
+}

package/src/materializer/execution/work-executor.ts

@@ -1,5 +1,6 @@
 import { MaterializationValidationError } from "../../materialization/errors"
 import { utf8JsonByteLength } from "../../materialization/refs"
+import type { Storage } from "../../storage"
 import type {
   MaterializationSession,
   MaterializationWorkRecord,
@@ -7,6 +8,8 @@ import type {
 } from "../../storage/ontology"
 import type { MaterializerContext } from "../context"
 import { sequenceMaterializationEvent } from "../effective/build-events"
+import { scheduleVectorChanges } from "../effective/vector-indexing"
+import { invalidateVectorChanges } from "../effective/vectors"
 import { throwIfAborted } from "../shared/abort"
 import { chunkBySize } from "../shared/chunking"
 import type { TimedCommitIdentity } from "../shared/identity"
@@ -46,15 +49,20 @@ export async function stageWorkBounded(
 }
 
 export async function drainStagedWork(
-  context: BatchingContext,
-  storage: OntologyMaterializationStorage,
+  context: Pick<MaterializerContext, "batching" | "projectId" | "ontology" | "clock">,
+  transactionStorage: Storage,
   session: MaterializationSession,
-  signal?: AbortSignal
+  signal?: AbortSignal,
+  projection = false
 ): Promise<void> {
+  const ontologyStorage = transactionStorage.ontology
+  const storage = ontologyStorage.materializations
   let phase: number | null = null
   let pending: MaterializationPlanItem[] = []
   const flush = async () => {
     if (pending.length === 0) return
+    await scheduleVectorChanges(context, transactionStorage, pending, session, projection)
+    await invalidateVectorChanges(context.projectId, ontologyStorage.vectors, pending, session)
     await applyItems(context, storage, session, pending, signal)
     pending = []
   }

package/src/materializer/projections/replace.ts

@@ -347,7 +347,7 @@ async function executeProjectionTransaction(
     ready,
     origin
   )
-  await drainStagedWork(context, storage.ontology.materializations, session, command.signal)
+  await drainStagedWork(context, storage, session, command.signal, true)
   const eventCount = await drainStagedEvents(
     context,
     storage.ontology.materializations,

package/src/materializer/shared/normalize.ts

@@ -269,6 +269,8 @@ export function normalizeOntologyEditCommit(input: OntologyEditCommit): Ontology
   })
 
   if (input.mode === "continue") {
+    if ("vectorWrites" in input)
+      throw new MaterializationValidationError("Vector writes require an atomic commit.")
     return Object.freeze({
       mode: "continue",
       source: Object.freeze({
@@ -293,12 +295,21 @@ export function normalizeOntologyEditCommit(input: OntologyEditCommit): Ontology
           kind: "runtime" as const,
           requestId: normalizeNonblank(input.source.requestId, "Runtime request id"),
         })
+  if (input.vectorWrites?.length && (operations.length || input.source.kind !== "runtime")) {
+    throw new MaterializationValidationError(
+      "Vector writes require a dedicated atomic runtime commit."
+    )
+  }
+  if (input.vectorWrites !== undefined) assertJsonValue(input.vectorWrites, "Vector writes")
   const expectedObjects = input.expectedObjects.map(normalizeExpectedObject)
   const expectedLinks = input.expectedLinks.map(normalizeExpectedLink)
   const expectedLinkScopes = input.expectedLinkScopes.map(normalizeExpectedLinkScope)
 
   return Object.freeze({
     mode: "atomic",
+    ...(input.vectorWrites === undefined
+      ? {}
+      : { vectorWrites: structuredClone(input.vectorWrites) }),
     source,
     operations: Object.freeze(operations),
     expectedObjects: deduplicateExpectations(

package/src/materializer/telemetry/append.ts

@@ -404,7 +404,7 @@ async function executeTelemetryTransaction(
       ...(command.execution.actor === undefined ? {} : { actor: command.execution.actor }),
     }
   )
-  await drainStagedWork(context, storage.ontology.materializations, session)
+  await drainStagedWork(context, storage, session)
   const eventCount = await drainStagedEvents(
     context,
     storage.ontology.materializations,

package/src/models/catalog-index.ts

@@ -0,0 +1,38 @@
+import { RuntimeError } from "../runtime/errors"
+import type { ModelRef } from "./catalog"
+
+interface ModelBinding {
+  readonly providerId: string
+  readonly modelId: string
+}
+
+interface ModelEntry<TModel> extends ModelRef {
+  readonly model: TModel
+}
+
+/** Private index shared by model families; default selection belongs to each family. */
+export function indexModelBindings<TModel extends ModelBinding>(
+  models: readonly TModel[],
+  kind: string,
+  validate: (model: TModel, index: number) => void
+) {
+  const byRef = new Map<string, ModelEntry<TModel>>()
+
+  for (const [index, model] of models.entries()) {
+    validate(model, index)
+    const key = JSON.stringify([model.providerId, model.modelId])
+    if (byRef.has(key)) {
+      throw new RuntimeError(
+        `[Sixb] Duplicate ${kind} model '${model.providerId}/${model.modelId}' in 'models.${kind}'. Each provider and model id pair may be configured once.`
+      )
+    }
+
+    byRef.set(key, Object.freeze({ provider: model.providerId, modelId: model.modelId, model }))
+  }
+
+  const entries = Object.freeze([...byRef.values()])
+  return Object.freeze({
+    list: () => entries,
+    getByRef: (ref: ModelRef) => byRef.get(JSON.stringify([ref.provider, ref.modelId])) ?? null,
+  })
+}

package/src/models/catalog.ts

@@ -1,17 +1,32 @@
 import { RuntimeError } from "../runtime/errors"
+import { indexModelBindings } from "./catalog-index"
+import { createDecisionCatalog, type DecisionModelCatalog } from "./decision/catalog"
+import type { DecisionModel } from "./decision/types"
 import {
   defineLanguageModel,
   type LanguageModelDefinition,
   type ModelDefinition,
 } from "./definitions"
+import { assertEmbeddingModel, type EmbeddingModel } from "./embedding-model"
 import type { LanguageModel } from "./language-model"
 
-/** Stable identity of one configured language-model binding. */
-export interface LanguageModelRef {
+/** Stable identity of a configured provider binding. */
+export interface ModelRef {
   readonly provider: string
   readonly modelId: string
 }
 
+export type LanguageModelRef = ModelRef
+
+export interface EmbeddingModelEntry extends ModelRef {
+  readonly model: EmbeddingModel
+}
+
+export interface EmbeddingModelCatalog {
+  list(): readonly EmbeddingModelEntry[]
+  getByRef(ref: ModelRef): EmbeddingModelEntry | null
+}
+
 /** The configured language models, with the project default. */
 export interface LanguageModelCatalog {
   readonly default: LanguageModelEntry
@@ -32,61 +47,74 @@ export interface LanguageModelEntry extends LanguageModelRef {
 
 /** Models a project allows Sixb to use, organized by technical model kind. */
 export interface ModelCatalog {
-  readonly language: LanguageModelCatalog
+  readonly language?: LanguageModelCatalog
+  readonly decision?: DecisionModelCatalog
+  readonly embedding: EmbeddingModelCatalog
 }
 
 export interface ModelCatalogInput {
   /** Ordered; the first entry is the project default. */
-  readonly language: readonly LanguageModel[]
+  readonly language?: readonly LanguageModel[]
+  /** Ordered; the first entry is the project default decision model. */
+  readonly decision?: readonly DecisionModel[]
+  readonly embedding?: readonly EmbeddingModel[]
 }
 
-/** Build the immutable project model catalog. Rejects invalid, duplicate, and empty catalogs. */
+// Only statically required inputs make their corresponding catalogs required.
+type ConfiguredModelKinds<TInput> = {
+  [Kind in keyof ModelCatalog]-?: [TInput] extends [Record<Kind, readonly unknown[]>] ? Kind : never
+}[keyof ModelCatalog]
+
+export type ModelCatalogFor<TInput extends ModelCatalogInput> = ModelCatalog &
+  Required<Pick<ModelCatalog, ConfiguredModelKinds<TInput>>>
+
+/** Build an immutable catalog while preserving each family's default and empty-list rules. */
+export function createModelCatalog<TInput extends ModelCatalogInput>(
+  input: TInput
+): ModelCatalogFor<TInput>
 export function createModelCatalog(input: ModelCatalogInput): ModelCatalog {
-  if (!Array.isArray(input?.language)) {
-    throw new RuntimeError("[Sixb] 'models.language' must be an array of Sixb language models.")
+  if (!input || typeof input !== "object" || Array.isArray(input)) {
+    throw new RuntimeError("[Sixb] 'models' must be a model catalog configuration.")
   }
 
-  const entries: LanguageModelEntry[] = []
-  const byProvider = new Map<string, Map<string, LanguageModelEntry>>()
+  const language = createLanguageCatalog(input.language)
+  const embedding = createEmbeddingCatalog(input.embedding)
+  const decision = createDecisionCatalog(input.decision)
 
-  for (const [index, model] of input.language.entries()) {
-    assertLanguageModel(model, index)
+  if (!language && !decision && embedding.list().length === 0) {
+    throw new RuntimeError("[Sixb] Configure at least one language, embedding or decision model.")
+  }
 
-    let byModelId = byProvider.get(model.providerId)
-    if (byModelId === undefined) {
-      byModelId = new Map()
-      byProvider.set(model.providerId, byModelId)
-    }
-    if (byModelId.has(model.modelId)) {
-      throw new RuntimeError(
-        `[Sixb] Duplicate language model '${model.providerId}/${model.modelId}' in 'models'. Each provider and model id pair may be configured once.`
-      )
-    }
+  return Object.freeze({ language, embedding, decision })
+}
 
-    const entry = Object.freeze({
-      provider: model.providerId,
-      modelId: model.modelId,
-      model,
-    })
-    byModelId.set(model.modelId, entry)
-    entries.push(entry)
+function createLanguageCatalog(
+  models: readonly LanguageModel[] | undefined
+): LanguageModelCatalog | undefined {
+  if (models === undefined) return undefined
+  if (!Array.isArray(models)) {
+    throw new RuntimeError("[Sixb] 'models.language' must be an array of Sixb language models.")
   }
 
-  const [defaultEntry] = entries
-  if (defaultEntry === undefined) {
+  const index = indexModelBindings(models, "language", assertLanguageModel)
+  const [defaultEntry] = index.list()
+  if (!defaultEntry) {
     throw new RuntimeError(
-      "[Sixb] 'models.language' needs at least one model. Configure one or omit 'models' from createSixb()."
+      "[Sixb] 'models.language' needs at least one model. Configure one or omit 'models.language'."
     )
   }
 
-  const listed = Object.freeze(entries.slice())
-  const language: LanguageModelCatalog = Object.freeze({
-    default: defaultEntry,
-    list: () => listed,
-    getByRef: (ref: LanguageModelRef) => byProvider.get(ref.provider)?.get(ref.modelId) ?? null,
-  })
+  return Object.freeze({ ...index, default: defaultEntry })
+}
+
+function createEmbeddingCatalog(
+  models: readonly EmbeddingModel[] | undefined
+): EmbeddingModelCatalog {
+  if (models !== undefined && !Array.isArray(models)) {
+    throw new RuntimeError("[Sixb] models.embedding must be an array of embedding models.")
+  }
 
-  return Object.freeze({ language })
+  return indexModelBindings(models ?? [], "embedding", assertEmbeddingModel)
 }
 
 function assertLanguageModel(model: unknown, index: number): asserts model is LanguageModel {

package/src/models/decision/catalog.ts

@@ -0,0 +1,32 @@
+import { RuntimeError } from "../../runtime/errors"
+import type { ModelRef } from "../catalog"
+import { indexModelBindings } from "../catalog-index"
+import type { DecisionModel } from "./types"
+import { assertDecisionModel } from "./validation"
+
+export interface DecisionModelEntry extends ModelRef {
+  readonly model: DecisionModel
+}
+
+export interface DecisionModelCatalog {
+  readonly default: DecisionModelEntry
+  list(): readonly DecisionModelEntry[]
+  getByRef(ref: ModelRef): DecisionModelEntry | null
+}
+
+export function createDecisionCatalog(
+  models: readonly DecisionModel[] | undefined
+): DecisionModelCatalog | undefined {
+  if (models === undefined) return undefined
+  if (!Array.isArray(models)) {
+    throw new RuntimeError("[Sixb] models.decision must be a nonempty array of decision models.")
+  }
+
+  const index = indexModelBindings(models, "decision", assertDecisionModel)
+  const [defaultEntry] = index.list()
+  if (!defaultEntry) {
+    throw new RuntimeError("[Sixb] models.decision must be a nonempty array of decision models.")
+  }
+
+  return Object.freeze({ ...index, default: defaultEntry })
+}

package/src/models/decision/errors.ts

@@ -0,0 +1,18 @@
+import { ModelProviderError } from "../errors"
+import type { DecisionModelResponseMetadata } from "./types"
+
+export class DecisionModelResponseError extends ModelProviderError {
+  constructor(
+    message: string,
+    providerId: string,
+    modelId: string,
+    readonly metadata: DecisionModelResponseMetadata,
+    options?: ErrorOptions
+  ) {
+    super(message, providerId, modelId, {
+      ...options,
+      code: "invalid_decision_response",
+      retryable: false,
+    })
+  }
+}