@sixb/core 0.1.11 → 0.1.13

package/src/objects/vectors/indexing-batch.ts

@@ -0,0 +1,163 @@
+import type { OntologyMaterializerContract } from "../../materializer/materializer"
+import type { EmbeddingBatchLimits } from "../../models/embedding-model"
+import type { SixbHostContext } from "../../runtime/types"
+import type { VectorIndexingWork } from "../../storage/ontology/vector-indexing"
+import { generateVectors, type PreparedIndexingWork } from "./indexing-generate"
+import { indexingScope, prepare, VectorIndexingDeferred } from "./indexing-shared"
+import { processVector } from "./indexing-single"
+import { storeVectorBatch } from "./indexing-store-batch"
+
+/** Membership was persisted by one projection commit; there is no fill timer or memory queue. */
+export async function processVectorBatch(
+  runtime: SixbHostContext,
+  materializer: OntologyMaterializerContract,
+  batchId: string,
+  batch: readonly VectorIndexingWork[],
+  attempt: number,
+  signal: AbortSignal
+): Promise<void> {
+  if (batch.some((work) => work.status !== "pending")) {
+    await resumeBatch(runtime, materializer, batchId, batch, attempt, signal)
+    return
+  }
+
+  const entries = await prepareBatch(runtime, batch, signal)
+  const first = entries[0]
+  if (!first) return
+
+  const limits = getBatchLimits(runtime, first.work)
+  if (!limits) {
+    // Unknown provider batch limits: embed each object/profile individually.
+    for (const { work } of entries) {
+      await processVector(runtime, materializer, work.id, attempt, signal)
+    }
+    return
+  }
+
+  assertCompatibleEntries(entries, first.work)
+  const scope = await indexingScope(runtime, first.work, batchId)
+
+  for (const group of partition(entries, limits)) {
+    if (group.length === 1) {
+      await processVector(runtime, materializer, group[0]!.work.id, attempt, signal)
+    } else {
+      await generateVectors(runtime, group, scope, attempt, signal)
+    }
+  }
+
+  await storeVectorBatch(runtime, materializer, batchId, scope, signal)
+}
+
+/** Resume storage or terminalize interrupted calls; never repeat paid inference. */
+async function resumeBatch(
+  runtime: SixbHostContext,
+  materializer: OntologyMaterializerContract,
+  batchId: string,
+  batch: readonly VectorIndexingWork[],
+  attempt: number,
+  signal: AbortSignal
+): Promise<void> {
+  const ready = batch.find((work) => work.status === "ready")
+  if (ready) {
+    const scope = await indexingScope(runtime, ready, batchId)
+    await storeVectorBatch(runtime, materializer, batchId, scope, signal)
+  }
+
+  for (const work of batch) {
+    if (work.status !== "ready") {
+      await processVector(runtime, materializer, work.id, attempt, signal)
+    }
+  }
+}
+
+async function prepareBatch(
+  runtime: SixbHostContext,
+  batch: readonly VectorIndexingWork[],
+  signal: AbortSignal
+): Promise<PreparedIndexingWork[]> {
+  const availableAt = batch.reduce(
+    (latest, work) => (work.availableAt > latest ? work.availableAt : latest),
+    ""
+  )
+  if (Date.parse(availableAt) > Date.now()) {
+    throw new VectorIndexingDeferred(availableAt)
+  }
+
+  const entries: PreparedIndexingWork[] = []
+  const indexing = runtime.storage.ontology.vectorIndexing!
+
+  for (const work of batch) {
+    signal.throwIfAborted()
+    const input = await prepare(runtime, work)
+
+    if (input) {
+      entries.push({ work, input })
+    } else {
+      await indexing.remove({ projectId: runtime.projectId, id: work.id })
+    }
+  }
+
+  return entries
+}
+
+function getBatchLimits(
+  runtime: SixbHostContext,
+  work: VectorIndexingWork
+): EmbeddingBatchLimits | undefined {
+  const profile = runtime.ontology.resolveObjectType(work.ref.objectTypeId).search!.vectors![
+    work.profile
+  ]!
+
+  return runtime.embeddingModels?.getByRef({
+    provider: profile.model.providerId,
+    modelId: profile.model.modelId,
+  })?.model.batching
+}
+
+/** Do not mix commits or configurations, even with malformed storage membership. */
+function assertCompatibleEntries(
+  entries: readonly PreparedIndexingWork[],
+  reference: VectorIndexingWork
+): void {
+  const incompatible = entries.some(
+    ({ work }) =>
+      work.sourceCommitId !== reference.sourceCommitId ||
+      work.ref.objectTypeId !== reference.ref.objectTypeId ||
+      work.profile !== reference.profile ||
+      work.configuration !== reference.configuration
+  )
+
+  if (incompatible) {
+    throw new Error("[Sixb] Incompatible vector batch membership.")
+  }
+}
+
+/** Split an already durable, bounded page; never wait for more work or truncate input. */
+function* partition(entries: readonly PreparedIndexingWork[], limits: EmbeddingBatchLimits) {
+  let group: PreparedIndexingWork[] = []
+  let bytes = 0
+
+  for (const entry of entries) {
+    const size = Buffer.byteLength(entry.input.text)
+    const exceedsGroupLimit =
+      group.length >= limits.maxInputs ||
+      bytes + size > limits.maxTotalInputBytes ||
+      size > limits.maxInputBytes
+
+    if (group.length > 0 && exceedsGroupLimit) {
+      yield group
+      group = []
+      bytes = 0
+    }
+
+    if (size > limits.maxInputBytes || size > limits.maxTotalInputBytes) {
+      yield [entry]
+      continue
+    }
+
+    group.push(entry)
+    bytes += size
+  }
+
+  if (group.length > 0) yield group
+}

package/src/objects/vectors/indexing-generate.ts

@@ -0,0 +1,184 @@
+import { createSixbError, isSixbError } from "../../errors/internal"
+import type { ExecutionScope } from "../../execution/types"
+import type { PreparedObjectVector } from "../../materialization/vectors"
+import {
+  EmbeddingModelResponseError,
+  type EmbeddingModelResult,
+  sameEmbeddingModel,
+} from "../../models/embedding-model"
+import { bindEmbeddingModels } from "../../models/execution/embedding"
+import { ModelExecutionSession } from "../../models/execution/session"
+import type { SixbHostContext } from "../../runtime/types"
+import type { VectorIndexingWork } from "../../storage/ontology/vector-indexing"
+import { fail, retryAt, VectorIndexingDeferred } from "./indexing-shared"
+import { normalizeVector } from "./profile"
+
+export interface PreparedIndexingWork {
+  readonly work: VectorIndexingWork
+  readonly input: PreparedObjectVector
+}
+
+interface GenerationAttempt {
+  readonly runtime: SixbHostContext
+  readonly entries: readonly PreparedIndexingWork[]
+  readonly signal: AbortSignal
+  /** Running means the durable claim succeeded; budget admission may still be pending. */
+  phase: "pending" | "claiming" | "running"
+}
+
+/** One accounted provider call, with durable admission and atomic result persistence. */
+export async function generateVectors(
+  runtime: SixbHostContext,
+  entries: readonly PreparedIndexingWork[],
+  scope: ExecutionScope,
+  attempt: number,
+  signal: AbortSignal
+): Promise<void> {
+  const first = entries[0]!
+  const profile = runtime.ontology.resolveObjectType(first.work.ref.objectTypeId).search!.vectors![
+    first.work.profile
+  ]!
+  const generation: GenerationAttempt = { runtime, entries, signal, phase: "pending" }
+
+  const session = new ModelExecutionSession(
+    { ...runtime, runtimeAuthorization: scope.authorization },
+    scope.execution
+  )
+  session.bind({
+    attempt,
+    signal,
+    embeddingAdmission: (admit) => admitGeneration(generation, admit),
+  })
+
+  try {
+    const model = bindEmbeddingModels(runtime.embeddingModels, session)?.getByRef({
+      provider: profile.model.providerId,
+      modelId: profile.model.modelId,
+    })?.model
+
+    if (!model || !sameEmbeddingModel(model, profile.model)) {
+      throw createSixbError(
+        "vector.model_unavailable",
+        "[Sixb] Vector indexing requires the profile's configured embedding model."
+      )
+    }
+
+    const result = await model.embed({ texts: entries.map((entry) => entry.input.text), signal })
+    const values = normalizeEmbeddingResult(
+      result,
+      entries.length,
+      profile.model.definition.dimensions
+    )
+
+    // Persist every surviving result atomically before any vector is published.
+    await runtime.storage.ontology.vectorIndexing!.updateBatch({
+      projectId: runtime.projectId,
+      updates: entries.map(({ work }, index) => ({
+        id: work.id,
+        expectedStatus: "running",
+        status: "ready",
+        values: values[index]!,
+        availableAt: new Date().toISOString(),
+      })),
+    })
+  } catch (error) {
+    await handleGenerationFailure(generation, error)
+  }
+}
+
+async function admitGeneration(
+  generation: GenerationAttempt,
+  admit: () => Promise<void>
+): Promise<void> {
+  generation.signal.throwIfAborted()
+  generation.phase = "claiming"
+
+  const claimed = await transitionWork(generation, "pending", "running", { requireAll: true })
+  generation.phase = claimed ? "running" : "pending"
+
+  if (!claimed) {
+    if (generation.entries.length === 1) {
+      throw new Error("[Sixb] Vector indexing request was superseded.")
+    }
+    throw new VectorIndexingDeferred(new Date().toISOString())
+  }
+
+  try {
+    await admit()
+  } catch (error) {
+    await transitionWork(generation, "running", "pending")
+    generation.phase = "pending"
+    throw error
+  }
+}
+
+function normalizeEmbeddingResult(
+  result: EmbeddingModelResult,
+  count: number,
+  dimensions: number
+): readonly (readonly number[])[] {
+  try {
+    if (!Array.isArray(result?.vectors) || result.vectors.length !== count) {
+      throw new Error("Unexpected vector count")
+    }
+
+    return result.vectors.map((vector) => normalizeVector(vector, dimensions))
+  } catch (cause) {
+    throw createSixbError("vector.response_invalid", "[Sixb] Invalid embedding batch response.", {
+      cause,
+    })
+  }
+}
+
+async function handleGenerationFailure(
+  generation: GenerationAttempt,
+  error: unknown
+): Promise<void> {
+  // No provider call can start while the storage claim is unresolved. Retry delivery,
+  // not inference; an ambiguously committed claim will be fenced as running on redelivery.
+  if (generation.phase === "claiming" || error instanceof VectorIndexingDeferred) {
+    throw error
+  }
+
+  if (generation.phase === "pending") {
+    if (
+      isSixbError(error) &&
+      (error.code === "ai.usage_limit_exceeded" || error.code === "ai.usage_limit_unavailable")
+    ) {
+      const availableAt = retryAt(error.details)
+      await transitionWork(generation, "pending", "pending", { availableAt })
+      throw new VectorIndexingDeferred(availableAt)
+    }
+
+    if (generation.signal.aborted) throw error
+  }
+
+  const failure =
+    error instanceof EmbeddingModelResponseError
+      ? createSixbError("vector.response_invalid", "[Sixb] Invalid embedding batch response.", {
+          cause: error,
+        })
+      : error
+  const { runtime, entries } = generation
+  const indexing = runtime.storage.ontology.vectorIndexing!
+
+  for (const { work } of entries) {
+    await fail(runtime, indexing, { ...work, status: generation.phase }, failure)
+  }
+}
+
+function transitionWork(
+  generation: GenerationAttempt,
+  expectedStatus: VectorIndexingWork["status"],
+  status: VectorIndexingWork["status"],
+  options: { availableAt?: string; requireAll?: boolean } = {}
+): Promise<boolean> {
+  const { runtime, entries } = generation
+  const availableAt = options.availableAt ?? new Date().toISOString()
+
+  return runtime.storage.ontology.vectorIndexing!.updateBatch({
+    projectId: runtime.projectId,
+    requireAll: options.requireAll ?? false,
+    updates: entries.map(({ work }) => ({ id: work.id, expectedStatus, status, availableAt })),
+  })
+}

package/src/objects/vectors/indexing-runtime.ts

@@ -0,0 +1,17 @@
+/** Internal worker entry point; it does not expose the host context or domain SDK. */
+export interface VectorIndexingRuntime {
+  process(id: string, attempt: number, signal: AbortSignal): Promise<void>
+}
+
+// Associates each host with its service. Durable work lives in ontology storage, not this map.
+const runtimes = new WeakMap<object, VectorIndexingRuntime>()
+
+export function registerVectorIndexingRuntime(host: object, runtime: VectorIndexingRuntime): void {
+  runtimes.set(host, runtime)
+}
+
+export function getVectorIndexingRuntime(host: object): VectorIndexingRuntime {
+  const runtime = runtimes.get(host)
+  if (!runtime) throw new Error("[Sixb] Vector indexing requires a configured Sixb host.")
+  return runtime
+}

package/src/objects/vectors/indexing-shared.ts

@@ -0,0 +1,188 @@
+import { randomUUID } from "node:crypto"
+import { reportVectorIndexingFailure } from "../../error-reporting/capability"
+import { captureSixbFailure } from "../../errors/internal"
+import { createKernelRuntimeAuthorization } from "../../execution/authorization"
+import { ensureExecutionRecord, executionRecordInputFromRuntime } from "../../execution/durable"
+import type { ExecutionContext, ExecutionScope } from "../../execution/types"
+import { MaterializationConflictError } from "../../materialization/errors"
+import type { ObjectVectorWrite, PreparedObjectVector } from "../../materialization/vectors"
+import type { OntologyMaterializerContract } from "../../materializer/materializer"
+import type { SixbHostContext } from "../../runtime/types"
+import {
+  type OntologyVectorIndexingStorage,
+  VECTOR_INDEXING_FAILURE_CODES,
+  type VectorIndexingWork,
+} from "../../storage/ontology/vector-indexing"
+import { vectorConfiguration, vectorSources } from "./profile"
+
+/** Return null when the requested representation is obsolete or already stored. */
+export async function prepare(
+  runtime: SixbHostContext,
+  work: VectorIndexingWork
+): Promise<PreparedObjectVector | null> {
+  const profile = runtime.ontology.getObjectTypeById(work.ref.objectTypeId)?.search?.vectors?.[
+    work.profile
+  ]
+  if (!profile || vectorConfiguration(profile) !== work.configuration) return null
+
+  const row = await runtime.storage.objects.getByPrimaryId({
+    projectId: runtime.projectId,
+    ...work.ref,
+  })
+  if (!row) return null
+
+  const sources = vectorSources(profile.source, row.properties)
+  if (sources.sourceFingerprint !== work.sourceFingerprint) return null
+
+  const storedVectors = await runtime.storage.ontology.vectors!.list({
+    projectId: runtime.projectId,
+    ref: work.ref,
+  })
+  const currentVector = storedVectors.find((entry) => entry.profile === work.profile)
+  const alreadyIndexed =
+    currentVector?.configuration === work.configuration &&
+    currentVector.sourceFingerprint === work.sourceFingerprint
+
+  if (alreadyIndexed) return null
+
+  return {
+    projectId: runtime.projectId,
+    ref: work.ref,
+    profile: work.profile,
+    configuration: work.configuration,
+    ...sources,
+    expectedObject: {
+      ref: work.ref,
+      exists: true,
+      version: row.version,
+      lastCommitId: row.lastCommitId,
+    },
+    expectedVectorCommitId: currentVector?.lastCommitId ?? null,
+  }
+}
+
+export async function storeVector(
+  runtime: SixbHostContext,
+  materializer: OntologyMaterializerContract,
+  work: VectorIndexingWork,
+  scope: ExecutionScope,
+  signal: AbortSignal
+): Promise<void> {
+  const indexing = runtime.storage.ontology.vectorIndexing!
+  const identity = { projectId: runtime.projectId, id: work.id }
+
+  // Unrelated property changes renew the object fence without repeating a paid inference.
+  for (let attempt = 0; attempt < 3; attempt++) {
+    signal.throwIfAborted()
+    const prepared = await prepare(runtime, work)
+
+    if (!prepared) {
+      await indexing.remove(identity)
+      return
+    }
+
+    try {
+      await commitPreparedVectors(materializer, scope, [{ input: prepared, values: work.values! }])
+      await indexing.remove(identity)
+      return
+    } catch (error) {
+      if (!(error instanceof MaterializationConflictError)) throw error
+
+      const currentWork = await indexing.get(identity)
+      if (!currentWork) return
+    }
+  }
+
+  throw new VectorIndexingDeferred(new Date(Date.now() + 1000).toISOString())
+}
+
+export async function commitPreparedVectors(
+  materializer: OntologyMaterializerContract,
+  scope: ExecutionScope,
+  writes: readonly ObjectVectorWrite[]
+): Promise<void> {
+  await materializer.edits.commit({
+    scope,
+    input: {
+      mode: "atomic",
+      source: { kind: "runtime", requestId: randomUUID() },
+      operations: [],
+      vectorWrites: writes,
+      expectedObjects: writes.map((write) => write.input.expectedObject),
+      expectedLinks: [],
+      expectedLinkScopes: [],
+    },
+  })
+}
+
+export async function indexingScope(
+  runtime: SixbHostContext,
+  work: VectorIndexingWork,
+  indexingId = work.id
+): Promise<ExecutionScope> {
+  const operation = { type: "ontology.indexVectors", indexingId } as const
+  const execution: ExecutionContext = {
+    id: `exec_vector_${indexingId}`,
+    projectId: runtime.projectId,
+    executor: { type: "kernel", operation },
+    source: { type: "ontologyCommit", commitId: work.sourceCommitId },
+    correlationId: `vector_${indexingId}`,
+  }
+  const authorization = createKernelRuntimeAuthorization({ execution, operation })
+
+  await ensureExecutionRecord(runtime.storage.executions, {
+    ...executionRecordInputFromRuntime({ execution, runtimeAuthorization: authorization }),
+    requesterGroupIds: [],
+  })
+
+  return { execution, authorization }
+}
+
+export async function fail(
+  runtime: SixbHostContext,
+  indexing: OntologyVectorIndexingStorage,
+  work: VectorIndexingWork,
+  error: unknown
+): Promise<void> {
+  const failure = captureSixbFailure(error, {
+    allowedCodes: VECTOR_INDEXING_FAILURE_CODES,
+    defaultCode: "internal.unexpected",
+  })
+
+  const recorded = await indexing.update({
+    projectId: runtime.projectId,
+    id: work.id,
+    expectedStatus: work.status,
+    status: "failed",
+    availableAt: new Date().toISOString(),
+    error: failure,
+  })
+
+  if (!recorded) return
+
+  reportVectorIndexingFailure(runtime, error, {
+    projectId: runtime.projectId,
+    indexingId: work.id,
+    failure,
+    ...work.ref,
+    profile: work.profile,
+  })
+}
+
+export function retryAt(details: unknown): string {
+  const resetAt =
+    details && typeof details === "object" && "resetAt" in details ? details.resetAt : undefined
+  const resetTimestamp = typeof resetAt === "string" ? Date.parse(resetAt) : Number.NaN
+
+  const minimumRetryTimestamp = Date.now() + 60_000
+  const budgetResetTimestamp = Number.isFinite(resetTimestamp) ? resetTimestamp : 0
+  const retryTimestamp = Math.max(minimumRetryTimestamp, budgetResetTimestamp)
+
+  return new Date(retryTimestamp).toISOString()
+}
+
+export class VectorIndexingDeferred extends Error {
+  constructor(readonly availableAt: string) {
+    super("[Sixb] Vector indexing is deferred.")
+  }
+}

package/src/objects/vectors/indexing-single.ts

@@ -0,0 +1,51 @@
+import { createSixbError } from "../../errors/internal"
+import type { OntologyMaterializerContract } from "../../materializer/materializer"
+import type { SixbHostContext } from "../../runtime/types"
+import { generateVectors } from "./indexing-generate"
+import {
+  fail,
+  indexingScope,
+  prepare,
+  storeVector,
+  VectorIndexingDeferred,
+} from "./indexing-shared"
+
+export async function processVector(
+  runtime: SixbHostContext,
+  materializer: OntologyMaterializerContract,
+  id: string,
+  attempt: number,
+  signal: AbortSignal
+): Promise<void> {
+  const indexing = runtime.storage.ontology.vectorIndexing
+  if (!indexing) throw new Error("[Sixb] Storage does not support automatic vector indexing.")
+  const projectId = runtime.projectId
+  const work = await indexing.get({ projectId, id })
+  if (!work || work.status === "failed") return
+  if (work.status === "running") {
+    await fail(
+      runtime,
+      indexing,
+      work,
+      createSixbError(
+        "vector.outcome_unknown",
+        "[Sixb] Interrupted embedding call has an unknown outcome; inference was not repeated."
+      )
+    )
+    return
+  }
+  if (Date.parse(work.availableAt) > Date.now()) throw new VectorIndexingDeferred(work.availableAt)
+  const prepared = await prepare(runtime, work)
+  if (!prepared) {
+    await indexing.remove({ projectId, id })
+    return
+  }
+  const scope = await indexingScope(runtime, work, work.batchId ?? work.id)
+  if (work.status === "ready") {
+    await storeVector(runtime, materializer, work, scope, signal)
+    return
+  }
+  await generateVectors(runtime, [{ work, input: prepared }], scope, attempt, signal)
+  const ready = await indexing.get({ projectId, id })
+  if (ready?.status === "ready") await storeVector(runtime, materializer, ready, scope, signal)
+}

package/src/objects/vectors/indexing-store-batch.ts

@@ -0,0 +1,62 @@
+import type { ExecutionScope } from "../../execution/types"
+import { MaterializationConflictError } from "../../materialization/errors"
+import type { ObjectVectorWrite } from "../../materialization/vectors"
+import type { OntologyMaterializerContract } from "../../materializer/materializer"
+import type { SixbHostContext } from "../../runtime/types"
+import {
+  commitPreparedVectors,
+  prepare,
+  storeVector,
+  VectorIndexingDeferred,
+} from "./indexing-shared"
+
+/** Publish a current subset atomically; reprepare conflicts without repeating inference. */
+export async function storeVectorBatch(
+  runtime: SixbHostContext,
+  materializer: OntologyMaterializerContract,
+  batchId: string,
+  scope: ExecutionScope,
+  signal: AbortSignal
+): Promise<void> {
+  const indexing = runtime.storage.ontology.vectorIndexing!
+  const identity = { projectId: runtime.projectId, batchId }
+
+  for (let attempt = 0; attempt < 3; attempt++) {
+    const ready = (await indexing.getBatch(identity)).filter((work) => work.status === "ready")
+    const writes: ObjectVectorWrite[] = []
+
+    for (const work of ready) {
+      signal.throwIfAborted()
+      const input = await prepare(runtime, work)
+      if (input) {
+        writes.push({ input, values: work.values! })
+      } else {
+        await indexing.remove({ projectId: runtime.projectId, id: work.id })
+      }
+    }
+
+    if (!writes.length) return
+
+    try {
+      signal.throwIfAborted()
+      await commitPreparedVectors(materializer, scope, writes)
+      return
+    } catch (error) {
+      if (!(error instanceof MaterializationConflictError)) throw error
+    }
+  }
+
+  // A continuously changing object must not keep stable peers from becoming searchable.
+  let deferred = false
+  for (const work of await indexing.getBatch(identity)) {
+    if (work.status !== "ready") continue
+    try {
+      await storeVector(runtime, materializer, work, scope, signal)
+    } catch (error) {
+      if (!(error instanceof VectorIndexingDeferred)) throw error
+      deferred = true
+    }
+  }
+
+  if (deferred) throw new VectorIndexingDeferred(new Date(Date.now() + 1000).toISOString())
+}

package/src/objects/vectors/indexing.ts

@@ -0,0 +1,23 @@
+import type { OntologyMaterializerContract } from "../../materializer/materializer"
+import type { SixbHostContext } from "../../runtime/types"
+import { processVectorBatch } from "./indexing-batch"
+import type { VectorIndexingRuntime } from "./indexing-runtime"
+import { processVector } from "./indexing-single"
+
+export { VectorIndexingDeferred } from "./indexing-shared"
+
+/** Internal processing port; durable projection groups and individual requests share delivery. */
+export function createVectorIndexingRuntime(
+  runtime: SixbHostContext,
+  materializer: OntologyMaterializerContract
+): VectorIndexingRuntime {
+  return {
+    async process(id, attempt, signal) {
+      const indexing = runtime.storage.ontology.vectorIndexing
+      if (!indexing) throw new Error("[Sixb] Storage does not support automatic vector indexing.")
+      const batch = await indexing.getBatch({ projectId: runtime.projectId, batchId: id })
+      if (batch.length) await processVectorBatch(runtime, materializer, id, batch, attempt, signal)
+      else await processVector(runtime, materializer, id, attempt, signal)
+    },
+  }
+}