@sixb/core 0.1.11 → 0.1.12

package/src/objects/vectors/indexing-generate.ts

@@ -0,0 +1,184 @@
+import { createSixbError, isSixbError } from "../../errors/internal"
+import type { ExecutionScope } from "../../execution/types"
+import type { PreparedObjectVector } from "../../materialization/vectors"
+import {
+  EmbeddingModelResponseError,
+  type EmbeddingModelResult,
+  sameEmbeddingModel,
+} from "../../models/embedding-model"
+import { bindEmbeddingModels } from "../../models/execution/embedding"
+import { ModelExecutionSession } from "../../models/execution/session"
+import type { SixbHostContext } from "../../runtime/types"
+import type { VectorIndexingWork } from "../../storage/ontology/vector-indexing"
+import { fail, retryAt, VectorIndexingDeferred } from "./indexing-shared"
+import { normalizeVector } from "./profile"
+
+export interface PreparedIndexingWork {
+  readonly work: VectorIndexingWork
+  readonly input: PreparedObjectVector
+}
+
+interface GenerationAttempt {
+  readonly runtime: SixbHostContext
+  readonly entries: readonly PreparedIndexingWork[]
+  readonly signal: AbortSignal
+  /** Running means the durable claim succeeded; budget admission may still be pending. */
+  phase: "pending" | "claiming" | "running"
+}
+
+/** One accounted provider call, with durable admission and atomic result persistence. */
+export async function generateVectors(
+  runtime: SixbHostContext,
+  entries: readonly PreparedIndexingWork[],
+  scope: ExecutionScope,
+  attempt: number,
+  signal: AbortSignal
+): Promise<void> {
+  const first = entries[0]!
+  const profile = runtime.ontology.resolveObjectType(first.work.ref.objectTypeId).search!.vectors![
+    first.work.profile
+  ]!
+  const generation: GenerationAttempt = { runtime, entries, signal, phase: "pending" }
+
+  const session = new ModelExecutionSession(
+    { ...runtime, runtimeAuthorization: scope.authorization },
+    scope.execution
+  )
+  session.bind({
+    attempt,
+    signal,
+    embeddingAdmission: (admit) => admitGeneration(generation, admit),
+  })
+
+  try {
+    const model = bindEmbeddingModels(runtime.embeddingModels, session)?.getByRef({
+      provider: profile.model.providerId,
+      modelId: profile.model.modelId,
+    })?.model
+
+    if (!model || !sameEmbeddingModel(model, profile.model)) {
+      throw createSixbError(
+        "vector.model_unavailable",
+        "[Sixb] Vector indexing requires the profile's configured embedding model."
+      )
+    }
+
+    const result = await model.embed({ texts: entries.map((entry) => entry.input.text), signal })
+    const values = normalizeEmbeddingResult(
+      result,
+      entries.length,
+      profile.model.definition.dimensions
+    )
+
+    // Persist every surviving result atomically before any vector is published.
+    await runtime.storage.ontology.vectorIndexing!.updateBatch({
+      projectId: runtime.projectId,
+      updates: entries.map(({ work }, index) => ({
+        id: work.id,
+        expectedStatus: "running",
+        status: "ready",
+        values: values[index]!,
+        availableAt: new Date().toISOString(),
+      })),
+    })
+  } catch (error) {
+    await handleGenerationFailure(generation, error)
+  }
+}
+
+async function admitGeneration(
+  generation: GenerationAttempt,
+  admit: () => Promise<void>
+): Promise<void> {
+  generation.signal.throwIfAborted()
+  generation.phase = "claiming"
+
+  const claimed = await transitionWork(generation, "pending", "running", { requireAll: true })
+  generation.phase = claimed ? "running" : "pending"
+
+  if (!claimed) {
+    if (generation.entries.length === 1) {
+      throw new Error("[Sixb] Vector indexing request was superseded.")
+    }
+    throw new VectorIndexingDeferred(new Date().toISOString())
+  }
+
+  try {
+    await admit()
+  } catch (error) {
+    await transitionWork(generation, "running", "pending")
+    generation.phase = "pending"
+    throw error
+  }
+}
+
+function normalizeEmbeddingResult(
+  result: EmbeddingModelResult,
+  count: number,
+  dimensions: number
+): readonly (readonly number[])[] {
+  try {
+    if (!Array.isArray(result?.vectors) || result.vectors.length !== count) {
+      throw new Error("Unexpected vector count")
+    }
+
+    return result.vectors.map((vector) => normalizeVector(vector, dimensions))
+  } catch (cause) {
+    throw createSixbError("vector.response_invalid", "[Sixb] Invalid embedding batch response.", {
+      cause,
+    })
+  }
+}
+
+async function handleGenerationFailure(
+  generation: GenerationAttempt,
+  error: unknown
+): Promise<void> {
+  // No provider call can start while the storage claim is unresolved. Retry delivery,
+  // not inference; an ambiguously committed claim will be fenced as running on redelivery.
+  if (generation.phase === "claiming" || error instanceof VectorIndexingDeferred) {
+    throw error
+  }
+
+  if (generation.phase === "pending") {
+    if (
+      isSixbError(error) &&
+      (error.code === "ai.usage_limit_exceeded" || error.code === "ai.usage_limit_unavailable")
+    ) {
+      const availableAt = retryAt(error.details)
+      await transitionWork(generation, "pending", "pending", { availableAt })
+      throw new VectorIndexingDeferred(availableAt)
+    }
+
+    if (generation.signal.aborted) throw error
+  }
+
+  const failure =
+    error instanceof EmbeddingModelResponseError
+      ? createSixbError("vector.response_invalid", "[Sixb] Invalid embedding batch response.", {
+          cause: error,
+        })
+      : error
+  const { runtime, entries } = generation
+  const indexing = runtime.storage.ontology.vectorIndexing!
+
+  for (const { work } of entries) {
+    await fail(runtime, indexing, { ...work, status: generation.phase }, failure)
+  }
+}
+
+function transitionWork(
+  generation: GenerationAttempt,
+  expectedStatus: VectorIndexingWork["status"],
+  status: VectorIndexingWork["status"],
+  options: { availableAt?: string; requireAll?: boolean } = {}
+): Promise<boolean> {
+  const { runtime, entries } = generation
+  const availableAt = options.availableAt ?? new Date().toISOString()
+
+  return runtime.storage.ontology.vectorIndexing!.updateBatch({
+    projectId: runtime.projectId,
+    requireAll: options.requireAll ?? false,
+    updates: entries.map(({ work }) => ({ id: work.id, expectedStatus, status, availableAt })),
+  })
+}

package/src/objects/vectors/indexing-runtime.ts

@@ -0,0 +1,17 @@
+/** Internal worker entry point; it does not expose the host context or domain SDK. */
+export interface VectorIndexingRuntime {
+  process(id: string, attempt: number, signal: AbortSignal): Promise<void>
+}
+
+// Associates each host with its service. Durable work lives in ontology storage, not this map.
+const runtimes = new WeakMap<object, VectorIndexingRuntime>()
+
+export function registerVectorIndexingRuntime(host: object, runtime: VectorIndexingRuntime): void {
+  runtimes.set(host, runtime)
+}
+
+export function getVectorIndexingRuntime(host: object): VectorIndexingRuntime {
+  const runtime = runtimes.get(host)
+  if (!runtime) throw new Error("[Sixb] Vector indexing requires a configured Sixb host.")
+  return runtime
+}

package/src/objects/vectors/indexing-shared.ts

@@ -0,0 +1,188 @@
+import { randomUUID } from "node:crypto"
+import { reportVectorIndexingFailure } from "../../error-reporting/capability"
+import { captureSixbFailure } from "../../errors/internal"
+import { createKernelRuntimeAuthorization } from "../../execution/authorization"
+import { ensureExecutionRecord, executionRecordInputFromRuntime } from "../../execution/durable"
+import type { ExecutionContext, ExecutionScope } from "../../execution/types"
+import { MaterializationConflictError } from "../../materialization/errors"
+import type { ObjectVectorWrite, PreparedObjectVector } from "../../materialization/vectors"
+import type { OntologyMaterializerContract } from "../../materializer/materializer"
+import type { SixbHostContext } from "../../runtime/types"
+import {
+  type OntologyVectorIndexingStorage,
+  VECTOR_INDEXING_FAILURE_CODES,
+  type VectorIndexingWork,
+} from "../../storage/ontology/vector-indexing"
+import { vectorConfiguration, vectorSources } from "./profile"
+
+/** Return null when the requested representation is obsolete or already stored. */
+export async function prepare(
+  runtime: SixbHostContext,
+  work: VectorIndexingWork
+): Promise<PreparedObjectVector | null> {
+  const profile = runtime.ontology.getObjectTypeById(work.ref.objectTypeId)?.search?.vectors?.[
+    work.profile
+  ]
+  if (!profile || vectorConfiguration(profile) !== work.configuration) return null
+
+  const row = await runtime.storage.objects.getByPrimaryId({
+    projectId: runtime.projectId,
+    ...work.ref,
+  })
+  if (!row) return null
+
+  const sources = vectorSources(profile.source, row.properties)
+  if (sources.sourceFingerprint !== work.sourceFingerprint) return null
+
+  const storedVectors = await runtime.storage.ontology.vectors!.list({
+    projectId: runtime.projectId,
+    ref: work.ref,
+  })
+  const currentVector = storedVectors.find((entry) => entry.profile === work.profile)
+  const alreadyIndexed =
+    currentVector?.configuration === work.configuration &&
+    currentVector.sourceFingerprint === work.sourceFingerprint
+
+  if (alreadyIndexed) return null
+
+  return {
+    projectId: runtime.projectId,
+    ref: work.ref,
+    profile: work.profile,
+    configuration: work.configuration,
+    ...sources,
+    expectedObject: {
+      ref: work.ref,
+      exists: true,
+      version: row.version,
+      lastCommitId: row.lastCommitId,
+    },
+    expectedVectorCommitId: currentVector?.lastCommitId ?? null,
+  }
+}
+
+export async function storeVector(
+  runtime: SixbHostContext,
+  materializer: OntologyMaterializerContract,
+  work: VectorIndexingWork,
+  scope: ExecutionScope,
+  signal: AbortSignal
+): Promise<void> {
+  const indexing = runtime.storage.ontology.vectorIndexing!
+  const identity = { projectId: runtime.projectId, id: work.id }
+
+  // Unrelated property changes renew the object fence without repeating a paid inference.
+  for (let attempt = 0; attempt < 3; attempt++) {
+    signal.throwIfAborted()
+    const prepared = await prepare(runtime, work)
+
+    if (!prepared) {
+      await indexing.remove(identity)
+      return
+    }
+
+    try {
+      await commitPreparedVectors(materializer, scope, [{ input: prepared, values: work.values! }])
+      await indexing.remove(identity)
+      return
+    } catch (error) {
+      if (!(error instanceof MaterializationConflictError)) throw error
+
+      const currentWork = await indexing.get(identity)
+      if (!currentWork) return
+    }
+  }
+
+  throw new VectorIndexingDeferred(new Date(Date.now() + 1000).toISOString())
+}
+
+export async function commitPreparedVectors(
+  materializer: OntologyMaterializerContract,
+  scope: ExecutionScope,
+  writes: readonly ObjectVectorWrite[]
+): Promise<void> {
+  await materializer.edits.commit({
+    scope,
+    input: {
+      mode: "atomic",
+      source: { kind: "runtime", requestId: randomUUID() },
+      operations: [],
+      vectorWrites: writes,
+      expectedObjects: writes.map((write) => write.input.expectedObject),
+      expectedLinks: [],
+      expectedLinkScopes: [],
+    },
+  })
+}
+
+export async function indexingScope(
+  runtime: SixbHostContext,
+  work: VectorIndexingWork,
+  indexingId = work.id
+): Promise<ExecutionScope> {
+  const operation = { type: "ontology.indexVectors", indexingId } as const
+  const execution: ExecutionContext = {
+    id: `exec_vector_${indexingId}`,
+    projectId: runtime.projectId,
+    executor: { type: "kernel", operation },
+    source: { type: "ontologyCommit", commitId: work.sourceCommitId },
+    correlationId: `vector_${indexingId}`,
+  }
+  const authorization = createKernelRuntimeAuthorization({ execution, operation })
+
+  await ensureExecutionRecord(runtime.storage.executions, {
+    ...executionRecordInputFromRuntime({ execution, runtimeAuthorization: authorization }),
+    requesterGroupIds: [],
+  })
+
+  return { execution, authorization }
+}
+
+export async function fail(
+  runtime: SixbHostContext,
+  indexing: OntologyVectorIndexingStorage,
+  work: VectorIndexingWork,
+  error: unknown
+): Promise<void> {
+  const failure = captureSixbFailure(error, {
+    allowedCodes: VECTOR_INDEXING_FAILURE_CODES,
+    defaultCode: "internal.unexpected",
+  })
+
+  const recorded = await indexing.update({
+    projectId: runtime.projectId,
+    id: work.id,
+    expectedStatus: work.status,
+    status: "failed",
+    availableAt: new Date().toISOString(),
+    error: failure,
+  })
+
+  if (!recorded) return
+
+  reportVectorIndexingFailure(runtime, error, {
+    projectId: runtime.projectId,
+    indexingId: work.id,
+    failure,
+    ...work.ref,
+    profile: work.profile,
+  })
+}
+
+export function retryAt(details: unknown): string {
+  const resetAt =
+    details && typeof details === "object" && "resetAt" in details ? details.resetAt : undefined
+  const resetTimestamp = typeof resetAt === "string" ? Date.parse(resetAt) : Number.NaN
+
+  const minimumRetryTimestamp = Date.now() + 60_000
+  const budgetResetTimestamp = Number.isFinite(resetTimestamp) ? resetTimestamp : 0
+  const retryTimestamp = Math.max(minimumRetryTimestamp, budgetResetTimestamp)
+
+  return new Date(retryTimestamp).toISOString()
+}
+
+export class VectorIndexingDeferred extends Error {
+  constructor(readonly availableAt: string) {
+    super("[Sixb] Vector indexing is deferred.")
+  }
+}

package/src/objects/vectors/indexing-single.ts

@@ -0,0 +1,51 @@
+import { createSixbError } from "../../errors/internal"
+import type { OntologyMaterializerContract } from "../../materializer/materializer"
+import type { SixbHostContext } from "../../runtime/types"
+import { generateVectors } from "./indexing-generate"
+import {
+  fail,
+  indexingScope,
+  prepare,
+  storeVector,
+  VectorIndexingDeferred,
+} from "./indexing-shared"
+
+export async function processVector(
+  runtime: SixbHostContext,
+  materializer: OntologyMaterializerContract,
+  id: string,
+  attempt: number,
+  signal: AbortSignal
+): Promise<void> {
+  const indexing = runtime.storage.ontology.vectorIndexing
+  if (!indexing) throw new Error("[Sixb] Storage does not support automatic vector indexing.")
+  const projectId = runtime.projectId
+  const work = await indexing.get({ projectId, id })
+  if (!work || work.status === "failed") return
+  if (work.status === "running") {
+    await fail(
+      runtime,
+      indexing,
+      work,
+      createSixbError(
+        "vector.outcome_unknown",
+        "[Sixb] Interrupted embedding call has an unknown outcome; inference was not repeated."
+      )
+    )
+    return
+  }
+  if (Date.parse(work.availableAt) > Date.now()) throw new VectorIndexingDeferred(work.availableAt)
+  const prepared = await prepare(runtime, work)
+  if (!prepared) {
+    await indexing.remove({ projectId, id })
+    return
+  }
+  const scope = await indexingScope(runtime, work, work.batchId ?? work.id)
+  if (work.status === "ready") {
+    await storeVector(runtime, materializer, work, scope, signal)
+    return
+  }
+  await generateVectors(runtime, [{ work, input: prepared }], scope, attempt, signal)
+  const ready = await indexing.get({ projectId, id })
+  if (ready?.status === "ready") await storeVector(runtime, materializer, ready, scope, signal)
+}

package/src/objects/vectors/indexing-store-batch.ts

@@ -0,0 +1,62 @@
+import type { ExecutionScope } from "../../execution/types"
+import { MaterializationConflictError } from "../../materialization/errors"
+import type { ObjectVectorWrite } from "../../materialization/vectors"
+import type { OntologyMaterializerContract } from "../../materializer/materializer"
+import type { SixbHostContext } from "../../runtime/types"
+import {
+  commitPreparedVectors,
+  prepare,
+  storeVector,
+  VectorIndexingDeferred,
+} from "./indexing-shared"
+
+/** Publish a current subset atomically; reprepare conflicts without repeating inference. */
+export async function storeVectorBatch(
+  runtime: SixbHostContext,
+  materializer: OntologyMaterializerContract,
+  batchId: string,
+  scope: ExecutionScope,
+  signal: AbortSignal
+): Promise<void> {
+  const indexing = runtime.storage.ontology.vectorIndexing!
+  const identity = { projectId: runtime.projectId, batchId }
+
+  for (let attempt = 0; attempt < 3; attempt++) {
+    const ready = (await indexing.getBatch(identity)).filter((work) => work.status === "ready")
+    const writes: ObjectVectorWrite[] = []
+
+    for (const work of ready) {
+      signal.throwIfAborted()
+      const input = await prepare(runtime, work)
+      if (input) {
+        writes.push({ input, values: work.values! })
+      } else {
+        await indexing.remove({ projectId: runtime.projectId, id: work.id })
+      }
+    }
+
+    if (!writes.length) return
+
+    try {
+      signal.throwIfAborted()
+      await commitPreparedVectors(materializer, scope, writes)
+      return
+    } catch (error) {
+      if (!(error instanceof MaterializationConflictError)) throw error
+    }
+  }
+
+  // A continuously changing object must not keep stable peers from becoming searchable.
+  let deferred = false
+  for (const work of await indexing.getBatch(identity)) {
+    if (work.status !== "ready") continue
+    try {
+      await storeVector(runtime, materializer, work, scope, signal)
+    } catch (error) {
+      if (!(error instanceof VectorIndexingDeferred)) throw error
+      deferred = true
+    }
+  }
+
+  if (deferred) throw new VectorIndexingDeferred(new Date(Date.now() + 1000).toISOString())
+}

package/src/objects/vectors/indexing.ts

@@ -0,0 +1,23 @@
+import type { OntologyMaterializerContract } from "../../materializer/materializer"
+import type { SixbHostContext } from "../../runtime/types"
+import { processVectorBatch } from "./indexing-batch"
+import type { VectorIndexingRuntime } from "./indexing-runtime"
+import { processVector } from "./indexing-single"
+
+export { VectorIndexingDeferred } from "./indexing-shared"
+
+/** Internal processing port; durable projection groups and individual requests share delivery. */
+export function createVectorIndexingRuntime(
+  runtime: SixbHostContext,
+  materializer: OntologyMaterializerContract
+): VectorIndexingRuntime {
+  return {
+    async process(id, attempt, signal) {
+      const indexing = runtime.storage.ontology.vectorIndexing
+      if (!indexing) throw new Error("[Sixb] Storage does not support automatic vector indexing.")
+      const batch = await indexing.getBatch({ projectId: runtime.projectId, batchId: id })
+      if (batch.length) await processVectorBatch(runtime, materializer, id, batch, attempt, signal)
+      else await processVector(runtime, materializer, id, attempt, signal)
+    },
+  }
+}

package/src/objects/vectors/profile.ts

@@ -0,0 +1,64 @@
+import { MaterializationValidationError } from "../../materialization/errors"
+import { sha256Canonical } from "../../materialization/identity"
+import type { ObjectVectorSearchProfile } from "../../ontology/types"
+
+export const MAX_VECTOR_K = 1000
+export const VECTOR_INPUT_VERSION = 1
+
+export function vectorConfiguration(profile: ObjectVectorSearchProfile): string {
+  return sha256Canonical({
+    format: VECTOR_INPUT_VERSION,
+    sources: [...profile.source],
+    provider: profile.model.providerId,
+    model: profile.model.modelId,
+    representation: profile.model.definition.representation ?? { name: profile.model.modelId },
+    dimensions: profile.model.definition.dimensions,
+    metric: "cosine",
+    precision: "float32",
+    normalization: "l2-v1",
+  })
+}
+
+export function vectorSources(
+  source: readonly string[],
+  properties: Readonly<Record<string, unknown>>
+) {
+  const values: Record<string, string | null> = Object.create(null)
+  for (const key of source) {
+    const value = properties[key]
+    if (value !== undefined && value !== null && typeof value !== "string") {
+      throw new MaterializationValidationError(`Vector source must be text: ${key}`)
+    }
+    values[key] = typeof value === "string" ? value : null
+  }
+  // Ordered JSON pairs preserve field boundaries, escaping and absent/empty distinction.
+  const text = JSON.stringify(source.map((key) => [key, values[key]]))
+  return { text, sourceFingerprint: sha256Canonical(text) }
+}
+
+/** Unit-length float32 representation keeps native cosine arithmetic within range. */
+export function normalizeVector(values: readonly number[], dimensions: number): readonly number[] {
+  if (!Array.isArray(values) || values.length !== dimensions) {
+    throw new MaterializationValidationError(`Vector dimension mismatch; expected ${dimensions}`)
+  }
+  const normalized = Array.from(values, (value) => {
+    if (
+      typeof value !== "number" ||
+      !Number.isFinite(value) ||
+      !Number.isFinite(Math.fround(value))
+    ) {
+      throw new MaterializationValidationError("Vector values must be finite float32 numbers.")
+    }
+    return Math.fround(value)
+  })
+  if (!normalized.some((value) => value !== 0)) {
+    throw new MaterializationValidationError("Cosine search requires a nonzero vector.")
+  }
+  const norm = Math.hypot(...normalized)
+  return Object.freeze(normalized.map((value) => Math.fround(value / norm)))
+}
+
+/** V1 envelope for exact SQL scoring; exceeding it must fail rather than truncate candidates. */
+export function vectorSearchCandidateLimit(dimensions: number): number {
+  return Math.min(10000, Math.floor(16000000 / dimensions))
+}

package/src/objects/vectors/query.ts

@@ -0,0 +1,16 @@
+import type { ObjectQuery } from "../query/ir"
+
+export function hasVectorProfile(query: ObjectQuery): boolean {
+  if (query.kind === "vector") return true
+  if (query.kind === "set") return query.inputs.some(hasVectorProfile)
+  return "input" in query && hasVectorProfile(query.input)
+}
+
+/** V1 supports one exact top-k, with filters applied before ranking. */
+export function isVectorProfileQuery(query: ObjectQuery): boolean {
+  if (query.kind === "limit" || query.kind === "project") return isVectorProfileQuery(query.input)
+  if (query.kind !== "vector" || !query.profile) return false
+  let input = query.input
+  while (input.kind === "filter") input = input.input
+  return input.kind === "start" && !input.includeSubtypes
+}