@sixb/core 0.1.10 → 0.1.12

package/dist/{chunk-ftv470j8.js → chunk-m7cx05vp.js}

@@ -16,7 +16,10 @@ import {
   telemetryPointSortKey2,
   compareObjectRefs2,
   compareLinkRefs2
-} from "./chunk-c5rkjrgp.js";
+} from "./chunk-7ytf06d3.js";
+import {
+  withFailureMessage
+} from "./chunk-z3y5kg1x.js";
 import {
   normalizeObjectProperties2,
   normalizeSchemaValue2,
@@ -26,7 +29,7 @@ import {
   normalizeLinkProperties2,
   validateLinkProperties2,
   validateTelemetryUnit2
-} from "./chunk-ms2tmn1k.js";
+} from "./chunk-vsg0g7h9.js";
 import {
   materializationEventKindOrdinal2,
   sha256Canonical2,
@@ -34,32 +37,38 @@ import {
   createCommitId2,
   createEventId2,
   materializationApplyPhase
-} from "./chunk-gqh45d2n.js";
+} from "./chunk-hkw8rykk.js";
 import {
   ProjectionValidationError2,
   validateProjectionsAtStartup,
   isProjectionDefinition2,
   resolveExecutionScopeAuthorization
-} from "./chunk-dz5h08xv.js";
+} from "./chunk-bbaa1fet.js";
 import {
   isStorageSerializationFailure2
-} from "./chunk-8vj0rjgs.js";
+} from "./chunk-wjffn5m4.js";
+import {
+  objectBatchKey2,
+  vectorConfiguration,
+  vectorSources,
+  normalizeVector2
+} from "./chunk-ybhz3j9g.js";
 import {
   parseDatasetTimestamp2
 } from "./chunk-z667z3sg.js";
 import {
   executionRecordInputFromRuntime,
   ensureExecutionRecord2
-} from "./chunk-17kfctj6.js";
+} from "./chunk-0mza4xb2.js";
 import {
   assertPinnedDatasetWatermark2
-} from "./chunk-wjasrj67.js";
+} from "./chunk-3qpcrcj8.js";
 import {
   MaterializationValidationError2,
   MaterializationObjectNotFoundError2,
   MaterializationCancellationError2,
   MaterializationConflictError2
-} from "./chunk-yv79ncbv.js";
+} from "./chunk-96yw3pwd.js";
 import {
   assertJsonValue2,
   cloneJsonValue2,
@@ -589,6 +598,9 @@ function createMaterializerContext(input) {
   if (computeOntologyRevision2(input.ontology) !== input.projections.ontologyRevision) {
     throw new MaterializationValidationError2("Materializer Ontology does not match the Ontology pinned by its projection registry.");
   }
+  if ((!input.storage.ontology.vectors || !input.storage.ontology.vectorIndexing) && input.ontology.listObjectTypes().some((type) => Object.keys(type.search?.vectors ?? {}).length > 0)) {
+    throw new MaterializationValidationError2("Storage does not support transactional vector profiles and automatic indexing.");
+  }
   const dependencies = input.dependencies ?? {};
   const maxSerializationRetries = dependencies.maxSerializationRetries ?? 2;
   if (!Number.isSafeInteger(maxSerializationRetries) || maxSerializationRetries < 0) {
@@ -814,6 +826,100 @@ function sequenceMaterializationEvent(projectId, commitId, commitOrdinal, draft)
   return { ...draft, ...sequence };
 }
 
+// src/materializer/effective/vector-indexing.ts
+import { randomUUID as randomUUID2 } from "crypto";
+async function scheduleVectorChanges(context, storage, items, session, projection = false) {
+  const indexing = storage.ontology.vectorIndexing;
+  if (!indexing)
+    return;
+  const upserts = items.filter((item) => item.kind === "object-upsert").filter((item) => Object.keys(context.ontology.resolveObjectType(item.value.row.ref.objectTypeId).search?.vectors ?? {}).length > 0);
+  const previous = await storage.objects.getByPrimaryIdBatch({
+    projectId: context.projectId,
+    items: upserts.map((item) => item.value.row.ref)
+  });
+  const requests = [];
+  const deleted = [];
+  const groups = new Map;
+  for (const item of items) {
+    if (item.kind === "object-delete") {
+      deleted.push(item.value.ref);
+      continue;
+    }
+    if (item.kind !== "object-upsert")
+      continue;
+    const row = item.value.row;
+    const profiles = context.ontology.resolveObjectType(row.ref.objectTypeId).search?.vectors ?? {};
+    const before = previous.get(objectBatchKey2(row.ref.objectTypeId, row.ref.primaryId));
+    for (const [profile, definition] of Object.entries(profiles)) {
+      const { sourceFingerprint, text } = vectorSources(definition.source, row.properties);
+      if (before && vectorSources(definition.source, before.properties).sourceFingerprint === sourceFingerprint)
+        continue;
+      const configuration = vectorConfiguration(definition);
+      const bytes = Buffer.byteLength(text);
+      const key = JSON.stringify([row.ref.objectTypeId, profile, configuration, row.lastCommitId]);
+      let group = projection ? groups.get(key) : undefined;
+      if (projection && (!group || group.count === 32 || group.bytes + bytes > 32768)) {
+        group = { id: randomUUID2(), count: 0, bytes: 0 };
+        groups.set(key, group);
+      }
+      if (group) {
+        group.count++;
+        group.bytes += bytes;
+      }
+      requests.push({
+        ...group ? { batchId: group.id } : {},
+        id: randomUUID2(),
+        ref: row.ref,
+        profile,
+        sourceFingerprint,
+        configuration,
+        sourceCommitId: row.lastCommitId
+      });
+    }
+  }
+  if (requests.length || deleted.length)
+    await indexing.schedule({
+      projectId: context.projectId,
+      session,
+      requests,
+      deleted,
+      availableAt: context.clock().toISOString()
+    });
+}
+
+// src/materializer/effective/vectors.ts
+async function invalidateVectorChanges(projectId, vectors, items, session) {
+  if (!vectors)
+    return;
+  const changes = new Map(items.flatMap((item) => {
+    if (item.kind !== "object-upsert" && item.kind !== "object-delete")
+      return [];
+    const ref = item.kind === "object-upsert" ? item.value.row.ref : item.value.ref;
+    return [[objectRefKey2(ref), { ref, item }]];
+  }));
+  if (!changes.size)
+    return;
+  const stored = await vectors.listBatch({
+    projectId,
+    refs: [...changes.values()].map(({ ref }) => ref)
+  });
+  const removals = [];
+  for (const entry of stored) {
+    const change = changes.get(objectRefKey2(entry.ref));
+    if (!change)
+      continue;
+    const { item } = change;
+    if (item.kind === "object-upsert" && entry.source.every((id) => {
+      const value = item.value.row.properties[id];
+      return value === undefined || value === null || typeof value === "string";
+    }) && vectorSources(entry.source, item.value.row.properties).sourceFingerprint === entry.sourceFingerprint)
+      continue;
+    removals.push({ ref: entry.ref, profile: entry.profile, expectedCommitId: entry.lastCommitId });
+  }
+  if (removals.length)
+    await vectors.removeBatch({ session, projectId, entries: removals });
+}
+
 // src/materializer/shared/abort.ts
 function throwIfAborted(signal) {
   if (signal?.aborted) {
@@ -1084,12 +1190,16 @@ async function stageWorkBounded(context, storage, session, records) {
     await storage.stageWork({ session, records: chunk });
   }
 }
-async function drainStagedWork(context, storage, session, signal) {
+async function drainStagedWork(context, transactionStorage, session, signal, projection = false) {
+  const ontologyStorage = transactionStorage.ontology;
+  const storage = ontologyStorage.materializations;
   let phase = null;
   let pending = [];
   const flush = async () => {
     if (pending.length === 0)
       return;
+    await scheduleVectorChanges(context, transactionStorage, pending, session, projection);
+    await invalidateVectorChanges(context.projectId, ontologyStorage.vectors, pending, session);
     await applyItems(context, storage, session, pending, signal);
     pending = [];
   };
@@ -1351,6 +1461,8 @@ function normalizeOntologyEditCommit2(input) {
     return normalized;
   });
   if (input.mode === "continue") {
+    if ("vectorWrites" in input)
+      throw new MaterializationValidationError2("Vector writes require an atomic commit.");
     return Object.freeze({
       mode: "continue",
       source: Object.freeze({
@@ -1369,11 +1481,17 @@ function normalizeOntologyEditCommit2(input) {
     kind: "runtime",
     requestId: normalizeNonblank(input.source.requestId, "Runtime request id")
   });
+  if (input.vectorWrites?.length && (operations.length || input.source.kind !== "runtime")) {
+    throw new MaterializationValidationError2("Vector writes require a dedicated atomic runtime commit.");
+  }
+  if (input.vectorWrites !== undefined)
+    assertJsonValue2(input.vectorWrites, "Vector writes");
   const expectedObjects = input.expectedObjects.map(normalizeExpectedObject);
   const expectedLinks = input.expectedLinks.map(normalizeExpectedLink);
   const expectedLinkScopes = input.expectedLinkScopes.map(normalizeExpectedLinkScope);
   return Object.freeze({
     mode: "atomic",
+    ...input.vectorWrites === undefined ? {} : { vectorWrites: structuredClone(input.vectorWrites) },
     source,
     operations: Object.freeze(operations),
     expectedObjects: deduplicateExpectations(expectedObjects, (value) => objectRefKey2(value.ref), "Expected objects"),
@@ -2165,7 +2283,7 @@ function validateEffectiveObject(ontology, ref, properties) {
     assertKnownProperties2(objectType, properties);
     for (const property of objectType.properties) {
       if (property.mode !== "telemetry" && property.required && properties[property.id] === undefined) {
-        throw new Error(`[Sixb] Missing required property '${property.id}' for object type '${objectType.id}'`);
+        throw withFailureMessage(new Error(`[Sixb] Missing required property '${property.id}' for object type '${objectType.id}'`), `Missing required property '${objectType.id}.${property.id}'.`);
       }
     }
     validateObjectProperties2(objectType, properties, ontology.getValueTypesById());
@@ -2208,7 +2326,7 @@ function materializationValidation(run) {
   } catch (error) {
     if (error instanceof MaterializationValidationError2)
       throw error;
-    throw new MaterializationValidationError2(error instanceof Error ? error.message : String(error));
+    throw new MaterializationValidationError2(error instanceof Error ? error.message : String(error), { cause: error });
   }
 }
 
@@ -2744,6 +2862,115 @@ function sortedLinkSlots(slots) {
   return [...slots.values()].sort((left, right) => linkScopeKey(left.ref.source, left.ref.linkId).localeCompare(linkScopeKey(right.ref.source, right.ref.linkId)));
 }
 
+// src/materializer/edits/vector-authority.ts
+async function authorizeVectorIndexingCommit(storage, projectId, input, execution) {
+  const executor = execution.scope.execution.executor;
+  if (executor.type !== "kernel" || executor.operation.type !== "ontology.indexVectors")
+    return false;
+  if (input.mode !== "atomic" || input.source.kind !== "runtime" || input.operations.length || !input.vectorWrites?.length || input.expectedLinks.length || input.expectedLinkScopes.length) {
+    throw new MaterializationValidationError2("Vector indexing authority only permits its persisted vector writes.");
+  }
+  const writes = input.vectorWrites;
+  const refs = new Set(writes.map((write) => objectRefKey2(write.input.ref)));
+  if (input.expectedObjects.length !== refs.size || input.expectedObjects.some((expected) => !refs.has(objectRefKey2(expected.ref)))) {
+    throw new MaterializationValidationError2("Vector indexing must fence exactly its written objects.");
+  }
+  const indexing = storage.ontology.vectorIndexing;
+  const indexingId = executor.operation.indexingId;
+  const single = await indexing?.get({ projectId, id: indexingId });
+  if (single && writes.length !== 1) {
+    throw new MaterializationValidationError2("Individual indexing authority only permits one vector write.");
+  }
+  let members;
+  if (single) {
+    members = [single];
+  } else if (writes.length === 1) {
+    const member = await indexing?.getBatchMember({
+      projectId,
+      batchId: indexingId,
+      ref: writes[0].input.ref,
+      profile: writes[0].input.profile
+    });
+    members = member ? [member] : [];
+  } else {
+    members = await indexing?.getBatch({ projectId, batchId: indexingId }) ?? [];
+  }
+  const byProfile = new Map(members.map((work) => [JSON.stringify([objectRefKey2(work.ref), work.profile]), work]));
+  const source = execution.scope.execution.source;
+  for (const write of writes) {
+    const work = byProfile.get(JSON.stringify([objectRefKey2(write.input.ref), write.input.profile]));
+    if (!work || work.status !== "ready") {
+      throw new MaterializationConflictError2("effective-state", "Vector indexing work is no longer ready.");
+    }
+    if (source.type !== "ontologyCommit" || source.commitId !== work.sourceCommitId || !matchesStoredResult(write, work)) {
+      throw new MaterializationValidationError2("Vector indexing authority only permits its current prepared representation.");
+    }
+  }
+  return true;
+}
+function matchesStoredResult(write, work) {
+  const prepared = write.input;
+  return prepared.ref.objectTypeId === work.ref.objectTypeId && prepared.ref.primaryId === work.ref.primaryId && prepared.profile === work.profile && prepared.configuration === work.configuration && prepared.sourceFingerprint === work.sourceFingerprint && work.values !== undefined && work.values.length === write.values.length && work.values.every((value, index) => value === write.values[index]);
+}
+
+// src/materializer/edits/vectors.ts
+async function commitVectorWrites(context, storage, input, commitId, session) {
+  if (input.mode !== "atomic" || !input.vectorWrites?.length)
+    return;
+  const vectors = storage.ontology.vectors;
+  if (!vectors)
+    throw new MaterializationValidationError2("Storage does not support vector profiles.");
+  const rows = await storage.objects.getByPrimaryIdBatch({
+    projectId: context.projectId,
+    items: input.vectorWrites.map((write) => write.input.ref)
+  });
+  const expectations = new Map(input.expectedObjects.map((expected) => [objectRefKey2(expected.ref), expected]));
+  const entries = [];
+  const seen = new Set;
+  for (const write of input.vectorWrites) {
+    const prepared = write.input;
+    const key = JSON.stringify([objectRefKey2(prepared.ref), prepared.profile]);
+    if (seen.has(key))
+      throw new MaterializationValidationError2(`Duplicate vector write: ${key}`);
+    seen.add(key);
+    const profileDefinition = context.ontology.resolveObjectType(prepared.ref.objectTypeId).search?.vectors?.[prepared.profile];
+    if (prepared.projectId !== context.projectId || !profileDefinition || vectorConfiguration(profileDefinition) !== prepared.configuration) {
+      throw new MaterializationConflictError2("effective-state", "Vector profile configuration changed; index it again.");
+    }
+    const expected = expectations.get(objectRefKey2(prepared.ref));
+    if (!expected?.exists || !prepared.expectedObject?.exists || objectRefKey2(prepared.expectedObject.ref) !== objectRefKey2(prepared.ref) || expected.version !== prepared.expectedObject.version || expected.lastCommitId !== prepared.expectedObject.lastCommitId) {
+      throw new MaterializationValidationError2("Vector writes must fence their prepared object revision.");
+    }
+    const row = rows.get(objectBatchKey2(prepared.ref.objectTypeId, prepared.ref.primaryId));
+    if (!row || row.version !== expected.version || row.lastCommitId !== expected.lastCommitId || vectorSources(profileDefinition.source, row.properties).sourceFingerprint !== prepared.sourceFingerprint) {
+      throw new MaterializationConflictError2("effective-state", "Vector sources changed; index the profile again.");
+    }
+    entries.push({
+      expectedCommitId: prepared.expectedVectorCommitId,
+      value: {
+        ref: prepared.ref,
+        profile: prepared.profile,
+        configuration: prepared.configuration,
+        source: profileDefinition.source,
+        sourceFingerprint: prepared.sourceFingerprint,
+        values: normalizeVector2(write.values, profileDefinition.model.definition.dimensions),
+        lastCommitId: commitId
+      }
+    });
+  }
+  await vectors.writeBatch({ session, projectId: context.projectId, entries });
+  await storage.ontology.vectorIndexing?.complete({
+    session,
+    projectId: context.projectId,
+    entries: input.vectorWrites.map(({ input: prepared }) => ({
+      ref: prepared.ref,
+      profile: prepared.profile,
+      configuration: prepared.configuration,
+      sourceFingerprint: prepared.sourceFingerprint
+    }))
+  });
+}
+
 // src/materializer/edits/commit.ts
 async function commitEdits(context, raw) {
   const command = prepareEditCommit(context, raw);
@@ -2785,6 +3012,8 @@ async function lockActionRunForMaterialization(storage, projectId, input, execut
   assertMaterializerRunExecution(execution, run.executionId, `Action run '${run.id}'`);
 }
 async function validateMutationExecution(storage, projectId, input, execution) {
+  if (await authorizeVectorIndexingCommit(storage, projectId, input, execution))
+    return;
   if (input.source.kind !== "action") {
     assertRuntimeMutationExecution(execution);
     return;
@@ -2822,7 +3051,8 @@ async function executeEditTransaction(context, storage, command) {
   const workingState = await loadEditWorkingState(context, storage.ontology.materializations, session, command.input.operations);
   const outcomes = applyEditOperations(context, workingState, command);
   const changes = await stageEditPlan(context, storage.ontology.materializations, session, workingState, editPlanContext(command));
-  await drainStagedWork(context, storage.ontology.materializations, session);
+  await drainStagedWork(context, storage, session);
+  await commitVectorWrites(context, storage, command.input, command.identity.commitId, session);
   const eventCount = await drainStagedEvents(context, storage.ontology.materializations, session, command.identity);
   const result = {
     kind: "edit",
@@ -2858,7 +3088,8 @@ function buildEditCommit(context, command) {
     intent: {
       kind: "edit",
       mode: command.input.mode,
-      operationCount: command.input.operations.length
+      operationCount: command.input.operations.length,
+      ...command.input.mode === "atomic" && command.input.vectorWrites?.length ? { vectorWriteCount: command.input.vectorWrites.length } : {}
     },
     committedAt: command.identity.committedAt
   };
@@ -3707,7 +3938,7 @@ async function executeProjectionTransaction(context, storage, command, ready) {
     }
   });
   const counts = await planReadyProjection(context, storage.ontology.materializations, session, command, ready, origin);
-  await drainStagedWork(context, storage.ontology.materializations, session, command.signal);
+  await drainStagedWork(context, storage, session, command.signal, true);
   const eventCount = await drainStagedEvents(context, storage.ontology.materializations, session, ready.identity, command.signal);
   const result = {
     kind: "projection",
@@ -4206,7 +4437,7 @@ async function executeTelemetryTransaction(context, storage, command) {
     correlationId: command.execution.correlationId,
     ...command.execution.actor === undefined ? {} : { actor: command.execution.actor }
   });
-  await drainStagedWork(context, storage.ontology.materializations, session);
+  await drainStagedWork(context, storage, session);
   const eventCount = await drainStagedEvents(context, storage.ontology.materializations, session, command.identity);
   const result = {
     kind: "telemetry",
@@ -4366,4 +4597,4 @@ function createOntologyMaterializer2(input) {
 }
 export { computeOntologyRevision2, computeProjectionRevision2, computeProjectionOwnershipHash2, ProjectionRegistry2, createActionIdempotencyKey2, createRuntimeIdempotencyKey2, createProjectionIdempotencyKey2, createRuntimeTelemetryIdempotencyKey2, createProjectionTelemetryIdempotencyKey2, createCommitIdentity2, timestampCommitIdentity2, createTimedCommitIdentity2, createProjectionMaterializationId2, createLinkScopeFingerprint2, normalizeJsonProperties2, normalizeOntologyEditCommit2, normalizeProjectionExecution2, normalizeProjectionSourceEntry2, normalizeTelemetryAppend2, OntologyMaterializer2, createOntologyMaterializer2 };
 
-//# debugId=45B690D58E462D3C64756E2164756E21
+//# debugId=07774B0E8444369164756E2164756E21