@sixb/core 0.1.15 → 0.1.17

package/dist/{chunk-dqkd3kdj.js → chunk-dam456ve.js}

@@ -1,11 +1,11 @@
 // @bun
 import {
   OntologyRegistry2
-} from "./chunk-3nx8vj5a.js";
+} from "./chunk-xyvy73tj.js";
 import {
   ProjectionRegistry2,
   createOntologyMaterializer2
-} from "./chunk-z708gemf.js";
+} from "./chunk-fggq3d9k.js";
 import {
   captureSixbFailure2,
   isSixbError2
@@ -13,37 +13,42 @@ import {
 import {
   SecurityRegistry,
   AuthRuntime2
-} from "./chunk-qne45xqq.js";
+} from "./chunk-svabwqdq.js";
 import {
   captureExecutionScope,
   resolveExecutionScopeAuthorization,
   AuthorizationError2
-} from "./chunk-8fjmqjyx.js";
+} from "./chunk-5be2ath4.js";
 import {
   AuthRuntimeError2,
   isMagicLinkAuthStrategy2,
   isOidcAuthStrategy2
 } from "./chunk-2jntasmh.js";
 import {
-  createModelCatalog2
-} from "./chunk-ky7zpbtt.js";
+  indexModelBindings,
+  assertRerankingModel,
+  validateRerankingResults,
+  RerankingModelResponseError2,
+  createModelCatalog2,
+  estimateModelCall
+} from "./chunk-s7ftxrdr.js";
 import {
   FileUploadSessionError2
-} from "./chunk-p0nbkys4.js";
+} from "./chunk-33saed7g.js";
 import {
   isMigrationCapableStorage2,
   checkStorageSchema2
-} from "./chunk-fjw417jf.js";
+} from "./chunk-5yd8cchh.js";
 import {
   createAuthorizedObjectReader
-} from "./chunk-wem5vqjw.js";
+} from "./chunk-10styjgk.js";
 import {
   attachSixbErrorReporter2,
   shareSixbErrorReporter2
 } from "./chunk-vjcd8d27.js";
 import {
   ActionRegistry2
-} from "./chunk-1jvq82rp.js";
+} from "./chunk-9g6jz9kk.js";
 import {
   assertDatasetDefinition2
 } from "./chunk-cd234ba4.js";
@@ -56,7 +61,7 @@ import {
 } from "./chunk-e1c18xjx.js";
 import {
   createSandboxDefinition
-} from "./chunk-vef6qa14.js";
+} from "./chunk-2am3db31.js";
 import {
   ConnectorError2,
   ConnectorNotFoundError2,
@@ -68,16 +73,16 @@ import {
   isConnectorStorageError,
   storageBoundaryError,
   recoverConnectorFailure
-} from "./chunk-r5sc1d2b.js";
+} from "./chunk-jnfxmr3f.js";
 import {
   toolsFromProjectConfig
-} from "./chunk-dsfw1bdr.js";
+} from "./chunk-cnb5kv05.js";
 import {
   assertNoAgentDefinitions,
   validateWorkflowsAtStartup2,
   validateWorkflowAgentStepGroupReferences2,
   createBoundSixb
-} from "./chunk-2tz5vnvz.js";
+} from "./chunk-6wbm6ymz.js";
 import {
   registerConnectorConnectionCallbackProcess
 } from "./chunk-5a4a52tq.js";
@@ -93,30 +98,31 @@ import {
   validateSharesAtStartup,
   discoverOntologySources2,
   discoverProjectDefinitions2
-} from "./chunk-wd34wnsy.js";
+} from "./chunk-xcy5ddha.js";
 import {
   LoggingService2
 } from "./chunk-3qms4nax.js";
 import {
   bindEmbeddingModels,
   createVectorIndexingRuntime,
-  registerVectorIndexingRuntime
-} from "./chunk-ex8s8wsk.js";
+  registerVectorIndexingRuntime,
+  getApiPublicOrigin
+} from "./chunk-vp3z1ckh.js";
 import {
   registerProjectionRegistry2
-} from "./chunk-6q7c68yx.js";
+} from "./chunk-f1xh7a32.js";
 import {
   SyncValidationError2
-} from "./chunk-y0zmae99.js";
+} from "./chunk-k4g51s47.js";
 import {
   ModelExecutionSession
-} from "./chunk-mmac1mp6.js";
+} from "./chunk-52c5zb0z.js";
 import {
   isOAuthConnectorDefinition2
 } from "./chunk-9x8dybsb.js";
 import {
   ensureExecutionRecord2
-} from "./chunk-8w27xf9e.js";
+} from "./chunk-a7er86jd.js";
 import {
   ConnectorConnectionStorageError2,
   CONNECTOR_CONNECTION_RUN_FAILURE_CODES2
@@ -129,7 +135,7 @@ import {
 } from "./chunk-kkqwzy6h.js";
 import {
   sameEmbeddingModel
-} from "./chunk-p1683c0v.js";
+} from "./chunk-kyjcwqz6.js";
 import {
   supportsDirectUpload2
 } from "./chunk-h3v76ams.js";
@@ -2701,11 +2707,13 @@ var DEGRADED_FAILURE_THRESHOLD = 2;
 var ABANDONED_UPLOAD_SWEEP_DELAY_MS = 5 * 60000;
 var ABANDONED_UPLOAD_ABORT_GRACE_MS = 15 * 60000;
 var ABANDONED_SOURCE_PURGE_BATCH = 50000;
+var EXPIRED_FILE_DOWNLOAD_RETENTION_MS = 7 * 24 * 60 * 60000;
 
 class OntologyMaintenance {
   projectId;
   storage;
   dispatcher;
+  vectorIndexing;
   blobStorage;
   intervalMs;
   publishedOutboxRetentionMs;
@@ -2725,6 +2733,7 @@ class OntologyMaintenance {
     this.projectId = dependencies.projectId;
     this.storage = dependencies.storage;
     this.dispatcher = dependencies.dispatcher;
+    this.vectorIndexing = dependencies.vectorIndexing;
     this.blobStorage = dependencies.blobStorage;
     this.intervalMs = positiveInteger(options.intervalMs ?? DEFAULT_INTERVAL_MS, "intervalMs");
     this.publishedOutboxRetentionMs = nonnegativeInteger(options.publishedOutboxRetentionMs ?? DEFAULT_RETENTION_MS, "publishedOutboxRetentionMs");
@@ -2788,9 +2797,12 @@ class OntologyMaintenance {
     this.snapshot = { ...this.snapshot, running: true, lastStartedAt: started.toISOString() };
     const failures = [];
     await captureFailure(() => this.dispatcher.drain(), failures);
+    const vectorIndexing = this.vectorIndexing;
+    if (vectorIndexing)
+      await captureFailure(() => vectorIndexing.drain(), failures);
     const cleanup = await this.cleanupExpiredRows(started, failures);
-    const { outbox, terminalSources } = await this.readOperationalSummaries(failures);
-    this.completePass({ started, failures, cleanup, outbox, terminalSources });
+    const outbox = await captureFailure(() => this.storage.ontology.outbox.summarize({ projectId: this.projectId }), failures, this.snapshot.outbox);
+    this.completePass({ started, failures, cleanup, outbox });
   }
   async cleanupExpiredRows(started, failures) {
     const publishedOutboxRowsDeleted = await captureFailure(() => this.storage.ontology.outbox.purgePublished({
@@ -2798,6 +2810,7 @@ class OntologyMaintenance {
       publishedBefore: new Date(started.getTime() - this.publishedOutboxRetentionMs).toISOString(),
       limit: this.cleanupLimit
     }), failures, 0);
+    const spentPlanRows = await captureFailure(() => this.purgeSpentPlans(started), failures, 0);
     const terminalSources = await captureFailure(() => this.storage.ontology.sources.cleanupTerminal({
       projectId: this.projectId,
       terminalBefore: new Date(started.getTime() - this.terminalSourceRetentionMs).toISOString(),
@@ -2809,9 +2822,17 @@ class OntologyMaintenance {
       await captureFailure(() => this.retireAbandonedUploads(uploads, started, failures), failures);
       await captureFailure(() => uploads.cleanupExpired(started), failures, 0);
     }
+    const downloads = this.storage.fileDownloadGrants;
+    if (downloads) {
+      await captureFailure(() => downloads.deleteExpired({
+        projectId: this.projectId,
+        expiredBefore: new Date(started.getTime() - EXPIRED_FILE_DOWNLOAD_RETENTION_MS),
+        limit: this.cleanupLimit
+      }), failures, 0);
+    }
     return {
       publishedOutboxRowsDeleted,
-      terminalSourceRowsDeleted: terminalSources.rowsDeleted + abandonedSources.rowsDeleted,
+      terminalSourceRowsDeleted: terminalSources.rowsDeleted + abandonedSources.rowsDeleted + spentPlanRows,
       terminalSourceMaterializationsDeleted: terminalSources.materializationsDeleted + abandonedSources.materializationsDeleted
     };
   }
@@ -2831,6 +2852,19 @@ class OntologyMaintenance {
       }
     }
   }
+  async purgeSpentPlans(started) {
+    let rowsDeleted = 0;
+    while (true) {
+      const purged = await this.storage.ontology.replacementPlans.purge({
+        projectId: this.projectId,
+        limit: ABANDONED_SOURCE_PURGE_BATCH
+      });
+      rowsDeleted += purged;
+      if (purged < ABANDONED_SOURCE_PURGE_BATCH || this.now().getTime() - started.getTime() >= this.intervalMs) {
+        return rowsDeleted;
+      }
+    }
+  }
   async retireAbandonedUploads(uploads, now, failures) {
     const abandoned = await uploads.listAbandoned({
       projectId: this.projectId,
@@ -2868,11 +2902,6 @@ class OntologyMaintenance {
     if (abortFailure !== undefined)
       throw abortFailure;
   }
-  async readOperationalSummaries(failures) {
-    const outbox = await captureFailure(() => this.storage.ontology.outbox.summarize({ projectId: this.projectId }), failures, this.snapshot.outbox);
-    const terminalSummary = await captureFailure(() => this.storage.ontology.sources.summarizeTerminal({ projectId: this.projectId }), failures, this.snapshot.terminalSources);
-    return { outbox, terminalSources: terminalSummary };
-  }
   completePass(input) {
     const completed = this.now();
     const failure = input.failures.length > 0 ? new AggregateError(input.failures, "Maintenance pass failed.") : null;
@@ -2885,7 +2914,6 @@ class OntologyMaintenance {
       consecutiveFailures: failure ? this.snapshot.consecutiveFailures + 1 : 0,
       lastError: failure ? errorMessage(failure) : null,
       outbox: input.outbox,
-      terminalSources: input.terminalSources,
       cleanup: input.cleanup
     };
     if (failure)
@@ -2937,7 +2965,6 @@ function emptySnapshot(intervalMs) {
     consecutiveFailures: 0,
     lastError: null,
     outbox: null,
-    terminalSources: null,
     cleanup: null
   };
 }
@@ -2993,6 +3020,163 @@ async function captureFailure(run, failures, fallback) {
     return fallback;
   }
 }
+// src/models/execution/reranking.ts
+import { randomUUID as randomUUID6 } from "crypto";
+function bindRerankingModels(catalog, session) {
+  if (!catalog)
+    return;
+  return indexModelBindings(catalog.list().map(({ model }) => Object.freeze({
+    providerId: model.providerId,
+    modelId: model.modelId,
+    definition: model.definition,
+    rerank: (input) => executeReranking(session, model, input)
+  })), "reranking", assertRerankingModel);
+}
+async function executeReranking(session, binding, input) {
+  const query = input.query;
+  if (typeof query !== "string" || !query.trim() || !Array.isArray(input.documents) || input.documents.some((text) => typeof text !== "string" || !text.trim())) {
+    throw new TypeError("[SixbModels] Reranking requires a query and nonempty document strings.");
+  }
+  const documents = [...input.documents];
+  const signal = AbortSignal.any([session.signal(input.signal), AbortSignal.timeout(30000)]);
+  signal.throwIfAborted();
+  if (documents.length === 0)
+    return { results: [] };
+  const accounting = await session.accounting();
+  accounting.assertHealthy();
+  const model = await binding.resolve?.() ?? binding;
+  assertRerankingModel(model);
+  if (model.providerId !== binding.providerId || model.modelId !== binding.modelId) {
+    throw new TypeError("[SixbModels] Resolved reranking model identity does not match its binding.");
+  }
+  signal.throwIfAborted();
+  const encoder = new TextEncoder;
+  const queryBytes = encoder.encode(query).byteLength;
+  const tokens = documents.reduce((sum, text) => sum + Math.ceil((queryBytes + encoder.encode(text).byteLength) / 4), 0);
+  const callId = `call_${randomUUID6()}`;
+  await accounting.admitCall({
+    callId,
+    providerId: model.providerId,
+    modelId: model.modelId,
+    modelKind: "reranking",
+    costEstimator: model.costEstimator,
+    inputTokens: { status: "estimated", tokens, method: "utf8BytesDividedByFour" },
+    outputTokenAllowance: 0,
+    estimatedTotalTokens: tokens
+  });
+  let result;
+  try {
+    signal.throwIfAborted();
+    result = await model.rerank({ query, documents, signal });
+  } catch (error) {
+    const metadata = error instanceof RerankingModelResponseError2 ? error.metadata : {};
+    await recordReranking(accounting, model, callId, metadata);
+    throw error;
+  }
+  await recordReranking(accounting, model, callId, result ?? {});
+  signal.throwIfAborted();
+  try {
+    return { ...result, results: validateRerankingResults(result?.results, documents.length) };
+  } catch (cause) {
+    throw new RerankingModelResponseError2("[SixbModels] Invalid reranking response.", model.providerId, model.modelId, result ?? {}, { cause });
+  }
+}
+async function recordReranking(accounting, model, callId, metadata) {
+  const usage = { ...metadata.usage, outputTokens: metadata.usage?.outputTokens ?? 0 };
+  const estimate = estimateModelCall(model, {
+    usage,
+    route: metadata.route,
+    responseModelId: metadata.responseModelId
+  });
+  await accounting.onModelCallEnd({
+    callId,
+    modelKind: "reranking",
+    providerId: model.providerId,
+    modelId: model.modelId,
+    responseId: metadata.providerIds?.responseId ?? callId,
+    responseModelId: metadata.responseModelId,
+    providerIds: metadata.providerIds,
+    usage,
+    cost: metadata.reportedCost ? { status: "reported", ...metadata.reportedCost } : estimate,
+    ...metadata.reportedCost ? { estimate } : {},
+    route: metadata.route
+  });
+}
+
+// src/objects/vectors/indexing-dispatch.ts
+var PAGE_SIZE = 100;
+var REDISPATCH_DELAY_MS = 30000;
+
+class VectorIndexingDispatcher {
+  projectId;
+  indexing;
+  queue;
+  running = null;
+  next = null;
+  stopped = false;
+  constructor(projectId, indexing, queue) {
+    this.projectId = projectId;
+    this.indexing = indexing;
+    this.queue = queue;
+  }
+  notify() {
+    this.drain().catch((error) => console.error("[Sixb] Could not dispatch vector indexing; maintenance will retry:", error));
+  }
+  drain() {
+    this.next ??= settled(this.running).then(() => {
+      this.next = null;
+      if (this.stopped)
+        return;
+      this.running = this.pass().finally(() => {
+        this.running = null;
+      });
+      return this.running;
+    });
+    return this.next;
+  }
+  async stop() {
+    this.stopped = true;
+    await settled(this.next);
+    await settled(this.running);
+  }
+  async pass() {
+    for (;; ) {
+      const now = new Date;
+      const work = await this.indexing.listDue({
+        projectId: this.projectId,
+        now: now.toISOString(),
+        limit: PAGE_SIZE
+      });
+      if (work.length) {
+        const jobs = [...new Map(work.map((item) => [item.batchId ?? item.id, item])).entries()];
+        await this.queue.enqueue({
+          projectId: this.projectId,
+          jobs: jobs.map(([id, item]) => ({
+            id,
+            type: "vector.index.requested",
+            payload: { indexingId: id },
+            availableAt: item.availableAt
+          }))
+        });
+        await this.indexing.dispatched({
+          projectId: this.projectId,
+          ids: work.map((item) => item.id),
+          nextDispatchAt: new Date(now.getTime() + REDISPATCH_DELAY_MS).toISOString()
+        });
+      }
+      if (work.length < PAGE_SIZE || this.stopped)
+        return;
+    }
+  }
+}
+function settled(promise) {
+  return (promise ?? Promise.resolve()).then(() => {
+    return;
+  }, () => {
+    return;
+  });
+}
+
 // src/agents/tool-catalog.ts
 function createAgentToolCatalog(definitions) {
   const tools = toolsFromProjectConfig(definitions);
@@ -3280,6 +3464,7 @@ class SixbHost2 {
   webhookRegistry;
   hostContext;
   committedFacts;
+  vectorIndexing;
   eventService;
   ontologyMaintenance;
   storageReadiness;
@@ -3359,15 +3544,21 @@ class SixbHost2 {
         ...failure
       })
     });
+    const vectorIndexing = this.storage.ontology.vectorIndexing;
+    if (vectorIndexing && definitions.ontology.listObjectTypes().some((type) => Object.keys(type.search?.vectors ?? {}).length > 0)) {
+      this.vectorIndexing = new VectorIndexingDispatcher(this.projectId, vectorIndexing, this.queues.vectorIndexing);
+    }
     this.ontologyMaintenance = new OntologyMaintenance({
       projectId: this.projectId,
       storage: this.storage,
       dispatcher: this.committedFacts,
+      vectorIndexing: this.vectorIndexing,
       blobStorage: this.blobStorage,
       options: options.ontologyMaintenance
     });
     this.hostContext = {
       embeddingModels: definitions.models?.embedding,
+      rerankingModels: definitions.models?.reranking,
       projectId: this.projectId,
       broker: this.broker,
       ontology: definitions.ontology,
@@ -3392,15 +3583,18 @@ class SixbHost2 {
     }
     const modelExecution = new ModelExecutionSession({ ...this.hostContext, runtimeAuthorization: capturedScope.authorization }, capturedScope.execution);
     const embeddingModels = bindEmbeddingModels(this.hostContext.embeddingModels, modelExecution);
+    const rerankingModels = bindRerankingModels(this.hostContext.rerankingModels, modelExecution);
     const objectReader = createAuthorizedObjectReader({
       scope: capturedScope,
       ontology: this.hostContext.ontology,
       objectStorage: this.storage.objects,
-      embeddingModels
+      embeddingModels,
+      rerankingModels
     });
     const runtime = {
       ...this.hostContext,
       embeddingModels,
+      rerankingModels,
       modelExecution,
       runtimeAuthorization: capturedScope.authorization,
       objectReader,
@@ -3408,7 +3602,10 @@ class SixbHost2 {
     };
     registerOntologyMutationRuntime2(runtime, createOntologyMutationRuntime2({
       materializer: this.materializer.withScope(capturedScope),
-      notifyCommittedFacts: () => this.committedFacts.notify()
+      notifyCommittedFacts: () => {
+        this.committedFacts.notify();
+        this.vectorIndexing?.notify();
+      }
     }));
     return createBoundSixb(runtime, this.sixbDependencies(), capturedScope.execution);
   }
@@ -3442,6 +3639,7 @@ class SixbHost2 {
   }
   async closeBroker() {
     await this.committedFacts.stop();
+    await this.vectorIndexing?.stop();
     await this.ontologyMaintenance.stop();
     await this.broker.close?.();
   }
@@ -3456,6 +3654,7 @@ class SixbHost2 {
       connectorService: this.connectorService,
       ...this.connectorService.connectionProcess === undefined ? {} : { connectorConnections: this.connectorService.connectionProcess },
       blobStorage: this.blobStorage,
+      apiPublicOrigin: () => getApiPublicOrigin(this),
       lakeStorage: this.lakeStorage
     };
   }
@@ -3527,4 +3726,4 @@ async function createSixb2(options) {
 }
 export { WebhookValidationError2, defineWebhook2, webhookConnector2, UnverifiedWebhookError2, resolveWebhookVerification2, warnUnverifiedWebhook2, SixbHost2, createSixb2 };
 
-//# debugId=5D4C3D731AF4BC5E64756E2164756E21
+//# debugId=CC8274E37BA99FA964756E2164756E21