@delali/narsil 0.2.4-next.50 → 0.2.4-next.52

package/dist/chunk-U36IDJN7.mjs

@@ -0,0 +1,366 @@
+import { validateRequiredFields } from './chunk-ZT5BRDFW.mjs';
+import { ErrorCodes, NarsilError } from './chunk-P4ELXVJQ.mjs';
+
+// src/narsil/internals.ts
+var engineCores = /* @__PURE__ */ new WeakMap();
+function bindEngineCore(engine, core) {
+  engineCores.set(engine, core);
+}
+function engineCoreOf(engine) {
+  return engineCores.get(engine);
+}
+
+// src/distribution/constants.ts
+var WIRE_BATCH_BUDGET = {
+  maxCount: 1e3,
+  maxBytes: 8388608,
+  overheadBytes: 256
+};
+var MAX_PARTITION_COUNT = 65536;
+var MAX_REPLICATION_FACTOR = 255;
+
+// src/engine/embed.ts
+function assertDocumentCarriesMappedVectors(document, embeddingConfig, adapterName) {
+  for (const targetField of Object.keys(embeddingConfig.fields)) {
+    const existing = resolveFieldValue(document, targetField);
+    if (existing === void 0 || existing === null) {
+      throw new NarsilError(
+        ErrorCodes.EMBEDDING_CONFIG_INVALID,
+        adapterName ? `The index embeds "${targetField}" via adapter "${adapterName}", which is not registered on this engine. Register the adapter or supply the vector on the document.` : `The index embeds "${targetField}" automatically but no embedding adapter is bound. Bind an adapter or supply the vector on the document.`,
+        adapterName ? { field: targetField, adapter: adapterName } : { field: targetField }
+      );
+    }
+  }
+}
+function validateVector(vector, targetField, expectedDimensions) {
+  if (!(vector instanceof Float32Array)) {
+    throw new NarsilError(
+      ErrorCodes.EMBEDDING_FAILED,
+      `Adapter returned ${typeof vector} for field "${targetField}", expected Float32Array`,
+      { field: targetField, returnedType: typeof vector }
+    );
+  }
+  if (vector.length !== expectedDimensions) {
+    throw new NarsilError(
+      ErrorCodes.EMBEDDING_DIMENSION_MISMATCH,
+      `Adapter returned ${vector.length}-dimensional vector for field "${targetField}", expected ${expectedDimensions}`,
+      { field: targetField, expected: expectedDimensions, actual: vector.length }
+    );
+  }
+  for (let i = 0; i < vector.length; i++) {
+    if (Number.isNaN(vector[i])) {
+      throw new NarsilError(
+        ErrorCodes.EMBEDDING_FAILED,
+        `Adapter returned vector with NaN at index ${i} for field "${targetField}"`,
+        { field: targetField, index: i }
+      );
+    }
+  }
+  return vector;
+}
+function resolveFieldValue(document, path) {
+  if (!path.includes(".")) return document[path];
+  const segments = path.split(".");
+  let current = document;
+  for (const segment of segments) {
+    if (current === null || current === void 0 || typeof current !== "object") return void 0;
+    current = current[segment];
+  }
+  return current;
+}
+function setFieldValue(document, path, value) {
+  if (!path.includes(".")) {
+    document[path] = value;
+    return;
+  }
+  const segments = path.split(".");
+  let current = document;
+  for (let i = 0; i < segments.length - 1; i++) {
+    let next = current[segments[i]];
+    if (next === null || next === void 0 || typeof next !== "object") {
+      next = {};
+      current[segments[i]] = next;
+    }
+    current = next;
+  }
+  current[segments[segments.length - 1]] = value;
+}
+function deleteFieldValue(document, path) {
+  if (!path.includes(".")) {
+    delete document[path];
+    return;
+  }
+  const segments = path.split(".");
+  let current = document;
+  for (let i = 0; i < segments.length - 1; i++) {
+    const next = current[segments[i]];
+    if (next === null || next === void 0 || typeof next !== "object") return;
+    current = next;
+  }
+  delete current[segments[segments.length - 1]];
+}
+function collectSourceText(document, sourceFields, targetField, documentIndex) {
+  const sources = Array.isArray(sourceFields) ? sourceFields : [sourceFields];
+  const parts = [];
+  for (const source of sources) {
+    const value = resolveFieldValue(document, source);
+    if (value === void 0 || value === null || value === "") continue;
+    if (typeof value !== "string") {
+      throw new NarsilError(
+        ErrorCodes.DOC_VALIDATION_FAILED,
+        `Embedding source field "${source}" must be a string, got ${typeof value}`,
+        { field: source, targetField }
+      );
+    }
+    parts.push(value);
+  }
+  if (parts.length === 0) {
+    const details = { field: targetField };
+    if (documentIndex !== void 0) {
+      details.documentIndex = documentIndex;
+    }
+    throw new NarsilError(
+      ErrorCodes.EMBEDDING_NO_SOURCE,
+      `All source fields for embedding target "${targetField}" are missing or empty`,
+      details
+    );
+  }
+  return parts.join("\n");
+}
+function wrapAdapterError(err, targetField) {
+  if (err instanceof NarsilError) return err;
+  const message = err instanceof Error ? err.message : String(err);
+  return new NarsilError(ErrorCodes.EMBEDDING_FAILED, `Embedding failed for field "${targetField}": ${message}`, {
+    field: targetField,
+    cause: message
+  });
+}
+async function embedDocumentFields(document, embeddingConfig, adapter, signal) {
+  const embeddedFields = /* @__PURE__ */ new Set();
+  try {
+    for (const [targetField, sourceFields] of Object.entries(embeddingConfig.fields)) {
+      const existing = resolveFieldValue(document, targetField);
+      if (existing !== void 0 && existing !== null) continue;
+      const text = collectSourceText(document, sourceFields, targetField);
+      let vector;
+      try {
+        const raw = await adapter.embed(text, "document", signal);
+        vector = validateVector(raw, targetField, adapter.dimensions);
+      } catch (err) {
+        throw wrapAdapterError(err, targetField);
+      }
+      setFieldValue(document, targetField, vector);
+      embeddedFields.add(targetField);
+    }
+  } catch (err) {
+    for (const field of embeddedFields) {
+      deleteFieldValue(document, field);
+    }
+    throw err;
+  }
+  return embeddedFields;
+}
+var SOURCE_FAILURE_CODES = /* @__PURE__ */ new Set([
+  ErrorCodes.EMBEDDING_NO_SOURCE,
+  ErrorCodes.DOC_VALIDATION_FAILED
+]);
+function isDocumentSourceError(err) {
+  return err instanceof NarsilError && SOURCE_FAILURE_CODES.has(err.code);
+}
+async function embedBatchDocumentFields(documents, embeddingConfig, adapter, signal) {
+  const embedded = /* @__PURE__ */ new Map();
+  const failed = /* @__PURE__ */ new Map();
+  try {
+    for (const [targetField, sourceFields] of Object.entries(embeddingConfig.fields)) {
+      const needsEmbedding = [];
+      for (let i = 0; i < documents.length; i++) {
+        if (failed.has(i)) continue;
+        const doc = documents[i];
+        const existing = resolveFieldValue(doc, targetField);
+        if (existing !== void 0 && existing !== null) continue;
+        try {
+          const text = collectSourceText(doc, sourceFields, targetField, i);
+          needsEmbedding.push({ docIndex: i, text });
+        } catch (err) {
+          if (isDocumentSourceError(err)) {
+            failed.set(i, err);
+          } else {
+            throw err;
+          }
+        }
+      }
+      if (needsEmbedding.length === 0) continue;
+      const texts = needsEmbedding.map((entry) => entry.text);
+      let vectors;
+      try {
+        if (adapter.embedBatch) {
+          vectors = await adapter.embedBatch(texts, "document", signal);
+        } else {
+          vectors = [];
+          const concurrency = 8;
+          for (let start = 0; start < texts.length; start += concurrency) {
+            const chunk = texts.slice(start, start + concurrency);
+            const chunkResults = await Promise.all(chunk.map((t) => adapter.embed(t, "document", signal)));
+            vectors.push(...chunkResults);
+          }
+        }
+      } catch (err) {
+        throw wrapAdapterError(err, targetField);
+      }
+      if (vectors.length !== needsEmbedding.length) {
+        throw new NarsilError(
+          ErrorCodes.EMBEDDING_FAILED,
+          `Adapter returned ${vectors.length} vectors for ${needsEmbedding.length} inputs on field "${targetField}"`,
+          { field: targetField, expected: needsEmbedding.length, actual: vectors.length }
+        );
+      }
+      for (let j = 0; j < needsEmbedding.length; j++) {
+        const { docIndex } = needsEmbedding[j];
+        const validated = validateVector(vectors[j], targetField, adapter.dimensions);
+        setFieldValue(documents[docIndex], targetField, validated);
+        let fieldSet = embedded.get(docIndex);
+        if (!fieldSet) {
+          fieldSet = /* @__PURE__ */ new Set();
+          embedded.set(docIndex, fieldSet);
+        }
+        fieldSet.add(targetField);
+      }
+    }
+  } catch (err) {
+    for (const [docIndex, fields] of embedded) {
+      for (const field of fields) {
+        deleteFieldValue(documents[docIndex], field);
+      }
+    }
+    throw err;
+  }
+  return { embedded, failed };
+}
+
+// src/engine/mutations/insert-admission.ts
+function providedDocId(document) {
+  const id = document.id;
+  return typeof id === "string" && id.length > 0 ? id : void 0;
+}
+function asBatchInsertError(err) {
+  return err instanceof NarsilError ? err : new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, String(err));
+}
+function admitInsert(ctx, indexName, manager, docId, pendingAdmitted = 0) {
+  if (!ctx.isRebalancing(indexName)) {
+    manager.assertCapacity(pendingAdmitted);
+    return;
+  }
+  const bufferedState = ctx.bufferedDocState(indexName, docId);
+  const exists = bufferedState !== void 0 ? bufferedState === "present" : manager.has(docId);
+  if (exists) {
+    throw new NarsilError(ErrorCodes.DOC_ALREADY_EXISTS, `Document "${docId}" already exists`, { docId });
+  }
+  manager.assertCapacity(
+    ctx.pendingRebalanceWrites(indexName) + pendingAdmitted,
+    ctx.rebalanceTargetPartitionCount(indexName)
+  );
+}
+function collectRequiredFieldFailures(documents, chunkStart, chunkEnd, required, chunkFailedIndexes, failed) {
+  for (let i = chunkStart; i < chunkEnd; i++) {
+    try {
+      validateRequiredFields(documents[i], required);
+    } catch (err) {
+      chunkFailedIndexes.add(i);
+      failed.push({ docId: providedDocId(documents[i]) ?? "", error: asBatchInsertError(err) });
+    }
+  }
+}
+async function embedChunkDocuments(entry, documents, chunkStart, chunkEnd, signal, chunkFailedIndexes, failed) {
+  if (entry.config.embedding === void 0) return;
+  if (entry.embeddingAdapter === null) {
+    for (let i = chunkStart; i < chunkEnd; i++) {
+      if (chunkFailedIndexes.has(i)) continue;
+      try {
+        assertDocumentCarriesMappedVectors(
+          documents[i],
+          entry.config.embedding,
+          entry.embeddingAdapterName
+        );
+      } catch (err) {
+        chunkFailedIndexes.add(i);
+        failed.push({
+          docId: providedDocId(documents[i]) ?? "",
+          error: err instanceof NarsilError ? err : new NarsilError(ErrorCodes.EMBEDDING_FAILED, String(err))
+        });
+      }
+    }
+    return;
+  }
+  const embeddableSlice = [];
+  const embeddableOriginalIndexes = [];
+  for (let i = chunkStart; i < chunkEnd; i++) {
+    if (chunkFailedIndexes.has(i)) continue;
+    embeddableSlice.push(documents[i]);
+    embeddableOriginalIndexes.push(i);
+  }
+  if (embeddableSlice.length === 0) return;
+  try {
+    const embedResult = await embedBatchDocumentFields(
+      embeddableSlice,
+      entry.config.embedding,
+      entry.embeddingAdapter,
+      signal
+    );
+    for (const [sliceIndex, error] of embedResult.failed) {
+      const originalIdx = embeddableOriginalIndexes[sliceIndex];
+      chunkFailedIndexes.add(originalIdx);
+      failed.push({ docId: providedDocId(documents[originalIdx]) ?? "", error });
+    }
+  } catch (err) {
+    const embeddingError = err instanceof NarsilError ? err : new NarsilError(ErrorCodes.EMBEDDING_FAILED, String(err));
+    for (const originalIdx of embeddableOriginalIndexes) {
+      chunkFailedIndexes.add(originalIdx);
+      failed.push({ docId: providedDocId(documents[originalIdx]) ?? "", error: embeddingError });
+    }
+  }
+}
+
+// src/engine/resolve-vector-text.ts
+async function resolveVectorText(params, embeddingAdapter, signal, embeddingAdapterName) {
+  if (!params.vector) return params;
+  if (params.vector.text === void 0 && params.vector.value === void 0) return params;
+  if (params.vector.value !== void 0 && params.vector.text === void 0) return params;
+  if (params.vector.text !== void 0 && params.vector.value !== void 0) {
+    throw new NarsilError(ErrorCodes.EMBEDDING_CONFIG_INVALID, "Vector query cannot have both 'text' and 'value'");
+  }
+  if (!embeddingAdapter) {
+    throw new NarsilError(
+      ErrorCodes.EMBEDDING_CONFIG_INVALID,
+      embeddingAdapterName ? `Vector query with 'text' needs embedding adapter "${embeddingAdapterName}", which is not registered on this engine. Register it with registerEmbeddingAdapter or supply a raw vector.` : "Vector query with 'text' requires an embedding adapter on the index or instance",
+      embeddingAdapterName ? { adapter: embeddingAdapterName } : void 0
+    );
+  }
+  if (typeof params.vector.text !== "string") {
+    throw new NarsilError(
+      ErrorCodes.DOC_VALIDATION_FAILED,
+      `Vector query 'text' must be a string, got ${typeof params.vector.text}`
+    );
+  }
+  if (params.vector.text.length === 0) {
+    throw new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, "Vector query 'text' must not be empty");
+  }
+  const raw = await embeddingAdapter.embed(params.vector.text, "query", signal);
+  if (!(raw instanceof Float32Array)) {
+    throw new NarsilError(
+      ErrorCodes.EMBEDDING_FAILED,
+      `Adapter returned ${typeof raw} for query embedding, expected Float32Array`
+    );
+  }
+  if (raw.length !== embeddingAdapter.dimensions) {
+    throw new NarsilError(
+      ErrorCodes.EMBEDDING_DIMENSION_MISMATCH,
+      `Adapter returned ${raw.length}-dimensional vector for query, expected ${embeddingAdapter.dimensions}`,
+      { expected: embeddingAdapter.dimensions, actual: raw.length }
+    );
+  }
+  const resolved = { ...params.vector, value: Array.from(raw) };
+  delete resolved.text;
+  return { ...params, vector: resolved };
+}
+
+export { MAX_PARTITION_COUNT, MAX_REPLICATION_FACTOR, WIRE_BATCH_BUDGET, admitInsert, asBatchInsertError, assertDocumentCarriesMappedVectors, bindEngineCore, collectRequiredFieldFailures, embedChunkDocuments, embedDocumentFields, engineCoreOf, providedDocId, resolveVectorText };