@absolutejs/absolute 0.19.0-beta.641 → 0.19.0-beta.642

package/dist/ai/index.js

@@ -16711,6 +16711,13 @@ var prepareRAGDirectoryDocuments = async (input) => prepareRAGDocuments(await lo
 
 // src/ai/rag/collection.ts
 var DEFAULT_TOP_K2 = 6;
+var MULTIVECTOR_VARIANT_CHUNK_DELIMITER = "__mv__";
+var MULTIVECTOR_PARENT_CHUNK_ID = "absoluteMultivectorParentChunkId";
+var MULTIVECTOR_VARIANT_ID = "absoluteMultivectorVariantId";
+var MULTIVECTOR_VARIANT_LABEL = "absoluteMultivectorVariantLabel";
+var MULTIVECTOR_VARIANT_TEXT = "absoluteMultivectorVariantText";
+var MULTIVECTOR_VARIANT_METADATA = "absoluteMultivectorVariantMetadata";
+var MULTIVECTOR_PRIMARY = "absoluteMultivectorPrimary";
 var resolveNativeQueryProfile = (input) => {
   if (!input.profile) {
     return;
@@ -16826,6 +16833,108 @@ var mergeQueryResults = (results) => {
     return left.chunkId.localeCompare(right.chunkId);
   });
 };
+var getMultivectorParentChunkId = (result) => typeof result.metadata?.[MULTIVECTOR_PARENT_CHUNK_ID] === "string" ? result.metadata[MULTIVECTOR_PARENT_CHUNK_ID] : undefined;
+var stripMultivectorInternalMetadata = (metadata) => {
+  if (!metadata) {
+    return;
+  }
+  const cleaned = { ...metadata };
+  delete cleaned[MULTIVECTOR_PARENT_CHUNK_ID];
+  delete cleaned[MULTIVECTOR_VARIANT_ID];
+  delete cleaned[MULTIVECTOR_VARIANT_LABEL];
+  delete cleaned[MULTIVECTOR_VARIANT_TEXT];
+  delete cleaned[MULTIVECTOR_VARIANT_METADATA];
+  delete cleaned[MULTIVECTOR_PRIMARY];
+  return cleaned;
+};
+var collapseMultivectorResults = (results) => {
+  const grouped = new Map;
+  for (const result of results) {
+    const parentChunkId = getMultivectorParentChunkId(result) ?? result.chunkId;
+    const existing = grouped.get(parentChunkId);
+    const nextVariantHits = getMultivectorParentChunkId(result) ? 1 : 0;
+    if (existing) {
+      existing.results.push(result);
+      existing.variantHits += nextVariantHits;
+      continue;
+    }
+    grouped.set(parentChunkId, {
+      parentChunkId,
+      results: [result],
+      variantHits: nextVariantHits
+    });
+  }
+  let variantHits = 0;
+  let collapsedParents = 0;
+  const collapsed = [...grouped.values()].map((entry) => {
+    variantHits += entry.variantHits;
+    if (entry.variantHits > 0) {
+      collapsedParents += 1;
+    }
+    const preferred = entry.results.reduce((best, current) => current.score > best.score ? current : best);
+    const variantMatches = entry.results.map((result) => {
+      const variantId = result.metadata?.[MULTIVECTOR_VARIANT_ID];
+      if (typeof variantId !== "string") {
+        return;
+      }
+      return {
+        id: variantId,
+        label: typeof result.metadata?.[MULTIVECTOR_VARIANT_LABEL] === "string" ? String(result.metadata?.[MULTIVECTOR_VARIANT_LABEL]) : undefined,
+        score: result.score
+      };
+    }).filter((value) => value !== undefined).sort((left, right) => right.score - left.score);
+    const cleanedMetadata = stripMultivectorInternalMetadata(preferred.metadata);
+    return {
+      ...preferred,
+      chunkId: entry.parentChunkId,
+      metadata: {
+        ...cleanedMetadata ?? {},
+        multivectorMatchedVariantCount: variantMatches.length,
+        multivectorMatchedVariantId: variantMatches[0]?.id,
+        multivectorMatchedVariantLabel: variantMatches[0]?.label,
+        multivectorMatchedVariants: variantMatches.length > 0 ? variantMatches.map((match) => ({
+          id: match.id,
+          label: match.label,
+          score: match.score
+        })) : undefined
+      }
+    };
+  }).sort((left, right) => {
+    if (right.score !== left.score) {
+      return right.score - left.score;
+    }
+    return left.chunkId.localeCompare(right.chunkId);
+  });
+  return {
+    collapsed,
+    collapsedParents,
+    variantHits
+  };
+};
+var expandChunkForMultivectorStorage = (chunk) => {
+  const expanded = [{ ...chunk }];
+  for (const variant of chunk.embeddingVariants ?? []) {
+    if (!variant.id) {
+      continue;
+    }
+    expanded.push({
+      ...chunk,
+      chunkId: `${chunk.chunkId}${MULTIVECTOR_VARIANT_CHUNK_DELIMITER}${variant.id}`,
+      embedding: variant.embedding,
+      metadata: {
+        ...chunk.metadata ?? {},
+        [MULTIVECTOR_PARENT_CHUNK_ID]: chunk.chunkId,
+        [MULTIVECTOR_PRIMARY]: false,
+        [MULTIVECTOR_VARIANT_ID]: variant.id,
+        ...variant.label ? { [MULTIVECTOR_VARIANT_LABEL]: variant.label } : {},
+        ...variant.text ? { [MULTIVECTOR_VARIANT_TEXT]: variant.text } : {},
+        ...variant.metadata ? { [MULTIVECTOR_VARIANT_METADATA]: variant.metadata } : {}
+      },
+      text: variant.text ?? chunk.text
+    });
+  }
+  return expanded;
+};
 var getRAGSourceDiversityKey = (result) => {
   const documentId = typeof result.metadata?.documentId === "string" ? result.metadata.documentId : undefined;
   return result.source ?? documentId ?? result.title ?? result.chunkId;
@@ -17272,23 +17381,34 @@ var createRAGCollection = (options) => {
           topK: lexicalTopK
         }) ?? Promise.resolve([]) : Promise.resolve([])
       ]);
+      const annotatedLexicalResults = annotateRetrievalQueryOrigin({
+        inputQuery: input.query,
+        query,
+        queryIndex,
+        results: weightQueryResults(lexicalResults2, queryIndex),
+        transformedQuery: transformed.query
+      });
+      const annotatedVectorResults = annotateRetrievalQueryOrigin({
+        inputQuery: input.query,
+        query,
+        queryIndex,
+        results: weightQueryResults(vectorResults2, queryIndex),
+        transformedQuery: transformed.query
+      });
+      const collapsedLexicalResults = collapseMultivectorResults(annotatedLexicalResults);
+      const collapsedVectorResults = collapseMultivectorResults(annotatedVectorResults);
       return {
-        lexicalResults: annotateRetrievalQueryOrigin({
-          inputQuery: input.query,
-          query,
-          queryIndex,
-          results: weightQueryResults(lexicalResults2, queryIndex),
-          transformedQuery: transformed.query
-        }),
-        vectorResults: annotateRetrievalQueryOrigin({
-          inputQuery: input.query,
-          query,
-          queryIndex,
-          results: weightQueryResults(vectorResults2, queryIndex),
-          transformedQuery: transformed.query
-        })
+        lexicalResults: collapsedLexicalResults.collapsed,
+        lexicalVariantHits: collapsedLexicalResults.variantHits,
+        lexicalCollapsedParents: collapsedLexicalResults.collapsedParents,
+        vectorResults: collapsedVectorResults.collapsed,
+        vectorVariantHits: collapsedVectorResults.variantHits,
+        vectorCollapsedParents: collapsedVectorResults.collapsedParents
       };
     }));
+    const vectorVariantHits = resultGroups.reduce((total, group) => total + group.vectorVariantHits, 0);
+    const lexicalVariantHits = resultGroups.reduce((total, group) => total + group.lexicalVariantHits, 0);
+    const collapsedParents = resultGroups.reduce((total, group) => total + Math.max(group.vectorCollapsedParents, group.lexicalCollapsedParents), 0);
     const vectorResults = mergeQueryResults(resultGroups.flatMap((group) => group.vectorResults));
     if (runVector) {
       const vectorPlanMetadata = buildStoreQueryPlanTraceMetadata(options.store);
@@ -17296,6 +17416,8 @@ var createRAGCollection = (options) => {
         count: vectorResults.length,
         label: "Collected vector candidates",
         metadata: {
+          collapsedParents,
+          multiVectorVariantHits: vectorVariantHits,
           queryCount: searchQueries.length,
           topK: candidateTopK,
           ...vectorPlanMetadata
@@ -17311,6 +17433,8 @@ var createRAGCollection = (options) => {
         count: lexicalResults.length,
         label: "Collected lexical candidates",
         metadata: {
+          collapsedParents,
+          multiVectorVariantHits: lexicalVariantHits,
           queryCount: searchQueries.length,
           topK: lexicalTopK
         },
@@ -17419,6 +17543,12 @@ var createRAGCollection = (options) => {
           queryTransformLabel: transformed.label,
           queryTransformProvider: resolvedQueryTransform?.providerName,
           queryTransformReason: transformed.reason,
+          multiVector: {
+            collapsedParents,
+            configured: vectorVariantHits > 0 || lexicalVariantHits > 0,
+            lexicalVariantHits,
+            vectorVariantHits
+          },
           resultCounts: {
             final: limited.length,
             fused: results.length,
@@ -17475,6 +17605,12 @@ var createRAGCollection = (options) => {
         queryTransformLabel: transformed.label,
         queryTransformProvider: resolvedQueryTransform?.providerName,
         queryTransformReason: transformed.reason,
+        multiVector: {
+          collapsedParents,
+          configured: vectorVariantHits > 0 || lexicalVariantHits > 0,
+          lexicalVariantHits,
+          vectorVariantHits
+        },
         resultCounts: {
           final: filtered.length,
           fused: results.length,
@@ -17502,19 +17638,31 @@ var createRAGCollection = (options) => {
     return result.results;
   };
   const ingest = async (input) => {
-    const chunks = await Promise.all(input.chunks.map(async (chunk) => {
-      if (chunk.embedding) {
-        validateRAGEmbeddingDimensions(chunk.embedding, getExpectedDimensions(), "chunk");
-        return chunk;
-      }
-      return {
-        ...chunk,
-        embedding: await embed({
+    const chunks = (await Promise.all(input.chunks.map(async (chunk) => {
+      const normalizedEmbedding = chunk.embedding ? (validateRAGEmbeddingDimensions(chunk.embedding, getExpectedDimensions(), "chunk"), chunk.embedding) : await embed({
+        model: options.defaultModel,
+        text: chunk.text
+      }, "chunk");
+      const normalizedVariants = chunk.embeddingVariants ? await Promise.all(chunk.embeddingVariants.map(async (variant) => {
+        const embedding = variant.embedding ? (validateRAGEmbeddingDimensions(variant.embedding, getExpectedDimensions(), "chunk"), variant.embedding) : await embed({
           model: options.defaultModel,
-          text: chunk.text
-        }, "chunk")
-      };
-    }));
+          text: variant.text ?? chunk.text
+        }, "chunk");
+        return {
+          ...variant,
+          embedding
+        };
+      })) : undefined;
+      return expandChunkForMultivectorStorage({
+        ...chunk,
+        embedding: normalizedEmbedding,
+        embeddingVariants: normalizedVariants,
+        metadata: {
+          ...chunk.metadata ?? {},
+          [MULTIVECTOR_PRIMARY]: true
+        }
+      });
+    }))).flat();
     await options.store.upsert({ chunks });
   };
   return {
@@ -33909,5 +34057,5 @@ export {
   addRAGEvaluationSuiteCase
 };
 
-//# debugId=27461AB67370F55164756E2164756E21
+//# debugId=7958E184D4E3DB1764756E2164756E21
 //# sourceMappingURL=index.js.map