@infinityi/engine-lib 1.3.1 → 1.4.0

package/dist/retrieval/index.js

@@ -0,0 +1,747 @@
+import {
+  estimateTokens
+} from "../index-nvdsr51v.js";
+import"../index-1p6mb2vz.js";
+import {
+  CancelledError
+} from "../index-ajr3nk10.js";
+import"../index-37x76zdn.js";
+
+// src/retrieval/utils.ts
+function throwIfAborted(ctx) {
+  if (ctx === undefined)
+    return;
+  const signal = ctx.engine?.signal ?? ctx.signal;
+  if (signal?.aborted)
+    throw new CancelledError("retrieval cancelled");
+}
+function assertPositiveInteger(name, value) {
+  if (value === undefined)
+    return;
+  if (!Number.isFinite(value) || !Number.isInteger(value) || value <= 0) {
+    throw new TypeError(`${name} must be a positive integer`);
+  }
+}
+function assertNonNegativeInteger(name, value) {
+  if (!Number.isFinite(value) || !Number.isInteger(value) || value < 0) {
+    throw new TypeError(`${name} must be a non-negative integer`);
+  }
+}
+
+// src/retrieval/chunking.ts
+var DEFAULT_SEPARATORS = [`
+
+`, `
+`, ". ", " "];
+function lineStarts(text) {
+  const starts = [0];
+  for (let index = 0;index < text.length; index += 1) {
+    if (text.charCodeAt(index) === 10)
+      starts.push(index + 1);
+  }
+  return starts;
+}
+function lineNumberAt(starts, offset) {
+  let low = 0;
+  let high = starts.length - 1;
+  while (low <= high) {
+    const mid = Math.floor((low + high) / 2);
+    const start = starts[mid] ?? 0;
+    const next = starts[mid + 1] ?? Number.POSITIVE_INFINITY;
+    if (offset >= start && offset < next)
+      return mid + 1;
+    if (offset < start)
+      high = mid - 1;
+    else
+      low = mid + 1;
+  }
+  return Math.max(1, starts.length);
+}
+function trimRange(text, start, end) {
+  let trimmedStart = start;
+  let trimmedEnd = end;
+  while (trimmedStart < trimmedEnd && /\s/.test(text[trimmedStart] ?? ""))
+    trimmedStart += 1;
+  while (trimmedEnd > trimmedStart && /\s/.test(text[trimmedEnd - 1] ?? ""))
+    trimmedEnd -= 1;
+  return { start: trimmedStart, end: trimmedEnd };
+}
+function findChunkEnd(text, start, maxChars, separators) {
+  const maxEnd = Math.min(text.length, start + maxChars);
+  if (maxEnd >= text.length)
+    return text.length;
+  const minBreak = start + Math.floor(maxChars / 2);
+  for (const separator of separators) {
+    if (separator === "")
+      continue;
+    const index = text.lastIndexOf(separator, maxEnd);
+    if (index > minBreak)
+      return Math.min(text.length, index + separator.length);
+  }
+  return maxEnd;
+}
+function fixedRanges(text, offset, maxChars) {
+  const ranges = [];
+  for (let start = 0;start < text.length; start += maxChars) {
+    ranges.push({ start: offset + start, end: offset + Math.min(text.length, start + maxChars) });
+  }
+  return ranges;
+}
+function splitBySeparator(text, offset, separator) {
+  const ranges = [];
+  let start = 0;
+  while (start < text.length) {
+    const index = text.indexOf(separator, start);
+    const end = index === -1 ? text.length : index + separator.length;
+    ranges.push({ start: offset + start, end: offset + end });
+    start = end;
+  }
+  return ranges;
+}
+function recursiveUnits(text, offset, maxChars, separators, separatorIndex) {
+  if (text.length <= maxChars)
+    return [{ start: offset, end: offset + text.length }];
+  const separator = separators[separatorIndex];
+  if (separator === undefined || separator === "")
+    return fixedRanges(text, offset, maxChars);
+  const split = splitBySeparator(text, offset, separator);
+  if (split.length <= 1)
+    return recursiveUnits(text, offset, maxChars, separators, separatorIndex + 1);
+  const out = [];
+  for (const range of split) {
+    const length = range.end - range.start;
+    if (length <= maxChars)
+      out.push(range);
+    else
+      out.push(...recursiveUnits(text.slice(range.start - offset, range.end - offset), range.start, maxChars, separators, separatorIndex + 1));
+  }
+  return out;
+}
+function mergeRanges(ranges, maxChars) {
+  const merged = [];
+  let current;
+  for (const range of ranges) {
+    if (current === undefined) {
+      current = range;
+      continue;
+    }
+    if (range.end - current.start <= maxChars) {
+      current = { start: current.start, end: range.end };
+      continue;
+    }
+    merged.push(current);
+    current = range;
+  }
+  if (current !== undefined)
+    merged.push(current);
+  return merged;
+}
+function sourceForChunk(document, location) {
+  return {
+    ...document.source ?? {},
+    ...document.id !== undefined && document.source?.id === undefined ? { id: document.id } : {},
+    location
+  };
+}
+function chunkId(document, documentIndex, chunkIndex) {
+  const sourceId = document.id ?? document.source?.id ?? document.source?.uri ?? `document-${documentIndex + 1}`;
+  return `${sourceId}#chunk-${chunkIndex + 1}`;
+}
+function chunkFromRange(document, documentIndex, chunkIndex, starts, range) {
+  const trimmed = trimRange(document.content, range.start, range.end);
+  if (trimmed.end <= trimmed.start)
+    return;
+  const location = {
+    startOffset: trimmed.start,
+    endOffset: trimmed.end,
+    startLine: lineNumberAt(starts, trimmed.start),
+    endLine: lineNumberAt(starts, Math.max(trimmed.start, trimmed.end - 1))
+  };
+  return {
+    id: chunkId(document, documentIndex, chunkIndex),
+    text: document.content.slice(trimmed.start, trimmed.end),
+    source: sourceForChunk(document, location),
+    ...document.metadata !== undefined ? { metadata: document.metadata } : {}
+  };
+}
+function createChunker(name, options, recursive) {
+  const maxChars = options.maxChars;
+  const overlapChars = options.overlapChars ?? 0;
+  assertPositiveInteger("maxChars", maxChars);
+  assertNonNegativeInteger("overlapChars", overlapChars);
+  if (overlapChars >= maxChars)
+    throw new TypeError("overlapChars must be smaller than maxChars");
+  const separators = options.separators ?? DEFAULT_SEPARATORS;
+  return {
+    name,
+    chunk(document, ctx) {
+      throwIfAborted(ctx);
+      const text = document.content;
+      if (text.trim() === "")
+        return [];
+      const starts = lineStarts(text);
+      const baseRanges = recursive ? mergeRanges(recursiveUnits(text, 0, maxChars, separators, 0), maxChars) : (() => {
+        const ranges2 = [];
+        let start = 0;
+        while (start < text.length) {
+          const end = findChunkEnd(text, start, maxChars, separators);
+          ranges2.push({ start, end });
+          if (end >= text.length)
+            break;
+          start = Math.max(start + 1, end - overlapChars);
+        }
+        return ranges2;
+      })();
+      const ranges = recursive ? baseRanges.map((range, index) => ({
+        start: index === 0 ? range.start : Math.max(0, range.end - maxChars, range.start - overlapChars),
+        end: range.end
+      })) : baseRanges;
+      const chunks = [];
+      for (const range of ranges) {
+        throwIfAborted(ctx);
+        const chunk = chunkFromRange(document, ctx?.documentIndex ?? 0, chunks.length, starts, range);
+        if (chunk !== undefined)
+          chunks.push(chunk);
+      }
+      return chunks;
+    }
+  };
+}
+function createTextChunker(options) {
+  return createChunker("text-chunker", options, false);
+}
+function recursiveTextChunker(options) {
+  return createChunker("recursive-text-chunker", options, true);
+}
+// src/retrieval/context.ts
+function textOf(message) {
+  return message.content.filter((part) => part.type === "text").map((part) => part.text).join("");
+}
+function latestUserText(messages) {
+  if (messages === undefined)
+    return;
+  for (let index = messages.length - 1;index >= 0; index -= 1) {
+    const message = messages[index];
+    if (message?.role !== "user")
+      continue;
+    const text = textOf(message).trim();
+    if (text !== "")
+      return text;
+  }
+  return;
+}
+async function resolveQuery(options, ctx, run) {
+  if (typeof options.query === "string")
+    return options.query.trim() === "" ? undefined : options.query;
+  if (typeof options.query === "function") {
+    const query = await options.query(ctx, run);
+    return query === undefined || query.trim() === "" ? undefined : query;
+  }
+  return latestUserText(run?.input) ?? latestUserText(run?.messages);
+}
+function messageForCount(text) {
+  return { role: "system", content: [{ type: "text", text }] };
+}
+function countText(text, countTokens) {
+  return countTokens([messageForCount(text)]);
+}
+function locationLabel(location) {
+  if (location === undefined)
+    return;
+  if (location.startLine !== undefined && location.endLine !== undefined) {
+    return location.startLine === location.endLine ? `line ${location.startLine}` : `lines ${location.startLine}-${location.endLine}`;
+  }
+  if (location.startOffset !== undefined && location.endOffset !== undefined) {
+    return `offsets ${location.startOffset}-${location.endOffset}`;
+  }
+  return;
+}
+function sourceLabel(source) {
+  const name = source?.title ?? source?.uri ?? source?.id ?? "unknown source";
+  const location = locationLabel(source?.location);
+  return location === undefined ? name : `${name}, ${location}`;
+}
+function renderResult(result, citation, includeScores, text = result.text) {
+  const score = includeScores ? ` score=${result.score.toFixed(3)}` : "";
+  return `[${citation}] ${sourceLabel(result.source)}${score}
+${text}`;
+}
+function joinBlocks(blocks) {
+  return blocks.join(`
+
+`);
+}
+function appendBlock(content, block) {
+  return content === "" ? block : `${content}
+
+${block}`;
+}
+function availableBudget(options, run, countTokens) {
+  if (options.maxContextTokens !== undefined)
+    return Math.max(0, options.maxContextTokens);
+  if (run?.contextWindow === undefined)
+    return;
+  const reserveTokens = options.reserveTokens ?? Math.min(512, Math.floor(run.contextWindow.maxTokens * 0.1));
+  const used = countTokens(run.messages);
+  return Math.max(0, run.contextWindow.maxTokens - used - reserveTokens);
+}
+function truncateToFit(prefixContent, result, citation, budget, countTokens, includeScores) {
+  let low = 1;
+  let high = result.text.length;
+  let best;
+  while (low <= high) {
+    const mid = Math.floor((low + high) / 2);
+    const text = `${result.text.slice(0, mid).trimEnd()}
+[truncated]`;
+    const block = renderResult(result, citation, includeScores, text);
+    const candidate = appendBlock(prefixContent, block);
+    if (countText(candidate, countTokens) <= budget) {
+      best = block;
+      low = mid + 1;
+    } else {
+      high = mid - 1;
+    }
+  }
+  return best;
+}
+function selectResults(results, budget, countTokens, includeScores) {
+  if (budget === undefined) {
+    return {
+      content: joinBlocks(results.map((result, index) => renderResult(result, index + 1, includeScores))),
+      selected: results
+    };
+  }
+  if (budget <= 0)
+    return { content: "", selected: [] };
+  let content = "";
+  const selected = [];
+  for (let index = 0;index < results.length; index += 1) {
+    const result = results[index];
+    if (result === undefined)
+      continue;
+    const citation = selected.length + 1;
+    const block = renderResult(result, citation, includeScores);
+    const candidate = appendBlock(content, block);
+    if (countText(candidate, countTokens) <= budget) {
+      content = candidate;
+      selected.push(result);
+      continue;
+    }
+    const truncated = truncateToFit(content, result, citation, budget, countTokens, includeScores);
+    if (truncated !== undefined) {
+      content = appendBlock(content, truncated);
+      selected.push(result);
+    }
+    break;
+  }
+  return { content, selected };
+}
+function retrieverContext(options) {
+  return {
+    name: options.name ?? `retriever:${options.retriever.name}`,
+    async resolve(ctx, run) {
+      const query = await resolveQuery(options, ctx, run);
+      if (query === undefined)
+        return [];
+      const results = await options.retriever.retrieve({
+        query,
+        ...options.topK !== undefined ? { topK: options.topK } : {},
+        ...options.minScore !== undefined ? { minScore: options.minScore } : {}
+      }, ctx);
+      if (results.length === 0)
+        return [];
+      const countTokens = options.countTokens ?? run?.contextWindow?.countTokens ?? estimateTokens;
+      const budget = availableBudget(options, run, countTokens);
+      const rendered = selectResults(results, budget, countTokens, options.includeScores ?? false);
+      if (rendered.content === "")
+        return [];
+      await options.onResults?.(rendered.selected, query, ctx, run);
+      return [{ title: options.title ?? "Retrieved Context", content: rendered.content }];
+    }
+  };
+}
+// src/retrieval/loaders.ts
+function isAsyncIterable(value) {
+  return Symbol.asyncIterator in Object(value);
+}
+function isIterable(value) {
+  return Symbol.iterator in Object(value);
+}
+async function collect(output, ctx) {
+  const documents = [];
+  if (isAsyncIterable(output)) {
+    for await (const document of output) {
+      throwIfAborted(ctx);
+      documents.push(document);
+    }
+    return documents;
+  }
+  if (isIterable(output)) {
+    for (const document of output) {
+      throwIfAborted(ctx);
+      documents.push(document);
+    }
+    return documents;
+  }
+  return documents;
+}
+function createDocumentLoader(name, load) {
+  return { name, load };
+}
+function staticDocumentLoader(documents, name = "static-documents") {
+  return createDocumentLoader(name, () => documents.map((document) => ({ ...document })));
+}
+async function loadDocuments(loaders, ctx) {
+  const documents = [];
+  for (const loader of loaders) {
+    throwIfAborted(ctx);
+    const output = await loader.load(ctx);
+    const loaded = await collect(output, ctx);
+    documents.push(...loaded);
+  }
+  return { documents, loaders: loaders.length };
+}
+// src/retrieval/vector-store.ts
+function assertVector(vector, label = "vector") {
+  if (vector.length === 0)
+    throw new TypeError(`${label} must not be empty`);
+  for (let index = 0;index < vector.length; index += 1) {
+    const value = vector[index];
+    if (typeof value !== "number" || !Number.isFinite(value)) {
+      throw new TypeError(`${label} contains a non-finite value at index ${index}`);
+    }
+  }
+}
+function dotProduct(a, b) {
+  let score = 0;
+  for (let index = 0;index < a.length; index += 1)
+    score += (a[index] ?? 0) * (b[index] ?? 0);
+  return score;
+}
+function cosineSimilarity(a, b) {
+  let aNorm = 0;
+  let bNorm = 0;
+  for (let index = 0;index < a.length; index += 1) {
+    const av = a[index] ?? 0;
+    const bv = b[index] ?? 0;
+    aNorm += av * av;
+    bNorm += bv * bv;
+  }
+  if (aNorm === 0 || bNorm === 0)
+    return 0;
+  return dotProduct(a, b) / (Math.sqrt(aNorm) * Math.sqrt(bNorm));
+}
+function euclideanSimilarity(a, b) {
+  let sum = 0;
+  for (let index = 0;index < a.length; index += 1) {
+    const delta = (a[index] ?? 0) - (b[index] ?? 0);
+    sum += delta * delta;
+  }
+  return 1 / (1 + Math.sqrt(sum));
+}
+function scoreVectors(a, b, similarity) {
+  switch (similarity) {
+    case "cosine":
+      return cosineSimilarity(a, b);
+    case "dot":
+      return dotProduct(a, b);
+    case "euclidean":
+      return euclideanSimilarity(a, b);
+  }
+}
+
+// src/retrieval/memory.ts
+function cloneVector(vector) {
+  return vector.map((value) => value);
+}
+function cloneRecord(record) {
+  return {
+    id: record.id,
+    vector: cloneVector(record.vector),
+    text: record.text,
+    ...record.source !== undefined ? { source: record.source } : {},
+    ...record.metadata !== undefined ? { metadata: record.metadata } : {}
+  };
+}
+
+class InMemoryVectorStore {
+  name;
+  records = new Map;
+  similarity;
+  recordDimensions;
+  constructor(options = {}) {
+    assertPositiveInteger("dimensions", options.dimensions);
+    this.name = options.name ?? "memory-vector-store";
+    this.recordDimensions = options.dimensions;
+    this.similarity = options.similarity ?? "cosine";
+  }
+  get dimensions() {
+    return this.recordDimensions;
+  }
+  ensureDimensions(vector, label) {
+    assertVector(vector, label);
+    if (this.recordDimensions === undefined) {
+      this.recordDimensions = vector.length;
+      return;
+    }
+    if (vector.length !== this.recordDimensions) {
+      throw new TypeError(`${label} has ${vector.length} dimensions; expected ${this.recordDimensions}`);
+    }
+  }
+  ensureQueryDimensions(vector) {
+    assertVector(vector, "query vector");
+    if (this.recordDimensions !== undefined && vector.length !== this.recordDimensions) {
+      throw new TypeError(`query vector has ${vector.length} dimensions; expected ${this.recordDimensions}`);
+    }
+  }
+  async upsert(records, ctx) {
+    for (const record of records) {
+      throwIfAborted(ctx);
+      if (record.id === "")
+        throw new TypeError("record id must not be empty");
+      this.ensureDimensions(record.vector, `record ${record.id} vector`);
+      this.records.set(record.id, cloneRecord(record));
+    }
+  }
+  async query(query, ctx) {
+    throwIfAborted(ctx);
+    this.ensureQueryDimensions(query.vector);
+    const topK = query.topK ?? 10;
+    assertPositiveInteger("topK", topK);
+    const hits = [];
+    for (const record of this.records.values()) {
+      throwIfAborted(ctx);
+      if (query.filter !== undefined && !query.filter(record))
+        continue;
+      const score = scoreVectors(query.vector, record.vector, this.similarity);
+      if (query.minScore !== undefined && score < query.minScore)
+        continue;
+      hits.push({
+        id: record.id,
+        score,
+        text: record.text,
+        ...record.source !== undefined ? { source: record.source } : {},
+        ...record.metadata !== undefined ? { metadata: record.metadata } : {},
+        ...query.includeVectors === true ? { vector: cloneVector(record.vector) } : {}
+      });
+    }
+    hits.sort((a, b) => b.score - a.score || a.id.localeCompare(b.id));
+    return hits.slice(0, topK);
+  }
+  async delete(ids, ctx) {
+    for (const id of ids) {
+      throwIfAborted(ctx);
+      this.records.delete(id);
+    }
+  }
+  async clear(ctx) {
+    throwIfAborted(ctx);
+    this.records.clear();
+  }
+  async stats(ctx) {
+    throwIfAborted(ctx);
+    return {
+      records: this.records.size,
+      ...this.recordDimensions !== undefined ? { dimensions: this.recordDimensions } : {}
+    };
+  }
+}
+// src/retrieval/retriever.ts
+function defaultModel(provider, model) {
+  return model ?? provider.defaultModel;
+}
+function assertEmbeddingCount(expected, actual) {
+  if (actual.length !== expected) {
+    throw new Error(`embedding provider returned ${actual.length} vectors for ${expected} inputs`);
+  }
+}
+function fromVectorHit(hit, index) {
+  return {
+    id: hit.id,
+    rank: index + 1,
+    score: hit.score,
+    scores: { vector: hit.score },
+    text: hit.text,
+    ...hit.source !== undefined ? { source: hit.source } : {},
+    ...hit.metadata !== undefined ? { metadata: hit.metadata } : {}
+  };
+}
+function normalizeScore(score, min, max) {
+  if (max === min)
+    return score > 0 ? 1 : 0;
+  return (score - min) / (max - min);
+}
+function scoreBounds(results) {
+  if (results.length === 0)
+    return { min: 0, max: 0 };
+  let min = Number.POSITIVE_INFINITY;
+  let max = Number.NEGATIVE_INFINITY;
+  for (const result of results) {
+    min = Math.min(min, result.score);
+    max = Math.max(max, result.score);
+  }
+  return { min, max };
+}
+function mergeHybridResults(vectorResults, keywordResults, options = {}) {
+  const inferredTopK = Math.max(vectorResults.length, keywordResults.length);
+  if (options.topK === undefined && inferredTopK === 0)
+    return [];
+  const topK = options.topK ?? inferredTopK;
+  assertPositiveInteger("topK", topK);
+  const vectorWeight = options.vectorWeight ?? 0.7;
+  const keywordWeight = options.keywordWeight ?? 0.3;
+  const vectorBounds = scoreBounds(vectorResults);
+  const keywordBounds = scoreBounds(keywordResults);
+  const entries = new Map;
+  for (const result of vectorResults) {
+    const previous = entries.get(result.id);
+    entries.set(result.id, {
+      id: result.id,
+      text: previous?.text ?? result.text,
+      ...previous?.source !== undefined || result.source !== undefined ? { source: previous?.source ?? result.source } : {},
+      ...previous?.metadata !== undefined || result.metadata !== undefined ? { metadata: previous?.metadata ?? result.metadata } : {},
+      ...previous?.keywordRaw !== undefined ? { keywordRaw: previous.keywordRaw } : {},
+      ...previous?.keywordNormalized !== undefined ? { keywordNormalized: previous.keywordNormalized } : {},
+      vectorRaw: result.score,
+      vectorNormalized: normalizeScore(result.score, vectorBounds.min, vectorBounds.max)
+    });
+  }
+  for (const result of keywordResults) {
+    const previous = entries.get(result.id);
+    const text = previous?.text === undefined || previous.text === "" ? result.text : previous.text;
+    entries.set(result.id, {
+      id: result.id,
+      text,
+      ...previous?.source !== undefined || result.source !== undefined ? { source: previous?.source ?? result.source } : {},
+      ...previous?.metadata !== undefined || result.metadata !== undefined ? { metadata: previous?.metadata ?? result.metadata } : {},
+      ...previous?.vectorRaw !== undefined ? { vectorRaw: previous.vectorRaw } : {},
+      ...previous?.vectorNormalized !== undefined ? { vectorNormalized: previous.vectorNormalized } : {},
+      keywordRaw: result.score,
+      keywordNormalized: normalizeScore(result.score, keywordBounds.min, keywordBounds.max)
+    });
+  }
+  const merged = [...entries.values()].map((entry) => {
+    const combined = (entry.vectorNormalized ?? 0) * vectorWeight + (entry.keywordNormalized ?? 0) * keywordWeight;
+    return {
+      id: entry.id,
+      rank: 0,
+      score: combined,
+      scores: {
+        ...entry.vectorRaw !== undefined ? { vector: entry.vectorRaw } : {},
+        ...entry.keywordRaw !== undefined ? { keyword: entry.keywordRaw } : {},
+        combined
+      },
+      text: entry.text,
+      ...entry.source !== undefined ? { source: entry.source } : {},
+      ...entry.metadata !== undefined ? { metadata: entry.metadata } : {}
+    };
+  }).filter((result) => options.minScore === undefined || result.score >= options.minScore).sort((a, b) => b.score - a.score || a.id.localeCompare(b.id)).slice(0, topK);
+  return merged.map((result, index) => ({ ...result, rank: index + 1 }));
+}
+async function indexDocuments(options, ctx) {
+  const batchSize = options.batchSize ?? 64;
+  assertPositiveInteger("batchSize", batchSize);
+  const loaded = await loadDocuments(options.loaders, ctx);
+  const chunks = [];
+  for (let documentIndex = 0;documentIndex < loaded.documents.length; documentIndex += 1) {
+    throwIfAborted(ctx);
+    const document = loaded.documents[documentIndex];
+    if (document === undefined)
+      continue;
+    const documentChunks = await options.chunker.chunk(document, { documentIndex, ...ctx !== undefined ? { engine: ctx } : {} });
+    chunks.push(...documentChunks);
+  }
+  let records = 0;
+  let model;
+  let dimensions;
+  for (let start = 0;start < chunks.length; start += batchSize) {
+    throwIfAborted(ctx);
+    const batch = chunks.slice(start, start + batchSize);
+    const result = await options.embeddings.embed({
+      input: batch.map((chunk) => chunk.text),
+      ...defaultModel(options.embeddings, options.embeddingModel) !== undefined ? { model: defaultModel(options.embeddings, options.embeddingModel) } : {}
+    }, ctx);
+    assertEmbeddingCount(batch.length, result.vectors);
+    model = result.model;
+    if (result.dimensions !== undefined)
+      dimensions = result.dimensions;
+    const vectorRecords = [];
+    for (let index = 0;index < batch.length; index += 1) {
+      const chunk = batch[index];
+      const vector = result.vectors[index];
+      if (chunk === undefined || vector === undefined)
+        continue;
+      assertVector(vector, `embedding ${chunk.id}`);
+      if (dimensions === undefined)
+        dimensions = vector.length;
+      vectorRecords.push({
+        id: chunk.id,
+        vector,
+        text: chunk.text,
+        ...chunk.source !== undefined ? { source: chunk.source } : {},
+        ...chunk.metadata !== undefined ? { metadata: chunk.metadata } : {}
+      });
+    }
+    await options.store.upsert(vectorRecords, ctx);
+    records += vectorRecords.length;
+  }
+  return {
+    documents: loaded.documents.length,
+    chunks: chunks.length,
+    records,
+    ...model !== undefined ? { model } : {},
+    ...dimensions !== undefined ? { dimensions } : {}
+  };
+}
+function createVectorRetriever(options) {
+  assertPositiveInteger("topK", options.topK);
+  return {
+    name: options.name ?? `vector:${options.store.name}`,
+    async retrieve(query, ctx) {
+      throwIfAborted(ctx);
+      const topK = query.topK ?? options.hybrid?.topK ?? options.topK ?? 5;
+      assertPositiveInteger("topK", topK);
+      const embed = await options.embeddings.embed({
+        input: [query.query],
+        ...defaultModel(options.embeddings, options.model) !== undefined ? { model: defaultModel(options.embeddings, options.model) } : {}
+      }, ctx);
+      assertEmbeddingCount(1, embed.vectors);
+      const vector = embed.vectors[0];
+      if (vector === undefined)
+        throw new Error("embedding provider returned no query vector");
+      assertVector(vector, "query embedding");
+      const vectorHits = await options.store.query({
+        vector,
+        topK,
+        ...query.filter !== undefined || options.filter !== undefined ? { filter: query.filter ?? options.filter } : {},
+        ...query.minScore !== undefined || options.minScore !== undefined ? { minScore: query.minScore ?? options.minScore } : {}
+      }, ctx);
+      const vectorResults = vectorHits.map(fromVectorHit);
+      if (options.keyword === undefined)
+        return vectorResults.map((result, index) => ({ ...result, rank: index + 1 }));
+      const keywordResults = await options.keyword.retrieve({ ...query, topK }, ctx);
+      return mergeHybridResults(vectorResults, keywordResults, {
+        ...options.hybrid,
+        topK,
+        ...query.minScore !== undefined || options.hybrid?.minScore !== undefined ? { minScore: query.minScore ?? options.hybrid?.minScore } : {}
+      });
+    }
+  };
+}
+export {
+  staticDocumentLoader,
+  scoreVectors,
+  retrieverContext,
+  recursiveTextChunker,
+  mergeHybridResults,
+  loadDocuments,
+  indexDocuments,
+  euclideanSimilarity,
+  dotProduct,
+  createVectorRetriever,
+  createTextChunker,
+  createDocumentLoader,
+  cosineSimilarity,
+  assertVector,
+  InMemoryVectorStore
+};