langchain 1.0.0-alpha.7 → 1.0.0-alpha.9

package/dist/retrievers/hyde.js

@@ -1,121 +0,0 @@
-import { __export } from "../_virtual/rolldown_runtime.js";
-import { PromptTemplate } from "@langchain/core/prompts";
-import { VectorStoreRetriever } from "@langchain/core/vectorstores";
-import { StringPromptValue } from "@langchain/core/prompt_values";
-
-//#region src/retrievers/hyde.ts
-var hyde_exports = {};
-__export(hyde_exports, {
-	HydeRetriever: () => HydeRetriever,
-	getPromptTemplateFromKey: () => getPromptTemplateFromKey
-});
-/**
-* A class for retrieving relevant documents based on a given query. It
-* extends the VectorStoreRetriever class and uses a BaseLanguageModel to
-* generate a hypothetical answer to the query, which is then used to
-* retrieve relevant documents.
-* @example
-* ```typescript
-* const retriever = new HydeRetriever({
-*   vectorStore: new MemoryVectorStore(new OpenAIEmbeddings()),
-*   llm: new ChatOpenAI({ model: "gpt-4o-mini" }),
-*   k: 1,
-* });
-* await vectorStore.addDocuments(
-*   [
-*     "My name is John.",
-*     "My name is Bob.",
-*     "My favourite food is pizza.",
-*     "My favourite food is pasta.",
-*   ].map((pageContent) => new Document({ pageContent })),
-* );
-* const results = await retriever.getRelevantDocuments(
-*   "What is my favourite food?",
-* );
-* ```
-*/
-var HydeRetriever = class extends VectorStoreRetriever {
-	static lc_name() {
-		return "HydeRetriever";
-	}
-	get lc_namespace() {
-		return [
-			"langchain",
-			"retrievers",
-			"hyde"
-		];
-	}
-	llm;
-	promptTemplate;
-	constructor(fields) {
-		super(fields);
-		this.llm = fields.llm;
-		this.promptTemplate = typeof fields.promptTemplate === "string" ? getPromptTemplateFromKey(fields.promptTemplate) : fields.promptTemplate;
-		if (this.promptTemplate) {
-			const { inputVariables } = this.promptTemplate;
-			if (inputVariables.length !== 1 && inputVariables[0] !== "question") throw new Error(`Prompt template must accept a single input variable 'question'. Invalid input variables for prompt template: ${inputVariables}`);
-		}
-	}
-	async _getRelevantDocuments(query, runManager) {
-		let value = new StringPromptValue(query);
-		if (this.promptTemplate) value = await this.promptTemplate.formatPromptValue({ question: query });
-		const res = await this.llm.generatePrompt([value]);
-		const answer = res.generations[0][0].text;
-		const results = await this.vectorStore.similaritySearch(answer, this.k, this.filter, runManager?.getChild("vectorstore"));
-		return results;
-	}
-};
-/**
-* Returns a BasePromptTemplate instance based on a given PromptKey.
-*/
-function getPromptTemplateFromKey(key) {
-	let template;
-	switch (key) {
-		case "websearch":
-			template = `Please write a passage to answer the question
-Question: {question}
-Passage:`;
-			break;
-		case "scifact":
-			template = `Please write a scientific paper passage to support/refute the claim
-Claim: {question}
-Passage:`;
-			break;
-		case "arguana":
-			template = `Please write a counter argument for the passage
-Passage: {question}
-Counter Argument:`;
-			break;
-		case "trec-covid":
-			template = `Please write a scientific paper passage to answer the question
-Question: {question}
-Passage:`;
-			break;
-		case "fiqa":
-			template = `Please write a financial article passage to answer the question
-Question: {question}
-Passage:`;
-			break;
-		case "dbpedia-entity":
-			template = `Please write a passage to answer the question.
-Question: {question}
-Passage:`;
-			break;
-		case "trec-news":
-			template = `Please write a news passage about the topic.
-Topic: {question}
-Passage:`;
-			break;
-		case "mr-tydi":
-			template = `Please write a passage in Swahili/Korean/Japanese/Bengali to answer the question in detail.
-Question: {question}
-Passage:`;
-			break;
-		default: throw new Error(`Invalid prompt key: ${key}`);
-	}
-	return PromptTemplate.fromTemplate(template);
-}
-
-//#endregion
-export { HydeRetriever, getPromptTemplateFromKey, hyde_exports };
-//# sourceMappingURL=hyde.js.map

package/dist/retrievers/hyde.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"hyde.js","names":["fields: HydeRetrieverOptions<V>","query: string","runManager?: CallbackManagerForRetrieverRun","value: BasePromptValue","key: PromptKey","template: string"],"sources":["../../src/retrievers/hyde.ts"],"sourcesContent":["import type { BaseLanguageModelInterface } from \"@langchain/core/language_models/base\";\nimport { Document } from \"@langchain/core/documents\";\nimport { PromptTemplate, BasePromptTemplate } from \"@langchain/core/prompts\";\nimport {\n  StringPromptValue,\n  BasePromptValue,\n} from \"@langchain/core/prompt_values\";\nimport {\n  VectorStore,\n  VectorStoreRetriever,\n  VectorStoreRetrieverInput,\n} from \"@langchain/core/vectorstores\";\nimport { CallbackManagerForRetrieverRun } from \"@langchain/core/callbacks/manager\";\n\n/**\n * A string that corresponds to a specific prompt template.\n */\nexport type PromptKey =\n  | \"websearch\"\n  | \"scifact\"\n  | \"arguana\"\n  | \"trec-covid\"\n  | \"fiqa\"\n  | \"dbpedia-entity\"\n  | \"trec-news\"\n  | \"mr-tydi\";\n\n/**\n * Options for the HydeRetriever class, which includes a BaseLanguageModel\n * instance, a VectorStore instance, and an optional promptTemplate which\n * can either be a BasePromptTemplate instance or a PromptKey.\n */\nexport type HydeRetrieverOptions<V extends VectorStore> =\n  VectorStoreRetrieverInput<V> & {\n    llm: BaseLanguageModelInterface;\n    promptTemplate?: BasePromptTemplate | PromptKey;\n  };\n\n/**\n * A class for retrieving relevant documents based on a given query. It\n * extends the VectorStoreRetriever class and uses a BaseLanguageModel to\n * generate a hypothetical answer to the query, which is then used to\n * retrieve relevant documents.\n * @example\n * ```typescript\n * const retriever = new HydeRetriever({\n *   vectorStore: new MemoryVectorStore(new OpenAIEmbeddings()),\n *   llm: new ChatOpenAI({ model: \"gpt-4o-mini\" }),\n *   k: 1,\n * });\n * await vectorStore.addDocuments(\n *   [\n *     \"My name is John.\",\n *     \"My name is Bob.\",\n *     \"My favourite food is pizza.\",\n *     \"My favourite food is pasta.\",\n *   ].map((pageContent) => new Document({ pageContent })),\n * );\n * const results = await retriever.getRelevantDocuments(\n *   \"What is my favourite food?\",\n * );\n * ```\n */\nexport class HydeRetriever<\n  V extends VectorStore = VectorStore\n> extends VectorStoreRetriever<V> {\n  static lc_name() {\n    return \"HydeRetriever\";\n  }\n\n  get lc_namespace(): string[] {\n    return [\"langchain\", \"retrievers\", \"hyde\"];\n  }\n\n  llm: BaseLanguageModelInterface;\n\n  promptTemplate?: BasePromptTemplate;\n\n  constructor(fields: HydeRetrieverOptions<V>) {\n    super(fields);\n    this.llm = fields.llm;\n    this.promptTemplate =\n      typeof fields.promptTemplate === \"string\"\n        ? getPromptTemplateFromKey(fields.promptTemplate)\n        : fields.promptTemplate;\n    if (this.promptTemplate) {\n      const { inputVariables } = this.promptTemplate;\n      if (inputVariables.length !== 1 && inputVariables[0] !== \"question\") {\n        throw new Error(\n          `Prompt template must accept a single input variable 'question'. Invalid input variables for prompt template: ${inputVariables}`\n        );\n      }\n    }\n  }\n\n  async _getRelevantDocuments(\n    query: string,\n    runManager?: CallbackManagerForRetrieverRun\n  ): Promise<Document[]> {\n    let value: BasePromptValue = new StringPromptValue(query);\n\n    // Use a custom template if provided\n    if (this.promptTemplate) {\n      value = await this.promptTemplate.formatPromptValue({ question: query });\n    }\n\n    // Get a hypothetical answer from the LLM\n    const res = await this.llm.generatePrompt([value]);\n    const answer = res.generations[0][0].text;\n\n    // Retrieve relevant documents based on the hypothetical answer\n    const results = await this.vectorStore.similaritySearch(\n      answer,\n      this.k,\n      this.filter,\n      runManager?.getChild(\"vectorstore\")\n    );\n\n    return results;\n  }\n}\n\n/**\n * Returns a BasePromptTemplate instance based on a given PromptKey.\n */\nexport function getPromptTemplateFromKey(key: PromptKey): BasePromptTemplate {\n  let template: string;\n\n  switch (key) {\n    case \"websearch\":\n      template = `Please write a passage to answer the question\nQuestion: {question}\nPassage:`;\n      break;\n    case \"scifact\":\n      template = `Please write a scientific paper passage to support/refute the claim\nClaim: {question}\nPassage:`;\n      break;\n    case \"arguana\":\n      template = `Please write a counter argument for the passage\nPassage: {question}\nCounter Argument:`;\n      break;\n    case \"trec-covid\":\n      template = `Please write a scientific paper passage to answer the question\nQuestion: {question}\nPassage:`;\n      break;\n    case \"fiqa\":\n      template = `Please write a financial article passage to answer the question\nQuestion: {question}\nPassage:`;\n      break;\n    case \"dbpedia-entity\":\n      template = `Please write a passage to answer the question.\nQuestion: {question}\nPassage:`;\n      break;\n    case \"trec-news\":\n      template = `Please write a news passage about the topic.\nTopic: {question}\nPassage:`;\n      break;\n    case \"mr-tydi\":\n      template = `Please write a passage in Swahili/Korean/Japanese/Bengali to answer the question in detail.\nQuestion: {question}\nPassage:`;\n      break;\n    default:\n      throw new Error(`Invalid prompt key: ${key}`);\n  }\n\n  return PromptTemplate.fromTemplate(template);\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA+DA,IAAa,gBAAb,cAEU,qBAAwB;CAChC,OAAO,UAAU;AACf,SAAO;CACR;CAED,IAAI,eAAyB;AAC3B,SAAO;GAAC;GAAa;GAAc;EAAO;CAC3C;CAED;CAEA;CAEA,YAAYA,QAAiC;EAC3C,MAAM,OAAO;EACb,KAAK,MAAM,OAAO;EAClB,KAAK,iBACH,OAAO,OAAO,mBAAmB,WAC7B,yBAAyB,OAAO,eAAe,GAC/C,OAAO;AACb,MAAI,KAAK,gBAAgB;GACvB,MAAM,EAAE,gBAAgB,GAAG,KAAK;AAChC,OAAI,eAAe,WAAW,KAAK,eAAe,OAAO,WACvD,OAAM,IAAI,MACR,CAAC,6GAA6G,EAAE,gBAAgB;EAGrI;CACF;CAED,MAAM,sBACJC,OACAC,YACqB;EACrB,IAAIC,QAAyB,IAAI,kBAAkB;AAGnD,MAAI,KAAK,gBACP,QAAQ,MAAM,KAAK,eAAe,kBAAkB,EAAE,UAAU,MAAO,EAAC;EAI1E,MAAM,MAAM,MAAM,KAAK,IAAI,eAAe,CAAC,KAAM,EAAC;EAClD,MAAM,SAAS,IAAI,YAAY,GAAG,GAAG;EAGrC,MAAM,UAAU,MAAM,KAAK,YAAY,iBACrC,QACA,KAAK,GACL,KAAK,QACL,YAAY,SAAS,cAAc,CACpC;AAED,SAAO;CACR;AACF;;;;AAKD,SAAgB,yBAAyBC,KAAoC;CAC3E,IAAIC;AAEJ,SAAQ,KAAR;EACE,KAAK;GACH,WAAW,CAAC;;QAEV,CAAC;AACH;EACF,KAAK;GACH,WAAW,CAAC;;QAEV,CAAC;AACH;EACF,KAAK;GACH,WAAW,CAAC;;iBAED,CAAC;AACZ;EACF,KAAK;GACH,WAAW,CAAC;;QAEV,CAAC;AACH;EACF,KAAK;GACH,WAAW,CAAC;;QAEV,CAAC;AACH;EACF,KAAK;GACH,WAAW,CAAC;;QAEV,CAAC;AACH;EACF,KAAK;GACH,WAAW,CAAC;;QAEV,CAAC;AACH;EACF,KAAK;GACH,WAAW,CAAC;;QAEV,CAAC;AACH;EACF,QACE,OAAM,IAAI,MAAM,CAAC,oBAAoB,EAAE,KAAK;CAC/C;AAED,QAAO,eAAe,aAAa,SAAS;AAC7C"}

package/dist/retrievers/matryoshka_retriever.cjs

@@ -1,116 +0,0 @@
-const require_rolldown_runtime = require('../_virtual/rolldown_runtime.cjs');
-const __langchain_core_vectorstores = require_rolldown_runtime.__toESM(require("@langchain/core/vectorstores"));
-const __langchain_core_utils_math = require_rolldown_runtime.__toESM(require("@langchain/core/utils/math"));
-
-//#region src/retrievers/matryoshka_retriever.ts
-var matryoshka_retriever_exports = {};
-require_rolldown_runtime.__export(matryoshka_retriever_exports, { MatryoshkaRetriever: () => MatryoshkaRetriever });
-/**
-* A retriever that uses two sets of embeddings to perform adaptive retrieval. Based
-* off of the "Matryoshka embeddings: faster OpenAI vector search using Adaptive Retrieval"
-* blog post {@link https://supabase.com/blog/matryoshka-embeddings}.
-*
-*
-* This class performs "Adaptive Retrieval" for searching text embeddings efficiently using the
-* Matryoshka Representation Learning (MRL) technique. It retrieves documents similar to a query
-* embedding in two steps:
-*
-* First-pass: Uses a lower dimensional sub-vector from the MRL embedding for an initial, fast,
-* but less accurate search.
-*
-* Second-pass: Re-ranks the top results from the first pass using the full, high-dimensional
-* embedding for higher accuracy.
-*
-*
-* This code implements MRL embeddings for efficient vector search by combining faster,
-* lower-dimensional initial search with accurate, high-dimensional re-ranking.
-*/
-var MatryoshkaRetriever = class extends __langchain_core_vectorstores.VectorStoreRetriever {
-	smallK = 50;
-	largeK = 8;
-	largeEmbeddingKey = "lc_large_embedding";
-	largeEmbeddingModel;
-	searchType = "cosine";
-	constructor(fields) {
-		super(fields);
-		this.smallK = fields.smallK ?? this.smallK;
-		this.largeK = fields.largeK ?? this.largeK;
-		this.largeEmbeddingKey = fields.largeEmbeddingKey ?? this.largeEmbeddingKey;
-		this.largeEmbeddingModel = fields.largeEmbeddingModel;
-		this.searchType = fields.searchType ?? this.searchType;
-	}
-	/**
-	* Ranks documents based on their similarity to a query embedding using larger embeddings.
-	*
-	* This method takes a query embedding and a list of documents (smallResults) as input. Each document
-	* in the smallResults array has previously been associated with a large embedding stored in its metadata.
-	* Depending on the `searchType` (cosine, innerProduct, or euclidean), it calculates the similarity scores
-	* between the query embedding and each document's large embedding. It then ranks the documents based on
-	* these similarity scores, from the most similar to the least similar.
-	*
-	* The method returns a promise that resolves to an array of the top `largeK` documents, where `largeK`
-	* is a class property defining the number of documents to return. This subset of documents is determined
-	* by sorting the entire list of documents based on their similarity scores and then selecting the top
-	* `largeK` documents.
-	*
-	* @param {number[]} embeddedQuery The embedding of the query, represented as an array of numbers.
-	* @param {DocumentInterface[]} smallResults An array of documents, each with metadata that includes a large embedding for similarity comparison.
-	* @returns {Promise<DocumentInterface[]>} A promise that resolves to an array of the top `largeK` ranked documents based on their similarity to the query embedding.
-	*/
-	_rankByLargeEmbeddings(embeddedQuery, smallResults) {
-		const largeEmbeddings = smallResults.map((doc) => JSON.parse(doc.metadata[this.largeEmbeddingKey]));
-		let func;
-		switch (this.searchType) {
-			case "cosine":
-				func = () => (0, __langchain_core_utils_math.cosineSimilarity)([embeddedQuery], largeEmbeddings);
-				break;
-			case "innerProduct":
-				func = () => (0, __langchain_core_utils_math.innerProduct)([embeddedQuery], largeEmbeddings);
-				break;
-			case "euclidean":
-				func = () => (0, __langchain_core_utils_math.euclideanDistance)([embeddedQuery], largeEmbeddings);
-				break;
-			default: throw new Error(`Unknown search type: ${this.searchType}`);
-		}
-		const [similarityScores] = func();
-		let indices = Array.from({ length: smallResults.length }, (_, index) => index);
-		indices = indices.map((v, i) => [similarityScores[i], v]).sort(([a], [b]) => b - a).slice(0, this.largeK).map(([, i]) => i);
-		return indices.map((i) => smallResults[i]);
-	}
-	async _getRelevantDocuments(query) {
-		const [embeddedQuery, smallResults] = await Promise.all([this.largeEmbeddingModel.embedQuery(query), this.vectorStore.similaritySearch(query, this.smallK, this.filter)]);
-		return this._rankByLargeEmbeddings(embeddedQuery, smallResults);
-	}
-	/**
-	* Override the default `addDocuments` method to embed the documents twice,
-	* once using the larger embeddings model, and then again using the default
-	* embedding model linked to the vector store.
-	*
-	* @param {DocumentInterface[]} documents - An array of documents to add to the vector store.
-	* @param {AddDocumentOptions} options - An optional object containing additional options for adding documents.
-	* @returns {Promise<string[] | void>} A promise that resolves to an array of the document IDs that were added to the vector store.
-	*/
-	addDocuments = async (documents, options) => {
-		if (documents.some((doc) => this.largeEmbeddingKey in doc.metadata)) throw new Error(`All documents must not contain the large embedding key: ${this.largeEmbeddingKey} in their metadata.`);
-		const allDocPageContent = documents.map((doc) => doc.pageContent);
-		const allDocLargeEmbeddings = await this.largeEmbeddingModel.embedDocuments(allDocPageContent);
-		const newDocuments = documents.map((doc, idx) => ({
-			...doc,
-			metadata: {
-				...doc.metadata,
-				[this.largeEmbeddingKey]: JSON.stringify(allDocLargeEmbeddings[idx])
-			}
-		}));
-		return this.vectorStore.addDocuments(newDocuments, options);
-	};
-};
-
-//#endregion
-exports.MatryoshkaRetriever = MatryoshkaRetriever;
-Object.defineProperty(exports, 'matryoshka_retriever_exports', {
-  enumerable: true,
-  get: function () {
-    return matryoshka_retriever_exports;
-  }
-});
-//# sourceMappingURL=matryoshka_retriever.cjs.map

package/dist/retrievers/matryoshka_retriever.cjs.map

@@ -1 +0,0 @@
-{"version":3,"file":"matryoshka_retriever.cjs","names":["VectorStoreRetriever","fields: MatryoshkaRetrieverFields & VectorStoreRetrieverInput<Store>","embeddedQuery: number[]","smallResults: DocumentInterface[]","largeEmbeddings: Array<number[]>","func: () => Array<number[]>","query: string","documents: DocumentInterface[]","options?: AddDocumentOptions","newDocuments: Array<DocumentInterface>"],"sources":["../../src/retrievers/matryoshka_retriever.ts"],"sourcesContent":["import { DocumentInterface } from \"@langchain/core/documents\";\nimport { Embeddings } from \"@langchain/core/embeddings\";\nimport {\n  cosineSimilarity,\n  euclideanDistance,\n  innerProduct,\n} from \"@langchain/core/utils/math\";\nimport {\n  VectorStore,\n  VectorStoreRetriever,\n  VectorStoreRetrieverInput,\n} from \"@langchain/core/vectorstores\";\n\n/**\n * Type for options when adding a document to the VectorStore.\n */\n// eslint-disable-next-line @typescript-eslint/no-explicit-any\ntype AddDocumentOptions = Record<string, any>;\n\nexport interface MatryoshkaRetrieverFields {\n  /**\n   * The number of documents to retrieve from the small store.\n   * @default 50\n   */\n  smallK?: number;\n  /**\n   * The number of documents to retrieve from the large store.\n   * @default 8\n   */\n  largeK?: number;\n  /**\n   * The metadata key to store the larger embeddings.\n   * @default \"lc_large_embedding\"\n   */\n  largeEmbeddingKey?: string;\n  /**\n   * The embedding model to use when generating the large\n   * embeddings.\n   */\n  largeEmbeddingModel: Embeddings;\n  /**\n   * The type of search to perform using the large embeddings.\n   * @default \"cosine\"\n   */\n  searchType?: \"cosine\" | \"innerProduct\" | \"euclidean\";\n}\n\n/**\n * A retriever that uses two sets of embeddings to perform adaptive retrieval. Based\n * off of the \"Matryoshka embeddings: faster OpenAI vector search using Adaptive Retrieval\"\n * blog post {@link https://supabase.com/blog/matryoshka-embeddings}.\n *\n *\n * This class performs \"Adaptive Retrieval\" for searching text embeddings efficiently using the\n * Matryoshka Representation Learning (MRL) technique. It retrieves documents similar to a query\n * embedding in two steps:\n *\n * First-pass: Uses a lower dimensional sub-vector from the MRL embedding for an initial, fast,\n * but less accurate search.\n *\n * Second-pass: Re-ranks the top results from the first pass using the full, high-dimensional\n * embedding for higher accuracy.\n *\n *\n * This code implements MRL embeddings for efficient vector search by combining faster,\n * lower-dimensional initial search with accurate, high-dimensional re-ranking.\n */\nexport class MatryoshkaRetriever<\n  Store extends VectorStore = VectorStore\n> extends VectorStoreRetriever<Store> {\n  smallK = 50;\n\n  largeK = 8;\n\n  largeEmbeddingKey = \"lc_large_embedding\";\n\n  largeEmbeddingModel: Embeddings;\n\n  searchType: \"cosine\" | \"innerProduct\" | \"euclidean\" = \"cosine\";\n\n  constructor(\n    fields: MatryoshkaRetrieverFields & VectorStoreRetrieverInput<Store>\n  ) {\n    super(fields);\n    this.smallK = fields.smallK ?? this.smallK;\n    this.largeK = fields.largeK ?? this.largeK;\n    this.largeEmbeddingKey = fields.largeEmbeddingKey ?? this.largeEmbeddingKey;\n    this.largeEmbeddingModel = fields.largeEmbeddingModel;\n    this.searchType = fields.searchType ?? this.searchType;\n  }\n\n  /**\n   * Ranks documents based on their similarity to a query embedding using larger embeddings.\n   *\n   * This method takes a query embedding and a list of documents (smallResults) as input. Each document\n   * in the smallResults array has previously been associated with a large embedding stored in its metadata.\n   * Depending on the `searchType` (cosine, innerProduct, or euclidean), it calculates the similarity scores\n   * between the query embedding and each document's large embedding. It then ranks the documents based on\n   * these similarity scores, from the most similar to the least similar.\n   *\n   * The method returns a promise that resolves to an array of the top `largeK` documents, where `largeK`\n   * is a class property defining the number of documents to return. This subset of documents is determined\n   * by sorting the entire list of documents based on their similarity scores and then selecting the top\n   * `largeK` documents.\n   *\n   * @param {number[]} embeddedQuery The embedding of the query, represented as an array of numbers.\n   * @param {DocumentInterface[]} smallResults An array of documents, each with metadata that includes a large embedding for similarity comparison.\n   * @returns {Promise<DocumentInterface[]>} A promise that resolves to an array of the top `largeK` ranked documents based on their similarity to the query embedding.\n   */\n  private _rankByLargeEmbeddings(\n    embeddedQuery: number[],\n    smallResults: DocumentInterface[]\n  ): DocumentInterface[] {\n    const largeEmbeddings: Array<number[]> = smallResults.map((doc) =>\n      JSON.parse(doc.metadata[this.largeEmbeddingKey])\n    );\n    let func: () => Array<number[]>;\n\n    switch (this.searchType) {\n      case \"cosine\":\n        func = () => cosineSimilarity([embeddedQuery], largeEmbeddings);\n        break;\n      case \"innerProduct\":\n        func = () => innerProduct([embeddedQuery], largeEmbeddings);\n        break;\n      case \"euclidean\":\n        func = () => euclideanDistance([embeddedQuery], largeEmbeddings);\n        break;\n      default:\n        throw new Error(`Unknown search type: ${this.searchType}`);\n    }\n\n    // Calculate the similarity scores between the query embedding and the large embeddings\n    const [similarityScores] = func();\n\n    // Create an array of indices from 0 to N-1, where N is the number of documents\n    let indices = Array.from(\n      { length: smallResults.length },\n      (_, index) => index\n    );\n\n    indices = indices\n      .map((v, i) => [similarityScores[i], v])\n      .sort(([a], [b]) => b - a)\n      .slice(0, this.largeK)\n      .map(([, i]) => i);\n\n    return indices.map((i) => smallResults[i]);\n  }\n\n  async _getRelevantDocuments(query: string): Promise<DocumentInterface[]> {\n    const [embeddedQuery, smallResults] = await Promise.all([\n      this.largeEmbeddingModel.embedQuery(query),\n      this.vectorStore.similaritySearch(query, this.smallK, this.filter),\n    ]);\n\n    return this._rankByLargeEmbeddings(embeddedQuery, smallResults);\n  }\n\n  /**\n   * Override the default `addDocuments` method to embed the documents twice,\n   * once using the larger embeddings model, and then again using the default\n   * embedding model linked to the vector store.\n   *\n   * @param {DocumentInterface[]} documents - An array of documents to add to the vector store.\n   * @param {AddDocumentOptions} options - An optional object containing additional options for adding documents.\n   * @returns {Promise<string[] | void>} A promise that resolves to an array of the document IDs that were added to the vector store.\n   */\n  override addDocuments = async (\n    documents: DocumentInterface[],\n    options?: AddDocumentOptions\n  ): Promise<string[] | void> => {\n    // Insure documents metadata does not contain the large embedding key\n    if (documents.some((doc) => this.largeEmbeddingKey in doc.metadata)) {\n      throw new Error(\n        `All documents must not contain the large embedding key: ${this.largeEmbeddingKey} in their metadata.`\n      );\n    }\n\n    const allDocPageContent = documents.map((doc) => doc.pageContent);\n    const allDocLargeEmbeddings = await this.largeEmbeddingModel.embedDocuments(\n      allDocPageContent\n    );\n\n    const newDocuments: Array<DocumentInterface> = documents.map(\n      (doc, idx) => ({\n        ...doc,\n        metadata: {\n          ...doc.metadata,\n          [this.largeEmbeddingKey]: JSON.stringify(allDocLargeEmbeddings[idx]),\n        },\n      })\n    );\n\n    return this.vectorStore.addDocuments(newDocuments, options);\n  };\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;AAmEA,IAAa,sBAAb,cAEUA,mDAA4B;CACpC,SAAS;CAET,SAAS;CAET,oBAAoB;CAEpB;CAEA,aAAsD;CAEtD,YACEC,QACA;EACA,MAAM,OAAO;EACb,KAAK,SAAS,OAAO,UAAU,KAAK;EACpC,KAAK,SAAS,OAAO,UAAU,KAAK;EACpC,KAAK,oBAAoB,OAAO,qBAAqB,KAAK;EAC1D,KAAK,sBAAsB,OAAO;EAClC,KAAK,aAAa,OAAO,cAAc,KAAK;CAC7C;;;;;;;;;;;;;;;;;;;CAoBD,AAAQ,uBACNC,eACAC,cACqB;EACrB,MAAMC,kBAAmC,aAAa,IAAI,CAAC,QACzD,KAAK,MAAM,IAAI,SAAS,KAAK,mBAAmB,CACjD;EACD,IAAIC;AAEJ,UAAQ,KAAK,YAAb;GACE,KAAK;IACH,OAAO,wDAAuB,CAAC,aAAc,GAAE,gBAAgB;AAC/D;GACF,KAAK;IACH,OAAO,oDAAmB,CAAC,aAAc,GAAE,gBAAgB;AAC3D;GACF,KAAK;IACH,OAAO,yDAAwB,CAAC,aAAc,GAAE,gBAAgB;AAChE;GACF,QACE,OAAM,IAAI,MAAM,CAAC,qBAAqB,EAAE,KAAK,YAAY;EAC5D;EAGD,MAAM,CAAC,iBAAiB,GAAG,MAAM;EAGjC,IAAI,UAAU,MAAM,KAClB,EAAE,QAAQ,aAAa,OAAQ,GAC/B,CAAC,GAAG,UAAU,MACf;EAED,UAAU,QACP,IAAI,CAAC,GAAG,MAAM,CAAC,iBAAiB,IAAI,CAAE,EAAC,CACvC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,KAAK,IAAI,EAAE,CACzB,MAAM,GAAG,KAAK,OAAO,CACrB,IAAI,CAAC,GAAG,EAAE,KAAK,EAAE;AAEpB,SAAO,QAAQ,IAAI,CAAC,MAAM,aAAa,GAAG;CAC3C;CAED,MAAM,sBAAsBC,OAA6C;EACvE,MAAM,CAAC,eAAe,aAAa,GAAG,MAAM,QAAQ,IAAI,CACtD,KAAK,oBAAoB,WAAW,MAAM,EAC1C,KAAK,YAAY,iBAAiB,OAAO,KAAK,QAAQ,KAAK,OAAO,AACnE,EAAC;AAEF,SAAO,KAAK,uBAAuB,eAAe,aAAa;CAChE;;;;;;;;;;CAWD,AAAS,eAAe,OACtBC,WACAC,YAC6B;AAE7B,MAAI,UAAU,KAAK,CAAC,QAAQ,KAAK,qBAAqB,IAAI,SAAS,CACjE,OAAM,IAAI,MACR,CAAC,wDAAwD,EAAE,KAAK,kBAAkB,mBAAmB,CAAC;EAI1G,MAAM,oBAAoB,UAAU,IAAI,CAAC,QAAQ,IAAI,YAAY;EACjE,MAAM,wBAAwB,MAAM,KAAK,oBAAoB,eAC3D,kBACD;EAED,MAAMC,eAAyC,UAAU,IACvD,CAAC,KAAK,SAAS;GACb,GAAG;GACH,UAAU;IACR,GAAG,IAAI;KACN,KAAK,oBAAoB,KAAK,UAAU,sBAAsB,KAAK;GACrE;EACF,GACF;AAED,SAAO,KAAK,YAAY,aAAa,cAAc,QAAQ;CAC5D;AACF"}

package/dist/retrievers/matryoshka_retriever.d.cts

@@ -1,99 +0,0 @@
-import { DocumentInterface } from "@langchain/core/documents";
-import { VectorStore, VectorStoreRetriever, VectorStoreRetrieverInput } from "@langchain/core/vectorstores";
-import { Embeddings } from "@langchain/core/embeddings";
-
-//#region src/retrievers/matryoshka_retriever.d.ts
-
-/**
- * Type for options when adding a document to the VectorStore.
- */
-// eslint-disable-next-line @typescript-eslint/no-explicit-any
-type AddDocumentOptions = Record<string, any>;
-interface MatryoshkaRetrieverFields {
-  /**
-   * The number of documents to retrieve from the small store.
-   * @default 50
-   */
-  smallK?: number;
-  /**
-   * The number of documents to retrieve from the large store.
-   * @default 8
-   */
-  largeK?: number;
-  /**
-   * The metadata key to store the larger embeddings.
-   * @default "lc_large_embedding"
-   */
-  largeEmbeddingKey?: string;
-  /**
-   * The embedding model to use when generating the large
-   * embeddings.
-   */
-  largeEmbeddingModel: Embeddings;
-  /**
-   * The type of search to perform using the large embeddings.
-   * @default "cosine"
-   */
-  searchType?: "cosine" | "innerProduct" | "euclidean";
-}
-/**
- * A retriever that uses two sets of embeddings to perform adaptive retrieval. Based
- * off of the "Matryoshka embeddings: faster OpenAI vector search using Adaptive Retrieval"
- * blog post {@link https://supabase.com/blog/matryoshka-embeddings}.
- *
- *
- * This class performs "Adaptive Retrieval" for searching text embeddings efficiently using the
- * Matryoshka Representation Learning (MRL) technique. It retrieves documents similar to a query
- * embedding in two steps:
- *
- * First-pass: Uses a lower dimensional sub-vector from the MRL embedding for an initial, fast,
- * but less accurate search.
- *
- * Second-pass: Re-ranks the top results from the first pass using the full, high-dimensional
- * embedding for higher accuracy.
- *
- *
- * This code implements MRL embeddings for efficient vector search by combining faster,
- * lower-dimensional initial search with accurate, high-dimensional re-ranking.
- */
-declare class MatryoshkaRetriever<Store extends VectorStore = VectorStore> extends VectorStoreRetriever<Store> {
-  smallK: number;
-  largeK: number;
-  largeEmbeddingKey: string;
-  largeEmbeddingModel: Embeddings;
-  searchType: "cosine" | "innerProduct" | "euclidean";
-  constructor(fields: MatryoshkaRetrieverFields & VectorStoreRetrieverInput<Store>);
-  /**
-   * Ranks documents based on their similarity to a query embedding using larger embeddings.
-   *
-   * This method takes a query embedding and a list of documents (smallResults) as input. Each document
-   * in the smallResults array has previously been associated with a large embedding stored in its metadata.
-   * Depending on the `searchType` (cosine, innerProduct, or euclidean), it calculates the similarity scores
-   * between the query embedding and each document's large embedding. It then ranks the documents based on
-   * these similarity scores, from the most similar to the least similar.
-   *
-   * The method returns a promise that resolves to an array of the top `largeK` documents, where `largeK`
-   * is a class property defining the number of documents to return. This subset of documents is determined
-   * by sorting the entire list of documents based on their similarity scores and then selecting the top
-   * `largeK` documents.
-   *
-   * @param {number[]} embeddedQuery The embedding of the query, represented as an array of numbers.
-   * @param {DocumentInterface[]} smallResults An array of documents, each with metadata that includes a large embedding for similarity comparison.
-   * @returns {Promise<DocumentInterface[]>} A promise that resolves to an array of the top `largeK` ranked documents based on their similarity to the query embedding.
-   */
-  private _rankByLargeEmbeddings;
-  _getRelevantDocuments(query: string): Promise<DocumentInterface[]>;
-  /**
-   * Override the default `addDocuments` method to embed the documents twice,
-   * once using the larger embeddings model, and then again using the default
-   * embedding model linked to the vector store.
-   *
-   * @param {DocumentInterface[]} documents - An array of documents to add to the vector store.
-   * @param {AddDocumentOptions} options - An optional object containing additional options for adding documents.
-   * @returns {Promise<string[] | void>} A promise that resolves to an array of the document IDs that were added to the vector store.
-   */
-  addDocuments: (documents: DocumentInterface<Record<string, any>>[], options?: AddDocumentOptions | undefined) => Promise<void | string[]>;
-}
-//#endregion
-export { MatryoshkaRetriever, MatryoshkaRetrieverFields };
-//# sourceMappingURL=matryoshka_retriever.d.cts.map

package/dist/retrievers/matryoshka_retriever.d.cts.map

@@ -1 +0,0 @@
-{"version":3,"file":"matryoshka_retriever.d.cts","names":["DocumentInterface","Embeddings","VectorStore","VectorStoreRetriever","VectorStoreRetrieverInput","AddDocumentOptions","Record","MatryoshkaRetrieverFields","MatryoshkaRetriever","Store","Promise"],"sources":["../../src/retrievers/matryoshka_retriever.d.ts"],"sourcesContent":["import { DocumentInterface } from \"@langchain/core/documents\";\nimport { Embeddings } from \"@langchain/core/embeddings\";\nimport { VectorStore, VectorStoreRetriever, VectorStoreRetrieverInput } from \"@langchain/core/vectorstores\";\n/**\n * Type for options when adding a document to the VectorStore.\n */\n// eslint-disable-next-line @typescript-eslint/no-explicit-any\ntype AddDocumentOptions = Record<string, any>;\nexport interface MatryoshkaRetrieverFields {\n    /**\n     * The number of documents to retrieve from the small store.\n     * @default 50\n     */\n    smallK?: number;\n    /**\n     * The number of documents to retrieve from the large store.\n     * @default 8\n     */\n    largeK?: number;\n    /**\n     * The metadata key to store the larger embeddings.\n     * @default \"lc_large_embedding\"\n     */\n    largeEmbeddingKey?: string;\n    /**\n     * The embedding model to use when generating the large\n     * embeddings.\n     */\n    largeEmbeddingModel: Embeddings;\n    /**\n     * The type of search to perform using the large embeddings.\n     * @default \"cosine\"\n     */\n    searchType?: \"cosine\" | \"innerProduct\" | \"euclidean\";\n}\n/**\n * A retriever that uses two sets of embeddings to perform adaptive retrieval. Based\n * off of the \"Matryoshka embeddings: faster OpenAI vector search using Adaptive Retrieval\"\n * blog post {@link https://supabase.com/blog/matryoshka-embeddings}.\n *\n *\n * This class performs \"Adaptive Retrieval\" for searching text embeddings efficiently using the\n * Matryoshka Representation Learning (MRL) technique. It retrieves documents similar to a query\n * embedding in two steps:\n *\n * First-pass: Uses a lower dimensional sub-vector from the MRL embedding for an initial, fast,\n * but less accurate search.\n *\n * Second-pass: Re-ranks the top results from the first pass using the full, high-dimensional\n * embedding for higher accuracy.\n *\n *\n * This code implements MRL embeddings for efficient vector search by combining faster,\n * lower-dimensional initial search with accurate, high-dimensional re-ranking.\n */\nexport declare class MatryoshkaRetriever<Store extends VectorStore = VectorStore> extends VectorStoreRetriever<Store> {\n    smallK: number;\n    largeK: number;\n    largeEmbeddingKey: string;\n    largeEmbeddingModel: Embeddings;\n    searchType: \"cosine\" | \"innerProduct\" | \"euclidean\";\n    constructor(fields: MatryoshkaRetrieverFields & VectorStoreRetrieverInput<Store>);\n    /**\n     * Ranks documents based on their similarity to a query embedding using larger embeddings.\n     *\n     * This method takes a query embedding and a list of documents (smallResults) as input. Each document\n     * in the smallResults array has previously been associated with a large embedding stored in its metadata.\n     * Depending on the `searchType` (cosine, innerProduct, or euclidean), it calculates the similarity scores\n     * between the query embedding and each document's large embedding. It then ranks the documents based on\n     * these similarity scores, from the most similar to the least similar.\n     *\n     * The method returns a promise that resolves to an array of the top `largeK` documents, where `largeK`\n     * is a class property defining the number of documents to return. This subset of documents is determined\n     * by sorting the entire list of documents based on their similarity scores and then selecting the top\n     * `largeK` documents.\n     *\n     * @param {number[]} embeddedQuery The embedding of the query, represented as an array of numbers.\n     * @param {DocumentInterface[]} smallResults An array of documents, each with metadata that includes a large embedding for similarity comparison.\n     * @returns {Promise<DocumentInterface[]>} A promise that resolves to an array of the top `largeK` ranked documents based on their similarity to the query embedding.\n     */\n    private _rankByLargeEmbeddings;\n    _getRelevantDocuments(query: string): Promise<DocumentInterface[]>;\n    /**\n     * Override the default `addDocuments` method to embed the documents twice,\n     * once using the larger embeddings model, and then again using the default\n     * embedding model linked to the vector store.\n     *\n     * @param {DocumentInterface[]} documents - An array of documents to add to the vector store.\n     * @param {AddDocumentOptions} options - An optional object containing additional options for adding documents.\n     * @returns {Promise<string[] | void>} A promise that resolves to an array of the document IDs that were added to the vector store.\n     */\n    addDocuments: (documents: DocumentInterface<Record<string, any>>[], options?: AddDocumentOptions | undefined) => Promise<void | string[]>;\n}\nexport {};\n"],"mappings":";;;;;;;;AAE4G;AAM5G;AA+CA,KAhDKK,kBAAAA,GAAqBC,MAgDc,CAAA,MAAA,EAAA,GAAA,CAAA;AAAA,UA/CvBC,yBAAAA,CA+CuB;EAAA;;;;EAIL,MAEXA,CAAAA,EAAAA,MAAAA;EAAyB;;;;EAoBA,MAUDD,CAAAA,EAAAA,MAAAA;EAAM;;;;EApCwD,iBAAA,CAAA,EAAA,MAAA;;;;;uBA3BrFL;;;;;;;;;;;;;;;;;;;;;;;;;;;cA2BJO,kCAAkCN,cAAcA,qBAAqBC,qBAAqBM;;;;uBAItFR;;sBAEDM,4BAA4BH,0BAA0BK;;;;;;;;;;;;;;;;;;;;wCAoBpCC,QAAQV;;;;;;;;;;4BAUpBA,kBAAkBM,kCAAkCD,mCAAmCK"}

package/dist/retrievers/matryoshka_retriever.d.ts

@@ -1,99 +0,0 @@
-import { DocumentInterface } from "@langchain/core/documents";
-import { Embeddings } from "@langchain/core/embeddings";
-import { VectorStore, VectorStoreRetriever, VectorStoreRetrieverInput } from "@langchain/core/vectorstores";
-
-//#region src/retrievers/matryoshka_retriever.d.ts
-
-/**
- * Type for options when adding a document to the VectorStore.
- */
-// eslint-disable-next-line @typescript-eslint/no-explicit-any
-type AddDocumentOptions = Record<string, any>;
-interface MatryoshkaRetrieverFields {
-  /**
-   * The number of documents to retrieve from the small store.
-   * @default 50
-   */
-  smallK?: number;
-  /**
-   * The number of documents to retrieve from the large store.
-   * @default 8
-   */
-  largeK?: number;
-  /**
-   * The metadata key to store the larger embeddings.
-   * @default "lc_large_embedding"
-   */
-  largeEmbeddingKey?: string;
-  /**
-   * The embedding model to use when generating the large
-   * embeddings.
-   */
-  largeEmbeddingModel: Embeddings;
-  /**
-   * The type of search to perform using the large embeddings.
-   * @default "cosine"
-   */
-  searchType?: "cosine" | "innerProduct" | "euclidean";
-}
-/**
- * A retriever that uses two sets of embeddings to perform adaptive retrieval. Based
- * off of the "Matryoshka embeddings: faster OpenAI vector search using Adaptive Retrieval"
- * blog post {@link https://supabase.com/blog/matryoshka-embeddings}.
- *
- *
- * This class performs "Adaptive Retrieval" for searching text embeddings efficiently using the
- * Matryoshka Representation Learning (MRL) technique. It retrieves documents similar to a query
- * embedding in two steps:
- *
- * First-pass: Uses a lower dimensional sub-vector from the MRL embedding for an initial, fast,
- * but less accurate search.
- *
- * Second-pass: Re-ranks the top results from the first pass using the full, high-dimensional
- * embedding for higher accuracy.
- *
- *
- * This code implements MRL embeddings for efficient vector search by combining faster,
- * lower-dimensional initial search with accurate, high-dimensional re-ranking.
- */
-declare class MatryoshkaRetriever<Store extends VectorStore = VectorStore> extends VectorStoreRetriever<Store> {
-  smallK: number;
-  largeK: number;
-  largeEmbeddingKey: string;
-  largeEmbeddingModel: Embeddings;
-  searchType: "cosine" | "innerProduct" | "euclidean";
-  constructor(fields: MatryoshkaRetrieverFields & VectorStoreRetrieverInput<Store>);
-  /**
-   * Ranks documents based on their similarity to a query embedding using larger embeddings.
-   *
-   * This method takes a query embedding and a list of documents (smallResults) as input. Each document
-   * in the smallResults array has previously been associated with a large embedding stored in its metadata.
-   * Depending on the `searchType` (cosine, innerProduct, or euclidean), it calculates the similarity scores
-   * between the query embedding and each document's large embedding. It then ranks the documents based on
-   * these similarity scores, from the most similar to the least similar.
-   *
-   * The method returns a promise that resolves to an array of the top `largeK` documents, where `largeK`
-   * is a class property defining the number of documents to return. This subset of documents is determined
-   * by sorting the entire list of documents based on their similarity scores and then selecting the top
-   * `largeK` documents.
-   *
-   * @param {number[]} embeddedQuery The embedding of the query, represented as an array of numbers.
-   * @param {DocumentInterface[]} smallResults An array of documents, each with metadata that includes a large embedding for similarity comparison.
-   * @returns {Promise<DocumentInterface[]>} A promise that resolves to an array of the top `largeK` ranked documents based on their similarity to the query embedding.
-   */
-  private _rankByLargeEmbeddings;
-  _getRelevantDocuments(query: string): Promise<DocumentInterface[]>;
-  /**
-   * Override the default `addDocuments` method to embed the documents twice,
-   * once using the larger embeddings model, and then again using the default
-   * embedding model linked to the vector store.
-   *
-   * @param {DocumentInterface[]} documents - An array of documents to add to the vector store.
-   * @param {AddDocumentOptions} options - An optional object containing additional options for adding documents.
-   * @returns {Promise<string[] | void>} A promise that resolves to an array of the document IDs that were added to the vector store.
-   */
-  addDocuments: (documents: DocumentInterface<Record<string, any>>[], options?: AddDocumentOptions | undefined) => Promise<void | string[]>;
-}
-//#endregion
-export { MatryoshkaRetriever, MatryoshkaRetrieverFields };
-//# sourceMappingURL=matryoshka_retriever.d.ts.map

package/dist/retrievers/matryoshka_retriever.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"matryoshka_retriever.d.ts","names":["DocumentInterface","Embeddings","VectorStore","VectorStoreRetriever","VectorStoreRetrieverInput","AddDocumentOptions","Record","MatryoshkaRetrieverFields","MatryoshkaRetriever","Store","Promise"],"sources":["../../src/retrievers/matryoshka_retriever.d.ts"],"sourcesContent":["import { DocumentInterface } from \"@langchain/core/documents\";\nimport { Embeddings } from \"@langchain/core/embeddings\";\nimport { VectorStore, VectorStoreRetriever, VectorStoreRetrieverInput } from \"@langchain/core/vectorstores\";\n/**\n * Type for options when adding a document to the VectorStore.\n */\n// eslint-disable-next-line @typescript-eslint/no-explicit-any\ntype AddDocumentOptions = Record<string, any>;\nexport interface MatryoshkaRetrieverFields {\n    /**\n     * The number of documents to retrieve from the small store.\n     * @default 50\n     */\n    smallK?: number;\n    /**\n     * The number of documents to retrieve from the large store.\n     * @default 8\n     */\n    largeK?: number;\n    /**\n     * The metadata key to store the larger embeddings.\n     * @default \"lc_large_embedding\"\n     */\n    largeEmbeddingKey?: string;\n    /**\n     * The embedding model to use when generating the large\n     * embeddings.\n     */\n    largeEmbeddingModel: Embeddings;\n    /**\n     * The type of search to perform using the large embeddings.\n     * @default \"cosine\"\n     */\n    searchType?: \"cosine\" | \"innerProduct\" | \"euclidean\";\n}\n/**\n * A retriever that uses two sets of embeddings to perform adaptive retrieval. Based\n * off of the \"Matryoshka embeddings: faster OpenAI vector search using Adaptive Retrieval\"\n * blog post {@link https://supabase.com/blog/matryoshka-embeddings}.\n *\n *\n * This class performs \"Adaptive Retrieval\" for searching text embeddings efficiently using the\n * Matryoshka Representation Learning (MRL) technique. It retrieves documents similar to a query\n * embedding in two steps:\n *\n * First-pass: Uses a lower dimensional sub-vector from the MRL embedding for an initial, fast,\n * but less accurate search.\n *\n * Second-pass: Re-ranks the top results from the first pass using the full, high-dimensional\n * embedding for higher accuracy.\n *\n *\n * This code implements MRL embeddings for efficient vector search by combining faster,\n * lower-dimensional initial search with accurate, high-dimensional re-ranking.\n */\nexport declare class MatryoshkaRetriever<Store extends VectorStore = VectorStore> extends VectorStoreRetriever<Store> {\n    smallK: number;\n    largeK: number;\n    largeEmbeddingKey: string;\n    largeEmbeddingModel: Embeddings;\n    searchType: \"cosine\" | \"innerProduct\" | \"euclidean\";\n    constructor(fields: MatryoshkaRetrieverFields & VectorStoreRetrieverInput<Store>);\n    /**\n     * Ranks documents based on their similarity to a query embedding using larger embeddings.\n     *\n     * This method takes a query embedding and a list of documents (smallResults) as input. Each document\n     * in the smallResults array has previously been associated with a large embedding stored in its metadata.\n     * Depending on the `searchType` (cosine, innerProduct, or euclidean), it calculates the similarity scores\n     * between the query embedding and each document's large embedding. It then ranks the documents based on\n     * these similarity scores, from the most similar to the least similar.\n     *\n     * The method returns a promise that resolves to an array of the top `largeK` documents, where `largeK`\n     * is a class property defining the number of documents to return. This subset of documents is determined\n     * by sorting the entire list of documents based on their similarity scores and then selecting the top\n     * `largeK` documents.\n     *\n     * @param {number[]} embeddedQuery The embedding of the query, represented as an array of numbers.\n     * @param {DocumentInterface[]} smallResults An array of documents, each with metadata that includes a large embedding for similarity comparison.\n     * @returns {Promise<DocumentInterface[]>} A promise that resolves to an array of the top `largeK` ranked documents based on their similarity to the query embedding.\n     */\n    private _rankByLargeEmbeddings;\n    _getRelevantDocuments(query: string): Promise<DocumentInterface[]>;\n    /**\n     * Override the default `addDocuments` method to embed the documents twice,\n     * once using the larger embeddings model, and then again using the default\n     * embedding model linked to the vector store.\n     *\n     * @param {DocumentInterface[]} documents - An array of documents to add to the vector store.\n     * @param {AddDocumentOptions} options - An optional object containing additional options for adding documents.\n     * @returns {Promise<string[] | void>} A promise that resolves to an array of the document IDs that were added to the vector store.\n     */\n    addDocuments: (documents: DocumentInterface<Record<string, any>>[], options?: AddDocumentOptions | undefined) => Promise<void | string[]>;\n}\nexport {};\n"],"mappings":";;;;;;;;AAE4G;AAM5G;AA+CA,KAhDKK,kBAAAA,GAAqBC,MAgDc,CAAA,MAAA,EAAA,GAAA,CAAA;AAAA,UA/CvBC,yBAAAA,CA+CuB;EAAA;;;;EAIL,MAEXA,CAAAA,EAAAA,MAAAA;EAAyB;;;;EAoBA,MAUDD,CAAAA,EAAAA,MAAAA;EAAM;;;;EApCwD,iBAAA,CAAA,EAAA,MAAA;;;;;uBA3BrFL;;;;;;;;;;;;;;;;;;;;;;;;;;;cA2BJO,kCAAkCN,cAAcA,qBAAqBC,qBAAqBM;;;;uBAItFR;;sBAEDM,4BAA4BH,0BAA0BK;;;;;;;;;;;;;;;;;;;;wCAoBpCC,QAAQV;;;;;;;;;;4BAUpBA,kBAAkBM,kCAAkCD,mCAAmCK"}

package/dist/retrievers/matryoshka_retriever.js

@@ -1,110 +0,0 @@
-import { __export } from "../_virtual/rolldown_runtime.js";
-import { VectorStoreRetriever } from "@langchain/core/vectorstores";
-import { cosineSimilarity, euclideanDistance, innerProduct } from "@langchain/core/utils/math";
-
-//#region src/retrievers/matryoshka_retriever.ts
-var matryoshka_retriever_exports = {};
-__export(matryoshka_retriever_exports, { MatryoshkaRetriever: () => MatryoshkaRetriever });
-/**
-* A retriever that uses two sets of embeddings to perform adaptive retrieval. Based
-* off of the "Matryoshka embeddings: faster OpenAI vector search using Adaptive Retrieval"
-* blog post {@link https://supabase.com/blog/matryoshka-embeddings}.
-*
-*
-* This class performs "Adaptive Retrieval" for searching text embeddings efficiently using the
-* Matryoshka Representation Learning (MRL) technique. It retrieves documents similar to a query
-* embedding in two steps:
-*
-* First-pass: Uses a lower dimensional sub-vector from the MRL embedding for an initial, fast,
-* but less accurate search.
-*
-* Second-pass: Re-ranks the top results from the first pass using the full, high-dimensional
-* embedding for higher accuracy.
-*
-*
-* This code implements MRL embeddings for efficient vector search by combining faster,
-* lower-dimensional initial search with accurate, high-dimensional re-ranking.
-*/
-var MatryoshkaRetriever = class extends VectorStoreRetriever {
-	smallK = 50;
-	largeK = 8;
-	largeEmbeddingKey = "lc_large_embedding";
-	largeEmbeddingModel;
-	searchType = "cosine";
-	constructor(fields) {
-		super(fields);
-		this.smallK = fields.smallK ?? this.smallK;
-		this.largeK = fields.largeK ?? this.largeK;
-		this.largeEmbeddingKey = fields.largeEmbeddingKey ?? this.largeEmbeddingKey;
-		this.largeEmbeddingModel = fields.largeEmbeddingModel;
-		this.searchType = fields.searchType ?? this.searchType;
-	}
-	/**
-	* Ranks documents based on their similarity to a query embedding using larger embeddings.
-	*
-	* This method takes a query embedding and a list of documents (smallResults) as input. Each document
-	* in the smallResults array has previously been associated with a large embedding stored in its metadata.
-	* Depending on the `searchType` (cosine, innerProduct, or euclidean), it calculates the similarity scores
-	* between the query embedding and each document's large embedding. It then ranks the documents based on
-	* these similarity scores, from the most similar to the least similar.
-	*
-	* The method returns a promise that resolves to an array of the top `largeK` documents, where `largeK`
-	* is a class property defining the number of documents to return. This subset of documents is determined
-	* by sorting the entire list of documents based on their similarity scores and then selecting the top
-	* `largeK` documents.
-	*
-	* @param {number[]} embeddedQuery The embedding of the query, represented as an array of numbers.
-	* @param {DocumentInterface[]} smallResults An array of documents, each with metadata that includes a large embedding for similarity comparison.
-	* @returns {Promise<DocumentInterface[]>} A promise that resolves to an array of the top `largeK` ranked documents based on their similarity to the query embedding.
-	*/
-	_rankByLargeEmbeddings(embeddedQuery, smallResults) {
-		const largeEmbeddings = smallResults.map((doc) => JSON.parse(doc.metadata[this.largeEmbeddingKey]));
-		let func;
-		switch (this.searchType) {
-			case "cosine":
-				func = () => cosineSimilarity([embeddedQuery], largeEmbeddings);
-				break;
-			case "innerProduct":
-				func = () => innerProduct([embeddedQuery], largeEmbeddings);
-				break;
-			case "euclidean":
-				func = () => euclideanDistance([embeddedQuery], largeEmbeddings);
-				break;
-			default: throw new Error(`Unknown search type: ${this.searchType}`);
-		}
-		const [similarityScores] = func();
-		let indices = Array.from({ length: smallResults.length }, (_, index) => index);
-		indices = indices.map((v, i) => [similarityScores[i], v]).sort(([a], [b]) => b - a).slice(0, this.largeK).map(([, i]) => i);
-		return indices.map((i) => smallResults[i]);
-	}
-	async _getRelevantDocuments(query) {
-		const [embeddedQuery, smallResults] = await Promise.all([this.largeEmbeddingModel.embedQuery(query), this.vectorStore.similaritySearch(query, this.smallK, this.filter)]);
-		return this._rankByLargeEmbeddings(embeddedQuery, smallResults);
-	}
-	/**
-	* Override the default `addDocuments` method to embed the documents twice,
-	* once using the larger embeddings model, and then again using the default
-	* embedding model linked to the vector store.
-	*
-	* @param {DocumentInterface[]} documents - An array of documents to add to the vector store.
-	* @param {AddDocumentOptions} options - An optional object containing additional options for adding documents.
-	* @returns {Promise<string[] | void>} A promise that resolves to an array of the document IDs that were added to the vector store.
-	*/
-	addDocuments = async (documents, options) => {
-		if (documents.some((doc) => this.largeEmbeddingKey in doc.metadata)) throw new Error(`All documents must not contain the large embedding key: ${this.largeEmbeddingKey} in their metadata.`);
-		const allDocPageContent = documents.map((doc) => doc.pageContent);
-		const allDocLargeEmbeddings = await this.largeEmbeddingModel.embedDocuments(allDocPageContent);
-		const newDocuments = documents.map((doc, idx) => ({
-			...doc,
-			metadata: {
-				...doc.metadata,
-				[this.largeEmbeddingKey]: JSON.stringify(allDocLargeEmbeddings[idx])
-			}
-		}));
-		return this.vectorStore.addDocuments(newDocuments, options);
-	};
-};
-
-//#endregion
-export { MatryoshkaRetriever, matryoshka_retriever_exports };
-//# sourceMappingURL=matryoshka_retriever.js.map

package/dist/retrievers/matryoshka_retriever.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"matryoshka_retriever.js","names":["fields: MatryoshkaRetrieverFields & VectorStoreRetrieverInput<Store>","embeddedQuery: number[]","smallResults: DocumentInterface[]","largeEmbeddings: Array<number[]>","func: () => Array<number[]>","query: string","documents: DocumentInterface[]","options?: AddDocumentOptions","newDocuments: Array<DocumentInterface>"],"sources":["../../src/retrievers/matryoshka_retriever.ts"],"sourcesContent":["import { DocumentInterface } from \"@langchain/core/documents\";\nimport { Embeddings } from \"@langchain/core/embeddings\";\nimport {\n  cosineSimilarity,\n  euclideanDistance,\n  innerProduct,\n} from \"@langchain/core/utils/math\";\nimport {\n  VectorStore,\n  VectorStoreRetriever,\n  VectorStoreRetrieverInput,\n} from \"@langchain/core/vectorstores\";\n\n/**\n * Type for options when adding a document to the VectorStore.\n */\n// eslint-disable-next-line @typescript-eslint/no-explicit-any\ntype AddDocumentOptions = Record<string, any>;\n\nexport interface MatryoshkaRetrieverFields {\n  /**\n   * The number of documents to retrieve from the small store.\n   * @default 50\n   */\n  smallK?: number;\n  /**\n   * The number of documents to retrieve from the large store.\n   * @default 8\n   */\n  largeK?: number;\n  /**\n   * The metadata key to store the larger embeddings.\n   * @default \"lc_large_embedding\"\n   */\n  largeEmbeddingKey?: string;\n  /**\n   * The embedding model to use when generating the large\n   * embeddings.\n   */\n  largeEmbeddingModel: Embeddings;\n  /**\n   * The type of search to perform using the large embeddings.\n   * @default \"cosine\"\n   */\n  searchType?: \"cosine\" | \"innerProduct\" | \"euclidean\";\n}\n\n/**\n * A retriever that uses two sets of embeddings to perform adaptive retrieval. Based\n * off of the \"Matryoshka embeddings: faster OpenAI vector search using Adaptive Retrieval\"\n * blog post {@link https://supabase.com/blog/matryoshka-embeddings}.\n *\n *\n * This class performs \"Adaptive Retrieval\" for searching text embeddings efficiently using the\n * Matryoshka Representation Learning (MRL) technique. It retrieves documents similar to a query\n * embedding in two steps:\n *\n * First-pass: Uses a lower dimensional sub-vector from the MRL embedding for an initial, fast,\n * but less accurate search.\n *\n * Second-pass: Re-ranks the top results from the first pass using the full, high-dimensional\n * embedding for higher accuracy.\n *\n *\n * This code implements MRL embeddings for efficient vector search by combining faster,\n * lower-dimensional initial search with accurate, high-dimensional re-ranking.\n */\nexport class MatryoshkaRetriever<\n  Store extends VectorStore = VectorStore\n> extends VectorStoreRetriever<Store> {\n  smallK = 50;\n\n  largeK = 8;\n\n  largeEmbeddingKey = \"lc_large_embedding\";\n\n  largeEmbeddingModel: Embeddings;\n\n  searchType: \"cosine\" | \"innerProduct\" | \"euclidean\" = \"cosine\";\n\n  constructor(\n    fields: MatryoshkaRetrieverFields & VectorStoreRetrieverInput<Store>\n  ) {\n    super(fields);\n    this.smallK = fields.smallK ?? this.smallK;\n    this.largeK = fields.largeK ?? this.largeK;\n    this.largeEmbeddingKey = fields.largeEmbeddingKey ?? this.largeEmbeddingKey;\n    this.largeEmbeddingModel = fields.largeEmbeddingModel;\n    this.searchType = fields.searchType ?? this.searchType;\n  }\n\n  /**\n   * Ranks documents based on their similarity to a query embedding using larger embeddings.\n   *\n   * This method takes a query embedding and a list of documents (smallResults) as input. Each document\n   * in the smallResults array has previously been associated with a large embedding stored in its metadata.\n   * Depending on the `searchType` (cosine, innerProduct, or euclidean), it calculates the similarity scores\n   * between the query embedding and each document's large embedding. It then ranks the documents based on\n   * these similarity scores, from the most similar to the least similar.\n   *\n   * The method returns a promise that resolves to an array of the top `largeK` documents, where `largeK`\n   * is a class property defining the number of documents to return. This subset of documents is determined\n   * by sorting the entire list of documents based on their similarity scores and then selecting the top\n   * `largeK` documents.\n   *\n   * @param {number[]} embeddedQuery The embedding of the query, represented as an array of numbers.\n   * @param {DocumentInterface[]} smallResults An array of documents, each with metadata that includes a large embedding for similarity comparison.\n   * @returns {Promise<DocumentInterface[]>} A promise that resolves to an array of the top `largeK` ranked documents based on their similarity to the query embedding.\n   */\n  private _rankByLargeEmbeddings(\n    embeddedQuery: number[],\n    smallResults: DocumentInterface[]\n  ): DocumentInterface[] {\n    const largeEmbeddings: Array<number[]> = smallResults.map((doc) =>\n      JSON.parse(doc.metadata[this.largeEmbeddingKey])\n    );\n    let func: () => Array<number[]>;\n\n    switch (this.searchType) {\n      case \"cosine\":\n        func = () => cosineSimilarity([embeddedQuery], largeEmbeddings);\n        break;\n      case \"innerProduct\":\n        func = () => innerProduct([embeddedQuery], largeEmbeddings);\n        break;\n      case \"euclidean\":\n        func = () => euclideanDistance([embeddedQuery], largeEmbeddings);\n        break;\n      default:\n        throw new Error(`Unknown search type: ${this.searchType}`);\n    }\n\n    // Calculate the similarity scores between the query embedding and the large embeddings\n    const [similarityScores] = func();\n\n    // Create an array of indices from 0 to N-1, where N is the number of documents\n    let indices = Array.from(\n      { length: smallResults.length },\n      (_, index) => index\n    );\n\n    indices = indices\n      .map((v, i) => [similarityScores[i], v])\n      .sort(([a], [b]) => b - a)\n      .slice(0, this.largeK)\n      .map(([, i]) => i);\n\n    return indices.map((i) => smallResults[i]);\n  }\n\n  async _getRelevantDocuments(query: string): Promise<DocumentInterface[]> {\n    const [embeddedQuery, smallResults] = await Promise.all([\n      this.largeEmbeddingModel.embedQuery(query),\n      this.vectorStore.similaritySearch(query, this.smallK, this.filter),\n    ]);\n\n    return this._rankByLargeEmbeddings(embeddedQuery, smallResults);\n  }\n\n  /**\n   * Override the default `addDocuments` method to embed the documents twice,\n   * once using the larger embeddings model, and then again using the default\n   * embedding model linked to the vector store.\n   *\n   * @param {DocumentInterface[]} documents - An array of documents to add to the vector store.\n   * @param {AddDocumentOptions} options - An optional object containing additional options for adding documents.\n   * @returns {Promise<string[] | void>} A promise that resolves to an array of the document IDs that were added to the vector store.\n   */\n  override addDocuments = async (\n    documents: DocumentInterface[],\n    options?: AddDocumentOptions\n  ): Promise<string[] | void> => {\n    // Insure documents metadata does not contain the large embedding key\n    if (documents.some((doc) => this.largeEmbeddingKey in doc.metadata)) {\n      throw new Error(\n        `All documents must not contain the large embedding key: ${this.largeEmbeddingKey} in their metadata.`\n      );\n    }\n\n    const allDocPageContent = documents.map((doc) => doc.pageContent);\n    const allDocLargeEmbeddings = await this.largeEmbeddingModel.embedDocuments(\n      allDocPageContent\n    );\n\n    const newDocuments: Array<DocumentInterface> = documents.map(\n      (doc, idx) => ({\n        ...doc,\n        metadata: {\n          ...doc.metadata,\n          [this.largeEmbeddingKey]: JSON.stringify(allDocLargeEmbeddings[idx]),\n        },\n      })\n    );\n\n    return this.vectorStore.addDocuments(newDocuments, options);\n  };\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;AAmEA,IAAa,sBAAb,cAEU,qBAA4B;CACpC,SAAS;CAET,SAAS;CAET,oBAAoB;CAEpB;CAEA,aAAsD;CAEtD,YACEA,QACA;EACA,MAAM,OAAO;EACb,KAAK,SAAS,OAAO,UAAU,KAAK;EACpC,KAAK,SAAS,OAAO,UAAU,KAAK;EACpC,KAAK,oBAAoB,OAAO,qBAAqB,KAAK;EAC1D,KAAK,sBAAsB,OAAO;EAClC,KAAK,aAAa,OAAO,cAAc,KAAK;CAC7C;;;;;;;;;;;;;;;;;;;CAoBD,AAAQ,uBACNC,eACAC,cACqB;EACrB,MAAMC,kBAAmC,aAAa,IAAI,CAAC,QACzD,KAAK,MAAM,IAAI,SAAS,KAAK,mBAAmB,CACjD;EACD,IAAIC;AAEJ,UAAQ,KAAK,YAAb;GACE,KAAK;IACH,OAAO,MAAM,iBAAiB,CAAC,aAAc,GAAE,gBAAgB;AAC/D;GACF,KAAK;IACH,OAAO,MAAM,aAAa,CAAC,aAAc,GAAE,gBAAgB;AAC3D;GACF,KAAK;IACH,OAAO,MAAM,kBAAkB,CAAC,aAAc,GAAE,gBAAgB;AAChE;GACF,QACE,OAAM,IAAI,MAAM,CAAC,qBAAqB,EAAE,KAAK,YAAY;EAC5D;EAGD,MAAM,CAAC,iBAAiB,GAAG,MAAM;EAGjC,IAAI,UAAU,MAAM,KAClB,EAAE,QAAQ,aAAa,OAAQ,GAC/B,CAAC,GAAG,UAAU,MACf;EAED,UAAU,QACP,IAAI,CAAC,GAAG,MAAM,CAAC,iBAAiB,IAAI,CAAE,EAAC,CACvC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,KAAK,IAAI,EAAE,CACzB,MAAM,GAAG,KAAK,OAAO,CACrB,IAAI,CAAC,GAAG,EAAE,KAAK,EAAE;AAEpB,SAAO,QAAQ,IAAI,CAAC,MAAM,aAAa,GAAG;CAC3C;CAED,MAAM,sBAAsBC,OAA6C;EACvE,MAAM,CAAC,eAAe,aAAa,GAAG,MAAM,QAAQ,IAAI,CACtD,KAAK,oBAAoB,WAAW,MAAM,EAC1C,KAAK,YAAY,iBAAiB,OAAO,KAAK,QAAQ,KAAK,OAAO,AACnE,EAAC;AAEF,SAAO,KAAK,uBAAuB,eAAe,aAAa;CAChE;;;;;;;;;;CAWD,AAAS,eAAe,OACtBC,WACAC,YAC6B;AAE7B,MAAI,UAAU,KAAK,CAAC,QAAQ,KAAK,qBAAqB,IAAI,SAAS,CACjE,OAAM,IAAI,MACR,CAAC,wDAAwD,EAAE,KAAK,kBAAkB,mBAAmB,CAAC;EAI1G,MAAM,oBAAoB,UAAU,IAAI,CAAC,QAAQ,IAAI,YAAY;EACjE,MAAM,wBAAwB,MAAM,KAAK,oBAAoB,eAC3D,kBACD;EAED,MAAMC,eAAyC,UAAU,IACvD,CAAC,KAAK,SAAS;GACb,GAAG;GACH,UAAU;IACR,GAAG,IAAI;KACN,KAAK,oBAAoB,KAAK,UAAU,sBAAsB,KAAK;GACrE;EACF,GACF;AAED,SAAO,KAAK,YAAY,aAAa,cAAc,QAAQ;CAC5D;AACF"}