@langchain/core 1.1.48 → 1.1.49-dev-1781044436312

package/dist/example_selectors/conditional.cjs.map

@@ -1 +1 @@
-{"version":3,"file":"conditional.cjs","names":[],"sources":["../../src/example_selectors/conditional.ts"],"sourcesContent":["import type { BaseChatModel } from \"../language_models/chat_models.js\";\nimport type { BasePromptTemplate } from \"../prompts/base.js\";\nimport type { BaseLanguageModelInterface } from \"../language_models/base.js\";\nimport type { BaseLLM } from \"../language_models/llms.js\";\nimport type { PartialValues } from \"../utils/types/index.js\";\n\nexport type BaseGetPromptAsyncOptions = {\n  partialVariables?: PartialValues;\n};\n\n/**\n * Abstract class that defines the interface for selecting a prompt for a\n * given language model.\n */\nexport abstract class BasePromptSelector {\n  /**\n   * Abstract method that must be implemented by any class that extends\n   * `BasePromptSelector`. It takes a language model as an argument and\n   * returns a prompt template.\n   * @param llm The language model for which to get a prompt.\n   * @returns A prompt template.\n   */\n  abstract getPrompt(llm: BaseLanguageModelInterface): BasePromptTemplate;\n\n  /**\n   * Asynchronous version of `getPrompt` that also accepts an options object\n   * for partial variables.\n   * @param llm The language model for which to get a prompt.\n   * @param options Optional object for partial variables.\n   * @returns A Promise that resolves to a prompt template.\n   */\n  async getPromptAsync(\n    llm: BaseLanguageModelInterface,\n    options?: BaseGetPromptAsyncOptions\n  ): Promise<BasePromptTemplate> {\n    const prompt = this.getPrompt(llm);\n    return prompt.partial(options?.partialVariables ?? {});\n  }\n}\n\n/**\n * Concrete implementation of `BasePromptSelector` that selects a prompt\n * based on a set of conditions. It has a default prompt that it returns\n * if none of the conditions are met.\n */\nexport class ConditionalPromptSelector extends BasePromptSelector {\n  defaultPrompt: BasePromptTemplate;\n\n  conditionals: Array<\n    [\n      condition: (llm: BaseLanguageModelInterface) => boolean,\n      prompt: BasePromptTemplate,\n    ]\n  >;\n\n  constructor(\n    default_prompt: BasePromptTemplate,\n    conditionals: Array<\n      [\n        condition: (llm: BaseLanguageModelInterface) => boolean,\n        prompt: BasePromptTemplate,\n      ]\n    > = []\n  ) {\n    super();\n    this.defaultPrompt = default_prompt;\n    this.conditionals = conditionals;\n  }\n\n  /**\n   * Method that selects a prompt based on a set of conditions. If none of\n   * the conditions are met, it returns the default prompt.\n   * @param llm The language model for which to get a prompt.\n   * @returns A prompt template.\n   */\n  getPrompt(llm: BaseLanguageModelInterface): BasePromptTemplate {\n    for (const [condition, prompt] of this.conditionals) {\n      if (condition(llm)) {\n        return prompt;\n      }\n    }\n    return this.defaultPrompt;\n  }\n}\n\n/**\n * Type guard function that checks if a given language model is of type\n * `BaseLLM`.\n */\nexport function isLLM(llm: BaseLanguageModelInterface): llm is BaseLLM {\n  return llm._modelType() === \"base_llm\";\n}\n\n/**\n * Type guard function that checks if a given language model is of type\n * `BaseChatModel`.\n */\nexport function isChatModel(\n  llm: BaseLanguageModelInterface\n): llm is BaseChatModel {\n  return llm._modelType() === \"base_chat_model\";\n}\n"],"mappings":";;;;;AAcA,IAAsB,qBAAtB,MAAyC;;;;;;;;CAiBvC,MAAM,eACJ,KACA,SAC6B;AAE7B,SADe,KAAK,UAAU,IAAI,CACpB,QAAQ,SAAS,oBAAoB,EAAE,CAAC;;;;;;;;AAS1D,IAAa,4BAAb,cAA+C,mBAAmB;CAChE;CAEA;CAOA,YACE,gBACA,eAKI,EAAE,EACN;AACA,SAAO;AACP,OAAK,gBAAgB;AACrB,OAAK,eAAe;;;;;;;;CAStB,UAAU,KAAqD;AAC7D,OAAK,MAAM,CAAC,WAAW,WAAW,KAAK,aACrC,KAAI,UAAU,IAAI,CAChB,QAAO;AAGX,SAAO,KAAK;;;;;;;AAQhB,SAAgB,MAAM,KAAiD;AACrE,QAAO,IAAI,YAAY,KAAK;;;;;;AAO9B,SAAgB,YACd,KACsB;AACtB,QAAO,IAAI,YAAY,KAAK"}
+{"version":3,"file":"conditional.cjs","names":[],"sources":["../../src/example_selectors/conditional.ts"],"sourcesContent":["import type { BaseChatModel } from \"../language_models/chat_models.js\";\nimport type { BasePromptTemplate } from \"../prompts/base.js\";\nimport type { BaseLanguageModelInterface } from \"../language_models/base.js\";\nimport type { BaseLLM } from \"../language_models/llms.js\";\nimport type { PartialValues } from \"../utils/types/index.js\";\n\nexport type BaseGetPromptAsyncOptions = {\n  partialVariables?: PartialValues;\n};\n\n/**\n * Abstract class that defines the interface for selecting a prompt for a\n * given language model.\n */\nexport abstract class BasePromptSelector {\n  /**\n   * Abstract method that must be implemented by any class that extends\n   * `BasePromptSelector`. It takes a language model as an argument and\n   * returns a prompt template.\n   * @param llm The language model for which to get a prompt.\n   * @returns A prompt template.\n   */\n  abstract getPrompt(llm: BaseLanguageModelInterface): BasePromptTemplate;\n\n  /**\n   * Asynchronous version of `getPrompt` that also accepts an options object\n   * for partial variables.\n   * @param llm The language model for which to get a prompt.\n   * @param options Optional object for partial variables.\n   * @returns A Promise that resolves to a prompt template.\n   */\n  async getPromptAsync(\n    llm: BaseLanguageModelInterface,\n    options?: BaseGetPromptAsyncOptions\n  ): Promise<BasePromptTemplate> {\n    const prompt = this.getPrompt(llm);\n    return prompt.partial(options?.partialVariables ?? {});\n  }\n}\n\n/**\n * Concrete implementation of `BasePromptSelector` that selects a prompt\n * based on a set of conditions. It has a default prompt that it returns\n * if none of the conditions are met.\n */\nexport class ConditionalPromptSelector extends BasePromptSelector {\n  defaultPrompt: BasePromptTemplate;\n\n  conditionals: Array<\n    [\n      condition: (llm: BaseLanguageModelInterface) => boolean,\n      prompt: BasePromptTemplate,\n    ]\n  >;\n\n  constructor(\n    default_prompt: BasePromptTemplate,\n    conditionals: Array<\n      [\n        condition: (llm: BaseLanguageModelInterface) => boolean,\n        prompt: BasePromptTemplate,\n      ]\n    > = []\n  ) {\n    super();\n    this.defaultPrompt = default_prompt;\n    this.conditionals = conditionals;\n  }\n\n  /**\n   * Method that selects a prompt based on a set of conditions. If none of\n   * the conditions are met, it returns the default prompt.\n   * @param llm The language model for which to get a prompt.\n   * @returns A prompt template.\n   */\n  getPrompt(llm: BaseLanguageModelInterface): BasePromptTemplate {\n    for (const [condition, prompt] of this.conditionals) {\n      if (condition(llm)) {\n        return prompt;\n      }\n    }\n    return this.defaultPrompt;\n  }\n}\n\n/**\n * Type guard function that checks if a given language model is of type\n * `BaseLLM`.\n */\nexport function isLLM(llm: BaseLanguageModelInterface): llm is BaseLLM {\n  return llm._modelType() === \"base_llm\";\n}\n\n/**\n * Type guard function that checks if a given language model is of type\n * `BaseChatModel`.\n */\nexport function isChatModel(\n  llm: BaseLanguageModelInterface\n): llm is BaseChatModel {\n  return llm._modelType() === \"base_chat_model\";\n}\n"],"mappings":";;;;;AAcA,IAAsB,qBAAtB,MAAyC;;;;;;;;CAiBvC,MAAM,eACJ,KACA,SAC6B;AAE7B,SADe,KAAK,UAAU,IACjB,CAAC,QAAQ,SAAS,oBAAoB,EAAE,CAAC;;;;;;;;AAS1D,IAAa,4BAAb,cAA+C,mBAAmB;CAChE;CAEA;CAOA,YACE,gBACA,eAKI,EAAE,EACN;AACA,SAAO;AACP,OAAK,gBAAgB;AACrB,OAAK,eAAe;;;;;;;;CAStB,UAAU,KAAqD;AAC7D,OAAK,MAAM,CAAC,WAAW,WAAW,KAAK,aACrC,KAAI,UAAU,IAAI,CAChB,QAAO;AAGX,SAAO,KAAK;;;;;;;AAQhB,SAAgB,MAAM,KAAiD;AACrE,QAAO,IAAI,YAAY,KAAK;;;;;;AAO9B,SAAgB,YACd,KACsB;AACtB,QAAO,IAAI,YAAY,KAAK"}

package/dist/example_selectors/conditional.js.map

@@ -1 +1 @@
-{"version":3,"file":"conditional.js","names":[],"sources":["../../src/example_selectors/conditional.ts"],"sourcesContent":["import type { BaseChatModel } from \"../language_models/chat_models.js\";\nimport type { BasePromptTemplate } from \"../prompts/base.js\";\nimport type { BaseLanguageModelInterface } from \"../language_models/base.js\";\nimport type { BaseLLM } from \"../language_models/llms.js\";\nimport type { PartialValues } from \"../utils/types/index.js\";\n\nexport type BaseGetPromptAsyncOptions = {\n  partialVariables?: PartialValues;\n};\n\n/**\n * Abstract class that defines the interface for selecting a prompt for a\n * given language model.\n */\nexport abstract class BasePromptSelector {\n  /**\n   * Abstract method that must be implemented by any class that extends\n   * `BasePromptSelector`. It takes a language model as an argument and\n   * returns a prompt template.\n   * @param llm The language model for which to get a prompt.\n   * @returns A prompt template.\n   */\n  abstract getPrompt(llm: BaseLanguageModelInterface): BasePromptTemplate;\n\n  /**\n   * Asynchronous version of `getPrompt` that also accepts an options object\n   * for partial variables.\n   * @param llm The language model for which to get a prompt.\n   * @param options Optional object for partial variables.\n   * @returns A Promise that resolves to a prompt template.\n   */\n  async getPromptAsync(\n    llm: BaseLanguageModelInterface,\n    options?: BaseGetPromptAsyncOptions\n  ): Promise<BasePromptTemplate> {\n    const prompt = this.getPrompt(llm);\n    return prompt.partial(options?.partialVariables ?? {});\n  }\n}\n\n/**\n * Concrete implementation of `BasePromptSelector` that selects a prompt\n * based on a set of conditions. It has a default prompt that it returns\n * if none of the conditions are met.\n */\nexport class ConditionalPromptSelector extends BasePromptSelector {\n  defaultPrompt: BasePromptTemplate;\n\n  conditionals: Array<\n    [\n      condition: (llm: BaseLanguageModelInterface) => boolean,\n      prompt: BasePromptTemplate,\n    ]\n  >;\n\n  constructor(\n    default_prompt: BasePromptTemplate,\n    conditionals: Array<\n      [\n        condition: (llm: BaseLanguageModelInterface) => boolean,\n        prompt: BasePromptTemplate,\n      ]\n    > = []\n  ) {\n    super();\n    this.defaultPrompt = default_prompt;\n    this.conditionals = conditionals;\n  }\n\n  /**\n   * Method that selects a prompt based on a set of conditions. If none of\n   * the conditions are met, it returns the default prompt.\n   * @param llm The language model for which to get a prompt.\n   * @returns A prompt template.\n   */\n  getPrompt(llm: BaseLanguageModelInterface): BasePromptTemplate {\n    for (const [condition, prompt] of this.conditionals) {\n      if (condition(llm)) {\n        return prompt;\n      }\n    }\n    return this.defaultPrompt;\n  }\n}\n\n/**\n * Type guard function that checks if a given language model is of type\n * `BaseLLM`.\n */\nexport function isLLM(llm: BaseLanguageModelInterface): llm is BaseLLM {\n  return llm._modelType() === \"base_llm\";\n}\n\n/**\n * Type guard function that checks if a given language model is of type\n * `BaseChatModel`.\n */\nexport function isChatModel(\n  llm: BaseLanguageModelInterface\n): llm is BaseChatModel {\n  return llm._modelType() === \"base_chat_model\";\n}\n"],"mappings":";;;;;AAcA,IAAsB,qBAAtB,MAAyC;;;;;;;;CAiBvC,MAAM,eACJ,KACA,SAC6B;AAE7B,SADe,KAAK,UAAU,IAAI,CACpB,QAAQ,SAAS,oBAAoB,EAAE,CAAC;;;;;;;;AAS1D,IAAa,4BAAb,cAA+C,mBAAmB;CAChE;CAEA;CAOA,YACE,gBACA,eAKI,EAAE,EACN;AACA,SAAO;AACP,OAAK,gBAAgB;AACrB,OAAK,eAAe;;;;;;;;CAStB,UAAU,KAAqD;AAC7D,OAAK,MAAM,CAAC,WAAW,WAAW,KAAK,aACrC,KAAI,UAAU,IAAI,CAChB,QAAO;AAGX,SAAO,KAAK;;;;;;;AAQhB,SAAgB,MAAM,KAAiD;AACrE,QAAO,IAAI,YAAY,KAAK;;;;;;AAO9B,SAAgB,YACd,KACsB;AACtB,QAAO,IAAI,YAAY,KAAK"}
+{"version":3,"file":"conditional.js","names":[],"sources":["../../src/example_selectors/conditional.ts"],"sourcesContent":["import type { BaseChatModel } from \"../language_models/chat_models.js\";\nimport type { BasePromptTemplate } from \"../prompts/base.js\";\nimport type { BaseLanguageModelInterface } from \"../language_models/base.js\";\nimport type { BaseLLM } from \"../language_models/llms.js\";\nimport type { PartialValues } from \"../utils/types/index.js\";\n\nexport type BaseGetPromptAsyncOptions = {\n  partialVariables?: PartialValues;\n};\n\n/**\n * Abstract class that defines the interface for selecting a prompt for a\n * given language model.\n */\nexport abstract class BasePromptSelector {\n  /**\n   * Abstract method that must be implemented by any class that extends\n   * `BasePromptSelector`. It takes a language model as an argument and\n   * returns a prompt template.\n   * @param llm The language model for which to get a prompt.\n   * @returns A prompt template.\n   */\n  abstract getPrompt(llm: BaseLanguageModelInterface): BasePromptTemplate;\n\n  /**\n   * Asynchronous version of `getPrompt` that also accepts an options object\n   * for partial variables.\n   * @param llm The language model for which to get a prompt.\n   * @param options Optional object for partial variables.\n   * @returns A Promise that resolves to a prompt template.\n   */\n  async getPromptAsync(\n    llm: BaseLanguageModelInterface,\n    options?: BaseGetPromptAsyncOptions\n  ): Promise<BasePromptTemplate> {\n    const prompt = this.getPrompt(llm);\n    return prompt.partial(options?.partialVariables ?? {});\n  }\n}\n\n/**\n * Concrete implementation of `BasePromptSelector` that selects a prompt\n * based on a set of conditions. It has a default prompt that it returns\n * if none of the conditions are met.\n */\nexport class ConditionalPromptSelector extends BasePromptSelector {\n  defaultPrompt: BasePromptTemplate;\n\n  conditionals: Array<\n    [\n      condition: (llm: BaseLanguageModelInterface) => boolean,\n      prompt: BasePromptTemplate,\n    ]\n  >;\n\n  constructor(\n    default_prompt: BasePromptTemplate,\n    conditionals: Array<\n      [\n        condition: (llm: BaseLanguageModelInterface) => boolean,\n        prompt: BasePromptTemplate,\n      ]\n    > = []\n  ) {\n    super();\n    this.defaultPrompt = default_prompt;\n    this.conditionals = conditionals;\n  }\n\n  /**\n   * Method that selects a prompt based on a set of conditions. If none of\n   * the conditions are met, it returns the default prompt.\n   * @param llm The language model for which to get a prompt.\n   * @returns A prompt template.\n   */\n  getPrompt(llm: BaseLanguageModelInterface): BasePromptTemplate {\n    for (const [condition, prompt] of this.conditionals) {\n      if (condition(llm)) {\n        return prompt;\n      }\n    }\n    return this.defaultPrompt;\n  }\n}\n\n/**\n * Type guard function that checks if a given language model is of type\n * `BaseLLM`.\n */\nexport function isLLM(llm: BaseLanguageModelInterface): llm is BaseLLM {\n  return llm._modelType() === \"base_llm\";\n}\n\n/**\n * Type guard function that checks if a given language model is of type\n * `BaseChatModel`.\n */\nexport function isChatModel(\n  llm: BaseLanguageModelInterface\n): llm is BaseChatModel {\n  return llm._modelType() === \"base_chat_model\";\n}\n"],"mappings":";;;;;AAcA,IAAsB,qBAAtB,MAAyC;;;;;;;;CAiBvC,MAAM,eACJ,KACA,SAC6B;AAE7B,SADe,KAAK,UAAU,IACjB,CAAC,QAAQ,SAAS,oBAAoB,EAAE,CAAC;;;;;;;;AAS1D,IAAa,4BAAb,cAA+C,mBAAmB;CAChE;CAEA;CAOA,YACE,gBACA,eAKI,EAAE,EACN;AACA,SAAO;AACP,OAAK,gBAAgB;AACrB,OAAK,eAAe;;;;;;;;CAStB,UAAU,KAAqD;AAC7D,OAAK,MAAM,CAAC,WAAW,WAAW,KAAK,aACrC,KAAI,UAAU,IAAI,CAChB,QAAO;AAGX,SAAO,KAAK;;;;;;;AAQhB,SAAgB,MAAM,KAAiD;AACrE,QAAO,IAAI,YAAY,KAAK;;;;;;AAO9B,SAAgB,YACd,KACsB;AACtB,QAAO,IAAI,YAAY,KAAK"}

package/dist/example_selectors/length_based.cjs.map

@@ -1 +1 @@
-{"version":3,"file":"length_based.cjs","names":["BaseExampleSelector"],"sources":["../../src/example_selectors/length_based.ts"],"sourcesContent":["import { Example } from \"../prompts/base.js\";\nimport { BaseExampleSelector } from \"./base.js\";\nimport { PromptTemplate } from \"../prompts/prompt.js\";\n\n/**\n * Calculates the length of a text based on the number of words and lines.\n */\nfunction getLengthBased(text: string): number {\n  return text.split(/\\n| /).length;\n}\n\n/**\n * Interface for the input parameters of the LengthBasedExampleSelector\n * class.\n */\nexport interface LengthBasedExampleSelectorInput {\n  examplePrompt: PromptTemplate;\n  maxLength?: number;\n  getTextLength?: (text: string) => number;\n}\n\n/**\n * A specialized example selector that selects examples based on their\n * length, ensuring that the total length of the selected examples does\n * not exceed a specified maximum length.\n * @example\n * ```typescript\n * const exampleSelector = new LengthBasedExampleSelector(\n *   [\n *     { input: \"happy\", output: \"sad\" },\n *     { input: \"tall\", output: \"short\" },\n *     { input: \"energetic\", output: \"lethargic\" },\n *     { input: \"sunny\", output: \"gloomy\" },\n *     { input: \"windy\", output: \"calm\" },\n *   ],\n *   {\n *     examplePrompt: new PromptTemplate({\n *       inputVariables: [\"input\", \"output\"],\n *       template: \"Input: {input}\\nOutput: {output}\",\n *     }),\n *     maxLength: 25,\n *   },\n * );\n * const dynamicPrompt = new FewShotPromptTemplate({\n *   exampleSelector,\n *   examplePrompt: new PromptTemplate({\n *     inputVariables: [\"input\", \"output\"],\n *     template: \"Input: {input}\\nOutput: {output}\",\n *   }),\n *   prefix: \"Give the antonym of every input\",\n *   suffix: \"Input: {adjective}\\nOutput:\",\n *   inputVariables: [\"adjective\"],\n * });\n * console.log(dynamicPrompt.format({ adjective: \"big\" }));\n * console.log(\n *   dynamicPrompt.format({\n *     adjective:\n *       \"big and huge and massive and large and gigantic and tall and much much much much much bigger than everything else\",\n *   }),\n * );\n * ```\n */\nexport class LengthBasedExampleSelector extends BaseExampleSelector {\n  protected examples: Example[] = [];\n\n  examplePrompt!: PromptTemplate;\n\n  getTextLength: (text: string) => number = getLengthBased;\n\n  maxLength = 2048;\n\n  exampleTextLengths: number[] = [];\n\n  constructor(data: LengthBasedExampleSelectorInput) {\n    super(data);\n    this.examplePrompt = data.examplePrompt;\n    this.maxLength = data.maxLength ?? 2048;\n    this.getTextLength = data.getTextLength ?? getLengthBased;\n  }\n\n  /**\n   * Adds an example to the list of examples and calculates its length.\n   * @param example The example to be added.\n   * @returns Promise that resolves when the example has been added and its length calculated.\n   */\n  async addExample(example: Example): Promise<void> {\n    this.examples.push(example);\n    const stringExample = await this.examplePrompt.format(example);\n    this.exampleTextLengths.push(this.getTextLength(stringExample));\n  }\n\n  /**\n   * Calculates the lengths of the examples.\n   * @param v Array of lengths of the examples.\n   * @param values Instance of LengthBasedExampleSelector.\n   * @returns Promise that resolves with an array of lengths of the examples.\n   */\n  async calculateExampleTextLengths(\n    v: number[],\n    values: LengthBasedExampleSelector\n  ): Promise<number[]> {\n    if (v.length > 0) {\n      return v;\n    }\n\n    const { examples, examplePrompt } = values;\n    const stringExamples = await Promise.all(\n      examples.map((eg: Example) => examplePrompt.format(eg))\n    );\n    return stringExamples.map((eg: string) => this.getTextLength(eg));\n  }\n\n  /**\n   * Selects examples until the total length of the selected examples\n   * reaches the maxLength.\n   * @param inputVariables The input variables for the examples.\n   * @returns Promise that resolves with an array of selected examples.\n   */\n  async selectExamples(inputVariables: Example): Promise<Example[]> {\n    const inputs = Object.values(inputVariables).join(\" \");\n    let remainingLength = this.maxLength - this.getTextLength(inputs);\n    let i = 0;\n    const examples: Example[] = [];\n\n    while (remainingLength > 0 && i < this.examples.length) {\n      const newLength = remainingLength - this.exampleTextLengths[i];\n      if (newLength < 0) {\n        break;\n      } else {\n        examples.push(this.examples[i]);\n        remainingLength = newLength;\n      }\n      i += 1;\n    }\n\n    return examples;\n  }\n\n  /**\n   * Creates a new instance of LengthBasedExampleSelector and adds a list of\n   * examples to it.\n   * @param examples Array of examples to be added.\n   * @param args Input parameters for the LengthBasedExampleSelector.\n   * @returns Promise that resolves with a new instance of LengthBasedExampleSelector with the examples added.\n   */\n  static async fromExamples(\n    examples: Example[],\n    args: LengthBasedExampleSelectorInput\n  ) {\n    const selector = new LengthBasedExampleSelector(args);\n    await Promise.all(examples.map((eg) => selector.addExample(eg)));\n    return selector;\n  }\n}\n"],"mappings":";;;;;AAOA,SAAS,eAAe,MAAsB;AAC5C,QAAO,KAAK,MAAM,OAAO,CAAC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAsD5B,IAAa,6BAAb,MAAa,mCAAmCA,aAAAA,oBAAoB;CAClE,WAAgC,EAAE;CAElC;CAEA,gBAA0C;CAE1C,YAAY;CAEZ,qBAA+B,EAAE;CAEjC,YAAY,MAAuC;AACjD,QAAM,KAAK;AACX,OAAK,gBAAgB,KAAK;AAC1B,OAAK,YAAY,KAAK,aAAa;AACnC,OAAK,gBAAgB,KAAK,iBAAiB;;;;;;;CAQ7C,MAAM,WAAW,SAAiC;AAChD,OAAK,SAAS,KAAK,QAAQ;EAC3B,MAAM,gBAAgB,MAAM,KAAK,cAAc,OAAO,QAAQ;AAC9D,OAAK,mBAAmB,KAAK,KAAK,cAAc,cAAc,CAAC;;;;;;;;CASjE,MAAM,4BACJ,GACA,QACmB;AACnB,MAAI,EAAE,SAAS,EACb,QAAO;EAGT,MAAM,EAAE,UAAU,kBAAkB;AAIpC,UAHuB,MAAM,QAAQ,IACnC,SAAS,KAAK,OAAgB,cAAc,OAAO,GAAG,CAAC,CACxD,EACqB,KAAK,OAAe,KAAK,cAAc,GAAG,CAAC;;;;;;;;CASnE,MAAM,eAAe,gBAA6C;EAChE,MAAM,SAAS,OAAO,OAAO,eAAe,CAAC,KAAK,IAAI;EACtD,IAAI,kBAAkB,KAAK,YAAY,KAAK,cAAc,OAAO;EACjE,IAAI,IAAI;EACR,MAAM,WAAsB,EAAE;AAE9B,SAAO,kBAAkB,KAAK,IAAI,KAAK,SAAS,QAAQ;GACtD,MAAM,YAAY,kBAAkB,KAAK,mBAAmB;AAC5D,OAAI,YAAY,EACd;QACK;AACL,aAAS,KAAK,KAAK,SAAS,GAAG;AAC/B,sBAAkB;;AAEpB,QAAK;;AAGP,SAAO;;;;;;;;;CAUT,aAAa,aACX,UACA,MACA;EACA,MAAM,WAAW,IAAI,2BAA2B,KAAK;AACrD,QAAM,QAAQ,IAAI,SAAS,KAAK,OAAO,SAAS,WAAW,GAAG,CAAC,CAAC;AAChE,SAAO"}
+{"version":3,"file":"length_based.cjs","names":["BaseExampleSelector"],"sources":["../../src/example_selectors/length_based.ts"],"sourcesContent":["import { Example } from \"../prompts/base.js\";\nimport { BaseExampleSelector } from \"./base.js\";\nimport { PromptTemplate } from \"../prompts/prompt.js\";\n\n/**\n * Calculates the length of a text based on the number of words and lines.\n */\nfunction getLengthBased(text: string): number {\n  return text.split(/\\n| /).length;\n}\n\n/**\n * Interface for the input parameters of the LengthBasedExampleSelector\n * class.\n */\nexport interface LengthBasedExampleSelectorInput {\n  examplePrompt: PromptTemplate;\n  maxLength?: number;\n  getTextLength?: (text: string) => number;\n}\n\n/**\n * A specialized example selector that selects examples based on their\n * length, ensuring that the total length of the selected examples does\n * not exceed a specified maximum length.\n * @example\n * ```typescript\n * const exampleSelector = new LengthBasedExampleSelector(\n *   [\n *     { input: \"happy\", output: \"sad\" },\n *     { input: \"tall\", output: \"short\" },\n *     { input: \"energetic\", output: \"lethargic\" },\n *     { input: \"sunny\", output: \"gloomy\" },\n *     { input: \"windy\", output: \"calm\" },\n *   ],\n *   {\n *     examplePrompt: new PromptTemplate({\n *       inputVariables: [\"input\", \"output\"],\n *       template: \"Input: {input}\\nOutput: {output}\",\n *     }),\n *     maxLength: 25,\n *   },\n * );\n * const dynamicPrompt = new FewShotPromptTemplate({\n *   exampleSelector,\n *   examplePrompt: new PromptTemplate({\n *     inputVariables: [\"input\", \"output\"],\n *     template: \"Input: {input}\\nOutput: {output}\",\n *   }),\n *   prefix: \"Give the antonym of every input\",\n *   suffix: \"Input: {adjective}\\nOutput:\",\n *   inputVariables: [\"adjective\"],\n * });\n * console.log(dynamicPrompt.format({ adjective: \"big\" }));\n * console.log(\n *   dynamicPrompt.format({\n *     adjective:\n *       \"big and huge and massive and large and gigantic and tall and much much much much much bigger than everything else\",\n *   }),\n * );\n * ```\n */\nexport class LengthBasedExampleSelector extends BaseExampleSelector {\n  protected examples: Example[] = [];\n\n  examplePrompt!: PromptTemplate;\n\n  getTextLength: (text: string) => number = getLengthBased;\n\n  maxLength = 2048;\n\n  exampleTextLengths: number[] = [];\n\n  constructor(data: LengthBasedExampleSelectorInput) {\n    super(data);\n    this.examplePrompt = data.examplePrompt;\n    this.maxLength = data.maxLength ?? 2048;\n    this.getTextLength = data.getTextLength ?? getLengthBased;\n  }\n\n  /**\n   * Adds an example to the list of examples and calculates its length.\n   * @param example The example to be added.\n   * @returns Promise that resolves when the example has been added and its length calculated.\n   */\n  async addExample(example: Example): Promise<void> {\n    this.examples.push(example);\n    const stringExample = await this.examplePrompt.format(example);\n    this.exampleTextLengths.push(this.getTextLength(stringExample));\n  }\n\n  /**\n   * Calculates the lengths of the examples.\n   * @param v Array of lengths of the examples.\n   * @param values Instance of LengthBasedExampleSelector.\n   * @returns Promise that resolves with an array of lengths of the examples.\n   */\n  async calculateExampleTextLengths(\n    v: number[],\n    values: LengthBasedExampleSelector\n  ): Promise<number[]> {\n    if (v.length > 0) {\n      return v;\n    }\n\n    const { examples, examplePrompt } = values;\n    const stringExamples = await Promise.all(\n      examples.map((eg: Example) => examplePrompt.format(eg))\n    );\n    return stringExamples.map((eg: string) => this.getTextLength(eg));\n  }\n\n  /**\n   * Selects examples until the total length of the selected examples\n   * reaches the maxLength.\n   * @param inputVariables The input variables for the examples.\n   * @returns Promise that resolves with an array of selected examples.\n   */\n  async selectExamples(inputVariables: Example): Promise<Example[]> {\n    const inputs = Object.values(inputVariables).join(\" \");\n    let remainingLength = this.maxLength - this.getTextLength(inputs);\n    let i = 0;\n    const examples: Example[] = [];\n\n    while (remainingLength > 0 && i < this.examples.length) {\n      const newLength = remainingLength - this.exampleTextLengths[i];\n      if (newLength < 0) {\n        break;\n      } else {\n        examples.push(this.examples[i]);\n        remainingLength = newLength;\n      }\n      i += 1;\n    }\n\n    return examples;\n  }\n\n  /**\n   * Creates a new instance of LengthBasedExampleSelector and adds a list of\n   * examples to it.\n   * @param examples Array of examples to be added.\n   * @param args Input parameters for the LengthBasedExampleSelector.\n   * @returns Promise that resolves with a new instance of LengthBasedExampleSelector with the examples added.\n   */\n  static async fromExamples(\n    examples: Example[],\n    args: LengthBasedExampleSelectorInput\n  ) {\n    const selector = new LengthBasedExampleSelector(args);\n    await Promise.all(examples.map((eg) => selector.addExample(eg)));\n    return selector;\n  }\n}\n"],"mappings":";;;;;AAOA,SAAS,eAAe,MAAsB;AAC5C,QAAO,KAAK,MAAM,OAAO,CAAC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAsD5B,IAAa,6BAAb,MAAa,mCAAmCA,aAAAA,oBAAoB;CAClE,WAAgC,EAAE;CAElC;CAEA,gBAA0C;CAE1C,YAAY;CAEZ,qBAA+B,EAAE;CAEjC,YAAY,MAAuC;AACjD,QAAM,KAAK;AACX,OAAK,gBAAgB,KAAK;AAC1B,OAAK,YAAY,KAAK,aAAa;AACnC,OAAK,gBAAgB,KAAK,iBAAiB;;;;;;;CAQ7C,MAAM,WAAW,SAAiC;AAChD,OAAK,SAAS,KAAK,QAAQ;EAC3B,MAAM,gBAAgB,MAAM,KAAK,cAAc,OAAO,QAAQ;AAC9D,OAAK,mBAAmB,KAAK,KAAK,cAAc,cAAc,CAAC;;;;;;;;CASjE,MAAM,4BACJ,GACA,QACmB;AACnB,MAAI,EAAE,SAAS,EACb,QAAO;EAGT,MAAM,EAAE,UAAU,kBAAkB;AAIpC,UAAO,MAHsB,QAAQ,IACnC,SAAS,KAAK,OAAgB,cAAc,OAAO,GAAG,CAAC,CACxD,EACqB,KAAK,OAAe,KAAK,cAAc,GAAG,CAAC;;;;;;;;CASnE,MAAM,eAAe,gBAA6C;EAChE,MAAM,SAAS,OAAO,OAAO,eAAe,CAAC,KAAK,IAAI;EACtD,IAAI,kBAAkB,KAAK,YAAY,KAAK,cAAc,OAAO;EACjE,IAAI,IAAI;EACR,MAAM,WAAsB,EAAE;AAE9B,SAAO,kBAAkB,KAAK,IAAI,KAAK,SAAS,QAAQ;GACtD,MAAM,YAAY,kBAAkB,KAAK,mBAAmB;AAC5D,OAAI,YAAY,EACd;QACK;AACL,aAAS,KAAK,KAAK,SAAS,GAAG;AAC/B,sBAAkB;;AAEpB,QAAK;;AAGP,SAAO;;;;;;;;;CAUT,aAAa,aACX,UACA,MACA;EACA,MAAM,WAAW,IAAI,2BAA2B,KAAK;AACrD,QAAM,QAAQ,IAAI,SAAS,KAAK,OAAO,SAAS,WAAW,GAAG,CAAC,CAAC;AAChE,SAAO"}

package/dist/example_selectors/length_based.js.map

@@ -1 +1 @@
-{"version":3,"file":"length_based.js","names":[],"sources":["../../src/example_selectors/length_based.ts"],"sourcesContent":["import { Example } from \"../prompts/base.js\";\nimport { BaseExampleSelector } from \"./base.js\";\nimport { PromptTemplate } from \"../prompts/prompt.js\";\n\n/**\n * Calculates the length of a text based on the number of words and lines.\n */\nfunction getLengthBased(text: string): number {\n  return text.split(/\\n| /).length;\n}\n\n/**\n * Interface for the input parameters of the LengthBasedExampleSelector\n * class.\n */\nexport interface LengthBasedExampleSelectorInput {\n  examplePrompt: PromptTemplate;\n  maxLength?: number;\n  getTextLength?: (text: string) => number;\n}\n\n/**\n * A specialized example selector that selects examples based on their\n * length, ensuring that the total length of the selected examples does\n * not exceed a specified maximum length.\n * @example\n * ```typescript\n * const exampleSelector = new LengthBasedExampleSelector(\n *   [\n *     { input: \"happy\", output: \"sad\" },\n *     { input: \"tall\", output: \"short\" },\n *     { input: \"energetic\", output: \"lethargic\" },\n *     { input: \"sunny\", output: \"gloomy\" },\n *     { input: \"windy\", output: \"calm\" },\n *   ],\n *   {\n *     examplePrompt: new PromptTemplate({\n *       inputVariables: [\"input\", \"output\"],\n *       template: \"Input: {input}\\nOutput: {output}\",\n *     }),\n *     maxLength: 25,\n *   },\n * );\n * const dynamicPrompt = new FewShotPromptTemplate({\n *   exampleSelector,\n *   examplePrompt: new PromptTemplate({\n *     inputVariables: [\"input\", \"output\"],\n *     template: \"Input: {input}\\nOutput: {output}\",\n *   }),\n *   prefix: \"Give the antonym of every input\",\n *   suffix: \"Input: {adjective}\\nOutput:\",\n *   inputVariables: [\"adjective\"],\n * });\n * console.log(dynamicPrompt.format({ adjective: \"big\" }));\n * console.log(\n *   dynamicPrompt.format({\n *     adjective:\n *       \"big and huge and massive and large and gigantic and tall and much much much much much bigger than everything else\",\n *   }),\n * );\n * ```\n */\nexport class LengthBasedExampleSelector extends BaseExampleSelector {\n  protected examples: Example[] = [];\n\n  examplePrompt!: PromptTemplate;\n\n  getTextLength: (text: string) => number = getLengthBased;\n\n  maxLength = 2048;\n\n  exampleTextLengths: number[] = [];\n\n  constructor(data: LengthBasedExampleSelectorInput) {\n    super(data);\n    this.examplePrompt = data.examplePrompt;\n    this.maxLength = data.maxLength ?? 2048;\n    this.getTextLength = data.getTextLength ?? getLengthBased;\n  }\n\n  /**\n   * Adds an example to the list of examples and calculates its length.\n   * @param example The example to be added.\n   * @returns Promise that resolves when the example has been added and its length calculated.\n   */\n  async addExample(example: Example): Promise<void> {\n    this.examples.push(example);\n    const stringExample = await this.examplePrompt.format(example);\n    this.exampleTextLengths.push(this.getTextLength(stringExample));\n  }\n\n  /**\n   * Calculates the lengths of the examples.\n   * @param v Array of lengths of the examples.\n   * @param values Instance of LengthBasedExampleSelector.\n   * @returns Promise that resolves with an array of lengths of the examples.\n   */\n  async calculateExampleTextLengths(\n    v: number[],\n    values: LengthBasedExampleSelector\n  ): Promise<number[]> {\n    if (v.length > 0) {\n      return v;\n    }\n\n    const { examples, examplePrompt } = values;\n    const stringExamples = await Promise.all(\n      examples.map((eg: Example) => examplePrompt.format(eg))\n    );\n    return stringExamples.map((eg: string) => this.getTextLength(eg));\n  }\n\n  /**\n   * Selects examples until the total length of the selected examples\n   * reaches the maxLength.\n   * @param inputVariables The input variables for the examples.\n   * @returns Promise that resolves with an array of selected examples.\n   */\n  async selectExamples(inputVariables: Example): Promise<Example[]> {\n    const inputs = Object.values(inputVariables).join(\" \");\n    let remainingLength = this.maxLength - this.getTextLength(inputs);\n    let i = 0;\n    const examples: Example[] = [];\n\n    while (remainingLength > 0 && i < this.examples.length) {\n      const newLength = remainingLength - this.exampleTextLengths[i];\n      if (newLength < 0) {\n        break;\n      } else {\n        examples.push(this.examples[i]);\n        remainingLength = newLength;\n      }\n      i += 1;\n    }\n\n    return examples;\n  }\n\n  /**\n   * Creates a new instance of LengthBasedExampleSelector and adds a list of\n   * examples to it.\n   * @param examples Array of examples to be added.\n   * @param args Input parameters for the LengthBasedExampleSelector.\n   * @returns Promise that resolves with a new instance of LengthBasedExampleSelector with the examples added.\n   */\n  static async fromExamples(\n    examples: Example[],\n    args: LengthBasedExampleSelectorInput\n  ) {\n    const selector = new LengthBasedExampleSelector(args);\n    await Promise.all(examples.map((eg) => selector.addExample(eg)));\n    return selector;\n  }\n}\n"],"mappings":";;;;;AAOA,SAAS,eAAe,MAAsB;AAC5C,QAAO,KAAK,MAAM,OAAO,CAAC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAsD5B,IAAa,6BAAb,MAAa,mCAAmC,oBAAoB;CAClE,WAAgC,EAAE;CAElC;CAEA,gBAA0C;CAE1C,YAAY;CAEZ,qBAA+B,EAAE;CAEjC,YAAY,MAAuC;AACjD,QAAM,KAAK;AACX,OAAK,gBAAgB,KAAK;AAC1B,OAAK,YAAY,KAAK,aAAa;AACnC,OAAK,gBAAgB,KAAK,iBAAiB;;;;;;;CAQ7C,MAAM,WAAW,SAAiC;AAChD,OAAK,SAAS,KAAK,QAAQ;EAC3B,MAAM,gBAAgB,MAAM,KAAK,cAAc,OAAO,QAAQ;AAC9D,OAAK,mBAAmB,KAAK,KAAK,cAAc,cAAc,CAAC;;;;;;;;CASjE,MAAM,4BACJ,GACA,QACmB;AACnB,MAAI,EAAE,SAAS,EACb,QAAO;EAGT,MAAM,EAAE,UAAU,kBAAkB;AAIpC,UAHuB,MAAM,QAAQ,IACnC,SAAS,KAAK,OAAgB,cAAc,OAAO,GAAG,CAAC,CACxD,EACqB,KAAK,OAAe,KAAK,cAAc,GAAG,CAAC;;;;;;;;CASnE,MAAM,eAAe,gBAA6C;EAChE,MAAM,SAAS,OAAO,OAAO,eAAe,CAAC,KAAK,IAAI;EACtD,IAAI,kBAAkB,KAAK,YAAY,KAAK,cAAc,OAAO;EACjE,IAAI,IAAI;EACR,MAAM,WAAsB,EAAE;AAE9B,SAAO,kBAAkB,KAAK,IAAI,KAAK,SAAS,QAAQ;GACtD,MAAM,YAAY,kBAAkB,KAAK,mBAAmB;AAC5D,OAAI,YAAY,EACd;QACK;AACL,aAAS,KAAK,KAAK,SAAS,GAAG;AAC/B,sBAAkB;;AAEpB,QAAK;;AAGP,SAAO;;;;;;;;;CAUT,aAAa,aACX,UACA,MACA;EACA,MAAM,WAAW,IAAI,2BAA2B,KAAK;AACrD,QAAM,QAAQ,IAAI,SAAS,KAAK,OAAO,SAAS,WAAW,GAAG,CAAC,CAAC;AAChE,SAAO"}
+{"version":3,"file":"length_based.js","names":[],"sources":["../../src/example_selectors/length_based.ts"],"sourcesContent":["import { Example } from \"../prompts/base.js\";\nimport { BaseExampleSelector } from \"./base.js\";\nimport { PromptTemplate } from \"../prompts/prompt.js\";\n\n/**\n * Calculates the length of a text based on the number of words and lines.\n */\nfunction getLengthBased(text: string): number {\n  return text.split(/\\n| /).length;\n}\n\n/**\n * Interface for the input parameters of the LengthBasedExampleSelector\n * class.\n */\nexport interface LengthBasedExampleSelectorInput {\n  examplePrompt: PromptTemplate;\n  maxLength?: number;\n  getTextLength?: (text: string) => number;\n}\n\n/**\n * A specialized example selector that selects examples based on their\n * length, ensuring that the total length of the selected examples does\n * not exceed a specified maximum length.\n * @example\n * ```typescript\n * const exampleSelector = new LengthBasedExampleSelector(\n *   [\n *     { input: \"happy\", output: \"sad\" },\n *     { input: \"tall\", output: \"short\" },\n *     { input: \"energetic\", output: \"lethargic\" },\n *     { input: \"sunny\", output: \"gloomy\" },\n *     { input: \"windy\", output: \"calm\" },\n *   ],\n *   {\n *     examplePrompt: new PromptTemplate({\n *       inputVariables: [\"input\", \"output\"],\n *       template: \"Input: {input}\\nOutput: {output}\",\n *     }),\n *     maxLength: 25,\n *   },\n * );\n * const dynamicPrompt = new FewShotPromptTemplate({\n *   exampleSelector,\n *   examplePrompt: new PromptTemplate({\n *     inputVariables: [\"input\", \"output\"],\n *     template: \"Input: {input}\\nOutput: {output}\",\n *   }),\n *   prefix: \"Give the antonym of every input\",\n *   suffix: \"Input: {adjective}\\nOutput:\",\n *   inputVariables: [\"adjective\"],\n * });\n * console.log(dynamicPrompt.format({ adjective: \"big\" }));\n * console.log(\n *   dynamicPrompt.format({\n *     adjective:\n *       \"big and huge and massive and large and gigantic and tall and much much much much much bigger than everything else\",\n *   }),\n * );\n * ```\n */\nexport class LengthBasedExampleSelector extends BaseExampleSelector {\n  protected examples: Example[] = [];\n\n  examplePrompt!: PromptTemplate;\n\n  getTextLength: (text: string) => number = getLengthBased;\n\n  maxLength = 2048;\n\n  exampleTextLengths: number[] = [];\n\n  constructor(data: LengthBasedExampleSelectorInput) {\n    super(data);\n    this.examplePrompt = data.examplePrompt;\n    this.maxLength = data.maxLength ?? 2048;\n    this.getTextLength = data.getTextLength ?? getLengthBased;\n  }\n\n  /**\n   * Adds an example to the list of examples and calculates its length.\n   * @param example The example to be added.\n   * @returns Promise that resolves when the example has been added and its length calculated.\n   */\n  async addExample(example: Example): Promise<void> {\n    this.examples.push(example);\n    const stringExample = await this.examplePrompt.format(example);\n    this.exampleTextLengths.push(this.getTextLength(stringExample));\n  }\n\n  /**\n   * Calculates the lengths of the examples.\n   * @param v Array of lengths of the examples.\n   * @param values Instance of LengthBasedExampleSelector.\n   * @returns Promise that resolves with an array of lengths of the examples.\n   */\n  async calculateExampleTextLengths(\n    v: number[],\n    values: LengthBasedExampleSelector\n  ): Promise<number[]> {\n    if (v.length > 0) {\n      return v;\n    }\n\n    const { examples, examplePrompt } = values;\n    const stringExamples = await Promise.all(\n      examples.map((eg: Example) => examplePrompt.format(eg))\n    );\n    return stringExamples.map((eg: string) => this.getTextLength(eg));\n  }\n\n  /**\n   * Selects examples until the total length of the selected examples\n   * reaches the maxLength.\n   * @param inputVariables The input variables for the examples.\n   * @returns Promise that resolves with an array of selected examples.\n   */\n  async selectExamples(inputVariables: Example): Promise<Example[]> {\n    const inputs = Object.values(inputVariables).join(\" \");\n    let remainingLength = this.maxLength - this.getTextLength(inputs);\n    let i = 0;\n    const examples: Example[] = [];\n\n    while (remainingLength > 0 && i < this.examples.length) {\n      const newLength = remainingLength - this.exampleTextLengths[i];\n      if (newLength < 0) {\n        break;\n      } else {\n        examples.push(this.examples[i]);\n        remainingLength = newLength;\n      }\n      i += 1;\n    }\n\n    return examples;\n  }\n\n  /**\n   * Creates a new instance of LengthBasedExampleSelector and adds a list of\n   * examples to it.\n   * @param examples Array of examples to be added.\n   * @param args Input parameters for the LengthBasedExampleSelector.\n   * @returns Promise that resolves with a new instance of LengthBasedExampleSelector with the examples added.\n   */\n  static async fromExamples(\n    examples: Example[],\n    args: LengthBasedExampleSelectorInput\n  ) {\n    const selector = new LengthBasedExampleSelector(args);\n    await Promise.all(examples.map((eg) => selector.addExample(eg)));\n    return selector;\n  }\n}\n"],"mappings":";;;;;AAOA,SAAS,eAAe,MAAsB;AAC5C,QAAO,KAAK,MAAM,OAAO,CAAC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAsD5B,IAAa,6BAAb,MAAa,mCAAmC,oBAAoB;CAClE,WAAgC,EAAE;CAElC;CAEA,gBAA0C;CAE1C,YAAY;CAEZ,qBAA+B,EAAE;CAEjC,YAAY,MAAuC;AACjD,QAAM,KAAK;AACX,OAAK,gBAAgB,KAAK;AAC1B,OAAK,YAAY,KAAK,aAAa;AACnC,OAAK,gBAAgB,KAAK,iBAAiB;;;;;;;CAQ7C,MAAM,WAAW,SAAiC;AAChD,OAAK,SAAS,KAAK,QAAQ;EAC3B,MAAM,gBAAgB,MAAM,KAAK,cAAc,OAAO,QAAQ;AAC9D,OAAK,mBAAmB,KAAK,KAAK,cAAc,cAAc,CAAC;;;;;;;;CASjE,MAAM,4BACJ,GACA,QACmB;AACnB,MAAI,EAAE,SAAS,EACb,QAAO;EAGT,MAAM,EAAE,UAAU,kBAAkB;AAIpC,UAAO,MAHsB,QAAQ,IACnC,SAAS,KAAK,OAAgB,cAAc,OAAO,GAAG,CAAC,CACxD,EACqB,KAAK,OAAe,KAAK,cAAc,GAAG,CAAC;;;;;;;;CASnE,MAAM,eAAe,gBAA6C;EAChE,MAAM,SAAS,OAAO,OAAO,eAAe,CAAC,KAAK,IAAI;EACtD,IAAI,kBAAkB,KAAK,YAAY,KAAK,cAAc,OAAO;EACjE,IAAI,IAAI;EACR,MAAM,WAAsB,EAAE;AAE9B,SAAO,kBAAkB,KAAK,IAAI,KAAK,SAAS,QAAQ;GACtD,MAAM,YAAY,kBAAkB,KAAK,mBAAmB;AAC5D,OAAI,YAAY,EACd;QACK;AACL,aAAS,KAAK,KAAK,SAAS,GAAG;AAC/B,sBAAkB;;AAEpB,QAAK;;AAGP,SAAO;;;;;;;;;CAUT,aAAa,aACX,UACA,MACA;EACA,MAAM,WAAW,IAAI,2BAA2B,KAAK;AACrD,QAAM,QAAQ,IAAI,SAAS,KAAK,OAAO,SAAS,WAAW,GAAG,CAAC,CAAC;AAChE,SAAO"}

package/dist/example_selectors/semantic_similarity.cjs.map

@@ -1 +1 @@
-{"version":3,"file":"semantic_similarity.cjs","names":["BaseExampleSelector","Document"],"sources":["../../src/example_selectors/semantic_similarity.ts"],"sourcesContent":["import type { Embeddings } from \"../embeddings.js\";\nimport type {\n  VectorStoreInterface,\n  VectorStoreRetrieverInterface,\n  VectorStore,\n} from \"../vectorstores.js\";\nimport type { Example } from \"../prompts/base.js\";\nimport { Document } from \"../documents/document.js\";\nimport { BaseExampleSelector } from \"./base.js\";\n\nfunction sortedValues<T>(values: Record<string, T>): T[] {\n  return Object.keys(values)\n    .sort()\n    .map((key) => values[key]);\n}\n\n/**\n * Interface for the input data of the SemanticSimilarityExampleSelector\n * class.\n */\nexport type SemanticSimilarityExampleSelectorInput<\n  V extends VectorStoreInterface = VectorStoreInterface,\n> =\n  | {\n      vectorStore: V;\n      k?: number;\n      filter?: V[\"FilterType\"];\n      exampleKeys?: string[];\n      inputKeys?: string[];\n      vectorStoreRetriever?: never;\n    }\n  | {\n      vectorStoreRetriever: VectorStoreRetrieverInterface<V>;\n      exampleKeys?: string[];\n      inputKeys?: string[];\n      vectorStore?: never;\n      k?: never;\n      filter?: never;\n    };\n\n/**\n * Class that selects examples based on semantic similarity. It extends\n * the BaseExampleSelector class.\n * @example\n * ```typescript\n * const exampleSelector = await SemanticSimilarityExampleSelector.fromExamples(\n *   [\n *     { input: \"happy\", output: \"sad\" },\n *     { input: \"tall\", output: \"short\" },\n *     { input: \"energetic\", output: \"lethargic\" },\n *     { input: \"sunny\", output: \"gloomy\" },\n *     { input: \"windy\", output: \"calm\" },\n *   ],\n *   new OpenAIEmbeddings(),\n *   HNSWLib,\n *   { k: 1 },\n * );\n * const dynamicPrompt = new FewShotPromptTemplate({\n *   exampleSelector,\n *   examplePrompt: PromptTemplate.fromTemplate(\n *     \"Input: {input}\\nOutput: {output}\",\n *   ),\n *   prefix: \"Give the antonym of every input\",\n *   suffix: \"Input: {adjective}\\nOutput:\",\n *   inputVariables: [\"adjective\"],\n * });\n * console.log(await dynamicPrompt.format({ adjective: \"rainy\" }));\n * ```\n */\nexport class SemanticSimilarityExampleSelector<\n  V extends VectorStoreInterface = VectorStoreInterface,\n> extends BaseExampleSelector {\n  vectorStoreRetriever: VectorStoreRetrieverInterface<V>;\n\n  exampleKeys?: string[];\n\n  inputKeys?: string[];\n\n  constructor(data: SemanticSimilarityExampleSelectorInput<V>) {\n    super(data);\n    this.exampleKeys = data.exampleKeys;\n    this.inputKeys = data.inputKeys;\n    if (data.vectorStore !== undefined) {\n      this.vectorStoreRetriever = data.vectorStore.asRetriever({\n        k: data.k ?? 4,\n        filter: data.filter,\n      });\n    } else if (data.vectorStoreRetriever) {\n      this.vectorStoreRetriever = data.vectorStoreRetriever;\n    } else {\n      throw new Error(\n        `You must specify one of \"vectorStore\" and \"vectorStoreRetriever\".`\n      );\n    }\n  }\n\n  /**\n   * Method that adds a new example to the vectorStore. The example is\n   * converted to a string and added to the vectorStore as a document.\n   * @param example The example to be added to the vectorStore.\n   * @returns Promise that resolves when the example has been added to the vectorStore.\n   */\n  async addExample(example: Example): Promise<void> {\n    const inputKeys = this.inputKeys ?? Object.keys(example);\n    const stringExample = sortedValues(\n      inputKeys.reduce(\n        (acc, key) => ({ ...acc, [key]: example[key] }),\n        {} as Example\n      )\n    ).join(\" \");\n\n    await this.vectorStoreRetriever.addDocuments([\n      new Document({\n        pageContent: stringExample,\n        metadata: example,\n      }),\n    ]);\n  }\n\n  /**\n   * Method that selects which examples to use based on semantic similarity.\n   * It performs a similarity search in the vectorStore using the input\n   * variables and returns the examples with the highest similarity.\n   * @param inputVariables The input variables used for the similarity search.\n   * @returns Promise that resolves with an array of the selected examples.\n   */\n  async selectExamples<T>(\n    inputVariables: Record<string, T>\n  ): Promise<Example[]> {\n    const inputKeys = this.inputKeys ?? Object.keys(inputVariables);\n    const query = sortedValues(\n      inputKeys.reduce(\n        (acc, key) => ({ ...acc, [key]: inputVariables[key] }),\n        {} as Record<string, T>\n      )\n    ).join(\" \");\n\n    const exampleDocs = await this.vectorStoreRetriever.invoke(query);\n\n    const examples = exampleDocs.map((doc) => doc.metadata);\n    if (this.exampleKeys) {\n      // If example keys are provided, filter examples to those keys.\n      return examples.map((example) =>\n        (this.exampleKeys as string[]).reduce(\n          (acc, key) => ({ ...acc, [key]: example[key] }),\n          {}\n        )\n      );\n    }\n    return examples;\n  }\n\n  /**\n   * Static method that creates a new instance of\n   * SemanticSimilarityExampleSelector. It takes a list of examples, an\n   * instance of Embeddings, a VectorStore class, and an options object as\n   * parameters. It converts the examples to strings, creates a VectorStore\n   * from the strings and the embeddings, and returns a new\n   * SemanticSimilarityExampleSelector with the created VectorStore and the\n   * options provided.\n   * @param examples The list of examples to be used.\n   * @param embeddings The instance of Embeddings to be used.\n   * @param vectorStoreCls The VectorStore class to be used.\n   * @param options The options object for the SemanticSimilarityExampleSelector.\n   * @returns Promise that resolves with a new instance of SemanticSimilarityExampleSelector.\n   */\n  static async fromExamples<C extends typeof VectorStore>(\n    examples: Record<string, string>[],\n    embeddings: Embeddings,\n    vectorStoreCls: C,\n    options: {\n      k?: number;\n      inputKeys?: string[];\n    } & Parameters<C[\"fromTexts\"]>[3] = {}\n  ): Promise<SemanticSimilarityExampleSelector> {\n    const inputKeys = options.inputKeys ?? null;\n    const stringExamples = examples.map((example) =>\n      sortedValues(\n        inputKeys\n          ? inputKeys.reduce(\n              (acc, key) => ({ ...acc, [key]: example[key] }),\n              {} as Record<string, string>\n            )\n          : example\n      ).join(\" \")\n    );\n\n    const vectorStore = await vectorStoreCls.fromTexts(\n      stringExamples,\n      examples, // metadatas\n      embeddings,\n      options\n    );\n\n    return new SemanticSimilarityExampleSelector({\n      vectorStore,\n      k: options.k ?? 4,\n      exampleKeys: options.exampleKeys,\n      inputKeys: options.inputKeys,\n    });\n  }\n}\n"],"mappings":";;;AAUA,SAAS,aAAgB,QAAgC;AACvD,QAAO,OAAO,KAAK,OAAO,CACvB,MAAM,CACN,KAAK,QAAQ,OAAO,KAAK;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAwD9B,IAAa,oCAAb,MAAa,0CAEHA,aAAAA,oBAAoB;CAC5B;CAEA;CAEA;CAEA,YAAY,MAAiD;AAC3D,QAAM,KAAK;AACX,OAAK,cAAc,KAAK;AACxB,OAAK,YAAY,KAAK;AACtB,MAAI,KAAK,gBAAgB,KAAA,EACvB,MAAK,uBAAuB,KAAK,YAAY,YAAY;GACvD,GAAG,KAAK,KAAK;GACb,QAAQ,KAAK;GACd,CAAC;WACO,KAAK,qBACd,MAAK,uBAAuB,KAAK;MAEjC,OAAM,IAAI,MACR,oEACD;;;;;;;;CAUL,MAAM,WAAW,SAAiC;EAEhD,MAAM,gBAAgB,cADJ,KAAK,aAAa,OAAO,KAAK,QAAQ,EAE5C,QACP,KAAK,SAAS;GAAE,GAAG;IAAM,MAAM,QAAQ;GAAM,GAC9C,EAAE,CACH,CACF,CAAC,KAAK,IAAI;AAEX,QAAM,KAAK,qBAAqB,aAAa,CAC3C,IAAIC,iBAAAA,SAAS;GACX,aAAa;GACb,UAAU;GACX,CAAC,CACH,CAAC;;;;;;;;;CAUJ,MAAM,eACJ,gBACoB;EAEpB,MAAM,QAAQ,cADI,KAAK,aAAa,OAAO,KAAK,eAAe,EAEnD,QACP,KAAK,SAAS;GAAE,GAAG;IAAM,MAAM,eAAe;GAAM,GACrD,EAAE,CACH,CACF,CAAC,KAAK,IAAI;EAIX,MAAM,YAFc,MAAM,KAAK,qBAAqB,OAAO,MAAM,EAEpC,KAAK,QAAQ,IAAI,SAAS;AACvD,MAAI,KAAK,YAEP,QAAO,SAAS,KAAK,YAClB,KAAK,YAAyB,QAC5B,KAAK,SAAS;GAAE,GAAG;IAAM,MAAM,QAAQ;GAAM,GAC9C,EAAE,CACH,CACF;AAEH,SAAO;;;;;;;;;;;;;;;;CAiBT,aAAa,aACX,UACA,YACA,gBACA,UAGoC,EAAE,EACM;EAC5C,MAAM,YAAY,QAAQ,aAAa;EACvC,MAAM,iBAAiB,SAAS,KAAK,YACnC,aACE,YACI,UAAU,QACP,KAAK,SAAS;GAAE,GAAG;IAAM,MAAM,QAAQ;GAAM,GAC9C,EAAE,CACH,GACD,QACL,CAAC,KAAK,IAAI,CACZ;AASD,SAAO,IAAI,kCAAkC;GAC3C,aARkB,MAAM,eAAe,UACvC,gBACA,UACA,YACA,QACD;GAIC,GAAG,QAAQ,KAAK;GAChB,aAAa,QAAQ;GACrB,WAAW,QAAQ;GACpB,CAAC"}
+{"version":3,"file":"semantic_similarity.cjs","names":["BaseExampleSelector","Document"],"sources":["../../src/example_selectors/semantic_similarity.ts"],"sourcesContent":["import type { Embeddings } from \"../embeddings.js\";\nimport type {\n  VectorStoreInterface,\n  VectorStoreRetrieverInterface,\n  VectorStore,\n} from \"../vectorstores.js\";\nimport type { Example } from \"../prompts/base.js\";\nimport { Document } from \"../documents/document.js\";\nimport { BaseExampleSelector } from \"./base.js\";\n\nfunction sortedValues<T>(values: Record<string, T>): T[] {\n  return Object.keys(values)\n    .sort()\n    .map((key) => values[key]);\n}\n\n/**\n * Interface for the input data of the SemanticSimilarityExampleSelector\n * class.\n */\nexport type SemanticSimilarityExampleSelectorInput<\n  V extends VectorStoreInterface = VectorStoreInterface,\n> =\n  | {\n      vectorStore: V;\n      k?: number;\n      filter?: V[\"FilterType\"];\n      exampleKeys?: string[];\n      inputKeys?: string[];\n      vectorStoreRetriever?: never;\n    }\n  | {\n      vectorStoreRetriever: VectorStoreRetrieverInterface<V>;\n      exampleKeys?: string[];\n      inputKeys?: string[];\n      vectorStore?: never;\n      k?: never;\n      filter?: never;\n    };\n\n/**\n * Class that selects examples based on semantic similarity. It extends\n * the BaseExampleSelector class.\n * @example\n * ```typescript\n * const exampleSelector = await SemanticSimilarityExampleSelector.fromExamples(\n *   [\n *     { input: \"happy\", output: \"sad\" },\n *     { input: \"tall\", output: \"short\" },\n *     { input: \"energetic\", output: \"lethargic\" },\n *     { input: \"sunny\", output: \"gloomy\" },\n *     { input: \"windy\", output: \"calm\" },\n *   ],\n *   new OpenAIEmbeddings(),\n *   HNSWLib,\n *   { k: 1 },\n * );\n * const dynamicPrompt = new FewShotPromptTemplate({\n *   exampleSelector,\n *   examplePrompt: PromptTemplate.fromTemplate(\n *     \"Input: {input}\\nOutput: {output}\",\n *   ),\n *   prefix: \"Give the antonym of every input\",\n *   suffix: \"Input: {adjective}\\nOutput:\",\n *   inputVariables: [\"adjective\"],\n * });\n * console.log(await dynamicPrompt.format({ adjective: \"rainy\" }));\n * ```\n */\nexport class SemanticSimilarityExampleSelector<\n  V extends VectorStoreInterface = VectorStoreInterface,\n> extends BaseExampleSelector {\n  vectorStoreRetriever: VectorStoreRetrieverInterface<V>;\n\n  exampleKeys?: string[];\n\n  inputKeys?: string[];\n\n  constructor(data: SemanticSimilarityExampleSelectorInput<V>) {\n    super(data);\n    this.exampleKeys = data.exampleKeys;\n    this.inputKeys = data.inputKeys;\n    if (data.vectorStore !== undefined) {\n      this.vectorStoreRetriever = data.vectorStore.asRetriever({\n        k: data.k ?? 4,\n        filter: data.filter,\n      });\n    } else if (data.vectorStoreRetriever) {\n      this.vectorStoreRetriever = data.vectorStoreRetriever;\n    } else {\n      throw new Error(\n        `You must specify one of \"vectorStore\" and \"vectorStoreRetriever\".`\n      );\n    }\n  }\n\n  /**\n   * Method that adds a new example to the vectorStore. The example is\n   * converted to a string and added to the vectorStore as a document.\n   * @param example The example to be added to the vectorStore.\n   * @returns Promise that resolves when the example has been added to the vectorStore.\n   */\n  async addExample(example: Example): Promise<void> {\n    const inputKeys = this.inputKeys ?? Object.keys(example);\n    const stringExample = sortedValues(\n      inputKeys.reduce(\n        (acc, key) => ({ ...acc, [key]: example[key] }),\n        {} as Example\n      )\n    ).join(\" \");\n\n    await this.vectorStoreRetriever.addDocuments([\n      new Document({\n        pageContent: stringExample,\n        metadata: example,\n      }),\n    ]);\n  }\n\n  /**\n   * Method that selects which examples to use based on semantic similarity.\n   * It performs a similarity search in the vectorStore using the input\n   * variables and returns the examples with the highest similarity.\n   * @param inputVariables The input variables used for the similarity search.\n   * @returns Promise that resolves with an array of the selected examples.\n   */\n  async selectExamples<T>(\n    inputVariables: Record<string, T>\n  ): Promise<Example[]> {\n    const inputKeys = this.inputKeys ?? Object.keys(inputVariables);\n    const query = sortedValues(\n      inputKeys.reduce(\n        (acc, key) => ({ ...acc, [key]: inputVariables[key] }),\n        {} as Record<string, T>\n      )\n    ).join(\" \");\n\n    const exampleDocs = await this.vectorStoreRetriever.invoke(query);\n\n    const examples = exampleDocs.map((doc) => doc.metadata);\n    if (this.exampleKeys) {\n      // If example keys are provided, filter examples to those keys.\n      return examples.map((example) =>\n        (this.exampleKeys as string[]).reduce(\n          (acc, key) => ({ ...acc, [key]: example[key] }),\n          {}\n        )\n      );\n    }\n    return examples;\n  }\n\n  /**\n   * Static method that creates a new instance of\n   * SemanticSimilarityExampleSelector. It takes a list of examples, an\n   * instance of Embeddings, a VectorStore class, and an options object as\n   * parameters. It converts the examples to strings, creates a VectorStore\n   * from the strings and the embeddings, and returns a new\n   * SemanticSimilarityExampleSelector with the created VectorStore and the\n   * options provided.\n   * @param examples The list of examples to be used.\n   * @param embeddings The instance of Embeddings to be used.\n   * @param vectorStoreCls The VectorStore class to be used.\n   * @param options The options object for the SemanticSimilarityExampleSelector.\n   * @returns Promise that resolves with a new instance of SemanticSimilarityExampleSelector.\n   */\n  static async fromExamples<C extends typeof VectorStore>(\n    examples: Record<string, string>[],\n    embeddings: Embeddings,\n    vectorStoreCls: C,\n    options: {\n      k?: number;\n      inputKeys?: string[];\n    } & Parameters<C[\"fromTexts\"]>[3] = {}\n  ): Promise<SemanticSimilarityExampleSelector> {\n    const inputKeys = options.inputKeys ?? null;\n    const stringExamples = examples.map((example) =>\n      sortedValues(\n        inputKeys\n          ? inputKeys.reduce(\n              (acc, key) => ({ ...acc, [key]: example[key] }),\n              {} as Record<string, string>\n            )\n          : example\n      ).join(\" \")\n    );\n\n    const vectorStore = await vectorStoreCls.fromTexts(\n      stringExamples,\n      examples, // metadatas\n      embeddings,\n      options\n    );\n\n    return new SemanticSimilarityExampleSelector({\n      vectorStore,\n      k: options.k ?? 4,\n      exampleKeys: options.exampleKeys,\n      inputKeys: options.inputKeys,\n    });\n  }\n}\n"],"mappings":";;;AAUA,SAAS,aAAgB,QAAgC;AACvD,QAAO,OAAO,KAAK,OAAO,CACvB,MAAM,CACN,KAAK,QAAQ,OAAO,KAAK;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAwD9B,IAAa,oCAAb,MAAa,0CAEHA,aAAAA,oBAAoB;CAC5B;CAEA;CAEA;CAEA,YAAY,MAAiD;AAC3D,QAAM,KAAK;AACX,OAAK,cAAc,KAAK;AACxB,OAAK,YAAY,KAAK;AACtB,MAAI,KAAK,gBAAgB,KAAA,EACvB,MAAK,uBAAuB,KAAK,YAAY,YAAY;GACvD,GAAG,KAAK,KAAK;GACb,QAAQ,KAAK;GACd,CAAC;WACO,KAAK,qBACd,MAAK,uBAAuB,KAAK;MAEjC,OAAM,IAAI,MACR,oEACD;;;;;;;;CAUL,MAAM,WAAW,SAAiC;EAEhD,MAAM,gBAAgB,cADJ,KAAK,aAAa,OAAO,KAAK,QAAQ,EAE5C,QACP,KAAK,SAAS;GAAE,GAAG;IAAM,MAAM,QAAQ;GAAM,GAC9C,EAAE,CACH,CACF,CAAC,KAAK,IAAI;AAEX,QAAM,KAAK,qBAAqB,aAAa,CAC3C,IAAIC,iBAAAA,SAAS;GACX,aAAa;GACb,UAAU;GACX,CAAC,CACH,CAAC;;;;;;;;;CAUJ,MAAM,eACJ,gBACoB;EAEpB,MAAM,QAAQ,cADI,KAAK,aAAa,OAAO,KAAK,eAAe,EAEnD,QACP,KAAK,SAAS;GAAE,GAAG;IAAM,MAAM,eAAe;GAAM,GACrD,EAAE,CACH,CACF,CAAC,KAAK,IAAI;EAIX,MAAM,YAAW,MAFS,KAAK,qBAAqB,OAAO,MAAM,EAEpC,KAAK,QAAQ,IAAI,SAAS;AACvD,MAAI,KAAK,YAEP,QAAO,SAAS,KAAK,YAClB,KAAK,YAAyB,QAC5B,KAAK,SAAS;GAAE,GAAG;IAAM,MAAM,QAAQ;GAAM,GAC9C,EAAE,CACH,CACF;AAEH,SAAO;;;;;;;;;;;;;;;;CAiBT,aAAa,aACX,UACA,YACA,gBACA,UAGoC,EAAE,EACM;EAC5C,MAAM,YAAY,QAAQ,aAAa;EACvC,MAAM,iBAAiB,SAAS,KAAK,YACnC,aACE,YACI,UAAU,QACP,KAAK,SAAS;GAAE,GAAG;IAAM,MAAM,QAAQ;GAAM,GAC9C,EAAE,CACH,GACD,QACL,CAAC,KAAK,IAAI,CACZ;AASD,SAAO,IAAI,kCAAkC;GAC3C,aAAA,MARwB,eAAe,UACvC,gBACA,UACA,YACA,QACD;GAIC,GAAG,QAAQ,KAAK;GAChB,aAAa,QAAQ;GACrB,WAAW,QAAQ;GACpB,CAAC"}

package/dist/example_selectors/semantic_similarity.js.map

@@ -1 +1 @@
-{"version":3,"file":"semantic_similarity.js","names":[],"sources":["../../src/example_selectors/semantic_similarity.ts"],"sourcesContent":["import type { Embeddings } from \"../embeddings.js\";\nimport type {\n  VectorStoreInterface,\n  VectorStoreRetrieverInterface,\n  VectorStore,\n} from \"../vectorstores.js\";\nimport type { Example } from \"../prompts/base.js\";\nimport { Document } from \"../documents/document.js\";\nimport { BaseExampleSelector } from \"./base.js\";\n\nfunction sortedValues<T>(values: Record<string, T>): T[] {\n  return Object.keys(values)\n    .sort()\n    .map((key) => values[key]);\n}\n\n/**\n * Interface for the input data of the SemanticSimilarityExampleSelector\n * class.\n */\nexport type SemanticSimilarityExampleSelectorInput<\n  V extends VectorStoreInterface = VectorStoreInterface,\n> =\n  | {\n      vectorStore: V;\n      k?: number;\n      filter?: V[\"FilterType\"];\n      exampleKeys?: string[];\n      inputKeys?: string[];\n      vectorStoreRetriever?: never;\n    }\n  | {\n      vectorStoreRetriever: VectorStoreRetrieverInterface<V>;\n      exampleKeys?: string[];\n      inputKeys?: string[];\n      vectorStore?: never;\n      k?: never;\n      filter?: never;\n    };\n\n/**\n * Class that selects examples based on semantic similarity. It extends\n * the BaseExampleSelector class.\n * @example\n * ```typescript\n * const exampleSelector = await SemanticSimilarityExampleSelector.fromExamples(\n *   [\n *     { input: \"happy\", output: \"sad\" },\n *     { input: \"tall\", output: \"short\" },\n *     { input: \"energetic\", output: \"lethargic\" },\n *     { input: \"sunny\", output: \"gloomy\" },\n *     { input: \"windy\", output: \"calm\" },\n *   ],\n *   new OpenAIEmbeddings(),\n *   HNSWLib,\n *   { k: 1 },\n * );\n * const dynamicPrompt = new FewShotPromptTemplate({\n *   exampleSelector,\n *   examplePrompt: PromptTemplate.fromTemplate(\n *     \"Input: {input}\\nOutput: {output}\",\n *   ),\n *   prefix: \"Give the antonym of every input\",\n *   suffix: \"Input: {adjective}\\nOutput:\",\n *   inputVariables: [\"adjective\"],\n * });\n * console.log(await dynamicPrompt.format({ adjective: \"rainy\" }));\n * ```\n */\nexport class SemanticSimilarityExampleSelector<\n  V extends VectorStoreInterface = VectorStoreInterface,\n> extends BaseExampleSelector {\n  vectorStoreRetriever: VectorStoreRetrieverInterface<V>;\n\n  exampleKeys?: string[];\n\n  inputKeys?: string[];\n\n  constructor(data: SemanticSimilarityExampleSelectorInput<V>) {\n    super(data);\n    this.exampleKeys = data.exampleKeys;\n    this.inputKeys = data.inputKeys;\n    if (data.vectorStore !== undefined) {\n      this.vectorStoreRetriever = data.vectorStore.asRetriever({\n        k: data.k ?? 4,\n        filter: data.filter,\n      });\n    } else if (data.vectorStoreRetriever) {\n      this.vectorStoreRetriever = data.vectorStoreRetriever;\n    } else {\n      throw new Error(\n        `You must specify one of \"vectorStore\" and \"vectorStoreRetriever\".`\n      );\n    }\n  }\n\n  /**\n   * Method that adds a new example to the vectorStore. The example is\n   * converted to a string and added to the vectorStore as a document.\n   * @param example The example to be added to the vectorStore.\n   * @returns Promise that resolves when the example has been added to the vectorStore.\n   */\n  async addExample(example: Example): Promise<void> {\n    const inputKeys = this.inputKeys ?? Object.keys(example);\n    const stringExample = sortedValues(\n      inputKeys.reduce(\n        (acc, key) => ({ ...acc, [key]: example[key] }),\n        {} as Example\n      )\n    ).join(\" \");\n\n    await this.vectorStoreRetriever.addDocuments([\n      new Document({\n        pageContent: stringExample,\n        metadata: example,\n      }),\n    ]);\n  }\n\n  /**\n   * Method that selects which examples to use based on semantic similarity.\n   * It performs a similarity search in the vectorStore using the input\n   * variables and returns the examples with the highest similarity.\n   * @param inputVariables The input variables used for the similarity search.\n   * @returns Promise that resolves with an array of the selected examples.\n   */\n  async selectExamples<T>(\n    inputVariables: Record<string, T>\n  ): Promise<Example[]> {\n    const inputKeys = this.inputKeys ?? Object.keys(inputVariables);\n    const query = sortedValues(\n      inputKeys.reduce(\n        (acc, key) => ({ ...acc, [key]: inputVariables[key] }),\n        {} as Record<string, T>\n      )\n    ).join(\" \");\n\n    const exampleDocs = await this.vectorStoreRetriever.invoke(query);\n\n    const examples = exampleDocs.map((doc) => doc.metadata);\n    if (this.exampleKeys) {\n      // If example keys are provided, filter examples to those keys.\n      return examples.map((example) =>\n        (this.exampleKeys as string[]).reduce(\n          (acc, key) => ({ ...acc, [key]: example[key] }),\n          {}\n        )\n      );\n    }\n    return examples;\n  }\n\n  /**\n   * Static method that creates a new instance of\n   * SemanticSimilarityExampleSelector. It takes a list of examples, an\n   * instance of Embeddings, a VectorStore class, and an options object as\n   * parameters. It converts the examples to strings, creates a VectorStore\n   * from the strings and the embeddings, and returns a new\n   * SemanticSimilarityExampleSelector with the created VectorStore and the\n   * options provided.\n   * @param examples The list of examples to be used.\n   * @param embeddings The instance of Embeddings to be used.\n   * @param vectorStoreCls The VectorStore class to be used.\n   * @param options The options object for the SemanticSimilarityExampleSelector.\n   * @returns Promise that resolves with a new instance of SemanticSimilarityExampleSelector.\n   */\n  static async fromExamples<C extends typeof VectorStore>(\n    examples: Record<string, string>[],\n    embeddings: Embeddings,\n    vectorStoreCls: C,\n    options: {\n      k?: number;\n      inputKeys?: string[];\n    } & Parameters<C[\"fromTexts\"]>[3] = {}\n  ): Promise<SemanticSimilarityExampleSelector> {\n    const inputKeys = options.inputKeys ?? null;\n    const stringExamples = examples.map((example) =>\n      sortedValues(\n        inputKeys\n          ? inputKeys.reduce(\n              (acc, key) => ({ ...acc, [key]: example[key] }),\n              {} as Record<string, string>\n            )\n          : example\n      ).join(\" \")\n    );\n\n    const vectorStore = await vectorStoreCls.fromTexts(\n      stringExamples,\n      examples, // metadatas\n      embeddings,\n      options\n    );\n\n    return new SemanticSimilarityExampleSelector({\n      vectorStore,\n      k: options.k ?? 4,\n      exampleKeys: options.exampleKeys,\n      inputKeys: options.inputKeys,\n    });\n  }\n}\n"],"mappings":";;;AAUA,SAAS,aAAgB,QAAgC;AACvD,QAAO,OAAO,KAAK,OAAO,CACvB,MAAM,CACN,KAAK,QAAQ,OAAO,KAAK;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAwD9B,IAAa,oCAAb,MAAa,0CAEH,oBAAoB;CAC5B;CAEA;CAEA;CAEA,YAAY,MAAiD;AAC3D,QAAM,KAAK;AACX,OAAK,cAAc,KAAK;AACxB,OAAK,YAAY,KAAK;AACtB,MAAI,KAAK,gBAAgB,KAAA,EACvB,MAAK,uBAAuB,KAAK,YAAY,YAAY;GACvD,GAAG,KAAK,KAAK;GACb,QAAQ,KAAK;GACd,CAAC;WACO,KAAK,qBACd,MAAK,uBAAuB,KAAK;MAEjC,OAAM,IAAI,MACR,oEACD;;;;;;;;CAUL,MAAM,WAAW,SAAiC;EAEhD,MAAM,gBAAgB,cADJ,KAAK,aAAa,OAAO,KAAK,QAAQ,EAE5C,QACP,KAAK,SAAS;GAAE,GAAG;IAAM,MAAM,QAAQ;GAAM,GAC9C,EAAE,CACH,CACF,CAAC,KAAK,IAAI;AAEX,QAAM,KAAK,qBAAqB,aAAa,CAC3C,IAAI,SAAS;GACX,aAAa;GACb,UAAU;GACX,CAAC,CACH,CAAC;;;;;;;;;CAUJ,MAAM,eACJ,gBACoB;EAEpB,MAAM,QAAQ,cADI,KAAK,aAAa,OAAO,KAAK,eAAe,EAEnD,QACP,KAAK,SAAS;GAAE,GAAG;IAAM,MAAM,eAAe;GAAM,GACrD,EAAE,CACH,CACF,CAAC,KAAK,IAAI;EAIX,MAAM,YAFc,MAAM,KAAK,qBAAqB,OAAO,MAAM,EAEpC,KAAK,QAAQ,IAAI,SAAS;AACvD,MAAI,KAAK,YAEP,QAAO,SAAS,KAAK,YAClB,KAAK,YAAyB,QAC5B,KAAK,SAAS;GAAE,GAAG;IAAM,MAAM,QAAQ;GAAM,GAC9C,EAAE,CACH,CACF;AAEH,SAAO;;;;;;;;;;;;;;;;CAiBT,aAAa,aACX,UACA,YACA,gBACA,UAGoC,EAAE,EACM;EAC5C,MAAM,YAAY,QAAQ,aAAa;EACvC,MAAM,iBAAiB,SAAS,KAAK,YACnC,aACE,YACI,UAAU,QACP,KAAK,SAAS;GAAE,GAAG;IAAM,MAAM,QAAQ;GAAM,GAC9C,EAAE,CACH,GACD,QACL,CAAC,KAAK,IAAI,CACZ;AASD,SAAO,IAAI,kCAAkC;GAC3C,aARkB,MAAM,eAAe,UACvC,gBACA,UACA,YACA,QACD;GAIC,GAAG,QAAQ,KAAK;GAChB,aAAa,QAAQ;GACrB,WAAW,QAAQ;GACpB,CAAC"}
+{"version":3,"file":"semantic_similarity.js","names":[],"sources":["../../src/example_selectors/semantic_similarity.ts"],"sourcesContent":["import type { Embeddings } from \"../embeddings.js\";\nimport type {\n  VectorStoreInterface,\n  VectorStoreRetrieverInterface,\n  VectorStore,\n} from \"../vectorstores.js\";\nimport type { Example } from \"../prompts/base.js\";\nimport { Document } from \"../documents/document.js\";\nimport { BaseExampleSelector } from \"./base.js\";\n\nfunction sortedValues<T>(values: Record<string, T>): T[] {\n  return Object.keys(values)\n    .sort()\n    .map((key) => values[key]);\n}\n\n/**\n * Interface for the input data of the SemanticSimilarityExampleSelector\n * class.\n */\nexport type SemanticSimilarityExampleSelectorInput<\n  V extends VectorStoreInterface = VectorStoreInterface,\n> =\n  | {\n      vectorStore: V;\n      k?: number;\n      filter?: V[\"FilterType\"];\n      exampleKeys?: string[];\n      inputKeys?: string[];\n      vectorStoreRetriever?: never;\n    }\n  | {\n      vectorStoreRetriever: VectorStoreRetrieverInterface<V>;\n      exampleKeys?: string[];\n      inputKeys?: string[];\n      vectorStore?: never;\n      k?: never;\n      filter?: never;\n    };\n\n/**\n * Class that selects examples based on semantic similarity. It extends\n * the BaseExampleSelector class.\n * @example\n * ```typescript\n * const exampleSelector = await SemanticSimilarityExampleSelector.fromExamples(\n *   [\n *     { input: \"happy\", output: \"sad\" },\n *     { input: \"tall\", output: \"short\" },\n *     { input: \"energetic\", output: \"lethargic\" },\n *     { input: \"sunny\", output: \"gloomy\" },\n *     { input: \"windy\", output: \"calm\" },\n *   ],\n *   new OpenAIEmbeddings(),\n *   HNSWLib,\n *   { k: 1 },\n * );\n * const dynamicPrompt = new FewShotPromptTemplate({\n *   exampleSelector,\n *   examplePrompt: PromptTemplate.fromTemplate(\n *     \"Input: {input}\\nOutput: {output}\",\n *   ),\n *   prefix: \"Give the antonym of every input\",\n *   suffix: \"Input: {adjective}\\nOutput:\",\n *   inputVariables: [\"adjective\"],\n * });\n * console.log(await dynamicPrompt.format({ adjective: \"rainy\" }));\n * ```\n */\nexport class SemanticSimilarityExampleSelector<\n  V extends VectorStoreInterface = VectorStoreInterface,\n> extends BaseExampleSelector {\n  vectorStoreRetriever: VectorStoreRetrieverInterface<V>;\n\n  exampleKeys?: string[];\n\n  inputKeys?: string[];\n\n  constructor(data: SemanticSimilarityExampleSelectorInput<V>) {\n    super(data);\n    this.exampleKeys = data.exampleKeys;\n    this.inputKeys = data.inputKeys;\n    if (data.vectorStore !== undefined) {\n      this.vectorStoreRetriever = data.vectorStore.asRetriever({\n        k: data.k ?? 4,\n        filter: data.filter,\n      });\n    } else if (data.vectorStoreRetriever) {\n      this.vectorStoreRetriever = data.vectorStoreRetriever;\n    } else {\n      throw new Error(\n        `You must specify one of \"vectorStore\" and \"vectorStoreRetriever\".`\n      );\n    }\n  }\n\n  /**\n   * Method that adds a new example to the vectorStore. The example is\n   * converted to a string and added to the vectorStore as a document.\n   * @param example The example to be added to the vectorStore.\n   * @returns Promise that resolves when the example has been added to the vectorStore.\n   */\n  async addExample(example: Example): Promise<void> {\n    const inputKeys = this.inputKeys ?? Object.keys(example);\n    const stringExample = sortedValues(\n      inputKeys.reduce(\n        (acc, key) => ({ ...acc, [key]: example[key] }),\n        {} as Example\n      )\n    ).join(\" \");\n\n    await this.vectorStoreRetriever.addDocuments([\n      new Document({\n        pageContent: stringExample,\n        metadata: example,\n      }),\n    ]);\n  }\n\n  /**\n   * Method that selects which examples to use based on semantic similarity.\n   * It performs a similarity search in the vectorStore using the input\n   * variables and returns the examples with the highest similarity.\n   * @param inputVariables The input variables used for the similarity search.\n   * @returns Promise that resolves with an array of the selected examples.\n   */\n  async selectExamples<T>(\n    inputVariables: Record<string, T>\n  ): Promise<Example[]> {\n    const inputKeys = this.inputKeys ?? Object.keys(inputVariables);\n    const query = sortedValues(\n      inputKeys.reduce(\n        (acc, key) => ({ ...acc, [key]: inputVariables[key] }),\n        {} as Record<string, T>\n      )\n    ).join(\" \");\n\n    const exampleDocs = await this.vectorStoreRetriever.invoke(query);\n\n    const examples = exampleDocs.map((doc) => doc.metadata);\n    if (this.exampleKeys) {\n      // If example keys are provided, filter examples to those keys.\n      return examples.map((example) =>\n        (this.exampleKeys as string[]).reduce(\n          (acc, key) => ({ ...acc, [key]: example[key] }),\n          {}\n        )\n      );\n    }\n    return examples;\n  }\n\n  /**\n   * Static method that creates a new instance of\n   * SemanticSimilarityExampleSelector. It takes a list of examples, an\n   * instance of Embeddings, a VectorStore class, and an options object as\n   * parameters. It converts the examples to strings, creates a VectorStore\n   * from the strings and the embeddings, and returns a new\n   * SemanticSimilarityExampleSelector with the created VectorStore and the\n   * options provided.\n   * @param examples The list of examples to be used.\n   * @param embeddings The instance of Embeddings to be used.\n   * @param vectorStoreCls The VectorStore class to be used.\n   * @param options The options object for the SemanticSimilarityExampleSelector.\n   * @returns Promise that resolves with a new instance of SemanticSimilarityExampleSelector.\n   */\n  static async fromExamples<C extends typeof VectorStore>(\n    examples: Record<string, string>[],\n    embeddings: Embeddings,\n    vectorStoreCls: C,\n    options: {\n      k?: number;\n      inputKeys?: string[];\n    } & Parameters<C[\"fromTexts\"]>[3] = {}\n  ): Promise<SemanticSimilarityExampleSelector> {\n    const inputKeys = options.inputKeys ?? null;\n    const stringExamples = examples.map((example) =>\n      sortedValues(\n        inputKeys\n          ? inputKeys.reduce(\n              (acc, key) => ({ ...acc, [key]: example[key] }),\n              {} as Record<string, string>\n            )\n          : example\n      ).join(\" \")\n    );\n\n    const vectorStore = await vectorStoreCls.fromTexts(\n      stringExamples,\n      examples, // metadatas\n      embeddings,\n      options\n    );\n\n    return new SemanticSimilarityExampleSelector({\n      vectorStore,\n      k: options.k ?? 4,\n      exampleKeys: options.exampleKeys,\n      inputKeys: options.inputKeys,\n    });\n  }\n}\n"],"mappings":";;;AAUA,SAAS,aAAgB,QAAgC;AACvD,QAAO,OAAO,KAAK,OAAO,CACvB,MAAM,CACN,KAAK,QAAQ,OAAO,KAAK;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAwD9B,IAAa,oCAAb,MAAa,0CAEH,oBAAoB;CAC5B;CAEA;CAEA;CAEA,YAAY,MAAiD;AAC3D,QAAM,KAAK;AACX,OAAK,cAAc,KAAK;AACxB,OAAK,YAAY,KAAK;AACtB,MAAI,KAAK,gBAAgB,KAAA,EACvB,MAAK,uBAAuB,KAAK,YAAY,YAAY;GACvD,GAAG,KAAK,KAAK;GACb,QAAQ,KAAK;GACd,CAAC;WACO,KAAK,qBACd,MAAK,uBAAuB,KAAK;MAEjC,OAAM,IAAI,MACR,oEACD;;;;;;;;CAUL,MAAM,WAAW,SAAiC;EAEhD,MAAM,gBAAgB,cADJ,KAAK,aAAa,OAAO,KAAK,QAAQ,EAE5C,QACP,KAAK,SAAS;GAAE,GAAG;IAAM,MAAM,QAAQ;GAAM,GAC9C,EAAE,CACH,CACF,CAAC,KAAK,IAAI;AAEX,QAAM,KAAK,qBAAqB,aAAa,CAC3C,IAAI,SAAS;GACX,aAAa;GACb,UAAU;GACX,CAAC,CACH,CAAC;;;;;;;;;CAUJ,MAAM,eACJ,gBACoB;EAEpB,MAAM,QAAQ,cADI,KAAK,aAAa,OAAO,KAAK,eAAe,EAEnD,QACP,KAAK,SAAS;GAAE,GAAG;IAAM,MAAM,eAAe;GAAM,GACrD,EAAE,CACH,CACF,CAAC,KAAK,IAAI;EAIX,MAAM,YAAW,MAFS,KAAK,qBAAqB,OAAO,MAAM,EAEpC,KAAK,QAAQ,IAAI,SAAS;AACvD,MAAI,KAAK,YAEP,QAAO,SAAS,KAAK,YAClB,KAAK,YAAyB,QAC5B,KAAK,SAAS;GAAE,GAAG;IAAM,MAAM,QAAQ;GAAM,GAC9C,EAAE,CACH,CACF;AAEH,SAAO;;;;;;;;;;;;;;;;CAiBT,aAAa,aACX,UACA,YACA,gBACA,UAGoC,EAAE,EACM;EAC5C,MAAM,YAAY,QAAQ,aAAa;EACvC,MAAM,iBAAiB,SAAS,KAAK,YACnC,aACE,YACI,UAAU,QACP,KAAK,SAAS;GAAE,GAAG;IAAM,MAAM,QAAQ;GAAM,GAC9C,EAAE,CACH,GACD,QACL,CAAC,KAAK,IAAI,CACZ;AASD,SAAO,IAAI,kCAAkC;GAC3C,aAAA,MARwB,eAAe,UACvC,gBACA,UACA,YACA,QACD;GAIC,GAAG,QAAQ,KAAK;GAChB,aAAa,QAAQ;GACrB,WAAW,QAAQ;GACpB,CAAC"}

package/dist/indexing/base.cjs.map

@@ -1 +1 @@
-{"version":3,"file":"base.cjs","names":["sha256","Document","uuidv5","UUIDV5_NAMESPACE"],"sources":["../../src/indexing/base.ts"],"sourcesContent":["import { v5 as uuidv5 } from \"../utils/uuid/index.js\";\nimport { VectorStore } from \"../vectorstores.js\";\nimport { RecordManagerInterface, UUIDV5_NAMESPACE } from \"./record_manager.js\";\nimport { sha256, type HashKeyEncoder } from \"../utils/hash.js\";\nimport { DocumentInterface, Document } from \"../documents/document.js\";\nimport { BaseDocumentLoader } from \"../document_loaders/base.js\";\n\ntype Metadata = Record<string, unknown>;\n\ntype IndexingResult = {\n  numAdded: number;\n  numDeleted: number;\n  numUpdated: number;\n  numSkipped: number;\n};\n\ntype StringOrDocFunc = string | ((doc: DocumentInterface) => string);\n\nexport interface HashedDocumentInterface extends DocumentInterface {\n  uid: string;\n  hash_?: string;\n  contentHash?: string;\n  metadataHash?: string;\n  pageContent: string;\n  metadata: Metadata;\n  calculateHashes(): void;\n  toDocument(): DocumentInterface;\n}\n\ninterface HashedDocumentArgs {\n  pageContent: string;\n  metadata: Metadata;\n  uid: string;\n}\n\n/**\n * HashedDocument is a Document with hashes calculated.\n * Hashes are calculated based on page content and metadata.\n * It is used for indexing.\n */\nexport class _HashedDocument implements HashedDocumentInterface {\n  uid: string;\n\n  hash_?: string;\n\n  contentHash?: string;\n\n  metadataHash?: string;\n\n  pageContent: string;\n\n  metadata: Metadata;\n\n  private keyEncoder: HashKeyEncoder = sha256;\n\n  constructor(fields: HashedDocumentArgs) {\n    this.uid = fields.uid;\n    this.pageContent = fields.pageContent;\n    this.metadata = fields.metadata;\n  }\n\n  makeDefaultKeyEncoder(keyEncoderFn: HashKeyEncoder): void {\n    this.keyEncoder = keyEncoderFn;\n  }\n\n  calculateHashes(): void {\n    const forbiddenKeys = [\"hash_\", \"content_hash\", \"metadata_hash\"];\n\n    for (const key of forbiddenKeys) {\n      if (key in this.metadata) {\n        throw new Error(\n          `Metadata cannot contain key ${key} as it is reserved for internal use. Restricted keys: [${forbiddenKeys.join(\n            \", \"\n          )}]`\n        );\n      }\n    }\n\n    const contentHash = this._hashStringToUUID(this.pageContent);\n\n    try {\n      const metadataHash = this._hashNestedDictToUUID(this.metadata);\n      this.contentHash = contentHash;\n      this.metadataHash = metadataHash;\n    } catch (e) {\n      throw new Error(\n        `Failed to hash metadata: ${e}. Please use a dict that can be serialized using json.`\n      );\n    }\n\n    this.hash_ = this._hashStringToUUID(this.contentHash + this.metadataHash);\n\n    if (!this.uid) {\n      this.uid = this.hash_;\n    }\n  }\n\n  toDocument(): DocumentInterface {\n    return new Document({\n      pageContent: this.pageContent,\n      metadata: this.metadata,\n    });\n  }\n\n  static fromDocument(\n    document: DocumentInterface,\n    uid?: string\n  ): _HashedDocument {\n    const doc = new this({\n      pageContent: document.pageContent,\n      metadata: document.metadata,\n      uid: uid || (document as DocumentInterface & { uid: string }).uid,\n    });\n    doc.calculateHashes();\n    return doc;\n  }\n\n  private _hashStringToUUID(inputString: string): string {\n    const hash_value = this.keyEncoder(inputString);\n    return uuidv5(hash_value, UUIDV5_NAMESPACE);\n  }\n\n  private _hashNestedDictToUUID(data: Record<string, unknown>): string {\n    const serialized_data = JSON.stringify(data, Object.keys(data).sort());\n    const hash_value = this.keyEncoder(serialized_data);\n    return uuidv5(hash_value, UUIDV5_NAMESPACE);\n  }\n}\n\nexport type CleanupMode = \"full\" | \"incremental\";\n\nexport type IndexOptions = {\n  /**\n   * The number of documents to index in one batch.\n   */\n  batchSize?: number;\n  /**\n   * The cleanup mode to use. Can be \"full\", \"incremental\" or undefined.\n   * - **Incremental**: Cleans up all documents that haven't been updated AND\n   *   that are associated with source ids that were seen\n   *   during indexing.\n   *   Clean up is done continuously during indexing helping\n   *   to minimize the probability of users seeing duplicated\n   *   content.\n   * - **Full**: Delete all documents that haven to been returned by the loader.\n   *   Clean up runs after all documents have been indexed.\n   *   This means that users may see duplicated content during indexing.\n   * - **undefined**: Do not delete any documents.\n   */\n  cleanup?: CleanupMode;\n  /**\n   * Optional key that helps identify the original source of the document.\n   * Must either be a string representing the key of the source in the metadata\n   * or a function that takes a document and returns a string representing the source.\n   * **Required when cleanup is incremental**.\n   */\n  sourceIdKey?: StringOrDocFunc;\n  /**\n   * Batch size to use when cleaning up documents.\n   */\n  cleanupBatchSize?: number;\n  /**\n   * Force update documents even if they are present in the\n   * record manager. Useful if you are re-indexing with updated embeddings.\n   */\n  forceUpdate?: boolean;\n};\n\nexport function _batch<T>(size: number, iterable: T[]): T[][] {\n  const batches: T[][] = [];\n  let currentBatch: T[] = [];\n\n  iterable.forEach((item) => {\n    currentBatch.push(item);\n\n    if (currentBatch.length >= size) {\n      batches.push(currentBatch);\n      currentBatch = [];\n    }\n  });\n\n  if (currentBatch.length > 0) {\n    batches.push(currentBatch);\n  }\n\n  return batches;\n}\n\nexport function _deduplicateInOrder(\n  hashedDocuments: HashedDocumentInterface[]\n): HashedDocumentInterface[] {\n  const seen = new Set<string>();\n  const deduplicated: HashedDocumentInterface[] = [];\n\n  for (const hashedDoc of hashedDocuments) {\n    if (!hashedDoc.hash_) {\n      throw new Error(\"Hashed document does not have a hash\");\n    }\n\n    if (!seen.has(hashedDoc.hash_)) {\n      seen.add(hashedDoc.hash_);\n      deduplicated.push(hashedDoc);\n    }\n  }\n  return deduplicated;\n}\n\nexport function _getSourceIdAssigner(\n  sourceIdKey: StringOrDocFunc | null\n): (doc: DocumentInterface) => string | null {\n  if (sourceIdKey === null) {\n    return (_doc: DocumentInterface) => null;\n  } else if (typeof sourceIdKey === \"string\") {\n    return (doc: DocumentInterface) => doc.metadata[sourceIdKey];\n  } else if (typeof sourceIdKey === \"function\") {\n    return sourceIdKey;\n  } else {\n    throw new Error(\n      `sourceIdKey should be null, a string or a function, got ${typeof sourceIdKey}`\n    );\n  }\n}\n\n// oxlint-disable-next-line @typescript-eslint/no-explicit-any\nexport const _isBaseDocumentLoader = (arg: any): arg is BaseDocumentLoader => {\n  if (\n    \"load\" in arg &&\n    typeof arg.load === \"function\" &&\n    \"loadAndSplit\" in arg &&\n    typeof arg.loadAndSplit === \"function\"\n  ) {\n    return true;\n  }\n  return false;\n};\n\ninterface IndexArgs {\n  docsSource: BaseDocumentLoader | DocumentInterface[];\n  recordManager: RecordManagerInterface;\n  vectorStore: VectorStore;\n  options?: IndexOptions;\n}\n\n/**\n * Index data from the doc source into the vector store.\n *\n * Indexing functionality uses a manager to keep track of which documents\n * are in the vector store.\n *\n * This allows us to keep track of which documents were updated, and which\n * documents were deleted, which documents should be skipped.\n *\n * For the time being, documents are indexed using their hashes, and users\n *  are not able to specify the uid of the document.\n *\n * @param {IndexArgs} args\n * @param {BaseDocumentLoader | DocumentInterface[]} args.docsSource The source of documents to index. Can be a DocumentLoader or a list of Documents.\n * @param {RecordManagerInterface} args.recordManager The record manager to use for keeping track of indexed documents.\n * @param {VectorStore} args.vectorStore The vector store to use for storing the documents.\n * @param {IndexOptions | undefined} args.options Options for indexing.\n * @returns {Promise<IndexingResult>}\n */\nexport async function index(args: IndexArgs): Promise<IndexingResult> {\n  const { docsSource, recordManager, vectorStore, options } = args;\n  const {\n    batchSize = 100,\n    cleanup,\n    sourceIdKey,\n    cleanupBatchSize = 1000,\n    forceUpdate = false,\n  } = options ?? {};\n\n  if (cleanup === \"incremental\" && !sourceIdKey) {\n    throw new Error(\n      \"sourceIdKey is required when cleanup mode is incremental. Please provide through 'options.sourceIdKey'.\"\n    );\n  }\n\n  const docs = _isBaseDocumentLoader(docsSource)\n    ? await docsSource.load()\n    : docsSource;\n\n  const sourceIdAssigner = _getSourceIdAssigner(sourceIdKey ?? null);\n\n  const indexStartDt = await recordManager.getTime();\n  let numAdded = 0;\n  let numDeleted = 0;\n  let numUpdated = 0;\n  let numSkipped = 0;\n\n  const batches = _batch<DocumentInterface>(batchSize ?? 100, docs);\n\n  for (const batch of batches) {\n    const hashedDocs = _deduplicateInOrder(\n      batch.map((doc) => _HashedDocument.fromDocument(doc))\n    );\n\n    const sourceIds = hashedDocs.map((doc) => sourceIdAssigner(doc));\n\n    if (cleanup === \"incremental\") {\n      hashedDocs.forEach((_hashedDoc, index) => {\n        const source = sourceIds[index];\n        if (source === null) {\n          throw new Error(\n            \"sourceIdKey must be provided when cleanup is incremental\"\n          );\n        }\n      });\n    }\n\n    const batchExists = await recordManager.exists(\n      hashedDocs.map((doc) => doc.uid)\n    );\n\n    const uids: string[] = [];\n    const docsToIndex: DocumentInterface[] = [];\n    const docsToUpdate: string[] = [];\n    const seenDocs = new Set<string>();\n    hashedDocs.forEach((hashedDoc, i) => {\n      const docExists = batchExists[i];\n      if (docExists) {\n        if (forceUpdate) {\n          seenDocs.add(hashedDoc.uid);\n        } else {\n          docsToUpdate.push(hashedDoc.uid);\n          return;\n        }\n      }\n      uids.push(hashedDoc.uid);\n      docsToIndex.push(hashedDoc.toDocument());\n    });\n\n    if (docsToUpdate.length > 0) {\n      await recordManager.update(docsToUpdate, { timeAtLeast: indexStartDt });\n      numSkipped += docsToUpdate.length;\n    }\n\n    if (docsToIndex.length > 0) {\n      await vectorStore.addDocuments(docsToIndex, { ids: uids });\n      numAdded += docsToIndex.length - seenDocs.size;\n      numUpdated += seenDocs.size;\n    }\n\n    await recordManager.update(\n      hashedDocs.map((doc) => doc.uid),\n      { timeAtLeast: indexStartDt, groupIds: sourceIds }\n    );\n\n    if (cleanup === \"incremental\") {\n      sourceIds.forEach((sourceId) => {\n        if (!sourceId) throw new Error(\"Source id cannot be null\");\n      });\n      const uidsToDelete = await recordManager.listKeys({\n        before: indexStartDt,\n        groupIds: sourceIds,\n      });\n\n      if (uidsToDelete.length > 0) {\n        await vectorStore.delete({ ids: uidsToDelete });\n        await recordManager.deleteKeys(uidsToDelete);\n        numDeleted += uidsToDelete.length;\n      }\n    }\n  }\n\n  if (cleanup === \"full\") {\n    let uidsToDelete = await recordManager.listKeys({\n      before: indexStartDt,\n      limit: cleanupBatchSize,\n    });\n    while (uidsToDelete.length > 0) {\n      await vectorStore.delete({ ids: uidsToDelete });\n      await recordManager.deleteKeys(uidsToDelete);\n      numDeleted += uidsToDelete.length;\n      uidsToDelete = await recordManager.listKeys({\n        before: indexStartDt,\n        limit: cleanupBatchSize,\n      });\n    }\n  }\n\n  return {\n    numAdded,\n    numDeleted,\n    numUpdated,\n    numSkipped,\n  };\n}\n"],"mappings":";;;;;;;;;;;AAwCA,IAAa,kBAAb,MAAgE;CAC9D;CAEA;CAEA;CAEA;CAEA;CAEA;CAEA,aAAqCA,aAAAA;CAErC,YAAY,QAA4B;AACtC,OAAK,MAAM,OAAO;AAClB,OAAK,cAAc,OAAO;AAC1B,OAAK,WAAW,OAAO;;CAGzB,sBAAsB,cAAoC;AACxD,OAAK,aAAa;;CAGpB,kBAAwB;EACtB,MAAM,gBAAgB;GAAC;GAAS;GAAgB;GAAgB;AAEhE,OAAK,MAAM,OAAO,cAChB,KAAI,OAAO,KAAK,SACd,OAAM,IAAI,MACR,+BAA+B,IAAI,yDAAyD,cAAc,KACxG,KACD,CAAC,GACH;EAIL,MAAM,cAAc,KAAK,kBAAkB,KAAK,YAAY;AAE5D,MAAI;GACF,MAAM,eAAe,KAAK,sBAAsB,KAAK,SAAS;AAC9D,QAAK,cAAc;AACnB,QAAK,eAAe;WACb,GAAG;AACV,SAAM,IAAI,MACR,4BAA4B,EAAE,wDAC/B;;AAGH,OAAK,QAAQ,KAAK,kBAAkB,KAAK,cAAc,KAAK,aAAa;AAEzE,MAAI,CAAC,KAAK,IACR,MAAK,MAAM,KAAK;;CAIpB,aAAgC;AAC9B,SAAO,IAAIC,iBAAAA,SAAS;GAClB,aAAa,KAAK;GAClB,UAAU,KAAK;GAChB,CAAC;;CAGJ,OAAO,aACL,UACA,KACiB;EACjB,MAAM,MAAM,IAAI,KAAK;GACnB,aAAa,SAAS;GACtB,UAAU,SAAS;GACnB,KAAK,OAAQ,SAAiD;GAC/D,CAAC;AACF,MAAI,iBAAiB;AACrB,SAAO;;CAGT,kBAA0B,aAA6B;AAErD,SAAOC,yBAAAA,GADY,KAAK,WAAW,YAAY,EACrBC,uBAAAA,iBAAiB;;CAG7C,sBAA8B,MAAuC;EACnE,MAAM,kBAAkB,KAAK,UAAU,MAAM,OAAO,KAAK,KAAK,CAAC,MAAM,CAAC;AAEtE,SAAOD,yBAAAA,GADY,KAAK,WAAW,gBAAgB,EACzBC,uBAAAA,iBAAiB;;;AA2C/C,SAAgB,OAAU,MAAc,UAAsB;CAC5D,MAAM,UAAiB,EAAE;CACzB,IAAI,eAAoB,EAAE;AAE1B,UAAS,SAAS,SAAS;AACzB,eAAa,KAAK,KAAK;AAEvB,MAAI,aAAa,UAAU,MAAM;AAC/B,WAAQ,KAAK,aAAa;AAC1B,kBAAe,EAAE;;GAEnB;AAEF,KAAI,aAAa,SAAS,EACxB,SAAQ,KAAK,aAAa;AAG5B,QAAO;;AAGT,SAAgB,oBACd,iBAC2B;CAC3B,MAAM,uBAAO,IAAI,KAAa;CAC9B,MAAM,eAA0C,EAAE;AAElD,MAAK,MAAM,aAAa,iBAAiB;AACvC,MAAI,CAAC,UAAU,MACb,OAAM,IAAI,MAAM,uCAAuC;AAGzD,MAAI,CAAC,KAAK,IAAI,UAAU,MAAM,EAAE;AAC9B,QAAK,IAAI,UAAU,MAAM;AACzB,gBAAa,KAAK,UAAU;;;AAGhC,QAAO;;AAGT,SAAgB,qBACd,aAC2C;AAC3C,KAAI,gBAAgB,KAClB,SAAQ,SAA4B;UAC3B,OAAO,gBAAgB,SAChC,SAAQ,QAA2B,IAAI,SAAS;UACvC,OAAO,gBAAgB,WAChC,QAAO;KAEP,OAAM,IAAI,MACR,2DAA2D,OAAO,cACnE;;AAKL,MAAa,yBAAyB,QAAwC;AAC5E,KACE,UAAU,OACV,OAAO,IAAI,SAAS,cACpB,kBAAkB,OAClB,OAAO,IAAI,iBAAiB,WAE5B,QAAO;AAET,QAAO;;;;;;;;;;;;;;;;;;;;;AA6BT,eAAsB,MAAM,MAA0C;CACpE,MAAM,EAAE,YAAY,eAAe,aAAa,YAAY;CAC5D,MAAM,EACJ,YAAY,KACZ,SACA,aACA,mBAAmB,KACnB,cAAc,UACZ,WAAW,EAAE;AAEjB,KAAI,YAAY,iBAAiB,CAAC,YAChC,OAAM,IAAI,MACR,0GACD;CAGH,MAAM,OAAO,sBAAsB,WAAW,GAC1C,MAAM,WAAW,MAAM,GACvB;CAEJ,MAAM,mBAAmB,qBAAqB,eAAe,KAAK;CAElE,MAAM,eAAe,MAAM,cAAc,SAAS;CAClD,IAAI,WAAW;CACf,IAAI,aAAa;CACjB,IAAI,aAAa;CACjB,IAAI,aAAa;CAEjB,MAAM,UAAU,OAA0B,aAAa,KAAK,KAAK;AAEjE,MAAK,MAAM,SAAS,SAAS;EAC3B,MAAM,aAAa,oBACjB,MAAM,KAAK,QAAQ,gBAAgB,aAAa,IAAI,CAAC,CACtD;EAED,MAAM,YAAY,WAAW,KAAK,QAAQ,iBAAiB,IAAI,CAAC;AAEhE,MAAI,YAAY,cACd,YAAW,SAAS,YAAY,UAAU;AAExC,OADe,UAAU,WACV,KACb,OAAM,IAAI,MACR,2DACD;IAEH;EAGJ,MAAM,cAAc,MAAM,cAAc,OACtC,WAAW,KAAK,QAAQ,IAAI,IAAI,CACjC;EAED,MAAM,OAAiB,EAAE;EACzB,MAAM,cAAmC,EAAE;EAC3C,MAAM,eAAyB,EAAE;EACjC,MAAM,2BAAW,IAAI,KAAa;AAClC,aAAW,SAAS,WAAW,MAAM;AAEnC,OADkB,YAAY,GAE5B,KAAI,YACF,UAAS,IAAI,UAAU,IAAI;QACtB;AACL,iBAAa,KAAK,UAAU,IAAI;AAChC;;AAGJ,QAAK,KAAK,UAAU,IAAI;AACxB,eAAY,KAAK,UAAU,YAAY,CAAC;IACxC;AAEF,MAAI,aAAa,SAAS,GAAG;AAC3B,SAAM,cAAc,OAAO,cAAc,EAAE,aAAa,cAAc,CAAC;AACvE,iBAAc,aAAa;;AAG7B,MAAI,YAAY,SAAS,GAAG;AAC1B,SAAM,YAAY,aAAa,aAAa,EAAE,KAAK,MAAM,CAAC;AAC1D,eAAY,YAAY,SAAS,SAAS;AAC1C,iBAAc,SAAS;;AAGzB,QAAM,cAAc,OAClB,WAAW,KAAK,QAAQ,IAAI,IAAI,EAChC;GAAE,aAAa;GAAc,UAAU;GAAW,CACnD;AAED,MAAI,YAAY,eAAe;AAC7B,aAAU,SAAS,aAAa;AAC9B,QAAI,CAAC,SAAU,OAAM,IAAI,MAAM,2BAA2B;KAC1D;GACF,MAAM,eAAe,MAAM,cAAc,SAAS;IAChD,QAAQ;IACR,UAAU;IACX,CAAC;AAEF,OAAI,aAAa,SAAS,GAAG;AAC3B,UAAM,YAAY,OAAO,EAAE,KAAK,cAAc,CAAC;AAC/C,UAAM,cAAc,WAAW,aAAa;AAC5C,kBAAc,aAAa;;;;AAKjC,KAAI,YAAY,QAAQ;EACtB,IAAI,eAAe,MAAM,cAAc,SAAS;GAC9C,QAAQ;GACR,OAAO;GACR,CAAC;AACF,SAAO,aAAa,SAAS,GAAG;AAC9B,SAAM,YAAY,OAAO,EAAE,KAAK,cAAc,CAAC;AAC/C,SAAM,cAAc,WAAW,aAAa;AAC5C,iBAAc,aAAa;AAC3B,kBAAe,MAAM,cAAc,SAAS;IAC1C,QAAQ;IACR,OAAO;IACR,CAAC;;;AAIN,QAAO;EACL;EACA;EACA;EACA;EACD"}
+{"version":3,"file":"base.cjs","names":["sha256","Document","uuidv5","UUIDV5_NAMESPACE"],"sources":["../../src/indexing/base.ts"],"sourcesContent":["import { v5 as uuidv5 } from \"../utils/uuid/index.js\";\nimport { VectorStore } from \"../vectorstores.js\";\nimport { RecordManagerInterface, UUIDV5_NAMESPACE } from \"./record_manager.js\";\nimport { sha256, type HashKeyEncoder } from \"../utils/hash.js\";\nimport { DocumentInterface, Document } from \"../documents/document.js\";\nimport { BaseDocumentLoader } from \"../document_loaders/base.js\";\n\ntype Metadata = Record<string, unknown>;\n\ntype IndexingResult = {\n  numAdded: number;\n  numDeleted: number;\n  numUpdated: number;\n  numSkipped: number;\n};\n\ntype StringOrDocFunc = string | ((doc: DocumentInterface) => string);\n\nexport interface HashedDocumentInterface extends DocumentInterface {\n  uid: string;\n  hash_?: string;\n  contentHash?: string;\n  metadataHash?: string;\n  pageContent: string;\n  metadata: Metadata;\n  calculateHashes(): void;\n  toDocument(): DocumentInterface;\n}\n\ninterface HashedDocumentArgs {\n  pageContent: string;\n  metadata: Metadata;\n  uid: string;\n}\n\n/**\n * HashedDocument is a Document with hashes calculated.\n * Hashes are calculated based on page content and metadata.\n * It is used for indexing.\n */\nexport class _HashedDocument implements HashedDocumentInterface {\n  uid: string;\n\n  hash_?: string;\n\n  contentHash?: string;\n\n  metadataHash?: string;\n\n  pageContent: string;\n\n  metadata: Metadata;\n\n  private keyEncoder: HashKeyEncoder = sha256;\n\n  constructor(fields: HashedDocumentArgs) {\n    this.uid = fields.uid;\n    this.pageContent = fields.pageContent;\n    this.metadata = fields.metadata;\n  }\n\n  makeDefaultKeyEncoder(keyEncoderFn: HashKeyEncoder): void {\n    this.keyEncoder = keyEncoderFn;\n  }\n\n  calculateHashes(): void {\n    const forbiddenKeys = [\"hash_\", \"content_hash\", \"metadata_hash\"];\n\n    for (const key of forbiddenKeys) {\n      if (key in this.metadata) {\n        throw new Error(\n          `Metadata cannot contain key ${key} as it is reserved for internal use. Restricted keys: [${forbiddenKeys.join(\n            \", \"\n          )}]`\n        );\n      }\n    }\n\n    const contentHash = this._hashStringToUUID(this.pageContent);\n\n    try {\n      const metadataHash = this._hashNestedDictToUUID(this.metadata);\n      this.contentHash = contentHash;\n      this.metadataHash = metadataHash;\n    } catch (e) {\n      throw new Error(\n        `Failed to hash metadata: ${e}. Please use a dict that can be serialized using json.`\n      );\n    }\n\n    this.hash_ = this._hashStringToUUID(this.contentHash + this.metadataHash);\n\n    if (!this.uid) {\n      this.uid = this.hash_;\n    }\n  }\n\n  toDocument(): DocumentInterface {\n    return new Document({\n      pageContent: this.pageContent,\n      metadata: this.metadata,\n    });\n  }\n\n  static fromDocument(\n    document: DocumentInterface,\n    uid?: string\n  ): _HashedDocument {\n    const doc = new this({\n      pageContent: document.pageContent,\n      metadata: document.metadata,\n      uid: uid || (document as DocumentInterface & { uid: string }).uid,\n    });\n    doc.calculateHashes();\n    return doc;\n  }\n\n  private _hashStringToUUID(inputString: string): string {\n    const hash_value = this.keyEncoder(inputString);\n    return uuidv5(hash_value, UUIDV5_NAMESPACE);\n  }\n\n  private _hashNestedDictToUUID(data: Record<string, unknown>): string {\n    const serialized_data = JSON.stringify(data, Object.keys(data).sort());\n    const hash_value = this.keyEncoder(serialized_data);\n    return uuidv5(hash_value, UUIDV5_NAMESPACE);\n  }\n}\n\nexport type CleanupMode = \"full\" | \"incremental\";\n\nexport type IndexOptions = {\n  /**\n   * The number of documents to index in one batch.\n   */\n  batchSize?: number;\n  /**\n   * The cleanup mode to use. Can be \"full\", \"incremental\" or undefined.\n   * - **Incremental**: Cleans up all documents that haven't been updated AND\n   *   that are associated with source ids that were seen\n   *   during indexing.\n   *   Clean up is done continuously during indexing helping\n   *   to minimize the probability of users seeing duplicated\n   *   content.\n   * - **Full**: Delete all documents that haven to been returned by the loader.\n   *   Clean up runs after all documents have been indexed.\n   *   This means that users may see duplicated content during indexing.\n   * - **undefined**: Do not delete any documents.\n   */\n  cleanup?: CleanupMode;\n  /**\n   * Optional key that helps identify the original source of the document.\n   * Must either be a string representing the key of the source in the metadata\n   * or a function that takes a document and returns a string representing the source.\n   * **Required when cleanup is incremental**.\n   */\n  sourceIdKey?: StringOrDocFunc;\n  /**\n   * Batch size to use when cleaning up documents.\n   */\n  cleanupBatchSize?: number;\n  /**\n   * Force update documents even if they are present in the\n   * record manager. Useful if you are re-indexing with updated embeddings.\n   */\n  forceUpdate?: boolean;\n};\n\nexport function _batch<T>(size: number, iterable: T[]): T[][] {\n  const batches: T[][] = [];\n  let currentBatch: T[] = [];\n\n  iterable.forEach((item) => {\n    currentBatch.push(item);\n\n    if (currentBatch.length >= size) {\n      batches.push(currentBatch);\n      currentBatch = [];\n    }\n  });\n\n  if (currentBatch.length > 0) {\n    batches.push(currentBatch);\n  }\n\n  return batches;\n}\n\nexport function _deduplicateInOrder(\n  hashedDocuments: HashedDocumentInterface[]\n): HashedDocumentInterface[] {\n  const seen = new Set<string>();\n  const deduplicated: HashedDocumentInterface[] = [];\n\n  for (const hashedDoc of hashedDocuments) {\n    if (!hashedDoc.hash_) {\n      throw new Error(\"Hashed document does not have a hash\");\n    }\n\n    if (!seen.has(hashedDoc.hash_)) {\n      seen.add(hashedDoc.hash_);\n      deduplicated.push(hashedDoc);\n    }\n  }\n  return deduplicated;\n}\n\nexport function _getSourceIdAssigner(\n  sourceIdKey: StringOrDocFunc | null\n): (doc: DocumentInterface) => string | null {\n  if (sourceIdKey === null) {\n    return (_doc: DocumentInterface) => null;\n  } else if (typeof sourceIdKey === \"string\") {\n    return (doc: DocumentInterface) => doc.metadata[sourceIdKey];\n  } else if (typeof sourceIdKey === \"function\") {\n    return sourceIdKey;\n  } else {\n    throw new Error(\n      `sourceIdKey should be null, a string or a function, got ${typeof sourceIdKey}`\n    );\n  }\n}\n\n// oxlint-disable-next-line @typescript-eslint/no-explicit-any\nexport const _isBaseDocumentLoader = (arg: any): arg is BaseDocumentLoader => {\n  if (\n    \"load\" in arg &&\n    typeof arg.load === \"function\" &&\n    \"loadAndSplit\" in arg &&\n    typeof arg.loadAndSplit === \"function\"\n  ) {\n    return true;\n  }\n  return false;\n};\n\ninterface IndexArgs {\n  docsSource: BaseDocumentLoader | DocumentInterface[];\n  recordManager: RecordManagerInterface;\n  vectorStore: VectorStore;\n  options?: IndexOptions;\n}\n\n/**\n * Index data from the doc source into the vector store.\n *\n * Indexing functionality uses a manager to keep track of which documents\n * are in the vector store.\n *\n * This allows us to keep track of which documents were updated, and which\n * documents were deleted, which documents should be skipped.\n *\n * For the time being, documents are indexed using their hashes, and users\n *  are not able to specify the uid of the document.\n *\n * @param {IndexArgs} args\n * @param {BaseDocumentLoader | DocumentInterface[]} args.docsSource The source of documents to index. Can be a DocumentLoader or a list of Documents.\n * @param {RecordManagerInterface} args.recordManager The record manager to use for keeping track of indexed documents.\n * @param {VectorStore} args.vectorStore The vector store to use for storing the documents.\n * @param {IndexOptions | undefined} args.options Options for indexing.\n * @returns {Promise<IndexingResult>}\n */\nexport async function index(args: IndexArgs): Promise<IndexingResult> {\n  const { docsSource, recordManager, vectorStore, options } = args;\n  const {\n    batchSize = 100,\n    cleanup,\n    sourceIdKey,\n    cleanupBatchSize = 1000,\n    forceUpdate = false,\n  } = options ?? {};\n\n  if (cleanup === \"incremental\" && !sourceIdKey) {\n    throw new Error(\n      \"sourceIdKey is required when cleanup mode is incremental. Please provide through 'options.sourceIdKey'.\"\n    );\n  }\n\n  const docs = _isBaseDocumentLoader(docsSource)\n    ? await docsSource.load()\n    : docsSource;\n\n  const sourceIdAssigner = _getSourceIdAssigner(sourceIdKey ?? null);\n\n  const indexStartDt = await recordManager.getTime();\n  let numAdded = 0;\n  let numDeleted = 0;\n  let numUpdated = 0;\n  let numSkipped = 0;\n\n  const batches = _batch<DocumentInterface>(batchSize ?? 100, docs);\n\n  for (const batch of batches) {\n    const hashedDocs = _deduplicateInOrder(\n      batch.map((doc) => _HashedDocument.fromDocument(doc))\n    );\n\n    const sourceIds = hashedDocs.map((doc) => sourceIdAssigner(doc));\n\n    if (cleanup === \"incremental\") {\n      hashedDocs.forEach((_hashedDoc, index) => {\n        const source = sourceIds[index];\n        if (source === null) {\n          throw new Error(\n            \"sourceIdKey must be provided when cleanup is incremental\"\n          );\n        }\n      });\n    }\n\n    const batchExists = await recordManager.exists(\n      hashedDocs.map((doc) => doc.uid)\n    );\n\n    const uids: string[] = [];\n    const docsToIndex: DocumentInterface[] = [];\n    const docsToUpdate: string[] = [];\n    const seenDocs = new Set<string>();\n    hashedDocs.forEach((hashedDoc, i) => {\n      const docExists = batchExists[i];\n      if (docExists) {\n        if (forceUpdate) {\n          seenDocs.add(hashedDoc.uid);\n        } else {\n          docsToUpdate.push(hashedDoc.uid);\n          return;\n        }\n      }\n      uids.push(hashedDoc.uid);\n      docsToIndex.push(hashedDoc.toDocument());\n    });\n\n    if (docsToUpdate.length > 0) {\n      await recordManager.update(docsToUpdate, { timeAtLeast: indexStartDt });\n      numSkipped += docsToUpdate.length;\n    }\n\n    if (docsToIndex.length > 0) {\n      await vectorStore.addDocuments(docsToIndex, { ids: uids });\n      numAdded += docsToIndex.length - seenDocs.size;\n      numUpdated += seenDocs.size;\n    }\n\n    await recordManager.update(\n      hashedDocs.map((doc) => doc.uid),\n      { timeAtLeast: indexStartDt, groupIds: sourceIds }\n    );\n\n    if (cleanup === \"incremental\") {\n      sourceIds.forEach((sourceId) => {\n        if (!sourceId) throw new Error(\"Source id cannot be null\");\n      });\n      const uidsToDelete = await recordManager.listKeys({\n        before: indexStartDt,\n        groupIds: sourceIds,\n      });\n\n      if (uidsToDelete.length > 0) {\n        await vectorStore.delete({ ids: uidsToDelete });\n        await recordManager.deleteKeys(uidsToDelete);\n        numDeleted += uidsToDelete.length;\n      }\n    }\n  }\n\n  if (cleanup === \"full\") {\n    let uidsToDelete = await recordManager.listKeys({\n      before: indexStartDt,\n      limit: cleanupBatchSize,\n    });\n    while (uidsToDelete.length > 0) {\n      await vectorStore.delete({ ids: uidsToDelete });\n      await recordManager.deleteKeys(uidsToDelete);\n      numDeleted += uidsToDelete.length;\n      uidsToDelete = await recordManager.listKeys({\n        before: indexStartDt,\n        limit: cleanupBatchSize,\n      });\n    }\n  }\n\n  return {\n    numAdded,\n    numDeleted,\n    numUpdated,\n    numSkipped,\n  };\n}\n"],"mappings":";;;;;;;;;;;AAwCA,IAAa,kBAAb,MAAgE;CAC9D;CAEA;CAEA;CAEA;CAEA;CAEA;CAEA,aAAqCA,aAAAA;CAErC,YAAY,QAA4B;AACtC,OAAK,MAAM,OAAO;AAClB,OAAK,cAAc,OAAO;AAC1B,OAAK,WAAW,OAAO;;CAGzB,sBAAsB,cAAoC;AACxD,OAAK,aAAa;;CAGpB,kBAAwB;EACtB,MAAM,gBAAgB;GAAC;GAAS;GAAgB;GAAgB;AAEhE,OAAK,MAAM,OAAO,cAChB,KAAI,OAAO,KAAK,SACd,OAAM,IAAI,MACR,+BAA+B,IAAI,yDAAyD,cAAc,KACxG,KACD,CAAC,GACH;EAIL,MAAM,cAAc,KAAK,kBAAkB,KAAK,YAAY;AAE5D,MAAI;GACF,MAAM,eAAe,KAAK,sBAAsB,KAAK,SAAS;AAC9D,QAAK,cAAc;AACnB,QAAK,eAAe;WACb,GAAG;AACV,SAAM,IAAI,MACR,4BAA4B,EAAE,wDAC/B;;AAGH,OAAK,QAAQ,KAAK,kBAAkB,KAAK,cAAc,KAAK,aAAa;AAEzE,MAAI,CAAC,KAAK,IACR,MAAK,MAAM,KAAK;;CAIpB,aAAgC;AAC9B,SAAO,IAAIC,iBAAAA,SAAS;GAClB,aAAa,KAAK;GAClB,UAAU,KAAK;GAChB,CAAC;;CAGJ,OAAO,aACL,UACA,KACiB;EACjB,MAAM,MAAM,IAAI,KAAK;GACnB,aAAa,SAAS;GACtB,UAAU,SAAS;GACnB,KAAK,OAAQ,SAAiD;GAC/D,CAAC;AACF,MAAI,iBAAiB;AACrB,SAAO;;CAGT,kBAA0B,aAA6B;AAErD,SAAOC,yBAAAA,GADY,KAAK,WAAW,YACX,EAAEC,uBAAAA,iBAAiB;;CAG7C,sBAA8B,MAAuC;EACnE,MAAM,kBAAkB,KAAK,UAAU,MAAM,OAAO,KAAK,KAAK,CAAC,MAAM,CAAC;AAEtE,SAAOD,yBAAAA,GADY,KAAK,WAAW,gBACX,EAAEC,uBAAAA,iBAAiB;;;AA2C/C,SAAgB,OAAU,MAAc,UAAsB;CAC5D,MAAM,UAAiB,EAAE;CACzB,IAAI,eAAoB,EAAE;AAE1B,UAAS,SAAS,SAAS;AACzB,eAAa,KAAK,KAAK;AAEvB,MAAI,aAAa,UAAU,MAAM;AAC/B,WAAQ,KAAK,aAAa;AAC1B,kBAAe,EAAE;;GAEnB;AAEF,KAAI,aAAa,SAAS,EACxB,SAAQ,KAAK,aAAa;AAG5B,QAAO;;AAGT,SAAgB,oBACd,iBAC2B;CAC3B,MAAM,uBAAO,IAAI,KAAa;CAC9B,MAAM,eAA0C,EAAE;AAElD,MAAK,MAAM,aAAa,iBAAiB;AACvC,MAAI,CAAC,UAAU,MACb,OAAM,IAAI,MAAM,uCAAuC;AAGzD,MAAI,CAAC,KAAK,IAAI,UAAU,MAAM,EAAE;AAC9B,QAAK,IAAI,UAAU,MAAM;AACzB,gBAAa,KAAK,UAAU;;;AAGhC,QAAO;;AAGT,SAAgB,qBACd,aAC2C;AAC3C,KAAI,gBAAgB,KAClB,SAAQ,SAA4B;UAC3B,OAAO,gBAAgB,SAChC,SAAQ,QAA2B,IAAI,SAAS;UACvC,OAAO,gBAAgB,WAChC,QAAO;KAEP,OAAM,IAAI,MACR,2DAA2D,OAAO,cACnE;;AAKL,MAAa,yBAAyB,QAAwC;AAC5E,KACE,UAAU,OACV,OAAO,IAAI,SAAS,cACpB,kBAAkB,OAClB,OAAO,IAAI,iBAAiB,WAE5B,QAAO;AAET,QAAO;;;;;;;;;;;;;;;;;;;;;AA6BT,eAAsB,MAAM,MAA0C;CACpE,MAAM,EAAE,YAAY,eAAe,aAAa,YAAY;CAC5D,MAAM,EACJ,YAAY,KACZ,SACA,aACA,mBAAmB,KACnB,cAAc,UACZ,WAAW,EAAE;AAEjB,KAAI,YAAY,iBAAiB,CAAC,YAChC,OAAM,IAAI,MACR,0GACD;CAGH,MAAM,OAAO,sBAAsB,WAAW,GAC1C,MAAM,WAAW,MAAM,GACvB;CAEJ,MAAM,mBAAmB,qBAAqB,eAAe,KAAK;CAElE,MAAM,eAAe,MAAM,cAAc,SAAS;CAClD,IAAI,WAAW;CACf,IAAI,aAAa;CACjB,IAAI,aAAa;CACjB,IAAI,aAAa;CAEjB,MAAM,UAAU,OAA0B,aAAa,KAAK,KAAK;AAEjE,MAAK,MAAM,SAAS,SAAS;EAC3B,MAAM,aAAa,oBACjB,MAAM,KAAK,QAAQ,gBAAgB,aAAa,IAAI,CAAC,CACtD;EAED,MAAM,YAAY,WAAW,KAAK,QAAQ,iBAAiB,IAAI,CAAC;AAEhE,MAAI,YAAY,cACd,YAAW,SAAS,YAAY,UAAU;AAExC,OADe,UAAU,WACV,KACb,OAAM,IAAI,MACR,2DACD;IAEH;EAGJ,MAAM,cAAc,MAAM,cAAc,OACtC,WAAW,KAAK,QAAQ,IAAI,IAAI,CACjC;EAED,MAAM,OAAiB,EAAE;EACzB,MAAM,cAAmC,EAAE;EAC3C,MAAM,eAAyB,EAAE;EACjC,MAAM,2BAAW,IAAI,KAAa;AAClC,aAAW,SAAS,WAAW,MAAM;AAEnC,OADkB,YAAY,GAE5B,KAAI,YACF,UAAS,IAAI,UAAU,IAAI;QACtB;AACL,iBAAa,KAAK,UAAU,IAAI;AAChC;;AAGJ,QAAK,KAAK,UAAU,IAAI;AACxB,eAAY,KAAK,UAAU,YAAY,CAAC;IACxC;AAEF,MAAI,aAAa,SAAS,GAAG;AAC3B,SAAM,cAAc,OAAO,cAAc,EAAE,aAAa,cAAc,CAAC;AACvE,iBAAc,aAAa;;AAG7B,MAAI,YAAY,SAAS,GAAG;AAC1B,SAAM,YAAY,aAAa,aAAa,EAAE,KAAK,MAAM,CAAC;AAC1D,eAAY,YAAY,SAAS,SAAS;AAC1C,iBAAc,SAAS;;AAGzB,QAAM,cAAc,OAClB,WAAW,KAAK,QAAQ,IAAI,IAAI,EAChC;GAAE,aAAa;GAAc,UAAU;GAAW,CACnD;AAED,MAAI,YAAY,eAAe;AAC7B,aAAU,SAAS,aAAa;AAC9B,QAAI,CAAC,SAAU,OAAM,IAAI,MAAM,2BAA2B;KAC1D;GACF,MAAM,eAAe,MAAM,cAAc,SAAS;IAChD,QAAQ;IACR,UAAU;IACX,CAAC;AAEF,OAAI,aAAa,SAAS,GAAG;AAC3B,UAAM,YAAY,OAAO,EAAE,KAAK,cAAc,CAAC;AAC/C,UAAM,cAAc,WAAW,aAAa;AAC5C,kBAAc,aAAa;;;;AAKjC,KAAI,YAAY,QAAQ;EACtB,IAAI,eAAe,MAAM,cAAc,SAAS;GAC9C,QAAQ;GACR,OAAO;GACR,CAAC;AACF,SAAO,aAAa,SAAS,GAAG;AAC9B,SAAM,YAAY,OAAO,EAAE,KAAK,cAAc,CAAC;AAC/C,SAAM,cAAc,WAAW,aAAa;AAC5C,iBAAc,aAAa;AAC3B,kBAAe,MAAM,cAAc,SAAS;IAC1C,QAAQ;IACR,OAAO;IACR,CAAC;;;AAIN,QAAO;EACL;EACA;EACA;EACA;EACD"}

package/dist/indexing/base.js.map

@@ -1 +1 @@
-{"version":3,"file":"base.js","names":["uuidv5"],"sources":["../../src/indexing/base.ts"],"sourcesContent":["import { v5 as uuidv5 } from \"../utils/uuid/index.js\";\nimport { VectorStore } from \"../vectorstores.js\";\nimport { RecordManagerInterface, UUIDV5_NAMESPACE } from \"./record_manager.js\";\nimport { sha256, type HashKeyEncoder } from \"../utils/hash.js\";\nimport { DocumentInterface, Document } from \"../documents/document.js\";\nimport { BaseDocumentLoader } from \"../document_loaders/base.js\";\n\ntype Metadata = Record<string, unknown>;\n\ntype IndexingResult = {\n  numAdded: number;\n  numDeleted: number;\n  numUpdated: number;\n  numSkipped: number;\n};\n\ntype StringOrDocFunc = string | ((doc: DocumentInterface) => string);\n\nexport interface HashedDocumentInterface extends DocumentInterface {\n  uid: string;\n  hash_?: string;\n  contentHash?: string;\n  metadataHash?: string;\n  pageContent: string;\n  metadata: Metadata;\n  calculateHashes(): void;\n  toDocument(): DocumentInterface;\n}\n\ninterface HashedDocumentArgs {\n  pageContent: string;\n  metadata: Metadata;\n  uid: string;\n}\n\n/**\n * HashedDocument is a Document with hashes calculated.\n * Hashes are calculated based on page content and metadata.\n * It is used for indexing.\n */\nexport class _HashedDocument implements HashedDocumentInterface {\n  uid: string;\n\n  hash_?: string;\n\n  contentHash?: string;\n\n  metadataHash?: string;\n\n  pageContent: string;\n\n  metadata: Metadata;\n\n  private keyEncoder: HashKeyEncoder = sha256;\n\n  constructor(fields: HashedDocumentArgs) {\n    this.uid = fields.uid;\n    this.pageContent = fields.pageContent;\n    this.metadata = fields.metadata;\n  }\n\n  makeDefaultKeyEncoder(keyEncoderFn: HashKeyEncoder): void {\n    this.keyEncoder = keyEncoderFn;\n  }\n\n  calculateHashes(): void {\n    const forbiddenKeys = [\"hash_\", \"content_hash\", \"metadata_hash\"];\n\n    for (const key of forbiddenKeys) {\n      if (key in this.metadata) {\n        throw new Error(\n          `Metadata cannot contain key ${key} as it is reserved for internal use. Restricted keys: [${forbiddenKeys.join(\n            \", \"\n          )}]`\n        );\n      }\n    }\n\n    const contentHash = this._hashStringToUUID(this.pageContent);\n\n    try {\n      const metadataHash = this._hashNestedDictToUUID(this.metadata);\n      this.contentHash = contentHash;\n      this.metadataHash = metadataHash;\n    } catch (e) {\n      throw new Error(\n        `Failed to hash metadata: ${e}. Please use a dict that can be serialized using json.`\n      );\n    }\n\n    this.hash_ = this._hashStringToUUID(this.contentHash + this.metadataHash);\n\n    if (!this.uid) {\n      this.uid = this.hash_;\n    }\n  }\n\n  toDocument(): DocumentInterface {\n    return new Document({\n      pageContent: this.pageContent,\n      metadata: this.metadata,\n    });\n  }\n\n  static fromDocument(\n    document: DocumentInterface,\n    uid?: string\n  ): _HashedDocument {\n    const doc = new this({\n      pageContent: document.pageContent,\n      metadata: document.metadata,\n      uid: uid || (document as DocumentInterface & { uid: string }).uid,\n    });\n    doc.calculateHashes();\n    return doc;\n  }\n\n  private _hashStringToUUID(inputString: string): string {\n    const hash_value = this.keyEncoder(inputString);\n    return uuidv5(hash_value, UUIDV5_NAMESPACE);\n  }\n\n  private _hashNestedDictToUUID(data: Record<string, unknown>): string {\n    const serialized_data = JSON.stringify(data, Object.keys(data).sort());\n    const hash_value = this.keyEncoder(serialized_data);\n    return uuidv5(hash_value, UUIDV5_NAMESPACE);\n  }\n}\n\nexport type CleanupMode = \"full\" | \"incremental\";\n\nexport type IndexOptions = {\n  /**\n   * The number of documents to index in one batch.\n   */\n  batchSize?: number;\n  /**\n   * The cleanup mode to use. Can be \"full\", \"incremental\" or undefined.\n   * - **Incremental**: Cleans up all documents that haven't been updated AND\n   *   that are associated with source ids that were seen\n   *   during indexing.\n   *   Clean up is done continuously during indexing helping\n   *   to minimize the probability of users seeing duplicated\n   *   content.\n   * - **Full**: Delete all documents that haven to been returned by the loader.\n   *   Clean up runs after all documents have been indexed.\n   *   This means that users may see duplicated content during indexing.\n   * - **undefined**: Do not delete any documents.\n   */\n  cleanup?: CleanupMode;\n  /**\n   * Optional key that helps identify the original source of the document.\n   * Must either be a string representing the key of the source in the metadata\n   * or a function that takes a document and returns a string representing the source.\n   * **Required when cleanup is incremental**.\n   */\n  sourceIdKey?: StringOrDocFunc;\n  /**\n   * Batch size to use when cleaning up documents.\n   */\n  cleanupBatchSize?: number;\n  /**\n   * Force update documents even if they are present in the\n   * record manager. Useful if you are re-indexing with updated embeddings.\n   */\n  forceUpdate?: boolean;\n};\n\nexport function _batch<T>(size: number, iterable: T[]): T[][] {\n  const batches: T[][] = [];\n  let currentBatch: T[] = [];\n\n  iterable.forEach((item) => {\n    currentBatch.push(item);\n\n    if (currentBatch.length >= size) {\n      batches.push(currentBatch);\n      currentBatch = [];\n    }\n  });\n\n  if (currentBatch.length > 0) {\n    batches.push(currentBatch);\n  }\n\n  return batches;\n}\n\nexport function _deduplicateInOrder(\n  hashedDocuments: HashedDocumentInterface[]\n): HashedDocumentInterface[] {\n  const seen = new Set<string>();\n  const deduplicated: HashedDocumentInterface[] = [];\n\n  for (const hashedDoc of hashedDocuments) {\n    if (!hashedDoc.hash_) {\n      throw new Error(\"Hashed document does not have a hash\");\n    }\n\n    if (!seen.has(hashedDoc.hash_)) {\n      seen.add(hashedDoc.hash_);\n      deduplicated.push(hashedDoc);\n    }\n  }\n  return deduplicated;\n}\n\nexport function _getSourceIdAssigner(\n  sourceIdKey: StringOrDocFunc | null\n): (doc: DocumentInterface) => string | null {\n  if (sourceIdKey === null) {\n    return (_doc: DocumentInterface) => null;\n  } else if (typeof sourceIdKey === \"string\") {\n    return (doc: DocumentInterface) => doc.metadata[sourceIdKey];\n  } else if (typeof sourceIdKey === \"function\") {\n    return sourceIdKey;\n  } else {\n    throw new Error(\n      `sourceIdKey should be null, a string or a function, got ${typeof sourceIdKey}`\n    );\n  }\n}\n\n// oxlint-disable-next-line @typescript-eslint/no-explicit-any\nexport const _isBaseDocumentLoader = (arg: any): arg is BaseDocumentLoader => {\n  if (\n    \"load\" in arg &&\n    typeof arg.load === \"function\" &&\n    \"loadAndSplit\" in arg &&\n    typeof arg.loadAndSplit === \"function\"\n  ) {\n    return true;\n  }\n  return false;\n};\n\ninterface IndexArgs {\n  docsSource: BaseDocumentLoader | DocumentInterface[];\n  recordManager: RecordManagerInterface;\n  vectorStore: VectorStore;\n  options?: IndexOptions;\n}\n\n/**\n * Index data from the doc source into the vector store.\n *\n * Indexing functionality uses a manager to keep track of which documents\n * are in the vector store.\n *\n * This allows us to keep track of which documents were updated, and which\n * documents were deleted, which documents should be skipped.\n *\n * For the time being, documents are indexed using their hashes, and users\n *  are not able to specify the uid of the document.\n *\n * @param {IndexArgs} args\n * @param {BaseDocumentLoader | DocumentInterface[]} args.docsSource The source of documents to index. Can be a DocumentLoader or a list of Documents.\n * @param {RecordManagerInterface} args.recordManager The record manager to use for keeping track of indexed documents.\n * @param {VectorStore} args.vectorStore The vector store to use for storing the documents.\n * @param {IndexOptions | undefined} args.options Options for indexing.\n * @returns {Promise<IndexingResult>}\n */\nexport async function index(args: IndexArgs): Promise<IndexingResult> {\n  const { docsSource, recordManager, vectorStore, options } = args;\n  const {\n    batchSize = 100,\n    cleanup,\n    sourceIdKey,\n    cleanupBatchSize = 1000,\n    forceUpdate = false,\n  } = options ?? {};\n\n  if (cleanup === \"incremental\" && !sourceIdKey) {\n    throw new Error(\n      \"sourceIdKey is required when cleanup mode is incremental. Please provide through 'options.sourceIdKey'.\"\n    );\n  }\n\n  const docs = _isBaseDocumentLoader(docsSource)\n    ? await docsSource.load()\n    : docsSource;\n\n  const sourceIdAssigner = _getSourceIdAssigner(sourceIdKey ?? null);\n\n  const indexStartDt = await recordManager.getTime();\n  let numAdded = 0;\n  let numDeleted = 0;\n  let numUpdated = 0;\n  let numSkipped = 0;\n\n  const batches = _batch<DocumentInterface>(batchSize ?? 100, docs);\n\n  for (const batch of batches) {\n    const hashedDocs = _deduplicateInOrder(\n      batch.map((doc) => _HashedDocument.fromDocument(doc))\n    );\n\n    const sourceIds = hashedDocs.map((doc) => sourceIdAssigner(doc));\n\n    if (cleanup === \"incremental\") {\n      hashedDocs.forEach((_hashedDoc, index) => {\n        const source = sourceIds[index];\n        if (source === null) {\n          throw new Error(\n            \"sourceIdKey must be provided when cleanup is incremental\"\n          );\n        }\n      });\n    }\n\n    const batchExists = await recordManager.exists(\n      hashedDocs.map((doc) => doc.uid)\n    );\n\n    const uids: string[] = [];\n    const docsToIndex: DocumentInterface[] = [];\n    const docsToUpdate: string[] = [];\n    const seenDocs = new Set<string>();\n    hashedDocs.forEach((hashedDoc, i) => {\n      const docExists = batchExists[i];\n      if (docExists) {\n        if (forceUpdate) {\n          seenDocs.add(hashedDoc.uid);\n        } else {\n          docsToUpdate.push(hashedDoc.uid);\n          return;\n        }\n      }\n      uids.push(hashedDoc.uid);\n      docsToIndex.push(hashedDoc.toDocument());\n    });\n\n    if (docsToUpdate.length > 0) {\n      await recordManager.update(docsToUpdate, { timeAtLeast: indexStartDt });\n      numSkipped += docsToUpdate.length;\n    }\n\n    if (docsToIndex.length > 0) {\n      await vectorStore.addDocuments(docsToIndex, { ids: uids });\n      numAdded += docsToIndex.length - seenDocs.size;\n      numUpdated += seenDocs.size;\n    }\n\n    await recordManager.update(\n      hashedDocs.map((doc) => doc.uid),\n      { timeAtLeast: indexStartDt, groupIds: sourceIds }\n    );\n\n    if (cleanup === \"incremental\") {\n      sourceIds.forEach((sourceId) => {\n        if (!sourceId) throw new Error(\"Source id cannot be null\");\n      });\n      const uidsToDelete = await recordManager.listKeys({\n        before: indexStartDt,\n        groupIds: sourceIds,\n      });\n\n      if (uidsToDelete.length > 0) {\n        await vectorStore.delete({ ids: uidsToDelete });\n        await recordManager.deleteKeys(uidsToDelete);\n        numDeleted += uidsToDelete.length;\n      }\n    }\n  }\n\n  if (cleanup === \"full\") {\n    let uidsToDelete = await recordManager.listKeys({\n      before: indexStartDt,\n      limit: cleanupBatchSize,\n    });\n    while (uidsToDelete.length > 0) {\n      await vectorStore.delete({ ids: uidsToDelete });\n      await recordManager.deleteKeys(uidsToDelete);\n      numDeleted += uidsToDelete.length;\n      uidsToDelete = await recordManager.listKeys({\n        before: indexStartDt,\n        limit: cleanupBatchSize,\n      });\n    }\n  }\n\n  return {\n    numAdded,\n    numDeleted,\n    numUpdated,\n    numSkipped,\n  };\n}\n"],"mappings":";;;;;;;;;;;AAwCA,IAAa,kBAAb,MAAgE;CAC9D;CAEA;CAEA;CAEA;CAEA;CAEA;CAEA,aAAqC;CAErC,YAAY,QAA4B;AACtC,OAAK,MAAM,OAAO;AAClB,OAAK,cAAc,OAAO;AAC1B,OAAK,WAAW,OAAO;;CAGzB,sBAAsB,cAAoC;AACxD,OAAK,aAAa;;CAGpB,kBAAwB;EACtB,MAAM,gBAAgB;GAAC;GAAS;GAAgB;GAAgB;AAEhE,OAAK,MAAM,OAAO,cAChB,KAAI,OAAO,KAAK,SACd,OAAM,IAAI,MACR,+BAA+B,IAAI,yDAAyD,cAAc,KACxG,KACD,CAAC,GACH;EAIL,MAAM,cAAc,KAAK,kBAAkB,KAAK,YAAY;AAE5D,MAAI;GACF,MAAM,eAAe,KAAK,sBAAsB,KAAK,SAAS;AAC9D,QAAK,cAAc;AACnB,QAAK,eAAe;WACb,GAAG;AACV,SAAM,IAAI,MACR,4BAA4B,EAAE,wDAC/B;;AAGH,OAAK,QAAQ,KAAK,kBAAkB,KAAK,cAAc,KAAK,aAAa;AAEzE,MAAI,CAAC,KAAK,IACR,MAAK,MAAM,KAAK;;CAIpB,aAAgC;AAC9B,SAAO,IAAI,SAAS;GAClB,aAAa,KAAK;GAClB,UAAU,KAAK;GAChB,CAAC;;CAGJ,OAAO,aACL,UACA,KACiB;EACjB,MAAM,MAAM,IAAI,KAAK;GACnB,aAAa,SAAS;GACtB,UAAU,SAAS;GACnB,KAAK,OAAQ,SAAiD;GAC/D,CAAC;AACF,MAAI,iBAAiB;AACrB,SAAO;;CAGT,kBAA0B,aAA6B;AAErD,SAAOA,GADY,KAAK,WAAW,YAAY,EACrB,iBAAiB;;CAG7C,sBAA8B,MAAuC;EACnE,MAAM,kBAAkB,KAAK,UAAU,MAAM,OAAO,KAAK,KAAK,CAAC,MAAM,CAAC;AAEtE,SAAOA,GADY,KAAK,WAAW,gBAAgB,EACzB,iBAAiB;;;AA2C/C,SAAgB,OAAU,MAAc,UAAsB;CAC5D,MAAM,UAAiB,EAAE;CACzB,IAAI,eAAoB,EAAE;AAE1B,UAAS,SAAS,SAAS;AACzB,eAAa,KAAK,KAAK;AAEvB,MAAI,aAAa,UAAU,MAAM;AAC/B,WAAQ,KAAK,aAAa;AAC1B,kBAAe,EAAE;;GAEnB;AAEF,KAAI,aAAa,SAAS,EACxB,SAAQ,KAAK,aAAa;AAG5B,QAAO;;AAGT,SAAgB,oBACd,iBAC2B;CAC3B,MAAM,uBAAO,IAAI,KAAa;CAC9B,MAAM,eAA0C,EAAE;AAElD,MAAK,MAAM,aAAa,iBAAiB;AACvC,MAAI,CAAC,UAAU,MACb,OAAM,IAAI,MAAM,uCAAuC;AAGzD,MAAI,CAAC,KAAK,IAAI,UAAU,MAAM,EAAE;AAC9B,QAAK,IAAI,UAAU,MAAM;AACzB,gBAAa,KAAK,UAAU;;;AAGhC,QAAO;;AAGT,SAAgB,qBACd,aAC2C;AAC3C,KAAI,gBAAgB,KAClB,SAAQ,SAA4B;UAC3B,OAAO,gBAAgB,SAChC,SAAQ,QAA2B,IAAI,SAAS;UACvC,OAAO,gBAAgB,WAChC,QAAO;KAEP,OAAM,IAAI,MACR,2DAA2D,OAAO,cACnE;;AAKL,MAAa,yBAAyB,QAAwC;AAC5E,KACE,UAAU,OACV,OAAO,IAAI,SAAS,cACpB,kBAAkB,OAClB,OAAO,IAAI,iBAAiB,WAE5B,QAAO;AAET,QAAO;;;;;;;;;;;;;;;;;;;;;AA6BT,eAAsB,MAAM,MAA0C;CACpE,MAAM,EAAE,YAAY,eAAe,aAAa,YAAY;CAC5D,MAAM,EACJ,YAAY,KACZ,SACA,aACA,mBAAmB,KACnB,cAAc,UACZ,WAAW,EAAE;AAEjB,KAAI,YAAY,iBAAiB,CAAC,YAChC,OAAM,IAAI,MACR,0GACD;CAGH,MAAM,OAAO,sBAAsB,WAAW,GAC1C,MAAM,WAAW,MAAM,GACvB;CAEJ,MAAM,mBAAmB,qBAAqB,eAAe,KAAK;CAElE,MAAM,eAAe,MAAM,cAAc,SAAS;CAClD,IAAI,WAAW;CACf,IAAI,aAAa;CACjB,IAAI,aAAa;CACjB,IAAI,aAAa;CAEjB,MAAM,UAAU,OAA0B,aAAa,KAAK,KAAK;AAEjE,MAAK,MAAM,SAAS,SAAS;EAC3B,MAAM,aAAa,oBACjB,MAAM,KAAK,QAAQ,gBAAgB,aAAa,IAAI,CAAC,CACtD;EAED,MAAM,YAAY,WAAW,KAAK,QAAQ,iBAAiB,IAAI,CAAC;AAEhE,MAAI,YAAY,cACd,YAAW,SAAS,YAAY,UAAU;AAExC,OADe,UAAU,WACV,KACb,OAAM,IAAI,MACR,2DACD;IAEH;EAGJ,MAAM,cAAc,MAAM,cAAc,OACtC,WAAW,KAAK,QAAQ,IAAI,IAAI,CACjC;EAED,MAAM,OAAiB,EAAE;EACzB,MAAM,cAAmC,EAAE;EAC3C,MAAM,eAAyB,EAAE;EACjC,MAAM,2BAAW,IAAI,KAAa;AAClC,aAAW,SAAS,WAAW,MAAM;AAEnC,OADkB,YAAY,GAE5B,KAAI,YACF,UAAS,IAAI,UAAU,IAAI;QACtB;AACL,iBAAa,KAAK,UAAU,IAAI;AAChC;;AAGJ,QAAK,KAAK,UAAU,IAAI;AACxB,eAAY,KAAK,UAAU,YAAY,CAAC;IACxC;AAEF,MAAI,aAAa,SAAS,GAAG;AAC3B,SAAM,cAAc,OAAO,cAAc,EAAE,aAAa,cAAc,CAAC;AACvE,iBAAc,aAAa;;AAG7B,MAAI,YAAY,SAAS,GAAG;AAC1B,SAAM,YAAY,aAAa,aAAa,EAAE,KAAK,MAAM,CAAC;AAC1D,eAAY,YAAY,SAAS,SAAS;AAC1C,iBAAc,SAAS;;AAGzB,QAAM,cAAc,OAClB,WAAW,KAAK,QAAQ,IAAI,IAAI,EAChC;GAAE,aAAa;GAAc,UAAU;GAAW,CACnD;AAED,MAAI,YAAY,eAAe;AAC7B,aAAU,SAAS,aAAa;AAC9B,QAAI,CAAC,SAAU,OAAM,IAAI,MAAM,2BAA2B;KAC1D;GACF,MAAM,eAAe,MAAM,cAAc,SAAS;IAChD,QAAQ;IACR,UAAU;IACX,CAAC;AAEF,OAAI,aAAa,SAAS,GAAG;AAC3B,UAAM,YAAY,OAAO,EAAE,KAAK,cAAc,CAAC;AAC/C,UAAM,cAAc,WAAW,aAAa;AAC5C,kBAAc,aAAa;;;;AAKjC,KAAI,YAAY,QAAQ;EACtB,IAAI,eAAe,MAAM,cAAc,SAAS;GAC9C,QAAQ;GACR,OAAO;GACR,CAAC;AACF,SAAO,aAAa,SAAS,GAAG;AAC9B,SAAM,YAAY,OAAO,EAAE,KAAK,cAAc,CAAC;AAC/C,SAAM,cAAc,WAAW,aAAa;AAC5C,iBAAc,aAAa;AAC3B,kBAAe,MAAM,cAAc,SAAS;IAC1C,QAAQ;IACR,OAAO;IACR,CAAC;;;AAIN,QAAO;EACL;EACA;EACA;EACA;EACD"}
+{"version":3,"file":"base.js","names":["uuidv5"],"sources":["../../src/indexing/base.ts"],"sourcesContent":["import { v5 as uuidv5 } from \"../utils/uuid/index.js\";\nimport { VectorStore } from \"../vectorstores.js\";\nimport { RecordManagerInterface, UUIDV5_NAMESPACE } from \"./record_manager.js\";\nimport { sha256, type HashKeyEncoder } from \"../utils/hash.js\";\nimport { DocumentInterface, Document } from \"../documents/document.js\";\nimport { BaseDocumentLoader } from \"../document_loaders/base.js\";\n\ntype Metadata = Record<string, unknown>;\n\ntype IndexingResult = {\n  numAdded: number;\n  numDeleted: number;\n  numUpdated: number;\n  numSkipped: number;\n};\n\ntype StringOrDocFunc = string | ((doc: DocumentInterface) => string);\n\nexport interface HashedDocumentInterface extends DocumentInterface {\n  uid: string;\n  hash_?: string;\n  contentHash?: string;\n  metadataHash?: string;\n  pageContent: string;\n  metadata: Metadata;\n  calculateHashes(): void;\n  toDocument(): DocumentInterface;\n}\n\ninterface HashedDocumentArgs {\n  pageContent: string;\n  metadata: Metadata;\n  uid: string;\n}\n\n/**\n * HashedDocument is a Document with hashes calculated.\n * Hashes are calculated based on page content and metadata.\n * It is used for indexing.\n */\nexport class _HashedDocument implements HashedDocumentInterface {\n  uid: string;\n\n  hash_?: string;\n\n  contentHash?: string;\n\n  metadataHash?: string;\n\n  pageContent: string;\n\n  metadata: Metadata;\n\n  private keyEncoder: HashKeyEncoder = sha256;\n\n  constructor(fields: HashedDocumentArgs) {\n    this.uid = fields.uid;\n    this.pageContent = fields.pageContent;\n    this.metadata = fields.metadata;\n  }\n\n  makeDefaultKeyEncoder(keyEncoderFn: HashKeyEncoder): void {\n    this.keyEncoder = keyEncoderFn;\n  }\n\n  calculateHashes(): void {\n    const forbiddenKeys = [\"hash_\", \"content_hash\", \"metadata_hash\"];\n\n    for (const key of forbiddenKeys) {\n      if (key in this.metadata) {\n        throw new Error(\n          `Metadata cannot contain key ${key} as it is reserved for internal use. Restricted keys: [${forbiddenKeys.join(\n            \", \"\n          )}]`\n        );\n      }\n    }\n\n    const contentHash = this._hashStringToUUID(this.pageContent);\n\n    try {\n      const metadataHash = this._hashNestedDictToUUID(this.metadata);\n      this.contentHash = contentHash;\n      this.metadataHash = metadataHash;\n    } catch (e) {\n      throw new Error(\n        `Failed to hash metadata: ${e}. Please use a dict that can be serialized using json.`\n      );\n    }\n\n    this.hash_ = this._hashStringToUUID(this.contentHash + this.metadataHash);\n\n    if (!this.uid) {\n      this.uid = this.hash_;\n    }\n  }\n\n  toDocument(): DocumentInterface {\n    return new Document({\n      pageContent: this.pageContent,\n      metadata: this.metadata,\n    });\n  }\n\n  static fromDocument(\n    document: DocumentInterface,\n    uid?: string\n  ): _HashedDocument {\n    const doc = new this({\n      pageContent: document.pageContent,\n      metadata: document.metadata,\n      uid: uid || (document as DocumentInterface & { uid: string }).uid,\n    });\n    doc.calculateHashes();\n    return doc;\n  }\n\n  private _hashStringToUUID(inputString: string): string {\n    const hash_value = this.keyEncoder(inputString);\n    return uuidv5(hash_value, UUIDV5_NAMESPACE);\n  }\n\n  private _hashNestedDictToUUID(data: Record<string, unknown>): string {\n    const serialized_data = JSON.stringify(data, Object.keys(data).sort());\n    const hash_value = this.keyEncoder(serialized_data);\n    return uuidv5(hash_value, UUIDV5_NAMESPACE);\n  }\n}\n\nexport type CleanupMode = \"full\" | \"incremental\";\n\nexport type IndexOptions = {\n  /**\n   * The number of documents to index in one batch.\n   */\n  batchSize?: number;\n  /**\n   * The cleanup mode to use. Can be \"full\", \"incremental\" or undefined.\n   * - **Incremental**: Cleans up all documents that haven't been updated AND\n   *   that are associated with source ids that were seen\n   *   during indexing.\n   *   Clean up is done continuously during indexing helping\n   *   to minimize the probability of users seeing duplicated\n   *   content.\n   * - **Full**: Delete all documents that haven to been returned by the loader.\n   *   Clean up runs after all documents have been indexed.\n   *   This means that users may see duplicated content during indexing.\n   * - **undefined**: Do not delete any documents.\n   */\n  cleanup?: CleanupMode;\n  /**\n   * Optional key that helps identify the original source of the document.\n   * Must either be a string representing the key of the source in the metadata\n   * or a function that takes a document and returns a string representing the source.\n   * **Required when cleanup is incremental**.\n   */\n  sourceIdKey?: StringOrDocFunc;\n  /**\n   * Batch size to use when cleaning up documents.\n   */\n  cleanupBatchSize?: number;\n  /**\n   * Force update documents even if they are present in the\n   * record manager. Useful if you are re-indexing with updated embeddings.\n   */\n  forceUpdate?: boolean;\n};\n\nexport function _batch<T>(size: number, iterable: T[]): T[][] {\n  const batches: T[][] = [];\n  let currentBatch: T[] = [];\n\n  iterable.forEach((item) => {\n    currentBatch.push(item);\n\n    if (currentBatch.length >= size) {\n      batches.push(currentBatch);\n      currentBatch = [];\n    }\n  });\n\n  if (currentBatch.length > 0) {\n    batches.push(currentBatch);\n  }\n\n  return batches;\n}\n\nexport function _deduplicateInOrder(\n  hashedDocuments: HashedDocumentInterface[]\n): HashedDocumentInterface[] {\n  const seen = new Set<string>();\n  const deduplicated: HashedDocumentInterface[] = [];\n\n  for (const hashedDoc of hashedDocuments) {\n    if (!hashedDoc.hash_) {\n      throw new Error(\"Hashed document does not have a hash\");\n    }\n\n    if (!seen.has(hashedDoc.hash_)) {\n      seen.add(hashedDoc.hash_);\n      deduplicated.push(hashedDoc);\n    }\n  }\n  return deduplicated;\n}\n\nexport function _getSourceIdAssigner(\n  sourceIdKey: StringOrDocFunc | null\n): (doc: DocumentInterface) => string | null {\n  if (sourceIdKey === null) {\n    return (_doc: DocumentInterface) => null;\n  } else if (typeof sourceIdKey === \"string\") {\n    return (doc: DocumentInterface) => doc.metadata[sourceIdKey];\n  } else if (typeof sourceIdKey === \"function\") {\n    return sourceIdKey;\n  } else {\n    throw new Error(\n      `sourceIdKey should be null, a string or a function, got ${typeof sourceIdKey}`\n    );\n  }\n}\n\n// oxlint-disable-next-line @typescript-eslint/no-explicit-any\nexport const _isBaseDocumentLoader = (arg: any): arg is BaseDocumentLoader => {\n  if (\n    \"load\" in arg &&\n    typeof arg.load === \"function\" &&\n    \"loadAndSplit\" in arg &&\n    typeof arg.loadAndSplit === \"function\"\n  ) {\n    return true;\n  }\n  return false;\n};\n\ninterface IndexArgs {\n  docsSource: BaseDocumentLoader | DocumentInterface[];\n  recordManager: RecordManagerInterface;\n  vectorStore: VectorStore;\n  options?: IndexOptions;\n}\n\n/**\n * Index data from the doc source into the vector store.\n *\n * Indexing functionality uses a manager to keep track of which documents\n * are in the vector store.\n *\n * This allows us to keep track of which documents were updated, and which\n * documents were deleted, which documents should be skipped.\n *\n * For the time being, documents are indexed using their hashes, and users\n *  are not able to specify the uid of the document.\n *\n * @param {IndexArgs} args\n * @param {BaseDocumentLoader | DocumentInterface[]} args.docsSource The source of documents to index. Can be a DocumentLoader or a list of Documents.\n * @param {RecordManagerInterface} args.recordManager The record manager to use for keeping track of indexed documents.\n * @param {VectorStore} args.vectorStore The vector store to use for storing the documents.\n * @param {IndexOptions | undefined} args.options Options for indexing.\n * @returns {Promise<IndexingResult>}\n */\nexport async function index(args: IndexArgs): Promise<IndexingResult> {\n  const { docsSource, recordManager, vectorStore, options } = args;\n  const {\n    batchSize = 100,\n    cleanup,\n    sourceIdKey,\n    cleanupBatchSize = 1000,\n    forceUpdate = false,\n  } = options ?? {};\n\n  if (cleanup === \"incremental\" && !sourceIdKey) {\n    throw new Error(\n      \"sourceIdKey is required when cleanup mode is incremental. Please provide through 'options.sourceIdKey'.\"\n    );\n  }\n\n  const docs = _isBaseDocumentLoader(docsSource)\n    ? await docsSource.load()\n    : docsSource;\n\n  const sourceIdAssigner = _getSourceIdAssigner(sourceIdKey ?? null);\n\n  const indexStartDt = await recordManager.getTime();\n  let numAdded = 0;\n  let numDeleted = 0;\n  let numUpdated = 0;\n  let numSkipped = 0;\n\n  const batches = _batch<DocumentInterface>(batchSize ?? 100, docs);\n\n  for (const batch of batches) {\n    const hashedDocs = _deduplicateInOrder(\n      batch.map((doc) => _HashedDocument.fromDocument(doc))\n    );\n\n    const sourceIds = hashedDocs.map((doc) => sourceIdAssigner(doc));\n\n    if (cleanup === \"incremental\") {\n      hashedDocs.forEach((_hashedDoc, index) => {\n        const source = sourceIds[index];\n        if (source === null) {\n          throw new Error(\n            \"sourceIdKey must be provided when cleanup is incremental\"\n          );\n        }\n      });\n    }\n\n    const batchExists = await recordManager.exists(\n      hashedDocs.map((doc) => doc.uid)\n    );\n\n    const uids: string[] = [];\n    const docsToIndex: DocumentInterface[] = [];\n    const docsToUpdate: string[] = [];\n    const seenDocs = new Set<string>();\n    hashedDocs.forEach((hashedDoc, i) => {\n      const docExists = batchExists[i];\n      if (docExists) {\n        if (forceUpdate) {\n          seenDocs.add(hashedDoc.uid);\n        } else {\n          docsToUpdate.push(hashedDoc.uid);\n          return;\n        }\n      }\n      uids.push(hashedDoc.uid);\n      docsToIndex.push(hashedDoc.toDocument());\n    });\n\n    if (docsToUpdate.length > 0) {\n      await recordManager.update(docsToUpdate, { timeAtLeast: indexStartDt });\n      numSkipped += docsToUpdate.length;\n    }\n\n    if (docsToIndex.length > 0) {\n      await vectorStore.addDocuments(docsToIndex, { ids: uids });\n      numAdded += docsToIndex.length - seenDocs.size;\n      numUpdated += seenDocs.size;\n    }\n\n    await recordManager.update(\n      hashedDocs.map((doc) => doc.uid),\n      { timeAtLeast: indexStartDt, groupIds: sourceIds }\n    );\n\n    if (cleanup === \"incremental\") {\n      sourceIds.forEach((sourceId) => {\n        if (!sourceId) throw new Error(\"Source id cannot be null\");\n      });\n      const uidsToDelete = await recordManager.listKeys({\n        before: indexStartDt,\n        groupIds: sourceIds,\n      });\n\n      if (uidsToDelete.length > 0) {\n        await vectorStore.delete({ ids: uidsToDelete });\n        await recordManager.deleteKeys(uidsToDelete);\n        numDeleted += uidsToDelete.length;\n      }\n    }\n  }\n\n  if (cleanup === \"full\") {\n    let uidsToDelete = await recordManager.listKeys({\n      before: indexStartDt,\n      limit: cleanupBatchSize,\n    });\n    while (uidsToDelete.length > 0) {\n      await vectorStore.delete({ ids: uidsToDelete });\n      await recordManager.deleteKeys(uidsToDelete);\n      numDeleted += uidsToDelete.length;\n      uidsToDelete = await recordManager.listKeys({\n        before: indexStartDt,\n        limit: cleanupBatchSize,\n      });\n    }\n  }\n\n  return {\n    numAdded,\n    numDeleted,\n    numUpdated,\n    numSkipped,\n  };\n}\n"],"mappings":";;;;;;;;;;;AAwCA,IAAa,kBAAb,MAAgE;CAC9D;CAEA;CAEA;CAEA;CAEA;CAEA;CAEA,aAAqC;CAErC,YAAY,QAA4B;AACtC,OAAK,MAAM,OAAO;AAClB,OAAK,cAAc,OAAO;AAC1B,OAAK,WAAW,OAAO;;CAGzB,sBAAsB,cAAoC;AACxD,OAAK,aAAa;;CAGpB,kBAAwB;EACtB,MAAM,gBAAgB;GAAC;GAAS;GAAgB;GAAgB;AAEhE,OAAK,MAAM,OAAO,cAChB,KAAI,OAAO,KAAK,SACd,OAAM,IAAI,MACR,+BAA+B,IAAI,yDAAyD,cAAc,KACxG,KACD,CAAC,GACH;EAIL,MAAM,cAAc,KAAK,kBAAkB,KAAK,YAAY;AAE5D,MAAI;GACF,MAAM,eAAe,KAAK,sBAAsB,KAAK,SAAS;AAC9D,QAAK,cAAc;AACnB,QAAK,eAAe;WACb,GAAG;AACV,SAAM,IAAI,MACR,4BAA4B,EAAE,wDAC/B;;AAGH,OAAK,QAAQ,KAAK,kBAAkB,KAAK,cAAc,KAAK,aAAa;AAEzE,MAAI,CAAC,KAAK,IACR,MAAK,MAAM,KAAK;;CAIpB,aAAgC;AAC9B,SAAO,IAAI,SAAS;GAClB,aAAa,KAAK;GAClB,UAAU,KAAK;GAChB,CAAC;;CAGJ,OAAO,aACL,UACA,KACiB;EACjB,MAAM,MAAM,IAAI,KAAK;GACnB,aAAa,SAAS;GACtB,UAAU,SAAS;GACnB,KAAK,OAAQ,SAAiD;GAC/D,CAAC;AACF,MAAI,iBAAiB;AACrB,SAAO;;CAGT,kBAA0B,aAA6B;AAErD,SAAOA,GADY,KAAK,WAAW,YACX,EAAE,iBAAiB;;CAG7C,sBAA8B,MAAuC;EACnE,MAAM,kBAAkB,KAAK,UAAU,MAAM,OAAO,KAAK,KAAK,CAAC,MAAM,CAAC;AAEtE,SAAOA,GADY,KAAK,WAAW,gBACX,EAAE,iBAAiB;;;AA2C/C,SAAgB,OAAU,MAAc,UAAsB;CAC5D,MAAM,UAAiB,EAAE;CACzB,IAAI,eAAoB,EAAE;AAE1B,UAAS,SAAS,SAAS;AACzB,eAAa,KAAK,KAAK;AAEvB,MAAI,aAAa,UAAU,MAAM;AAC/B,WAAQ,KAAK,aAAa;AAC1B,kBAAe,EAAE;;GAEnB;AAEF,KAAI,aAAa,SAAS,EACxB,SAAQ,KAAK,aAAa;AAG5B,QAAO;;AAGT,SAAgB,oBACd,iBAC2B;CAC3B,MAAM,uBAAO,IAAI,KAAa;CAC9B,MAAM,eAA0C,EAAE;AAElD,MAAK,MAAM,aAAa,iBAAiB;AACvC,MAAI,CAAC,UAAU,MACb,OAAM,IAAI,MAAM,uCAAuC;AAGzD,MAAI,CAAC,KAAK,IAAI,UAAU,MAAM,EAAE;AAC9B,QAAK,IAAI,UAAU,MAAM;AACzB,gBAAa,KAAK,UAAU;;;AAGhC,QAAO;;AAGT,SAAgB,qBACd,aAC2C;AAC3C,KAAI,gBAAgB,KAClB,SAAQ,SAA4B;UAC3B,OAAO,gBAAgB,SAChC,SAAQ,QAA2B,IAAI,SAAS;UACvC,OAAO,gBAAgB,WAChC,QAAO;KAEP,OAAM,IAAI,MACR,2DAA2D,OAAO,cACnE;;AAKL,MAAa,yBAAyB,QAAwC;AAC5E,KACE,UAAU,OACV,OAAO,IAAI,SAAS,cACpB,kBAAkB,OAClB,OAAO,IAAI,iBAAiB,WAE5B,QAAO;AAET,QAAO;;;;;;;;;;;;;;;;;;;;;AA6BT,eAAsB,MAAM,MAA0C;CACpE,MAAM,EAAE,YAAY,eAAe,aAAa,YAAY;CAC5D,MAAM,EACJ,YAAY,KACZ,SACA,aACA,mBAAmB,KACnB,cAAc,UACZ,WAAW,EAAE;AAEjB,KAAI,YAAY,iBAAiB,CAAC,YAChC,OAAM,IAAI,MACR,0GACD;CAGH,MAAM,OAAO,sBAAsB,WAAW,GAC1C,MAAM,WAAW,MAAM,GACvB;CAEJ,MAAM,mBAAmB,qBAAqB,eAAe,KAAK;CAElE,MAAM,eAAe,MAAM,cAAc,SAAS;CAClD,IAAI,WAAW;CACf,IAAI,aAAa;CACjB,IAAI,aAAa;CACjB,IAAI,aAAa;CAEjB,MAAM,UAAU,OAA0B,aAAa,KAAK,KAAK;AAEjE,MAAK,MAAM,SAAS,SAAS;EAC3B,MAAM,aAAa,oBACjB,MAAM,KAAK,QAAQ,gBAAgB,aAAa,IAAI,CAAC,CACtD;EAED,MAAM,YAAY,WAAW,KAAK,QAAQ,iBAAiB,IAAI,CAAC;AAEhE,MAAI,YAAY,cACd,YAAW,SAAS,YAAY,UAAU;AAExC,OADe,UAAU,WACV,KACb,OAAM,IAAI,MACR,2DACD;IAEH;EAGJ,MAAM,cAAc,MAAM,cAAc,OACtC,WAAW,KAAK,QAAQ,IAAI,IAAI,CACjC;EAED,MAAM,OAAiB,EAAE;EACzB,MAAM,cAAmC,EAAE;EAC3C,MAAM,eAAyB,EAAE;EACjC,MAAM,2BAAW,IAAI,KAAa;AAClC,aAAW,SAAS,WAAW,MAAM;AAEnC,OADkB,YAAY,GAE5B,KAAI,YACF,UAAS,IAAI,UAAU,IAAI;QACtB;AACL,iBAAa,KAAK,UAAU,IAAI;AAChC;;AAGJ,QAAK,KAAK,UAAU,IAAI;AACxB,eAAY,KAAK,UAAU,YAAY,CAAC;IACxC;AAEF,MAAI,aAAa,SAAS,GAAG;AAC3B,SAAM,cAAc,OAAO,cAAc,EAAE,aAAa,cAAc,CAAC;AACvE,iBAAc,aAAa;;AAG7B,MAAI,YAAY,SAAS,GAAG;AAC1B,SAAM,YAAY,aAAa,aAAa,EAAE,KAAK,MAAM,CAAC;AAC1D,eAAY,YAAY,SAAS,SAAS;AAC1C,iBAAc,SAAS;;AAGzB,QAAM,cAAc,OAClB,WAAW,KAAK,QAAQ,IAAI,IAAI,EAChC;GAAE,aAAa;GAAc,UAAU;GAAW,CACnD;AAED,MAAI,YAAY,eAAe;AAC7B,aAAU,SAAS,aAAa;AAC9B,QAAI,CAAC,SAAU,OAAM,IAAI,MAAM,2BAA2B;KAC1D;GACF,MAAM,eAAe,MAAM,cAAc,SAAS;IAChD,QAAQ;IACR,UAAU;IACX,CAAC;AAEF,OAAI,aAAa,SAAS,GAAG;AAC3B,UAAM,YAAY,OAAO,EAAE,KAAK,cAAc,CAAC;AAC/C,UAAM,cAAc,WAAW,aAAa;AAC5C,kBAAc,aAAa;;;;AAKjC,KAAI,YAAY,QAAQ;EACtB,IAAI,eAAe,MAAM,cAAc,SAAS;GAC9C,QAAQ;GACR,OAAO;GACR,CAAC;AACF,SAAO,aAAa,SAAS,GAAG;AAC9B,SAAM,YAAY,OAAO,EAAE,KAAK,cAAc,CAAC;AAC/C,SAAM,cAAc,WAAW,aAAa;AAC5C,iBAAc,aAAa;AAC3B,kBAAe,MAAM,cAAc,SAAS;IAC1C,QAAQ;IACR,OAAO;IACR,CAAC;;;AAIN,QAAO;EACL;EACA;EACA;EACA;EACD"}

package/dist/language_models/base.cjs

@@ -139,7 +139,7 @@ var BaseLangChain = class extends require_base.Runnable {
 		this.callbacks = params.callbacks;
 		this.tags = params.tags ?? [];
 		this.metadata = params.metadata ?? {};
-		this._addVersion("@langchain/core", "1.1.48");
+		this._addVersion("@langchain/core", "1.1.49-dev-1781044436312");
 	}
 	_addVersion(pkg, version) {
 		const existing = this.metadata?.versions;

package/dist/language_models/base.cjs.map

@@ -1 +1 @@
-{"version":3,"file":"base.cjs","names":["encodingForModel","Runnable","InMemoryCache","AsyncCaller","StringPromptValue","ChatPromptValue","coerceMessageLikeToMessage"],"sources":["../../src/language_models/base.ts"],"sourcesContent":["import type { Tiktoken, TiktokenModel } from \"js-tiktoken/lite\";\nimport type { ZodV3Like, ZodV4Like } from \"../utils/types/zod.js\";\n\nimport { type BaseCache, InMemoryCache } from \"../caches/index.js\";\nimport {\n  type BasePromptValueInterface,\n  StringPromptValue,\n  ChatPromptValue,\n} from \"../prompt_values.js\";\nimport {\n  type BaseMessage,\n  type BaseMessageLike,\n  type MessageContent,\n} from \"../messages/base.js\";\nimport { coerceMessageLikeToMessage } from \"../messages/utils.js\";\nimport { type LLMResult } from \"../outputs.js\";\nimport { CallbackManager, Callbacks } from \"../callbacks/manager.js\";\nimport { AsyncCaller, AsyncCallerParams } from \"../utils/async_caller.js\";\nimport { encodingForModel } from \"../utils/tiktoken.js\";\nimport { Runnable, type RunnableInterface } from \"../runnables/base.js\";\nimport { RunnableConfig } from \"../runnables/config.js\";\nimport { JSONSchema } from \"../utils/json_schema.js\";\nimport {\n  InferInteropZodOutput,\n  InteropZodObject,\n  InteropZodType,\n} from \"../utils/types/zod.js\";\nimport { ModelProfile } from \"./profile.js\";\nimport { type SerializableSchema } from \"../utils/standard_schema.js\";\n\n// https://www.npmjs.com/package/js-tiktoken\n\nexport const getModelNameForTiktoken = (modelName: string): TiktokenModel => {\n  if (modelName.startsWith(\"gpt-5\")) {\n    return \"gpt-5\" as TiktokenModel;\n  }\n\n  if (modelName.startsWith(\"gpt-3.5-turbo-16k\")) {\n    return \"gpt-3.5-turbo-16k\";\n  }\n\n  if (modelName.startsWith(\"gpt-3.5-turbo-\")) {\n    return \"gpt-3.5-turbo\";\n  }\n\n  if (modelName.startsWith(\"gpt-4-32k\")) {\n    return \"gpt-4-32k\";\n  }\n\n  if (modelName.startsWith(\"gpt-4-\")) {\n    return \"gpt-4\";\n  }\n\n  if (modelName.startsWith(\"gpt-4o\")) {\n    return \"gpt-4o\";\n  }\n\n  return modelName as TiktokenModel;\n};\n\nexport const getEmbeddingContextSize = (modelName?: string): number => {\n  switch (modelName) {\n    case \"text-embedding-ada-002\":\n      return 8191;\n    default:\n      return 2046;\n  }\n};\n\n/**\n * Get the context window size (max input tokens) for a given model.\n *\n * Context window sizes are sourced from official model documentation:\n * - OpenAI: https://platform.openai.com/docs/models\n * - Anthropic: https://docs.anthropic.com/claude/docs/models-overview\n * - Google: https://ai.google.dev/gemini/docs/models/gemini\n *\n * @param modelName - The name of the model\n * @returns The context window size in tokens\n */\nexport const getModelContextSize = (modelName: string): number => {\n  const normalizedName = getModelNameForTiktoken(modelName) as string;\n\n  switch (normalizedName) {\n    // GPT-5 series\n    case \"gpt-5\":\n    case \"gpt-5-turbo\":\n    case \"gpt-5-turbo-preview\":\n      return 400000;\n\n    // GPT-4o series\n    case \"gpt-4o\":\n    case \"gpt-4o-mini\":\n    case \"gpt-4o-2024-05-13\":\n    case \"gpt-4o-2024-08-06\":\n      return 128000;\n\n    // GPT-4 Turbo series\n    case \"gpt-4-turbo\":\n    case \"gpt-4-turbo-preview\":\n    case \"gpt-4-turbo-2024-04-09\":\n    case \"gpt-4-0125-preview\":\n    case \"gpt-4-1106-preview\":\n      return 128000;\n\n    // GPT-4 series\n    case \"gpt-4-32k\":\n    case \"gpt-4-32k-0314\":\n    case \"gpt-4-32k-0613\":\n      return 32768;\n    case \"gpt-4\":\n    case \"gpt-4-0314\":\n    case \"gpt-4-0613\":\n      return 8192;\n\n    // GPT-3.5 Turbo series\n    case \"gpt-3.5-turbo-16k\":\n    case \"gpt-3.5-turbo-16k-0613\":\n      return 16384;\n    case \"gpt-3.5-turbo\":\n    case \"gpt-3.5-turbo-0301\":\n    case \"gpt-3.5-turbo-0613\":\n    case \"gpt-3.5-turbo-1106\":\n    case \"gpt-3.5-turbo-0125\":\n      return 4096;\n\n    // Legacy GPT-3 models\n    case \"text-davinci-003\":\n    case \"text-davinci-002\":\n      return 4097;\n    case \"text-davinci-001\":\n      return 2049;\n    case \"text-curie-001\":\n    case \"text-babbage-001\":\n    case \"text-ada-001\":\n      return 2048;\n\n    // Code models\n    case \"code-davinci-002\":\n    case \"code-davinci-001\":\n      return 8000;\n    case \"code-cushman-001\":\n      return 2048;\n\n    // Claude models (Anthropic)\n    case \"claude-3-5-sonnet-20241022\":\n    case \"claude-3-5-sonnet-20240620\":\n    case \"claude-3-opus-20240229\":\n    case \"claude-3-sonnet-20240229\":\n    case \"claude-3-haiku-20240307\":\n    case \"claude-2.1\":\n      return 200000;\n    case \"claude-2.0\":\n    case \"claude-instant-1.2\":\n      return 100000;\n\n    // Gemini models (Google)\n    case \"gemini-1.5-pro\":\n    case \"gemini-1.5-pro-latest\":\n    case \"gemini-1.5-flash\":\n    case \"gemini-1.5-flash-latest\":\n      return 1000000; // 1M tokens\n    case \"gemini-pro\":\n    case \"gemini-pro-vision\":\n      return 32768;\n\n    default:\n      return 4097;\n  }\n};\n\n/**\n * Whether or not the input matches the OpenAI tool definition.\n * @param {unknown} tool The input to check.\n * @returns {boolean} Whether the input is an OpenAI tool definition.\n */\nexport function isOpenAITool(tool: unknown): tool is ToolDefinition {\n  if (typeof tool !== \"object\" || !tool) return false;\n  if (\n    \"type\" in tool &&\n    tool.type === \"function\" &&\n    \"function\" in tool &&\n    typeof tool.function === \"object\" &&\n    tool.function &&\n    \"name\" in tool.function &&\n    \"parameters\" in tool.function\n  ) {\n    return true;\n  }\n  return false;\n}\n\ninterface CalculateMaxTokenProps {\n  prompt: string;\n  modelName: TiktokenModel;\n}\n\nexport const calculateMaxTokens = async ({\n  prompt,\n  modelName,\n}: CalculateMaxTokenProps) => {\n  let numTokens;\n\n  try {\n    numTokens = (\n      await encodingForModel(getModelNameForTiktoken(modelName))\n    ).encode(prompt).length;\n  } catch {\n    console.warn(\n      \"Failed to calculate number of tokens, falling back to approximate count\"\n    );\n\n    // fallback to approximate calculation if tiktoken is not available\n    // each token is ~4 characters: https://help.openai.com/en/articles/4936856-what-are-tokens-and-how-to-count-them#\n    numTokens = Math.ceil(prompt.length / 4);\n  }\n\n  const maxTokens = getModelContextSize(modelName);\n  return maxTokens - numTokens;\n};\n\nconst getVerbosity = () => false;\n\nexport type SerializedLLM = {\n  _model: string;\n  _type: string;\n  // oxlint-disable-next-line @typescript-eslint/no-explicit-any\n} & Record<string, any>;\n\nexport interface BaseLangChainParams {\n  verbose?: boolean;\n  callbacks?: Callbacks;\n  tags?: string[];\n  metadata?: Record<string, unknown>;\n}\n\n/**\n * Base class for language models, chains, tools.\n */\nexport abstract class BaseLangChain<\n  RunInput,\n  RunOutput,\n  CallOptions extends RunnableConfig = RunnableConfig,\n>\n  extends Runnable<RunInput, RunOutput, CallOptions>\n  implements BaseLangChainParams\n{\n  /**\n   * Whether to print out response text.\n   */\n  verbose: boolean;\n\n  callbacks?: Callbacks;\n\n  tags?: string[];\n\n  metadata?: Record<string, unknown>;\n\n  get lc_attributes(): { [key: string]: undefined } | undefined {\n    return {\n      callbacks: undefined,\n      verbose: undefined,\n    };\n  }\n\n  constructor(params: BaseLangChainParams) {\n    super(params);\n    this.verbose = params.verbose ?? getVerbosity();\n    this.callbacks = params.callbacks;\n    this.tags = params.tags ?? [];\n    this.metadata = params.metadata ?? {};\n    this._addVersion(\"@langchain/core\", __PKG_VERSION__);\n  }\n\n  protected _addVersion(pkg: string, version: string) {\n    const existing = this.metadata?.versions;\n    this.metadata = {\n      ...this.metadata,\n      versions: {\n        ...(typeof existing === \"object\" && existing !== null ? existing : {}),\n        [pkg]: version,\n      },\n    };\n  }\n}\n\n/**\n * Base interface for language model parameters.\n * A subclass of {@link BaseLanguageModel} should have a constructor that\n * takes in a parameter that extends this interface.\n */\nexport interface BaseLanguageModelParams\n  extends AsyncCallerParams, BaseLangChainParams {\n  /**\n   * @deprecated Use `callbacks` instead\n   */\n  callbackManager?: CallbackManager;\n\n  cache?: BaseCache | boolean;\n}\n\nexport interface BaseLanguageModelTracingCallOptions {\n  /**\n   * Describes the format of structured outputs.\n   * This should be provided if an output is considered to be structured\n   */\n  ls_structured_output_format?: {\n    /**\n     * An object containing the method used for structured output (e.g., \"jsonMode\").\n     */\n    kwargs: { method: string };\n    /**\n     * The JSON schema describing the expected output structure.\n     */\n    schema?: JSONSchema;\n  };\n}\n\nexport interface BaseLanguageModelCallOptions\n  extends RunnableConfig, BaseLanguageModelTracingCallOptions {\n  /**\n   * Stop tokens to use for this call.\n   * If not provided, the default stop tokens for the model will be used.\n   */\n  stop?: string[];\n}\n\nexport interface FunctionDefinition {\n  /**\n   * The name of the function to be called. Must be a-z, A-Z, 0-9, or contain\n   * underscores and dashes, with a maximum length of 64.\n   */\n  name: string;\n\n  /**\n   * The parameters the functions accepts, described as a JSON Schema object. See the\n   * [guide](https://platform.openai.com/docs/guides/gpt/function-calling) for\n   * examples, and the\n   * [JSON Schema reference](https://json-schema.org/understanding-json-schema/) for\n   * documentation about the format.\n   *\n   * To describe a function that accepts no parameters, provide the value\n   * `{\"type\": \"object\", \"properties\": {}}`.\n   */\n  parameters: Record<string, unknown> | JSONSchema;\n\n  /**\n   * A description of what the function does, used by the model to choose when and\n   * how to call the function.\n   */\n  description?: string;\n}\n\nexport interface ToolDefinition {\n  type: \"function\";\n  function: FunctionDefinition;\n}\n\nexport type FunctionCallOption = {\n  name: string;\n};\n\nexport interface BaseFunctionCallOptions extends BaseLanguageModelCallOptions {\n  function_call?: FunctionCallOption;\n  functions?: FunctionDefinition[];\n}\n\nexport type BaseLanguageModelInput =\n  | BasePromptValueInterface\n  | string\n  | BaseMessageLike[];\n\nexport type StructuredOutputType = InferInteropZodOutput<InteropZodObject>;\n\nexport type StructuredOutputMethodOptions<IncludeRaw extends boolean = false> =\n  {\n    name?: string;\n    method?: \"functionCalling\" | \"jsonMode\" | \"jsonSchema\" | string;\n    includeRaw?: IncludeRaw;\n    /** Whether to use strict mode. Currently only supported by OpenAI models. */\n    strict?: boolean;\n  };\n\n/** @deprecated Use StructuredOutputMethodOptions instead */\nexport type StructuredOutputMethodParams<\n  RunOutput,\n  IncludeRaw extends boolean = false,\n> = {\n  /** @deprecated Pass schema in as the first argument */\n  // oxlint-disable-next-line @typescript-eslint/no-explicit-any\n  schema: InteropZodType<RunOutput> | Record<string, any>;\n  name?: string;\n  method?: \"functionCalling\" | \"jsonMode\";\n  includeRaw?: IncludeRaw;\n};\n\nexport interface BaseLanguageModelInterface<\n  // oxlint-disable-next-line @typescript-eslint/no-explicit-any\n  RunOutput = any,\n  CallOptions extends BaseLanguageModelCallOptions =\n    BaseLanguageModelCallOptions,\n> extends RunnableInterface<BaseLanguageModelInput, RunOutput, CallOptions> {\n  get callKeys(): string[];\n\n  generatePrompt(\n    promptValues: BasePromptValueInterface[],\n    options?: string[] | Partial<CallOptions>,\n    callbacks?: Callbacks\n  ): Promise<LLMResult>;\n\n  _modelType(): string;\n\n  _llmType(): string;\n\n  getNumTokens(content: MessageContent): Promise<number>;\n\n  /**\n   * Get the identifying parameters of the LLM.\n   */\n  // oxlint-disable-next-line @typescript-eslint/no-explicit-any\n  _identifyingParams(): Record<string, any>;\n\n  serialize(): SerializedLLM;\n}\n\nexport type LanguageModelOutput = BaseMessage | string;\n\nexport type LanguageModelLike = RunnableInterface<\n  BaseLanguageModelInput,\n  LanguageModelOutput\n>;\n\n/**\n * Base class for language models.\n */\nexport abstract class BaseLanguageModel<\n  // oxlint-disable-next-line @typescript-eslint/no-explicit-any\n  RunOutput = any,\n  CallOptions extends BaseLanguageModelCallOptions =\n    BaseLanguageModelCallOptions,\n>\n  extends BaseLangChain<BaseLanguageModelInput, RunOutput, CallOptions>\n  implements\n    BaseLanguageModelParams,\n    BaseLanguageModelInterface<RunOutput, CallOptions>\n{\n  /**\n   * Keys that the language model accepts as call options.\n   */\n  get callKeys(): string[] {\n    return [\"stop\", \"timeout\", \"signal\", \"tags\", \"metadata\", \"callbacks\"];\n  }\n\n  /**\n   * The async caller should be used by subclasses to make any async calls,\n   * which will thus benefit from the concurrency and retry logic.\n   */\n  caller: AsyncCaller;\n\n  cache?: BaseCache;\n\n  constructor({\n    callbacks,\n    callbackManager,\n    ...params\n  }: BaseLanguageModelParams) {\n    const { cache, ...rest } = params;\n    super({\n      callbacks: callbacks ?? callbackManager,\n      ...rest,\n    });\n    if (typeof cache === \"object\") {\n      this.cache = cache;\n    } else if (cache) {\n      this.cache = InMemoryCache.global();\n    } else {\n      this.cache = undefined;\n    }\n    this.caller = new AsyncCaller(params ?? {});\n  }\n\n  abstract generatePrompt(\n    promptValues: BasePromptValueInterface[],\n    options?: string[] | CallOptions,\n    callbacks?: Callbacks\n  ): Promise<LLMResult>;\n\n  abstract _modelType(): string;\n\n  abstract _llmType(): string;\n\n  private _encoding?: Tiktoken;\n\n  /**\n   * Get the number of tokens in the content.\n   * @param content The content to get the number of tokens for.\n   * @returns The number of tokens in the content.\n   */\n  async getNumTokens(content: MessageContent) {\n    // Extract text content from MessageContent\n    let textContent: string;\n    if (typeof content === \"string\") {\n      textContent = content;\n    } else {\n      /**\n       * Content is an array of ContentBlock\n       *\n       * ToDo(@christian-bromann): This is a temporary fix to get the number of tokens for the content.\n       * We need to find a better way to do this.\n       * @see https://github.com/langchain-ai/langchainjs/pull/8341#pullrequestreview-2933713116\n       */\n      textContent = content\n        .map((item) => {\n          if (typeof item === \"string\") return item;\n          if (item.type === \"text\" && \"text\" in item) return item.text;\n          return \"\";\n        })\n        .join(\"\");\n    }\n\n    // fallback to approximate calculation if tiktoken is not available\n    let numTokens = Math.ceil(textContent.length / 4);\n\n    if (!this._encoding) {\n      try {\n        this._encoding = await encodingForModel(\n          \"modelName\" in this\n            ? getModelNameForTiktoken(this.modelName as string)\n            : \"gpt2\"\n        );\n      } catch (error) {\n        console.warn(\n          \"Failed to calculate number of tokens, falling back to approximate count\",\n          error\n        );\n      }\n    }\n\n    if (this._encoding) {\n      try {\n        numTokens = this._encoding.encode(textContent).length;\n      } catch (error) {\n        console.warn(\n          \"Failed to calculate number of tokens, falling back to approximate count\",\n          error\n        );\n      }\n    }\n\n    return numTokens;\n  }\n\n  protected static _convertInputToPromptValue(\n    input: BaseLanguageModelInput\n  ): BasePromptValueInterface {\n    if (typeof input === \"string\") {\n      return new StringPromptValue(input);\n    } else if (Array.isArray(input)) {\n      return new ChatPromptValue(input.map(coerceMessageLikeToMessage));\n    } else {\n      return input;\n    }\n  }\n\n  /**\n   * Get the identifying parameters of the LLM.\n   */\n  // oxlint-disable-next-line @typescript-eslint/no-explicit-any\n  _identifyingParams(): Record<string, any> {\n    return {};\n  }\n\n  /**\n   * Create a unique cache key for a specific call to a specific language model.\n   * @param callOptions Call options for the model\n   * @returns A unique cache key.\n   */\n  _getSerializedCacheKeyParametersForCall(\n    // TODO: Fix when we remove the RunnableLambda backwards compatibility shim.\n    {\n      config,\n      ...callOptions\n    }: CallOptions & { config?: RunnableConfig }\n  ): string {\n    // oxlint-disable-next-line @typescript-eslint/no-explicit-any\n    const params: Record<string, any> = {\n      ...this._identifyingParams(),\n      ...callOptions,\n      _type: this._llmType(),\n      _model: this._modelType(),\n    };\n    const filteredEntries = Object.entries(params).filter(\n      ([_, value]) => value !== undefined\n    );\n    const serializedEntries = filteredEntries\n      .map(([key, value]) => `${key}:${JSON.stringify(value)}`)\n      .sort()\n      .join(\",\");\n    return serializedEntries;\n  }\n\n  /**\n   * @deprecated\n   * Return a json-like object representing this LLM.\n   */\n  serialize(): SerializedLLM {\n    return {\n      ...this._identifyingParams(),\n      _type: this._llmType(),\n      _model: this._modelType(),\n    };\n  }\n\n  /**\n   * @deprecated\n   * Load an LLM from a json-like object describing it.\n   */\n  static async deserialize(_data: SerializedLLM): Promise<BaseLanguageModel> {\n    throw new Error(\"Use .toJSON() instead\");\n  }\n\n  /**\n   * Return profiling information for the model.\n   *\n   * @returns {ModelProfile} An object describing the model's capabilities and constraints\n   */\n  get profile(): ModelProfile {\n    return {};\n  }\n\n  withStructuredOutput?<\n    // oxlint-disable-next-line @typescript-eslint/no-explicit-any\n    RunOutput extends Record<string, any> = Record<string, any>,\n  >(\n    schema: SerializableSchema<RunOutput>,\n    config?: StructuredOutputMethodOptions<false>\n  ): Runnable<BaseLanguageModelInput, RunOutput>;\n\n  withStructuredOutput?<\n    // oxlint-disable-next-line @typescript-eslint/no-explicit-any\n    RunOutput extends Record<string, any> = Record<string, any>,\n  >(\n    schema: SerializableSchema<RunOutput>,\n    config?: StructuredOutputMethodOptions<true>\n  ): Runnable<BaseLanguageModelInput, { raw: BaseMessage; parsed: RunOutput }>;\n\n  withStructuredOutput?<\n    // oxlint-disable-next-line @typescript-eslint/no-explicit-any\n    RunOutput extends Record<string, any> = Record<string, any>,\n  >(\n    schema:\n      | ZodV3Like<RunOutput>\n      // oxlint-disable-next-line @typescript-eslint/no-explicit-any\n      | Record<string, any>,\n    config?: StructuredOutputMethodOptions<false>\n  ): Runnable<BaseLanguageModelInput, RunOutput>;\n\n  withStructuredOutput?<\n    // oxlint-disable-next-line @typescript-eslint/no-explicit-any\n    RunOutput extends Record<string, any> = Record<string, any>,\n  >(\n    schema:\n      | ZodV3Like<RunOutput>\n      // oxlint-disable-next-line @typescript-eslint/no-explicit-any\n      | Record<string, any>,\n    config?: StructuredOutputMethodOptions<true>\n  ): Runnable<BaseLanguageModelInput, { raw: BaseMessage; parsed: RunOutput }>;\n\n  withStructuredOutput?<\n    // oxlint-disable-next-line @typescript-eslint/no-explicit-any\n    RunOutput extends Record<string, any> = Record<string, any>,\n  >(\n    schema:\n      | ZodV4Like<RunOutput>\n      // oxlint-disable-next-line @typescript-eslint/no-explicit-any\n      | Record<string, any>,\n    config?: StructuredOutputMethodOptions<false>\n  ): Runnable<BaseLanguageModelInput, RunOutput>;\n\n  withStructuredOutput?<\n    // oxlint-disable-next-line @typescript-eslint/no-explicit-any\n    RunOutput extends Record<string, any> = Record<string, any>,\n  >(\n    schema:\n      | ZodV4Like<RunOutput>\n      // oxlint-disable-next-line @typescript-eslint/no-explicit-any\n      | Record<string, any>,\n    config?: StructuredOutputMethodOptions<true>\n  ): Runnable<BaseLanguageModelInput, { raw: BaseMessage; parsed: RunOutput }>;\n\n  /**\n   * Model wrapper that returns outputs formatted to match the given schema.\n   *\n   * @template {BaseLanguageModelInput} RunInput The input type for the Runnable, expected to be the same input for the LLM.\n   * @template {Record<string, any>} RunOutput The output type for the Runnable, expected to be a Zod schema object for structured output validation.\n   *\n   * @param {InteropZodType<RunOutput>} schema The schema for the structured output. Either as a Zod schema or a valid JSON schema object.\n   *   If a Zod schema is passed, the returned attributes will be validated, whereas with JSON schema they will not be.\n   * @param {string} name The name of the function to call.\n   * @param {\"functionCalling\" | \"jsonMode\"} [method=functionCalling] The method to use for getting the structured output. Defaults to \"functionCalling\".\n   * @param {boolean | undefined} [includeRaw=false] Whether to include the raw output in the result. Defaults to false.\n   * @returns {Runnable<RunInput, RunOutput> | Runnable<RunInput, { raw: BaseMessage; parsed: RunOutput }>} A new runnable that calls the LLM with structured output.\n   */\n  withStructuredOutput?<\n    // oxlint-disable-next-line @typescript-eslint/no-explicit-any\n    RunOutput extends Record<string, any> = Record<string, any>,\n  >(\n    schema:\n      | InteropZodType<RunOutput>\n      // oxlint-disable-next-line @typescript-eslint/no-explicit-any\n      | Record<string, any>,\n    config?: StructuredOutputMethodOptions<boolean>\n  ):\n    | Runnable<BaseLanguageModelInput, RunOutput>\n    | Runnable<\n        BaseLanguageModelInput,\n        {\n          raw: BaseMessage;\n          parsed: RunOutput;\n        }\n      >;\n\n  /**\n   * Filter out large/inappropriate fields from invocation params for tracing metadata.\n   * Removes fields like tools, functions, messages, response_format that can be large.\n   */\n  protected _filterInvocationParamsForTracing(\n    // oxlint-disable-next-line @typescript-eslint/no-explicit-any\n    params: Record<string, any>\n    // oxlint-disable-next-line @typescript-eslint/no-explicit-any\n  ): Record<string, any> {\n    const { tools, functions, messages, response_format, ...rest } = params;\n    return rest;\n  }\n}\n\n/**\n * Shared interface for token usage\n * return type from LLM calls.\n */\nexport interface TokenUsage {\n  completionTokens?: number;\n  promptTokens?: number;\n  totalTokens?: number;\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;AAgCA,MAAa,2BAA2B,cAAqC;AAC3E,KAAI,UAAU,WAAW,QAAQ,CAC/B,QAAO;AAGT,KAAI,UAAU,WAAW,oBAAoB,CAC3C,QAAO;AAGT,KAAI,UAAU,WAAW,iBAAiB,CACxC,QAAO;AAGT,KAAI,UAAU,WAAW,YAAY,CACnC,QAAO;AAGT,KAAI,UAAU,WAAW,SAAS,CAChC,QAAO;AAGT,KAAI,UAAU,WAAW,SAAS,CAChC,QAAO;AAGT,QAAO;;AAGT,MAAa,2BAA2B,cAA+B;AACrE,SAAQ,WAAR;EACE,KAAK,yBACH,QAAO;EACT,QACE,QAAO;;;;;;;;;;;;;;AAeb,MAAa,uBAAuB,cAA8B;AAGhE,SAFuB,wBAAwB,UAAU,EAEzD;EAEE,KAAK;EACL,KAAK;EACL,KAAK,sBACH,QAAO;EAGT,KAAK;EACL,KAAK;EACL,KAAK;EACL,KAAK,oBACH,QAAO;EAGT,KAAK;EACL,KAAK;EACL,KAAK;EACL,KAAK;EACL,KAAK,qBACH,QAAO;EAGT,KAAK;EACL,KAAK;EACL,KAAK,iBACH,QAAO;EACT,KAAK;EACL,KAAK;EACL,KAAK,aACH,QAAO;EAGT,KAAK;EACL,KAAK,yBACH,QAAO;EACT,KAAK;EACL,KAAK;EACL,KAAK;EACL,KAAK;EACL,KAAK,qBACH,QAAO;EAGT,KAAK;EACL,KAAK,mBACH,QAAO;EACT,KAAK,mBACH,QAAO;EACT,KAAK;EACL,KAAK;EACL,KAAK,eACH,QAAO;EAGT,KAAK;EACL,KAAK,mBACH,QAAO;EACT,KAAK,mBACH,QAAO;EAGT,KAAK;EACL,KAAK;EACL,KAAK;EACL,KAAK;EACL,KAAK;EACL,KAAK,aACH,QAAO;EACT,KAAK;EACL,KAAK,qBACH,QAAO;EAGT,KAAK;EACL,KAAK;EACL,KAAK;EACL,KAAK,0BACH,QAAO;EACT,KAAK;EACL,KAAK,oBACH,QAAO;EAET,QACE,QAAO;;;;;;;;AASb,SAAgB,aAAa,MAAuC;AAClE,KAAI,OAAO,SAAS,YAAY,CAAC,KAAM,QAAO;AAC9C,KACE,UAAU,QACV,KAAK,SAAS,cACd,cAAc,QACd,OAAO,KAAK,aAAa,YACzB,KAAK,YACL,UAAU,KAAK,YACf,gBAAgB,KAAK,SAErB,QAAO;AAET,QAAO;;AAQT,MAAa,qBAAqB,OAAO,EACvC,QACA,gBAC4B;CAC5B,IAAI;AAEJ,KAAI;AACF,eACE,MAAMA,uBAAAA,iBAAiB,wBAAwB,UAAU,CAAC,EAC1D,OAAO,OAAO,CAAC;SACX;AACN,UAAQ,KACN,0EACD;AAID,cAAY,KAAK,KAAK,OAAO,SAAS,EAAE;;AAI1C,QADkB,oBAAoB,UAAU,GAC7B;;AAGrB,MAAM,qBAAqB;;;;AAkB3B,IAAsB,gBAAtB,cAKUC,aAAAA,SAEV;;;;CAIE;CAEA;CAEA;CAEA;CAEA,IAAI,gBAA0D;AAC5D,SAAO;GACL,WAAW,KAAA;GACX,SAAS,KAAA;GACV;;CAGH,YAAY,QAA6B;AACvC,QAAM,OAAO;AACb,OAAK,UAAU,OAAO,WAAW,cAAc;AAC/C,OAAK,YAAY,OAAO;AACxB,OAAK,OAAO,OAAO,QAAQ,EAAE;AAC7B,OAAK,WAAW,OAAO,YAAY,EAAE;AACrC,OAAK,YAAY,mBAAA,SAAmC;;CAGtD,YAAsB,KAAa,SAAiB;EAClD,MAAM,WAAW,KAAK,UAAU;AAChC,OAAK,WAAW;GACd,GAAG,KAAK;GACR,UAAU;IACR,GAAI,OAAO,aAAa,YAAY,aAAa,OAAO,WAAW,EAAE;KACpE,MAAM;IACR;GACF;;;;;;AAyJL,IAAsB,oBAAtB,cAMU,cAIV;;;;CAIE,IAAI,WAAqB;AACvB,SAAO;GAAC;GAAQ;GAAW;GAAU;GAAQ;GAAY;GAAY;;;;;;CAOvE;CAEA;CAEA,YAAY,EACV,WACA,iBACA,GAAG,UACuB;EAC1B,MAAM,EAAE,OAAO,GAAG,SAAS;AAC3B,QAAM;GACJ,WAAW,aAAa;GACxB,GAAG;GACJ,CAAC;AACF,MAAI,OAAO,UAAU,SACnB,MAAK,QAAQ;WACJ,MACT,MAAK,QAAQC,qBAAAA,cAAc,QAAQ;MAEnC,MAAK,QAAQ,KAAA;AAEf,OAAK,SAAS,IAAIC,2BAAAA,YAAY,UAAU,EAAE,CAAC;;CAa7C;;;;;;CAOA,MAAM,aAAa,SAAyB;EAE1C,IAAI;AACJ,MAAI,OAAO,YAAY,SACrB,eAAc;;;;;;;;;AASd,gBAAc,QACX,KAAK,SAAS;AACb,OAAI,OAAO,SAAS,SAAU,QAAO;AACrC,OAAI,KAAK,SAAS,UAAU,UAAU,KAAM,QAAO,KAAK;AACxD,UAAO;IACP,CACD,KAAK,GAAG;EAIb,IAAI,YAAY,KAAK,KAAK,YAAY,SAAS,EAAE;AAEjD,MAAI,CAAC,KAAK,UACR,KAAI;AACF,QAAK,YAAY,MAAMH,uBAAAA,iBACrB,eAAe,OACX,wBAAwB,KAAK,UAAoB,GACjD,OACL;WACM,OAAO;AACd,WAAQ,KACN,2EACA,MACD;;AAIL,MAAI,KAAK,UACP,KAAI;AACF,eAAY,KAAK,UAAU,OAAO,YAAY,CAAC;WACxC,OAAO;AACd,WAAQ,KACN,2EACA,MACD;;AAIL,SAAO;;CAGT,OAAiB,2BACf,OAC0B;AAC1B,MAAI,OAAO,UAAU,SACnB,QAAO,IAAII,sBAAAA,kBAAkB,MAAM;WAC1B,MAAM,QAAQ,MAAM,CAC7B,QAAO,IAAIC,sBAAAA,gBAAgB,MAAM,IAAIC,cAAAA,2BAA2B,CAAC;MAEjE,QAAO;;;;;CAQX,qBAA0C;AACxC,SAAO,EAAE;;;;;;;CAQX,wCAEE,EACE,QACA,GAAG,eAEG;EAER,MAAM,SAA8B;GAClC,GAAG,KAAK,oBAAoB;GAC5B,GAAG;GACH,OAAO,KAAK,UAAU;GACtB,QAAQ,KAAK,YAAY;GAC1B;AAQD,SAPwB,OAAO,QAAQ,OAAO,CAAC,QAC5C,CAAC,GAAG,WAAW,UAAU,KAAA,EAC3B,CAEE,KAAK,CAAC,KAAK,WAAW,GAAG,IAAI,GAAG,KAAK,UAAU,MAAM,GAAG,CACxD,MAAM,CACN,KAAK,IAAI;;;;;;CAQd,YAA2B;AACzB,SAAO;GACL,GAAG,KAAK,oBAAoB;GAC5B,OAAO,KAAK,UAAU;GACtB,QAAQ,KAAK,YAAY;GAC1B;;;;;;CAOH,aAAa,YAAY,OAAkD;AACzE,QAAM,IAAI,MAAM,wBAAwB;;;;;;;CAQ1C,IAAI,UAAwB;AAC1B,SAAO,EAAE;;;;;;CAmGX,kCAEE,QAEqB;EACrB,MAAM,EAAE,OAAO,WAAW,UAAU,iBAAiB,GAAG,SAAS;AACjE,SAAO"}
+{"version":3,"file":"base.cjs","names":["encodingForModel","Runnable","InMemoryCache","AsyncCaller","StringPromptValue","ChatPromptValue","coerceMessageLikeToMessage"],"sources":["../../src/language_models/base.ts"],"sourcesContent":["import type { Tiktoken, TiktokenModel } from \"js-tiktoken/lite\";\nimport type { ZodV3Like, ZodV4Like } from \"../utils/types/zod.js\";\n\nimport { type BaseCache, InMemoryCache } from \"../caches/index.js\";\nimport {\n  type BasePromptValueInterface,\n  StringPromptValue,\n  ChatPromptValue,\n} from \"../prompt_values.js\";\nimport {\n  type BaseMessage,\n  type BaseMessageLike,\n  type MessageContent,\n} from \"../messages/base.js\";\nimport { coerceMessageLikeToMessage } from \"../messages/utils.js\";\nimport { type LLMResult } from \"../outputs.js\";\nimport { CallbackManager, Callbacks } from \"../callbacks/manager.js\";\nimport { AsyncCaller, AsyncCallerParams } from \"../utils/async_caller.js\";\nimport { encodingForModel } from \"../utils/tiktoken.js\";\nimport { Runnable, type RunnableInterface } from \"../runnables/base.js\";\nimport { RunnableConfig } from \"../runnables/config.js\";\nimport { JSONSchema } from \"../utils/json_schema.js\";\nimport {\n  InferInteropZodOutput,\n  InteropZodObject,\n  InteropZodType,\n} from \"../utils/types/zod.js\";\nimport { ModelProfile } from \"./profile.js\";\nimport { type SerializableSchema } from \"../utils/standard_schema.js\";\n\n// https://www.npmjs.com/package/js-tiktoken\n\nexport const getModelNameForTiktoken = (modelName: string): TiktokenModel => {\n  if (modelName.startsWith(\"gpt-5\")) {\n    return \"gpt-5\" as TiktokenModel;\n  }\n\n  if (modelName.startsWith(\"gpt-3.5-turbo-16k\")) {\n    return \"gpt-3.5-turbo-16k\";\n  }\n\n  if (modelName.startsWith(\"gpt-3.5-turbo-\")) {\n    return \"gpt-3.5-turbo\";\n  }\n\n  if (modelName.startsWith(\"gpt-4-32k\")) {\n    return \"gpt-4-32k\";\n  }\n\n  if (modelName.startsWith(\"gpt-4-\")) {\n    return \"gpt-4\";\n  }\n\n  if (modelName.startsWith(\"gpt-4o\")) {\n    return \"gpt-4o\";\n  }\n\n  return modelName as TiktokenModel;\n};\n\nexport const getEmbeddingContextSize = (modelName?: string): number => {\n  switch (modelName) {\n    case \"text-embedding-ada-002\":\n      return 8191;\n    default:\n      return 2046;\n  }\n};\n\n/**\n * Get the context window size (max input tokens) for a given model.\n *\n * Context window sizes are sourced from official model documentation:\n * - OpenAI: https://platform.openai.com/docs/models\n * - Anthropic: https://docs.anthropic.com/claude/docs/models-overview\n * - Google: https://ai.google.dev/gemini/docs/models/gemini\n *\n * @param modelName - The name of the model\n * @returns The context window size in tokens\n */\nexport const getModelContextSize = (modelName: string): number => {\n  const normalizedName = getModelNameForTiktoken(modelName) as string;\n\n  switch (normalizedName) {\n    // GPT-5 series\n    case \"gpt-5\":\n    case \"gpt-5-turbo\":\n    case \"gpt-5-turbo-preview\":\n      return 400000;\n\n    // GPT-4o series\n    case \"gpt-4o\":\n    case \"gpt-4o-mini\":\n    case \"gpt-4o-2024-05-13\":\n    case \"gpt-4o-2024-08-06\":\n      return 128000;\n\n    // GPT-4 Turbo series\n    case \"gpt-4-turbo\":\n    case \"gpt-4-turbo-preview\":\n    case \"gpt-4-turbo-2024-04-09\":\n    case \"gpt-4-0125-preview\":\n    case \"gpt-4-1106-preview\":\n      return 128000;\n\n    // GPT-4 series\n    case \"gpt-4-32k\":\n    case \"gpt-4-32k-0314\":\n    case \"gpt-4-32k-0613\":\n      return 32768;\n    case \"gpt-4\":\n    case \"gpt-4-0314\":\n    case \"gpt-4-0613\":\n      return 8192;\n\n    // GPT-3.5 Turbo series\n    case \"gpt-3.5-turbo-16k\":\n    case \"gpt-3.5-turbo-16k-0613\":\n      return 16384;\n    case \"gpt-3.5-turbo\":\n    case \"gpt-3.5-turbo-0301\":\n    case \"gpt-3.5-turbo-0613\":\n    case \"gpt-3.5-turbo-1106\":\n    case \"gpt-3.5-turbo-0125\":\n      return 4096;\n\n    // Legacy GPT-3 models\n    case \"text-davinci-003\":\n    case \"text-davinci-002\":\n      return 4097;\n    case \"text-davinci-001\":\n      return 2049;\n    case \"text-curie-001\":\n    case \"text-babbage-001\":\n    case \"text-ada-001\":\n      return 2048;\n\n    // Code models\n    case \"code-davinci-002\":\n    case \"code-davinci-001\":\n      return 8000;\n    case \"code-cushman-001\":\n      return 2048;\n\n    // Claude models (Anthropic)\n    case \"claude-3-5-sonnet-20241022\":\n    case \"claude-3-5-sonnet-20240620\":\n    case \"claude-3-opus-20240229\":\n    case \"claude-3-sonnet-20240229\":\n    case \"claude-3-haiku-20240307\":\n    case \"claude-2.1\":\n      return 200000;\n    case \"claude-2.0\":\n    case \"claude-instant-1.2\":\n      return 100000;\n\n    // Gemini models (Google)\n    case \"gemini-1.5-pro\":\n    case \"gemini-1.5-pro-latest\":\n    case \"gemini-1.5-flash\":\n    case \"gemini-1.5-flash-latest\":\n      return 1000000; // 1M tokens\n    case \"gemini-pro\":\n    case \"gemini-pro-vision\":\n      return 32768;\n\n    default:\n      return 4097;\n  }\n};\n\n/**\n * Whether or not the input matches the OpenAI tool definition.\n * @param {unknown} tool The input to check.\n * @returns {boolean} Whether the input is an OpenAI tool definition.\n */\nexport function isOpenAITool(tool: unknown): tool is ToolDefinition {\n  if (typeof tool !== \"object\" || !tool) return false;\n  if (\n    \"type\" in tool &&\n    tool.type === \"function\" &&\n    \"function\" in tool &&\n    typeof tool.function === \"object\" &&\n    tool.function &&\n    \"name\" in tool.function &&\n    \"parameters\" in tool.function\n  ) {\n    return true;\n  }\n  return false;\n}\n\ninterface CalculateMaxTokenProps {\n  prompt: string;\n  modelName: TiktokenModel;\n}\n\nexport const calculateMaxTokens = async ({\n  prompt,\n  modelName,\n}: CalculateMaxTokenProps) => {\n  let numTokens;\n\n  try {\n    numTokens = (\n      await encodingForModel(getModelNameForTiktoken(modelName))\n    ).encode(prompt).length;\n  } catch {\n    console.warn(\n      \"Failed to calculate number of tokens, falling back to approximate count\"\n    );\n\n    // fallback to approximate calculation if tiktoken is not available\n    // each token is ~4 characters: https://help.openai.com/en/articles/4936856-what-are-tokens-and-how-to-count-them#\n    numTokens = Math.ceil(prompt.length / 4);\n  }\n\n  const maxTokens = getModelContextSize(modelName);\n  return maxTokens - numTokens;\n};\n\nconst getVerbosity = () => false;\n\nexport type SerializedLLM = {\n  _model: string;\n  _type: string;\n  // oxlint-disable-next-line @typescript-eslint/no-explicit-any\n} & Record<string, any>;\n\nexport interface BaseLangChainParams {\n  verbose?: boolean;\n  callbacks?: Callbacks;\n  tags?: string[];\n  metadata?: Record<string, unknown>;\n}\n\n/**\n * Base class for language models, chains, tools.\n */\nexport abstract class BaseLangChain<\n  RunInput,\n  RunOutput,\n  CallOptions extends RunnableConfig = RunnableConfig,\n>\n  extends Runnable<RunInput, RunOutput, CallOptions>\n  implements BaseLangChainParams\n{\n  /**\n   * Whether to print out response text.\n   */\n  verbose: boolean;\n\n  callbacks?: Callbacks;\n\n  tags?: string[];\n\n  metadata?: Record<string, unknown>;\n\n  get lc_attributes(): { [key: string]: undefined } | undefined {\n    return {\n      callbacks: undefined,\n      verbose: undefined,\n    };\n  }\n\n  constructor(params: BaseLangChainParams) {\n    super(params);\n    this.verbose = params.verbose ?? getVerbosity();\n    this.callbacks = params.callbacks;\n    this.tags = params.tags ?? [];\n    this.metadata = params.metadata ?? {};\n    this._addVersion(\"@langchain/core\", __PKG_VERSION__);\n  }\n\n  protected _addVersion(pkg: string, version: string) {\n    const existing = this.metadata?.versions;\n    this.metadata = {\n      ...this.metadata,\n      versions: {\n        ...(typeof existing === \"object\" && existing !== null ? existing : {}),\n        [pkg]: version,\n      },\n    };\n  }\n}\n\n/**\n * Base interface for language model parameters.\n * A subclass of {@link BaseLanguageModel} should have a constructor that\n * takes in a parameter that extends this interface.\n */\nexport interface BaseLanguageModelParams\n  extends AsyncCallerParams, BaseLangChainParams {\n  /**\n   * @deprecated Use `callbacks` instead\n   */\n  callbackManager?: CallbackManager;\n\n  cache?: BaseCache | boolean;\n}\n\nexport interface BaseLanguageModelTracingCallOptions {\n  /**\n   * Describes the format of structured outputs.\n   * This should be provided if an output is considered to be structured\n   */\n  ls_structured_output_format?: {\n    /**\n     * An object containing the method used for structured output (e.g., \"jsonMode\").\n     */\n    kwargs: { method: string };\n    /**\n     * The JSON schema describing the expected output structure.\n     */\n    schema?: JSONSchema;\n  };\n}\n\nexport interface BaseLanguageModelCallOptions\n  extends RunnableConfig, BaseLanguageModelTracingCallOptions {\n  /**\n   * Stop tokens to use for this call.\n   * If not provided, the default stop tokens for the model will be used.\n   */\n  stop?: string[];\n}\n\nexport interface FunctionDefinition {\n  /**\n   * The name of the function to be called. Must be a-z, A-Z, 0-9, or contain\n   * underscores and dashes, with a maximum length of 64.\n   */\n  name: string;\n\n  /**\n   * The parameters the functions accepts, described as a JSON Schema object. See the\n   * [guide](https://platform.openai.com/docs/guides/gpt/function-calling) for\n   * examples, and the\n   * [JSON Schema reference](https://json-schema.org/understanding-json-schema/) for\n   * documentation about the format.\n   *\n   * To describe a function that accepts no parameters, provide the value\n   * `{\"type\": \"object\", \"properties\": {}}`.\n   */\n  parameters: Record<string, unknown> | JSONSchema;\n\n  /**\n   * A description of what the function does, used by the model to choose when and\n   * how to call the function.\n   */\n  description?: string;\n}\n\nexport interface ToolDefinition {\n  type: \"function\";\n  function: FunctionDefinition;\n}\n\nexport type FunctionCallOption = {\n  name: string;\n};\n\nexport interface BaseFunctionCallOptions extends BaseLanguageModelCallOptions {\n  function_call?: FunctionCallOption;\n  functions?: FunctionDefinition[];\n}\n\nexport type BaseLanguageModelInput =\n  | BasePromptValueInterface\n  | string\n  | BaseMessageLike[];\n\nexport type StructuredOutputType = InferInteropZodOutput<InteropZodObject>;\n\nexport type StructuredOutputMethodOptions<IncludeRaw extends boolean = false> =\n  {\n    name?: string;\n    method?: \"functionCalling\" | \"jsonMode\" | \"jsonSchema\" | string;\n    includeRaw?: IncludeRaw;\n    /** Whether to use strict mode. Currently only supported by OpenAI models. */\n    strict?: boolean;\n  };\n\n/** @deprecated Use StructuredOutputMethodOptions instead */\nexport type StructuredOutputMethodParams<\n  RunOutput,\n  IncludeRaw extends boolean = false,\n> = {\n  /** @deprecated Pass schema in as the first argument */\n  // oxlint-disable-next-line @typescript-eslint/no-explicit-any\n  schema: InteropZodType<RunOutput> | Record<string, any>;\n  name?: string;\n  method?: \"functionCalling\" | \"jsonMode\";\n  includeRaw?: IncludeRaw;\n};\n\nexport interface BaseLanguageModelInterface<\n  // oxlint-disable-next-line @typescript-eslint/no-explicit-any\n  RunOutput = any,\n  CallOptions extends BaseLanguageModelCallOptions =\n    BaseLanguageModelCallOptions,\n> extends RunnableInterface<BaseLanguageModelInput, RunOutput, CallOptions> {\n  get callKeys(): string[];\n\n  generatePrompt(\n    promptValues: BasePromptValueInterface[],\n    options?: string[] | Partial<CallOptions>,\n    callbacks?: Callbacks\n  ): Promise<LLMResult>;\n\n  _modelType(): string;\n\n  _llmType(): string;\n\n  getNumTokens(content: MessageContent): Promise<number>;\n\n  /**\n   * Get the identifying parameters of the LLM.\n   */\n  // oxlint-disable-next-line @typescript-eslint/no-explicit-any\n  _identifyingParams(): Record<string, any>;\n\n  serialize(): SerializedLLM;\n}\n\nexport type LanguageModelOutput = BaseMessage | string;\n\nexport type LanguageModelLike = RunnableInterface<\n  BaseLanguageModelInput,\n  LanguageModelOutput\n>;\n\n/**\n * Base class for language models.\n */\nexport abstract class BaseLanguageModel<\n  // oxlint-disable-next-line @typescript-eslint/no-explicit-any\n  RunOutput = any,\n  CallOptions extends BaseLanguageModelCallOptions =\n    BaseLanguageModelCallOptions,\n>\n  extends BaseLangChain<BaseLanguageModelInput, RunOutput, CallOptions>\n  implements\n    BaseLanguageModelParams,\n    BaseLanguageModelInterface<RunOutput, CallOptions>\n{\n  /**\n   * Keys that the language model accepts as call options.\n   */\n  get callKeys(): string[] {\n    return [\"stop\", \"timeout\", \"signal\", \"tags\", \"metadata\", \"callbacks\"];\n  }\n\n  /**\n   * The async caller should be used by subclasses to make any async calls,\n   * which will thus benefit from the concurrency and retry logic.\n   */\n  caller: AsyncCaller;\n\n  cache?: BaseCache;\n\n  constructor({\n    callbacks,\n    callbackManager,\n    ...params\n  }: BaseLanguageModelParams) {\n    const { cache, ...rest } = params;\n    super({\n      callbacks: callbacks ?? callbackManager,\n      ...rest,\n    });\n    if (typeof cache === \"object\") {\n      this.cache = cache;\n    } else if (cache) {\n      this.cache = InMemoryCache.global();\n    } else {\n      this.cache = undefined;\n    }\n    this.caller = new AsyncCaller(params ?? {});\n  }\n\n  abstract generatePrompt(\n    promptValues: BasePromptValueInterface[],\n    options?: string[] | CallOptions,\n    callbacks?: Callbacks\n  ): Promise<LLMResult>;\n\n  abstract _modelType(): string;\n\n  abstract _llmType(): string;\n\n  private _encoding?: Tiktoken;\n\n  /**\n   * Get the number of tokens in the content.\n   * @param content The content to get the number of tokens for.\n   * @returns The number of tokens in the content.\n   */\n  async getNumTokens(content: MessageContent) {\n    // Extract text content from MessageContent\n    let textContent: string;\n    if (typeof content === \"string\") {\n      textContent = content;\n    } else {\n      /**\n       * Content is an array of ContentBlock\n       *\n       * ToDo(@christian-bromann): This is a temporary fix to get the number of tokens for the content.\n       * We need to find a better way to do this.\n       * @see https://github.com/langchain-ai/langchainjs/pull/8341#pullrequestreview-2933713116\n       */\n      textContent = content\n        .map((item) => {\n          if (typeof item === \"string\") return item;\n          if (item.type === \"text\" && \"text\" in item) return item.text;\n          return \"\";\n        })\n        .join(\"\");\n    }\n\n    // fallback to approximate calculation if tiktoken is not available\n    let numTokens = Math.ceil(textContent.length / 4);\n\n    if (!this._encoding) {\n      try {\n        this._encoding = await encodingForModel(\n          \"modelName\" in this\n            ? getModelNameForTiktoken(this.modelName as string)\n            : \"gpt2\"\n        );\n      } catch (error) {\n        console.warn(\n          \"Failed to calculate number of tokens, falling back to approximate count\",\n          error\n        );\n      }\n    }\n\n    if (this._encoding) {\n      try {\n        numTokens = this._encoding.encode(textContent).length;\n      } catch (error) {\n        console.warn(\n          \"Failed to calculate number of tokens, falling back to approximate count\",\n          error\n        );\n      }\n    }\n\n    return numTokens;\n  }\n\n  protected static _convertInputToPromptValue(\n    input: BaseLanguageModelInput\n  ): BasePromptValueInterface {\n    if (typeof input === \"string\") {\n      return new StringPromptValue(input);\n    } else if (Array.isArray(input)) {\n      return new ChatPromptValue(input.map(coerceMessageLikeToMessage));\n    } else {\n      return input;\n    }\n  }\n\n  /**\n   * Get the identifying parameters of the LLM.\n   */\n  // oxlint-disable-next-line @typescript-eslint/no-explicit-any\n  _identifyingParams(): Record<string, any> {\n    return {};\n  }\n\n  /**\n   * Create a unique cache key for a specific call to a specific language model.\n   * @param callOptions Call options for the model\n   * @returns A unique cache key.\n   */\n  _getSerializedCacheKeyParametersForCall(\n    // TODO: Fix when we remove the RunnableLambda backwards compatibility shim.\n    {\n      config,\n      ...callOptions\n    }: CallOptions & { config?: RunnableConfig }\n  ): string {\n    // oxlint-disable-next-line @typescript-eslint/no-explicit-any\n    const params: Record<string, any> = {\n      ...this._identifyingParams(),\n      ...callOptions,\n      _type: this._llmType(),\n      _model: this._modelType(),\n    };\n    const filteredEntries = Object.entries(params).filter(\n      ([_, value]) => value !== undefined\n    );\n    const serializedEntries = filteredEntries\n      .map(([key, value]) => `${key}:${JSON.stringify(value)}`)\n      .sort()\n      .join(\",\");\n    return serializedEntries;\n  }\n\n  /**\n   * @deprecated\n   * Return a json-like object representing this LLM.\n   */\n  serialize(): SerializedLLM {\n    return {\n      ...this._identifyingParams(),\n      _type: this._llmType(),\n      _model: this._modelType(),\n    };\n  }\n\n  /**\n   * @deprecated\n   * Load an LLM from a json-like object describing it.\n   */\n  static async deserialize(_data: SerializedLLM): Promise<BaseLanguageModel> {\n    throw new Error(\"Use .toJSON() instead\");\n  }\n\n  /**\n   * Return profiling information for the model.\n   *\n   * @returns {ModelProfile} An object describing the model's capabilities and constraints\n   */\n  get profile(): ModelProfile {\n    return {};\n  }\n\n  withStructuredOutput?<\n    // oxlint-disable-next-line @typescript-eslint/no-explicit-any\n    RunOutput extends Record<string, any> = Record<string, any>,\n  >(\n    schema: SerializableSchema<RunOutput>,\n    config?: StructuredOutputMethodOptions<false>\n  ): Runnable<BaseLanguageModelInput, RunOutput>;\n\n  withStructuredOutput?<\n    // oxlint-disable-next-line @typescript-eslint/no-explicit-any\n    RunOutput extends Record<string, any> = Record<string, any>,\n  >(\n    schema: SerializableSchema<RunOutput>,\n    config?: StructuredOutputMethodOptions<true>\n  ): Runnable<BaseLanguageModelInput, { raw: BaseMessage; parsed: RunOutput }>;\n\n  withStructuredOutput?<\n    // oxlint-disable-next-line @typescript-eslint/no-explicit-any\n    RunOutput extends Record<string, any> = Record<string, any>,\n  >(\n    schema:\n      | ZodV3Like<RunOutput>\n      // oxlint-disable-next-line @typescript-eslint/no-explicit-any\n      | Record<string, any>,\n    config?: StructuredOutputMethodOptions<false>\n  ): Runnable<BaseLanguageModelInput, RunOutput>;\n\n  withStructuredOutput?<\n    // oxlint-disable-next-line @typescript-eslint/no-explicit-any\n    RunOutput extends Record<string, any> = Record<string, any>,\n  >(\n    schema:\n      | ZodV3Like<RunOutput>\n      // oxlint-disable-next-line @typescript-eslint/no-explicit-any\n      | Record<string, any>,\n    config?: StructuredOutputMethodOptions<true>\n  ): Runnable<BaseLanguageModelInput, { raw: BaseMessage; parsed: RunOutput }>;\n\n  withStructuredOutput?<\n    // oxlint-disable-next-line @typescript-eslint/no-explicit-any\n    RunOutput extends Record<string, any> = Record<string, any>,\n  >(\n    schema:\n      | ZodV4Like<RunOutput>\n      // oxlint-disable-next-line @typescript-eslint/no-explicit-any\n      | Record<string, any>,\n    config?: StructuredOutputMethodOptions<false>\n  ): Runnable<BaseLanguageModelInput, RunOutput>;\n\n  withStructuredOutput?<\n    // oxlint-disable-next-line @typescript-eslint/no-explicit-any\n    RunOutput extends Record<string, any> = Record<string, any>,\n  >(\n    schema:\n      | ZodV4Like<RunOutput>\n      // oxlint-disable-next-line @typescript-eslint/no-explicit-any\n      | Record<string, any>,\n    config?: StructuredOutputMethodOptions<true>\n  ): Runnable<BaseLanguageModelInput, { raw: BaseMessage; parsed: RunOutput }>;\n\n  /**\n   * Model wrapper that returns outputs formatted to match the given schema.\n   *\n   * @template {BaseLanguageModelInput} RunInput The input type for the Runnable, expected to be the same input for the LLM.\n   * @template {Record<string, any>} RunOutput The output type for the Runnable, expected to be a Zod schema object for structured output validation.\n   *\n   * @param {InteropZodType<RunOutput>} schema The schema for the structured output. Either as a Zod schema or a valid JSON schema object.\n   *   If a Zod schema is passed, the returned attributes will be validated, whereas with JSON schema they will not be.\n   * @param {string} name The name of the function to call.\n   * @param {\"functionCalling\" | \"jsonMode\"} [method=functionCalling] The method to use for getting the structured output. Defaults to \"functionCalling\".\n   * @param {boolean | undefined} [includeRaw=false] Whether to include the raw output in the result. Defaults to false.\n   * @returns {Runnable<RunInput, RunOutput> | Runnable<RunInput, { raw: BaseMessage; parsed: RunOutput }>} A new runnable that calls the LLM with structured output.\n   */\n  withStructuredOutput?<\n    // oxlint-disable-next-line @typescript-eslint/no-explicit-any\n    RunOutput extends Record<string, any> = Record<string, any>,\n  >(\n    schema:\n      | InteropZodType<RunOutput>\n      // oxlint-disable-next-line @typescript-eslint/no-explicit-any\n      | Record<string, any>,\n    config?: StructuredOutputMethodOptions<boolean>\n  ):\n    | Runnable<BaseLanguageModelInput, RunOutput>\n    | Runnable<\n        BaseLanguageModelInput,\n        {\n          raw: BaseMessage;\n          parsed: RunOutput;\n        }\n      >;\n\n  /**\n   * Filter out large/inappropriate fields from invocation params for tracing metadata.\n   * Removes fields like tools, functions, messages, response_format that can be large.\n   */\n  protected _filterInvocationParamsForTracing(\n    // oxlint-disable-next-line @typescript-eslint/no-explicit-any\n    params: Record<string, any>\n    // oxlint-disable-next-line @typescript-eslint/no-explicit-any\n  ): Record<string, any> {\n    const { tools, functions, messages, response_format, ...rest } = params;\n    return rest;\n  }\n}\n\n/**\n * Shared interface for token usage\n * return type from LLM calls.\n */\nexport interface TokenUsage {\n  completionTokens?: number;\n  promptTokens?: number;\n  totalTokens?: number;\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;AAgCA,MAAa,2BAA2B,cAAqC;AAC3E,KAAI,UAAU,WAAW,QAAQ,CAC/B,QAAO;AAGT,KAAI,UAAU,WAAW,oBAAoB,CAC3C,QAAO;AAGT,KAAI,UAAU,WAAW,iBAAiB,CACxC,QAAO;AAGT,KAAI,UAAU,WAAW,YAAY,CACnC,QAAO;AAGT,KAAI,UAAU,WAAW,SAAS,CAChC,QAAO;AAGT,KAAI,UAAU,WAAW,SAAS,CAChC,QAAO;AAGT,QAAO;;AAGT,MAAa,2BAA2B,cAA+B;AACrE,SAAQ,WAAR;EACE,KAAK,yBACH,QAAO;EACT,QACE,QAAO;;;;;;;;;;;;;;AAeb,MAAa,uBAAuB,cAA8B;AAGhE,SAFuB,wBAAwB,UAEzB,EAAtB;EAEE,KAAK;EACL,KAAK;EACL,KAAK,sBACH,QAAO;EAGT,KAAK;EACL,KAAK;EACL,KAAK;EACL,KAAK,oBACH,QAAO;EAGT,KAAK;EACL,KAAK;EACL,KAAK;EACL,KAAK;EACL,KAAK,qBACH,QAAO;EAGT,KAAK;EACL,KAAK;EACL,KAAK,iBACH,QAAO;EACT,KAAK;EACL,KAAK;EACL,KAAK,aACH,QAAO;EAGT,KAAK;EACL,KAAK,yBACH,QAAO;EACT,KAAK;EACL,KAAK;EACL,KAAK;EACL,KAAK;EACL,KAAK,qBACH,QAAO;EAGT,KAAK;EACL,KAAK,mBACH,QAAO;EACT,KAAK,mBACH,QAAO;EACT,KAAK;EACL,KAAK;EACL,KAAK,eACH,QAAO;EAGT,KAAK;EACL,KAAK,mBACH,QAAO;EACT,KAAK,mBACH,QAAO;EAGT,KAAK;EACL,KAAK;EACL,KAAK;EACL,KAAK;EACL,KAAK;EACL,KAAK,aACH,QAAO;EACT,KAAK;EACL,KAAK,qBACH,QAAO;EAGT,KAAK;EACL,KAAK;EACL,KAAK;EACL,KAAK,0BACH,QAAO;EACT,KAAK;EACL,KAAK,oBACH,QAAO;EAET,QACE,QAAO;;;;;;;;AASb,SAAgB,aAAa,MAAuC;AAClE,KAAI,OAAO,SAAS,YAAY,CAAC,KAAM,QAAO;AAC9C,KACE,UAAU,QACV,KAAK,SAAS,cACd,cAAc,QACd,OAAO,KAAK,aAAa,YACzB,KAAK,YACL,UAAU,KAAK,YACf,gBAAgB,KAAK,SAErB,QAAO;AAET,QAAO;;AAQT,MAAa,qBAAqB,OAAO,EACvC,QACA,gBAC4B;CAC5B,IAAI;AAEJ,KAAI;AACF,eACE,MAAMA,uBAAAA,iBAAiB,wBAAwB,UAAU,CAAC,EAC1D,OAAO,OAAO,CAAC;SACX;AACN,UAAQ,KACN,0EACD;AAID,cAAY,KAAK,KAAK,OAAO,SAAS,EAAE;;AAI1C,QADkB,oBAAoB,UACtB,GAAG;;AAGrB,MAAM,qBAAqB;;;;AAkB3B,IAAsB,gBAAtB,cAKUC,aAAAA,SAEV;;;;CAIE;CAEA;CAEA;CAEA;CAEA,IAAI,gBAA0D;AAC5D,SAAO;GACL,WAAW,KAAA;GACX,SAAS,KAAA;GACV;;CAGH,YAAY,QAA6B;AACvC,QAAM,OAAO;AACb,OAAK,UAAU,OAAO,WAAW,cAAc;AAC/C,OAAK,YAAY,OAAO;AACxB,OAAK,OAAO,OAAO,QAAQ,EAAE;AAC7B,OAAK,WAAW,OAAO,YAAY,EAAE;AACrC,OAAK,YAAY,mBAAA,2BAAmC;;CAGtD,YAAsB,KAAa,SAAiB;EAClD,MAAM,WAAW,KAAK,UAAU;AAChC,OAAK,WAAW;GACd,GAAG,KAAK;GACR,UAAU;IACR,GAAI,OAAO,aAAa,YAAY,aAAa,OAAO,WAAW,EAAE;KACpE,MAAM;IACR;GACF;;;;;;AAyJL,IAAsB,oBAAtB,cAMU,cAIV;;;;CAIE,IAAI,WAAqB;AACvB,SAAO;GAAC;GAAQ;GAAW;GAAU;GAAQ;GAAY;GAAY;;;;;;CAOvE;CAEA;CAEA,YAAY,EACV,WACA,iBACA,GAAG,UACuB;EAC1B,MAAM,EAAE,OAAO,GAAG,SAAS;AAC3B,QAAM;GACJ,WAAW,aAAa;GACxB,GAAG;GACJ,CAAC;AACF,MAAI,OAAO,UAAU,SACnB,MAAK,QAAQ;WACJ,MACT,MAAK,QAAQC,qBAAAA,cAAc,QAAQ;MAEnC,MAAK,QAAQ,KAAA;AAEf,OAAK,SAAS,IAAIC,2BAAAA,YAAY,UAAU,EAAE,CAAC;;CAa7C;;;;;;CAOA,MAAM,aAAa,SAAyB;EAE1C,IAAI;AACJ,MAAI,OAAO,YAAY,SACrB,eAAc;;;;;;;;;AASd,gBAAc,QACX,KAAK,SAAS;AACb,OAAI,OAAO,SAAS,SAAU,QAAO;AACrC,OAAI,KAAK,SAAS,UAAU,UAAU,KAAM,QAAO,KAAK;AACxD,UAAO;IACP,CACD,KAAK,GAAG;EAIb,IAAI,YAAY,KAAK,KAAK,YAAY,SAAS,EAAE;AAEjD,MAAI,CAAC,KAAK,UACR,KAAI;AACF,QAAK,YAAY,MAAMH,uBAAAA,iBACrB,eAAe,OACX,wBAAwB,KAAK,UAAoB,GACjD,OACL;WACM,OAAO;AACd,WAAQ,KACN,2EACA,MACD;;AAIL,MAAI,KAAK,UACP,KAAI;AACF,eAAY,KAAK,UAAU,OAAO,YAAY,CAAC;WACxC,OAAO;AACd,WAAQ,KACN,2EACA,MACD;;AAIL,SAAO;;CAGT,OAAiB,2BACf,OAC0B;AAC1B,MAAI,OAAO,UAAU,SACnB,QAAO,IAAII,sBAAAA,kBAAkB,MAAM;WAC1B,MAAM,QAAQ,MAAM,CAC7B,QAAO,IAAIC,sBAAAA,gBAAgB,MAAM,IAAIC,cAAAA,2BAA2B,CAAC;MAEjE,QAAO;;;;;CAQX,qBAA0C;AACxC,SAAO,EAAE;;;;;;;CAQX,wCAEE,EACE,QACA,GAAG,eAEG;EAER,MAAM,SAA8B;GAClC,GAAG,KAAK,oBAAoB;GAC5B,GAAG;GACH,OAAO,KAAK,UAAU;GACtB,QAAQ,KAAK,YAAY;GAC1B;AAQD,SAPwB,OAAO,QAAQ,OAAO,CAAC,QAC5C,CAAC,GAAG,WAAW,UAAU,KAAA,EAEa,CACtC,KAAK,CAAC,KAAK,WAAW,GAAG,IAAI,GAAG,KAAK,UAAU,MAAM,GAAG,CACxD,MAAM,CACN,KAAK,IACgB;;;;;;CAO1B,YAA2B;AACzB,SAAO;GACL,GAAG,KAAK,oBAAoB;GAC5B,OAAO,KAAK,UAAU;GACtB,QAAQ,KAAK,YAAY;GAC1B;;;;;;CAOH,aAAa,YAAY,OAAkD;AACzE,QAAM,IAAI,MAAM,wBAAwB;;;;;;;CAQ1C,IAAI,UAAwB;AAC1B,SAAO,EAAE;;;;;;CAmGX,kCAEE,QAEqB;EACrB,MAAM,EAAE,OAAO,WAAW,UAAU,iBAAiB,GAAG,SAAS;AACjE,SAAO"}

package/dist/language_models/base.js

@@ -138,7 +138,7 @@ var BaseLangChain = class extends Runnable {
 		this.callbacks = params.callbacks;
 		this.tags = params.tags ?? [];
 		this.metadata = params.metadata ?? {};
-		this._addVersion("@langchain/core", "1.1.48");
+		this._addVersion("@langchain/core", "1.1.49-dev-1781044436312");
 	}
 	_addVersion(pkg, version) {
 		const existing = this.metadata?.versions;