@anvia/core 0.2.4 → 0.3.1

package/dist/loaders/index.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../../src/loaders/index.ts"],"sourcesContent":["import { readdir, readFile } from \"node:fs/promises\";\nimport { isAbsolute, join } from \"node:path\";\nimport { glob } from \"tinyglobby\";\nimport type { Document } from \"../completion\";\n\nexport type LoaderResult<T> = { ok: true; value: T } | { ok: false; error: unknown };\n\nexport type FileSource = { path: string } | { path: \"<memory>\"; bytes: Uint8Array };\nexport type FileReadWithPath = { path: string; text: string };\n\nexport type PdfSource = { path: string } | { path: \"<memory>\"; bytes: Uint8Array };\nexport type PdfReadWithPath = { path: string; text: string };\nexport type PdfPage = { pageNumber: number; text: string };\nexport type PdfPageWithPath = { path: string; pageNumber: number; text: string };\n\ntype FileMode = \"source\" | \"read\" | \"readWithPath\";\ntype PdfMode = \"source\" | \"read\" | \"readWithPath\" | \"byPage\" | \"byPageWithPath\";\n\nexport class FileLoader<T = LoaderResult<FileSource>> implements AsyncIterable<T> {\n  private constructor(\n    private readonly source: () => AsyncIterable<LoaderResult<FileSource>>,\n    private readonly mode: FileMode,\n    private readonly skipErrors: boolean,\n  ) {}\n\n  static withGlob(pattern: string): FileLoader<LoaderResult<FileSource>> {\n    return new FileLoader(\n      async function* () {\n        for (const path of await sortedGlob(pattern)) {\n          yield ok({ path });\n        }\n      },\n      \"source\",\n      false,\n    );\n  }\n\n  static withDir(directory: string): FileLoader<LoaderResult<FileSource>> {\n    return new FileLoader(\n      async function* () {\n        const entries = await readdir(directory, { withFileTypes: true });\n        const files = entries\n          .filter((entry) => entry.isFile())\n          .map((entry) => join(directory, entry.name))\n          .sort((left, right) => left.localeCompare(right));\n        for (const path of files) {\n          yield ok({ path });\n        }\n      },\n      \"source\",\n      false,\n    );\n  }\n\n  static fromBytes(bytes: Uint8Array | ArrayBuffer): FileLoader<LoaderResult<FileSource>> {\n    return FileLoader.fromBytesMany([bytes]);\n  }\n\n  static fromBytesMany(\n    bytes: Array<Uint8Array | ArrayBuffer>,\n  ): FileLoader<LoaderResult<FileSource>> {\n    return new FileLoader(\n      async function* () {\n        for (const value of bytes) {\n          yield ok({ path: \"<memory>\", bytes: toUint8Array(value) });\n        }\n      },\n      \"source\",\n      false,\n    );\n  }\n\n  read(): FileLoader<LoaderResult<string>> {\n    return new FileLoader(this.source, \"read\", this.skipErrors);\n  }\n\n  readWithPath(): FileLoader<LoaderResult<FileReadWithPath>> {\n    return new FileLoader(this.source, \"readWithPath\", this.skipErrors);\n  }\n\n  ignoreErrors(): FileLoader<UnwrapLoaderResult<T>> {\n    return new FileLoader(this.source, this.mode, true) as FileLoader<UnwrapLoaderResult<T>>;\n  }\n\n  async *[Symbol.asyncIterator](): AsyncIterator<T> {\n    for await (const item of this.source()) {\n      const result = await mapFileResult(item, this.mode);\n      if (!result.ok && this.skipErrors) {\n        continue;\n      }\n      yield (this.skipErrors && result.ok ? result.value : result) as T;\n    }\n  }\n}\n\nexport class PdfFileLoader<T = LoaderResult<PdfSource>> implements AsyncIterable<T> {\n  private constructor(\n    private readonly source: () => AsyncIterable<LoaderResult<PdfSource>>,\n    private readonly mode: PdfMode,\n    private readonly skipErrors: boolean,\n  ) {}\n\n  static withGlob(pattern: string): PdfFileLoader<LoaderResult<PdfSource>> {\n    return new PdfFileLoader(\n      async function* () {\n        for (const path of await sortedGlob(pattern)) {\n          yield ok({ path });\n        }\n      },\n      \"source\",\n      false,\n    );\n  }\n\n  static withDir(directory: string): PdfFileLoader<LoaderResult<PdfSource>> {\n    return new PdfFileLoader(\n      async function* () {\n        const entries = await readdir(directory, { withFileTypes: true });\n        const files = entries\n          .filter((entry) => entry.isFile())\n          .map((entry) => join(directory, entry.name))\n          .sort((left, right) => left.localeCompare(right));\n        for (const path of files) {\n          yield ok({ path });\n        }\n      },\n      \"source\",\n      false,\n    );\n  }\n\n  static fromBytes(bytes: Uint8Array | ArrayBuffer): PdfFileLoader<LoaderResult<PdfSource>> {\n    return PdfFileLoader.fromBytesMany([bytes]);\n  }\n\n  static fromBytesMany(\n    bytes: Array<Uint8Array | ArrayBuffer>,\n  ): PdfFileLoader<LoaderResult<PdfSource>> {\n    return new PdfFileLoader(\n      async function* () {\n        for (const value of bytes) {\n          yield ok({ path: \"<memory>\", bytes: toUint8Array(value) });\n        }\n      },\n      \"source\",\n      false,\n    );\n  }\n\n  read(): PdfFileLoader<LoaderResult<string>> {\n    return new PdfFileLoader(this.source, \"read\", this.skipErrors);\n  }\n\n  readWithPath(): PdfFileLoader<LoaderResult<PdfReadWithPath>> {\n    return new PdfFileLoader(this.source, \"readWithPath\", this.skipErrors);\n  }\n\n  byPage(\n    this: PdfFileLoader<LoaderResult<PdfReadWithPath>>,\n  ): PdfFileLoader<LoaderResult<PdfPageWithPath>>;\n  byPage(this: PdfFileLoader<PdfReadWithPath>): PdfFileLoader<PdfPageWithPath>;\n  byPage(): PdfFileLoader<LoaderResult<PdfPage>>;\n  byPage(): PdfFileLoader<LoaderResult<PdfPage> | LoaderResult<PdfPageWithPath> | PdfPageWithPath> {\n    const mode = this.mode === \"readWithPath\" ? \"byPageWithPath\" : \"byPage\";\n    return new PdfFileLoader(this.source, mode, this.skipErrors);\n  }\n\n  ignoreErrors(): PdfFileLoader<UnwrapLoaderResult<T>> {\n    return new PdfFileLoader(this.source, this.mode, true) as PdfFileLoader<UnwrapLoaderResult<T>>;\n  }\n\n  async *[Symbol.asyncIterator](): AsyncIterator<T> {\n    for await (const item of this.source()) {\n      const results = await mapPdfResult(item, this.mode);\n      for (const result of results) {\n        if (!result.ok && this.skipErrors) {\n          continue;\n        }\n        yield (this.skipErrors && result.ok ? result.value : result) as T;\n      }\n    }\n  }\n}\n\nexport function fileToDocument(file: FileReadWithPath): Document {\n  return {\n    id: file.path,\n    text: file.text,\n    additionalProps: {\n      source: file.path,\n      mediaType: \"text/plain\",\n    },\n  };\n}\n\nexport async function fileLoaderToDocuments(\n  loader: AsyncIterable<FileReadWithPath>,\n): Promise<Document[]> {\n  const documents: Document[] = [];\n  for await (const file of loader) {\n    documents.push(fileToDocument(file));\n  }\n  return documents;\n}\n\nexport function pdfToDocument(pdf: PdfReadWithPath): Document {\n  return {\n    id: pdf.path,\n    text: pdf.text,\n    additionalProps: {\n      source: pdf.path,\n      mediaType: \"application/pdf\",\n    },\n  };\n}\n\nexport async function pdfLoaderToDocuments(\n  loader: AsyncIterable<PdfReadWithPath>,\n): Promise<Document[]> {\n  const documents: Document[] = [];\n  for await (const pdf of loader) {\n    documents.push(pdfToDocument(pdf));\n  }\n  return documents;\n}\n\nexport function pdfPageToDocument(page: PdfPageWithPath): Document {\n  return {\n    id: `${page.path}#page=${page.pageNumber}`,\n    text: page.text,\n    additionalProps: {\n      source: page.path,\n      mediaType: \"application/pdf\",\n      pageNumber: String(page.pageNumber),\n    },\n  };\n}\n\nexport async function pdfPageLoaderToDocuments(\n  loader: AsyncIterable<PdfPageWithPath>,\n): Promise<Document[]> {\n  const documents: Document[] = [];\n  for await (const page of loader) {\n    documents.push(pdfPageToDocument(page));\n  }\n  return documents;\n}\n\ntype LoaderValue<T> = T extends { ok: true; value: infer Value } ? Value : never;\ntype UnwrapLoaderResult<T> = [LoaderValue<T>] extends [never] ? T : LoaderValue<T>;\n\nfunction ok<T>(value: T): LoaderResult<T> {\n  return { ok: true, value };\n}\n\nfunction err(error: unknown): LoaderResult<never> {\n  return { ok: false, error };\n}\n\nasync function sortedGlob(pattern: string): Promise<string[]> {\n  return (await glob(pattern, { absolute: isAbsolute(pattern), onlyFiles: true })).sort(\n    (left, right) => left.localeCompare(right),\n  );\n}\n\nasync function mapFileResult(\n  item: LoaderResult<FileSource>,\n  mode: FileMode,\n): Promise<LoaderResult<FileSource | string | FileReadWithPath>> {\n  if (!item.ok) {\n    return item;\n  }\n  if (mode === \"source\") {\n    return item;\n  }\n  try {\n    const text = await readFileSource(item.value);\n    return mode === \"read\" ? ok(text) : ok({ path: item.value.path, text });\n  } catch (error) {\n    return err(error);\n  }\n}\n\nasync function mapPdfResult(\n  item: LoaderResult<PdfSource>,\n  mode: PdfMode,\n): Promise<Array<LoaderResult<PdfSource | string | PdfReadWithPath | PdfPage | PdfPageWithPath>>> {\n  if (!item.ok) {\n    return [item];\n  }\n  if (mode === \"source\") {\n    return [item];\n  }\n  try {\n    const pages = await readPdfPages(item.value);\n    if (mode === \"read\") {\n      return [ok(pages.map((page) => page.text).join(\"\"))];\n    }\n    if (mode === \"readWithPath\") {\n      return [ok({ path: item.value.path, text: pages.map((page) => page.text).join(\"\") })];\n    }\n    if (mode === \"byPageWithPath\") {\n      return pages.map((page) => ok({ path: item.value.path, ...page }));\n    }\n    return pages.map(ok);\n  } catch (error) {\n    return [err(error)];\n  }\n}\n\nasync function readFileSource(source: FileSource): Promise<string> {\n  if (\"bytes\" in source) {\n    return new TextDecoder().decode(source.bytes);\n  }\n  return readFile(source.path, \"utf8\");\n}\n\nasync function readPdfPages(source: PdfSource): Promise<PdfPage[]> {\n  const pdfjs = await import(\"pdfjs-dist/legacy/build/pdf.mjs\");\n  const bytes = \"bytes\" in source ? source.bytes : toUint8Array(await readFile(source.path));\n  const document = await pdfjs.getDocument({ data: bytes }).promise;\n  const pages: PdfPage[] = [];\n  try {\n    for (let index = 1; index <= document.numPages; index += 1) {\n      const page = await document.getPage(index);\n      const content = await page.getTextContent();\n      const text = content.items\n        .flatMap((item) =>\n          typeof item === \"object\" && item !== null && \"str\" in item ? [String(item.str)] : [],\n        )\n        .join(\"\\n\");\n      pages.push({ pageNumber: index - 1, text: text.length > 0 ? `${text}\\n` : \"\" });\n    }\n  } finally {\n    await document.destroy();\n  }\n  return pages;\n}\n\nfunction toUint8Array(bytes: Uint8Array | ArrayBuffer): Uint8Array {\n  if (bytes instanceof Uint8Array) {\n    return new Uint8Array(\n      bytes.buffer.slice(bytes.byteOffset, bytes.byteOffset + bytes.byteLength),\n    );\n  }\n  return new Uint8Array(bytes);\n}\n"],"mappings":";AAAA,SAAS,SAAS,gBAAgB;AAClC,SAAS,YAAY,YAAY;AACjC,SAAS,YAAY;AAgBd,IAAM,aAAN,MAAM,YAAqE;AAAA,EACxE,YACW,QACA,MACA,YACjB;AAHiB;AACA;AACA;AAAA,EAChB;AAAA,EAHgB;AAAA,EACA;AAAA,EACA;AAAA,EAGnB,OAAO,SAAS,SAAuD;AACrE,WAAO,IAAI;AAAA,MACT,mBAAmB;AACjB,mBAAW,QAAQ,MAAM,WAAW,OAAO,GAAG;AAC5C,gBAAM,GAAG,EAAE,KAAK,CAAC;AAAA,QACnB;AAAA,MACF;AAAA,MACA;AAAA,MACA;AAAA,IACF;AAAA,EACF;AAAA,EAEA,OAAO,QAAQ,WAAyD;AACtE,WAAO,IAAI;AAAA,MACT,mBAAmB;AACjB,cAAM,UAAU,MAAM,QAAQ,WAAW,EAAE,eAAe,KAAK,CAAC;AAChE,cAAM,QAAQ,QACX,OAAO,CAAC,UAAU,MAAM,OAAO,CAAC,EAChC,IAAI,CAAC,UAAU,KAAK,WAAW,MAAM,IAAI,CAAC,EAC1C,KAAK,CAAC,MAAM,UAAU,KAAK,cAAc,KAAK,CAAC;AAClD,mBAAW,QAAQ,OAAO;AACxB,gBAAM,GAAG,EAAE,KAAK,CAAC;AAAA,QACnB;AAAA,MACF;AAAA,MACA;AAAA,MACA;AAAA,IACF;AAAA,EACF;AAAA,EAEA,OAAO,UAAU,OAAuE;AACtF,WAAO,YAAW,cAAc,CAAC,KAAK,CAAC;AAAA,EACzC;AAAA,EAEA,OAAO,cACL,OACsC;AACtC,WAAO,IAAI;AAAA,MACT,mBAAmB;AACjB,mBAAW,SAAS,OAAO;AACzB,gBAAM,GAAG,EAAE,MAAM,YAAY,OAAO,aAAa,KAAK,EAAE,CAAC;AAAA,QAC3D;AAAA,MACF;AAAA,MACA;AAAA,MACA;AAAA,IACF;AAAA,EACF;AAAA,EAEA,OAAyC;AACvC,WAAO,IAAI,YAAW,KAAK,QAAQ,QAAQ,KAAK,UAAU;AAAA,EAC5D;AAAA,EAEA,eAA2D;AACzD,WAAO,IAAI,YAAW,KAAK,QAAQ,gBAAgB,KAAK,UAAU;AAAA,EACpE;AAAA,EAEA,eAAkD;AAChD,WAAO,IAAI,YAAW,KAAK,QAAQ,KAAK,MAAM,IAAI;AAAA,EACpD;AAAA,EAEA,QAAQ,OAAO,aAAa,IAAsB;AAChD,qBAAiB,QAAQ,KAAK,OAAO,GAAG;AACtC,YAAM,SAAS,MAAM,cAAc,MAAM,KAAK,IAAI;AAClD,UAAI,CAAC,OAAO,MAAM,KAAK,YAAY;AACjC;AAAA,MACF;AACA,YAAO,KAAK,cAAc,OAAO,KAAK,OAAO,QAAQ;AAAA,IACvD;AAAA,EACF;AACF;AAEO,IAAM,gBAAN,MAAM,eAAuE;AAAA,EAC1E,YACW,QACA,MACA,YACjB;AAHiB;AACA;AACA;AAAA,EAChB;AAAA,EAHgB;AAAA,EACA;AAAA,EACA;AAAA,EAGnB,OAAO,SAAS,SAAyD;AACvE,WAAO,IAAI;AAAA,MACT,mBAAmB;AACjB,mBAAW,QAAQ,MAAM,WAAW,OAAO,GAAG;AAC5C,gBAAM,GAAG,EAAE,KAAK,CAAC;AAAA,QACnB;AAAA,MACF;AAAA,MACA;AAAA,MACA;AAAA,IACF;AAAA,EACF;AAAA,EAEA,OAAO,QAAQ,WAA2D;AACxE,WAAO,IAAI;AAAA,MACT,mBAAmB;AACjB,cAAM,UAAU,MAAM,QAAQ,WAAW,EAAE,eAAe,KAAK,CAAC;AAChE,cAAM,QAAQ,QACX,OAAO,CAAC,UAAU,MAAM,OAAO,CAAC,EAChC,IAAI,CAAC,UAAU,KAAK,WAAW,MAAM,IAAI,CAAC,EAC1C,KAAK,CAAC,MAAM,UAAU,KAAK,cAAc,KAAK,CAAC;AAClD,mBAAW,QAAQ,OAAO;AACxB,gBAAM,GAAG,EAAE,KAAK,CAAC;AAAA,QACnB;AAAA,MACF;AAAA,MACA;AAAA,MACA;AAAA,IACF;AAAA,EACF;AAAA,EAEA,OAAO,UAAU,OAAyE;AACxF,WAAO,eAAc,cAAc,CAAC,KAAK,CAAC;AAAA,EAC5C;AAAA,EAEA,OAAO,cACL,OACwC;AACxC,WAAO,IAAI;AAAA,MACT,mBAAmB;AACjB,mBAAW,SAAS,OAAO;AACzB,gBAAM,GAAG,EAAE,MAAM,YAAY,OAAO,aAAa,KAAK,EAAE,CAAC;AAAA,QAC3D;AAAA,MACF;AAAA,MACA;AAAA,MACA;AAAA,IACF;AAAA,EACF;AAAA,EAEA,OAA4C;AAC1C,WAAO,IAAI,eAAc,KAAK,QAAQ,QAAQ,KAAK,UAAU;AAAA,EAC/D;AAAA,EAEA,eAA6D;AAC3D,WAAO,IAAI,eAAc,KAAK,QAAQ,gBAAgB,KAAK,UAAU;AAAA,EACvE;AAAA,EAOA,SAAiG;AAC/F,UAAM,OAAO,KAAK,SAAS,iBAAiB,mBAAmB;AAC/D,WAAO,IAAI,eAAc,KAAK,QAAQ,MAAM,KAAK,UAAU;AAAA,EAC7D;AAAA,EAEA,eAAqD;AACnD,WAAO,IAAI,eAAc,KAAK,QAAQ,KAAK,MAAM,IAAI;AAAA,EACvD;AAAA,EAEA,QAAQ,OAAO,aAAa,IAAsB;AAChD,qBAAiB,QAAQ,KAAK,OAAO,GAAG;AACtC,YAAM,UAAU,MAAM,aAAa,MAAM,KAAK,IAAI;AAClD,iBAAW,UAAU,SAAS;AAC5B,YAAI,CAAC,OAAO,MAAM,KAAK,YAAY;AACjC;AAAA,QACF;AACA,cAAO,KAAK,cAAc,OAAO,KAAK,OAAO,QAAQ;AAAA,MACvD;AAAA,IACF;AAAA,EACF;AACF;AAEO,SAAS,eAAe,MAAkC;AAC/D,SAAO;AAAA,IACL,IAAI,KAAK;AAAA,IACT,MAAM,KAAK;AAAA,IACX,iBAAiB;AAAA,MACf,QAAQ,KAAK;AAAA,MACb,WAAW;AAAA,IACb;AAAA,EACF;AACF;AAEA,eAAsB,sBACpB,QACqB;AACrB,QAAM,YAAwB,CAAC;AAC/B,mBAAiB,QAAQ,QAAQ;AAC/B,cAAU,KAAK,eAAe,IAAI,CAAC;AAAA,EACrC;AACA,SAAO;AACT;AAEO,SAAS,cAAc,KAAgC;AAC5D,SAAO;AAAA,IACL,IAAI,IAAI;AAAA,IACR,MAAM,IAAI;AAAA,IACV,iBAAiB;AAAA,MACf,QAAQ,IAAI;AAAA,MACZ,WAAW;AAAA,IACb;AAAA,EACF;AACF;AAEA,eAAsB,qBACpB,QACqB;AACrB,QAAM,YAAwB,CAAC;AAC/B,mBAAiB,OAAO,QAAQ;AAC9B,cAAU,KAAK,cAAc,GAAG,CAAC;AAAA,EACnC;AACA,SAAO;AACT;AAEO,SAAS,kBAAkB,MAAiC;AACjE,SAAO;AAAA,IACL,IAAI,GAAG,KAAK,IAAI,SAAS,KAAK,UAAU;AAAA,IACxC,MAAM,KAAK;AAAA,IACX,iBAAiB;AAAA,MACf,QAAQ,KAAK;AAAA,MACb,WAAW;AAAA,MACX,YAAY,OAAO,KAAK,UAAU;AAAA,IACpC;AAAA,EACF;AACF;AAEA,eAAsB,yBACpB,QACqB;AACrB,QAAM,YAAwB,CAAC;AAC/B,mBAAiB,QAAQ,QAAQ;AAC/B,cAAU,KAAK,kBAAkB,IAAI,CAAC;AAAA,EACxC;AACA,SAAO;AACT;AAKA,SAAS,GAAM,OAA2B;AACxC,SAAO,EAAE,IAAI,MAAM,MAAM;AAC3B;AAEA,SAAS,IAAI,OAAqC;AAChD,SAAO,EAAE,IAAI,OAAO,MAAM;AAC5B;AAEA,eAAe,WAAW,SAAoC;AAC5D,UAAQ,MAAM,KAAK,SAAS,EAAE,UAAU,WAAW,OAAO,GAAG,WAAW,KAAK,CAAC,GAAG;AAAA,IAC/E,CAAC,MAAM,UAAU,KAAK,cAAc,KAAK;AAAA,EAC3C;AACF;AAEA,eAAe,cACb,MACA,MAC+D;AAC/D,MAAI,CAAC,KAAK,IAAI;AACZ,WAAO;AAAA,EACT;AACA,MAAI,SAAS,UAAU;AACrB,WAAO;AAAA,EACT;AACA,MAAI;AACF,UAAM,OAAO,MAAM,eAAe,KAAK,KAAK;AAC5C,WAAO,SAAS,SAAS,GAAG,IAAI,IAAI,GAAG,EAAE,MAAM,KAAK,MAAM,MAAM,KAAK,CAAC;AAAA,EACxE,SAAS,OAAO;AACd,WAAO,IAAI,KAAK;AAAA,EAClB;AACF;AAEA,eAAe,aACb,MACA,MACgG;AAChG,MAAI,CAAC,KAAK,IAAI;AACZ,WAAO,CAAC,IAAI;AAAA,EACd;AACA,MAAI,SAAS,UAAU;AACrB,WAAO,CAAC,IAAI;AAAA,EACd;AACA,MAAI;AACF,UAAM,QAAQ,MAAM,aAAa,KAAK,KAAK;AAC3C,QAAI,SAAS,QAAQ;AACnB,aAAO,CAAC,GAAG,MAAM,IAAI,CAAC,SAAS,KAAK,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;AAAA,IACrD;AACA,QAAI,SAAS,gBAAgB;AAC3B,aAAO,CAAC,GAAG,EAAE,MAAM,KAAK,MAAM,MAAM,MAAM,MAAM,IAAI,CAAC,SAAS,KAAK,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC;AAAA,IACtF;AACA,QAAI,SAAS,kBAAkB;AAC7B,aAAO,MAAM,IAAI,CAAC,SAAS,GAAG,EAAE,MAAM,KAAK,MAAM,MAAM,GAAG,KAAK,CAAC,CAAC;AAAA,IACnE;AACA,WAAO,MAAM,IAAI,EAAE;AAAA,EACrB,SAAS,OAAO;AACd,WAAO,CAAC,IAAI,KAAK,CAAC;AAAA,EACpB;AACF;AAEA,eAAe,eAAe,QAAqC;AACjE,MAAI,WAAW,QAAQ;AACrB,WAAO,IAAI,YAAY,EAAE,OAAO,OAAO,KAAK;AAAA,EAC9C;AACA,SAAO,SAAS,OAAO,MAAM,MAAM;AACrC;AAEA,eAAe,aAAa,QAAuC;AACjE,QAAM,QAAQ,MAAM,OAAO,iCAAiC;AAC5D,QAAM,QAAQ,WAAW,SAAS,OAAO,QAAQ,aAAa,MAAM,SAAS,OAAO,IAAI,CAAC;AACzF,QAAM,WAAW,MAAM,MAAM,YAAY,EAAE,MAAM,MAAM,CAAC,EAAE;AAC1D,QAAM,QAAmB,CAAC;AAC1B,MAAI;AACF,aAAS,QAAQ,GAAG,SAAS,SAAS,UAAU,SAAS,GAAG;AAC1D,YAAM,OAAO,MAAM,SAAS,QAAQ,KAAK;AACzC,YAAM,UAAU,MAAM,KAAK,eAAe;AAC1C,YAAM,OAAO,QAAQ,MAClB;AAAA,QAAQ,CAAC,SACR,OAAO,SAAS,YAAY,SAAS,QAAQ,SAAS,OAAO,CAAC,OAAO,KAAK,GAAG,CAAC,IAAI,CAAC;AAAA,MACrF,EACC,KAAK,IAAI;AACZ,YAAM,KAAK,EAAE,YAAY,QAAQ,GAAG,MAAM,KAAK,SAAS,IAAI,GAAG,IAAI;AAAA,IAAO,GAAG,CAAC;AAAA,IAChF;AAAA,EACF,UAAE;AACA,UAAM,SAAS,QAAQ;AAAA,EACzB;AACA,SAAO;AACT;AAEA,SAAS,aAAa,OAA6C;AACjE,MAAI,iBAAiB,YAAY;AAC/B,WAAO,IAAI;AAAA,MACT,MAAM,OAAO,MAAM,MAAM,YAAY,MAAM,aAAa,MAAM,UAAU;AAAA,IAC1E;AAAA,EACF;AACA,SAAO,IAAI,WAAW,KAAK;AAC7B;","names":[]}
+{"version":3,"sources":["../../src/loaders/index.ts"],"sourcesContent":["import { readdir, readFile } from \"node:fs/promises\";\nimport { isAbsolute, join } from \"node:path\";\nimport { glob } from \"tinyglobby\";\nimport type { Document } from \"../completion\";\n\nexport type LoaderResult<T> = { ok: true; value: T } | { ok: false; error: unknown };\n\nexport type FileSource = { path: string } | { path: \"<memory>\"; bytes: Uint8Array };\nexport type FileReadWithPath = { path: string; text: string };\n\nexport type PdfSource = { path: string } | { path: \"<memory>\"; bytes: Uint8Array };\nexport type PdfReadWithPath = { path: string; text: string };\nexport type PdfPage = { pageNumber: number; text: string };\nexport type PdfPageWithPath = { path: string; pageNumber: number; text: string };\n\ntype FileMode = \"source\" | \"read\" | \"readWithPath\";\ntype PdfMode = \"source\" | \"read\" | \"readWithPath\" | \"byPage\" | \"byPageWithPath\";\n\nexport class FileLoader<T = LoaderResult<FileSource>> implements AsyncIterable<T> {\n  private constructor(\n    private readonly source: () => AsyncIterable<LoaderResult<FileSource>>,\n    private readonly mode: FileMode,\n    private readonly skipErrors: boolean,\n  ) {}\n\n  static withGlob(pattern: string): FileLoader<LoaderResult<FileSource>> {\n    return new FileLoader(\n      async function* () {\n        for (const path of await sortedGlob(pattern)) {\n          yield ok({ path });\n        }\n      },\n      \"source\",\n      false,\n    );\n  }\n\n  static withDir(directory: string): FileLoader<LoaderResult<FileSource>> {\n    return new FileLoader(\n      async function* () {\n        const entries = await readdir(directory, { withFileTypes: true });\n        const files = entries\n          .filter((entry) => entry.isFile())\n          .map((entry) => join(directory, entry.name))\n          .sort((left, right) => left.localeCompare(right));\n        for (const path of files) {\n          yield ok({ path });\n        }\n      },\n      \"source\",\n      false,\n    );\n  }\n\n  static fromBytes(bytes: Uint8Array | ArrayBuffer): FileLoader<LoaderResult<FileSource>> {\n    return FileLoader.fromBytesMany([bytes]);\n  }\n\n  static fromBytesMany(\n    bytes: Array<Uint8Array | ArrayBuffer>,\n  ): FileLoader<LoaderResult<FileSource>> {\n    return new FileLoader(\n      async function* () {\n        for (const value of bytes) {\n          yield ok({ path: \"<memory>\", bytes: toUint8Array(value) });\n        }\n      },\n      \"source\",\n      false,\n    );\n  }\n\n  read(): FileLoader<LoaderResult<string>> {\n    return new FileLoader(this.source, \"read\", this.skipErrors);\n  }\n\n  readWithPath(): FileLoader<LoaderResult<FileReadWithPath>> {\n    return new FileLoader(this.source, \"readWithPath\", this.skipErrors);\n  }\n\n  ignoreErrors(): FileLoader<UnwrapLoaderResult<T>> {\n    return new FileLoader(this.source, this.mode, true) as FileLoader<UnwrapLoaderResult<T>>;\n  }\n\n  async *[Symbol.asyncIterator](): AsyncIterator<T> {\n    for await (const item of this.source()) {\n      const result = await mapFileResult(item, this.mode);\n      if (!result.ok && this.skipErrors) {\n        continue;\n      }\n      yield (this.skipErrors && result.ok ? result.value : result) as T;\n    }\n  }\n}\n\nexport class PdfFileLoader<T = LoaderResult<PdfSource>> implements AsyncIterable<T> {\n  private constructor(\n    private readonly source: () => AsyncIterable<LoaderResult<PdfSource>>,\n    private readonly mode: PdfMode,\n    private readonly skipErrors: boolean,\n  ) {}\n\n  static withGlob(pattern: string): PdfFileLoader<LoaderResult<PdfSource>> {\n    return new PdfFileLoader(\n      async function* () {\n        for (const path of await sortedGlob(pattern)) {\n          yield ok({ path });\n        }\n      },\n      \"source\",\n      false,\n    );\n  }\n\n  static withDir(directory: string): PdfFileLoader<LoaderResult<PdfSource>> {\n    return new PdfFileLoader(\n      async function* () {\n        const entries = await readdir(directory, { withFileTypes: true });\n        const files = entries\n          .filter((entry) => entry.isFile())\n          .map((entry) => join(directory, entry.name))\n          .sort((left, right) => left.localeCompare(right));\n        for (const path of files) {\n          yield ok({ path });\n        }\n      },\n      \"source\",\n      false,\n    );\n  }\n\n  static fromBytes(bytes: Uint8Array | ArrayBuffer): PdfFileLoader<LoaderResult<PdfSource>> {\n    return PdfFileLoader.fromBytesMany([bytes]);\n  }\n\n  static fromBytesMany(\n    bytes: Array<Uint8Array | ArrayBuffer>,\n  ): PdfFileLoader<LoaderResult<PdfSource>> {\n    return new PdfFileLoader(\n      async function* () {\n        for (const value of bytes) {\n          yield ok({ path: \"<memory>\", bytes: toUint8Array(value) });\n        }\n      },\n      \"source\",\n      false,\n    );\n  }\n\n  read(): PdfFileLoader<LoaderResult<string>> {\n    return new PdfFileLoader(this.source, \"read\", this.skipErrors);\n  }\n\n  readWithPath(): PdfFileLoader<LoaderResult<PdfReadWithPath>> {\n    return new PdfFileLoader(this.source, \"readWithPath\", this.skipErrors);\n  }\n\n  byPage(\n    this: PdfFileLoader<LoaderResult<PdfReadWithPath>>,\n  ): PdfFileLoader<LoaderResult<PdfPageWithPath>>;\n  byPage(this: PdfFileLoader<PdfReadWithPath>): PdfFileLoader<PdfPageWithPath>;\n  byPage(): PdfFileLoader<LoaderResult<PdfPage>>;\n  byPage(): PdfFileLoader<LoaderResult<PdfPage> | LoaderResult<PdfPageWithPath> | PdfPageWithPath> {\n    const mode = this.mode === \"readWithPath\" ? \"byPageWithPath\" : \"byPage\";\n    return new PdfFileLoader(this.source, mode, this.skipErrors);\n  }\n\n  ignoreErrors(): PdfFileLoader<UnwrapLoaderResult<T>> {\n    return new PdfFileLoader(this.source, this.mode, true) as PdfFileLoader<UnwrapLoaderResult<T>>;\n  }\n\n  async *[Symbol.asyncIterator](): AsyncIterator<T> {\n    for await (const item of this.source()) {\n      const results = await mapPdfResult(item, this.mode);\n      for (const result of results) {\n        if (!result.ok && this.skipErrors) {\n          continue;\n        }\n        yield (this.skipErrors && result.ok ? result.value : result) as T;\n      }\n    }\n  }\n}\n\nexport function fileToDocument(file: FileReadWithPath): Document {\n  return {\n    id: file.path,\n    text: file.text,\n    additionalProps: {\n      source: file.path,\n      mediaType: \"text/plain\",\n    },\n  };\n}\n\nexport async function fileLoaderToDocuments(\n  loader: AsyncIterable<FileReadWithPath>,\n): Promise<Document[]> {\n  const documents: Document[] = [];\n  for await (const file of loader) {\n    documents.push(fileToDocument(file));\n  }\n  return documents;\n}\n\nexport function pdfToDocument(pdf: PdfReadWithPath): Document {\n  return {\n    id: pdf.path,\n    text: pdf.text,\n    additionalProps: {\n      source: pdf.path,\n      mediaType: \"application/pdf\",\n    },\n  };\n}\n\nexport async function pdfLoaderToDocuments(\n  loader: AsyncIterable<PdfReadWithPath>,\n): Promise<Document[]> {\n  const documents: Document[] = [];\n  for await (const pdf of loader) {\n    documents.push(pdfToDocument(pdf));\n  }\n  return documents;\n}\n\nexport function pdfPageToDocument(page: PdfPageWithPath): Document {\n  return {\n    id: `${page.path}#page=${page.pageNumber}`,\n    text: page.text,\n    additionalProps: {\n      source: page.path,\n      mediaType: \"application/pdf\",\n      pageNumber: String(page.pageNumber),\n    },\n  };\n}\n\nexport async function pdfPageLoaderToDocuments(\n  loader: AsyncIterable<PdfPageWithPath>,\n): Promise<Document[]> {\n  const documents: Document[] = [];\n  for await (const page of loader) {\n    documents.push(pdfPageToDocument(page));\n  }\n  return documents;\n}\n\ntype LoaderValue<T> = T extends { ok: true; value: infer Value } ? Value : never;\ntype UnwrapLoaderResult<T> = [LoaderValue<T>] extends [never] ? T : LoaderValue<T>;\n\nfunction ok<T>(value: T): LoaderResult<T> {\n  return { ok: true, value };\n}\n\nfunction err(error: unknown): LoaderResult<never> {\n  return { ok: false, error };\n}\n\nasync function sortedGlob(pattern: string): Promise<string[]> {\n  return (await glob(pattern, { absolute: isAbsolute(pattern), onlyFiles: true })).sort(\n    (left, right) => left.localeCompare(right),\n  );\n}\n\nasync function mapFileResult(\n  item: LoaderResult<FileSource>,\n  mode: FileMode,\n): Promise<LoaderResult<FileSource | string | FileReadWithPath>> {\n  if (!item.ok) {\n    return item;\n  }\n  if (mode === \"source\") {\n    return item;\n  }\n  try {\n    const text = await readFileSource(item.value);\n    return mode === \"read\" ? ok(text) : ok({ path: item.value.path, text });\n  } catch (error) {\n    return err(error);\n  }\n}\n\nasync function mapPdfResult(\n  item: LoaderResult<PdfSource>,\n  mode: PdfMode,\n): Promise<Array<LoaderResult<PdfSource | string | PdfReadWithPath | PdfPage | PdfPageWithPath>>> {\n  if (!item.ok) {\n    return [item];\n  }\n  if (mode === \"source\") {\n    return [item];\n  }\n  try {\n    const pages = await readPdfPages(item.value);\n    if (mode === \"read\") {\n      return [ok(pages.map((page) => page.text).join(\"\"))];\n    }\n    if (mode === \"readWithPath\") {\n      return [ok({ path: item.value.path, text: pages.map((page) => page.text).join(\"\") })];\n    }\n    if (mode === \"byPageWithPath\") {\n      return pages.map((page) => ok({ path: item.value.path, ...page }));\n    }\n    return pages.map(ok);\n  } catch (error) {\n    return [err(error)];\n  }\n}\n\nasync function readFileSource(source: FileSource): Promise<string> {\n  if (\"bytes\" in source) {\n    return new TextDecoder().decode(source.bytes);\n  }\n  return readFile(source.path, \"utf8\");\n}\n\nasync function readPdfPages(source: PdfSource): Promise<PdfPage[]> {\n  const pdfjs = await import(\"pdfjs-dist/legacy/build/pdf.mjs\");\n  const bytes = \"bytes\" in source ? source.bytes : toUint8Array(await readFile(source.path));\n  const loadingTask = pdfjs.getDocument({ data: bytes });\n  const document = await loadingTask.promise;\n  const pages: PdfPage[] = [];\n  try {\n    for (let index = 1; index <= document.numPages; index += 1) {\n      const page = await document.getPage(index);\n      const content = await page.getTextContent();\n      const text = content.items\n        .flatMap((item) =>\n          typeof item === \"object\" && item !== null && \"str\" in item ? [String(item.str)] : [],\n        )\n        .join(\"\\n\");\n      pages.push({ pageNumber: index - 1, text: text.length > 0 ? `${text}\\n` : \"\" });\n    }\n  } finally {\n    await loadingTask.destroy();\n  }\n  return pages;\n}\n\nfunction toUint8Array(bytes: Uint8Array | ArrayBuffer): Uint8Array {\n  if (bytes instanceof Uint8Array) {\n    return new Uint8Array(\n      bytes.buffer.slice(bytes.byteOffset, bytes.byteOffset + bytes.byteLength),\n    );\n  }\n  return new Uint8Array(bytes);\n}\n"],"mappings":";AAAA,SAAS,SAAS,gBAAgB;AAClC,SAAS,YAAY,YAAY;AACjC,SAAS,YAAY;AAgBd,IAAM,aAAN,MAAM,YAAqE;AAAA,EACxE,YACW,QACA,MACA,YACjB;AAHiB;AACA;AACA;AAAA,EAChB;AAAA,EAHgB;AAAA,EACA;AAAA,EACA;AAAA,EAGnB,OAAO,SAAS,SAAuD;AACrE,WAAO,IAAI;AAAA,MACT,mBAAmB;AACjB,mBAAW,QAAQ,MAAM,WAAW,OAAO,GAAG;AAC5C,gBAAM,GAAG,EAAE,KAAK,CAAC;AAAA,QACnB;AAAA,MACF;AAAA,MACA;AAAA,MACA;AAAA,IACF;AAAA,EACF;AAAA,EAEA,OAAO,QAAQ,WAAyD;AACtE,WAAO,IAAI;AAAA,MACT,mBAAmB;AACjB,cAAM,UAAU,MAAM,QAAQ,WAAW,EAAE,eAAe,KAAK,CAAC;AAChE,cAAM,QAAQ,QACX,OAAO,CAAC,UAAU,MAAM,OAAO,CAAC,EAChC,IAAI,CAAC,UAAU,KAAK,WAAW,MAAM,IAAI,CAAC,EAC1C,KAAK,CAAC,MAAM,UAAU,KAAK,cAAc,KAAK,CAAC;AAClD,mBAAW,QAAQ,OAAO;AACxB,gBAAM,GAAG,EAAE,KAAK,CAAC;AAAA,QACnB;AAAA,MACF;AAAA,MACA;AAAA,MACA;AAAA,IACF;AAAA,EACF;AAAA,EAEA,OAAO,UAAU,OAAuE;AACtF,WAAO,YAAW,cAAc,CAAC,KAAK,CAAC;AAAA,EACzC;AAAA,EAEA,OAAO,cACL,OACsC;AACtC,WAAO,IAAI;AAAA,MACT,mBAAmB;AACjB,mBAAW,SAAS,OAAO;AACzB,gBAAM,GAAG,EAAE,MAAM,YAAY,OAAO,aAAa,KAAK,EAAE,CAAC;AAAA,QAC3D;AAAA,MACF;AAAA,MACA;AAAA,MACA;AAAA,IACF;AAAA,EACF;AAAA,EAEA,OAAyC;AACvC,WAAO,IAAI,YAAW,KAAK,QAAQ,QAAQ,KAAK,UAAU;AAAA,EAC5D;AAAA,EAEA,eAA2D;AACzD,WAAO,IAAI,YAAW,KAAK,QAAQ,gBAAgB,KAAK,UAAU;AAAA,EACpE;AAAA,EAEA,eAAkD;AAChD,WAAO,IAAI,YAAW,KAAK,QAAQ,KAAK,MAAM,IAAI;AAAA,EACpD;AAAA,EAEA,QAAQ,OAAO,aAAa,IAAsB;AAChD,qBAAiB,QAAQ,KAAK,OAAO,GAAG;AACtC,YAAM,SAAS,MAAM,cAAc,MAAM,KAAK,IAAI;AAClD,UAAI,CAAC,OAAO,MAAM,KAAK,YAAY;AACjC;AAAA,MACF;AACA,YAAO,KAAK,cAAc,OAAO,KAAK,OAAO,QAAQ;AAAA,IACvD;AAAA,EACF;AACF;AAEO,IAAM,gBAAN,MAAM,eAAuE;AAAA,EAC1E,YACW,QACA,MACA,YACjB;AAHiB;AACA;AACA;AAAA,EAChB;AAAA,EAHgB;AAAA,EACA;AAAA,EACA;AAAA,EAGnB,OAAO,SAAS,SAAyD;AACvE,WAAO,IAAI;AAAA,MACT,mBAAmB;AACjB,mBAAW,QAAQ,MAAM,WAAW,OAAO,GAAG;AAC5C,gBAAM,GAAG,EAAE,KAAK,CAAC;AAAA,QACnB;AAAA,MACF;AAAA,MACA;AAAA,MACA;AAAA,IACF;AAAA,EACF;AAAA,EAEA,OAAO,QAAQ,WAA2D;AACxE,WAAO,IAAI;AAAA,MACT,mBAAmB;AACjB,cAAM,UAAU,MAAM,QAAQ,WAAW,EAAE,eAAe,KAAK,CAAC;AAChE,cAAM,QAAQ,QACX,OAAO,CAAC,UAAU,MAAM,OAAO,CAAC,EAChC,IAAI,CAAC,UAAU,KAAK,WAAW,MAAM,IAAI,CAAC,EAC1C,KAAK,CAAC,MAAM,UAAU,KAAK,cAAc,KAAK,CAAC;AAClD,mBAAW,QAAQ,OAAO;AACxB,gBAAM,GAAG,EAAE,KAAK,CAAC;AAAA,QACnB;AAAA,MACF;AAAA,MACA;AAAA,MACA;AAAA,IACF;AAAA,EACF;AAAA,EAEA,OAAO,UAAU,OAAyE;AACxF,WAAO,eAAc,cAAc,CAAC,KAAK,CAAC;AAAA,EAC5C;AAAA,EAEA,OAAO,cACL,OACwC;AACxC,WAAO,IAAI;AAAA,MACT,mBAAmB;AACjB,mBAAW,SAAS,OAAO;AACzB,gBAAM,GAAG,EAAE,MAAM,YAAY,OAAO,aAAa,KAAK,EAAE,CAAC;AAAA,QAC3D;AAAA,MACF;AAAA,MACA;AAAA,MACA;AAAA,IACF;AAAA,EACF;AAAA,EAEA,OAA4C;AAC1C,WAAO,IAAI,eAAc,KAAK,QAAQ,QAAQ,KAAK,UAAU;AAAA,EAC/D;AAAA,EAEA,eAA6D;AAC3D,WAAO,IAAI,eAAc,KAAK,QAAQ,gBAAgB,KAAK,UAAU;AAAA,EACvE;AAAA,EAOA,SAAiG;AAC/F,UAAM,OAAO,KAAK,SAAS,iBAAiB,mBAAmB;AAC/D,WAAO,IAAI,eAAc,KAAK,QAAQ,MAAM,KAAK,UAAU;AAAA,EAC7D;AAAA,EAEA,eAAqD;AACnD,WAAO,IAAI,eAAc,KAAK,QAAQ,KAAK,MAAM,IAAI;AAAA,EACvD;AAAA,EAEA,QAAQ,OAAO,aAAa,IAAsB;AAChD,qBAAiB,QAAQ,KAAK,OAAO,GAAG;AACtC,YAAM,UAAU,MAAM,aAAa,MAAM,KAAK,IAAI;AAClD,iBAAW,UAAU,SAAS;AAC5B,YAAI,CAAC,OAAO,MAAM,KAAK,YAAY;AACjC;AAAA,QACF;AACA,cAAO,KAAK,cAAc,OAAO,KAAK,OAAO,QAAQ;AAAA,MACvD;AAAA,IACF;AAAA,EACF;AACF;AAEO,SAAS,eAAe,MAAkC;AAC/D,SAAO;AAAA,IACL,IAAI,KAAK;AAAA,IACT,MAAM,KAAK;AAAA,IACX,iBAAiB;AAAA,MACf,QAAQ,KAAK;AAAA,MACb,WAAW;AAAA,IACb;AAAA,EACF;AACF;AAEA,eAAsB,sBACpB,QACqB;AACrB,QAAM,YAAwB,CAAC;AAC/B,mBAAiB,QAAQ,QAAQ;AAC/B,cAAU,KAAK,eAAe,IAAI,CAAC;AAAA,EACrC;AACA,SAAO;AACT;AAEO,SAAS,cAAc,KAAgC;AAC5D,SAAO;AAAA,IACL,IAAI,IAAI;AAAA,IACR,MAAM,IAAI;AAAA,IACV,iBAAiB;AAAA,MACf,QAAQ,IAAI;AAAA,MACZ,WAAW;AAAA,IACb;AAAA,EACF;AACF;AAEA,eAAsB,qBACpB,QACqB;AACrB,QAAM,YAAwB,CAAC;AAC/B,mBAAiB,OAAO,QAAQ;AAC9B,cAAU,KAAK,cAAc,GAAG,CAAC;AAAA,EACnC;AACA,SAAO;AACT;AAEO,SAAS,kBAAkB,MAAiC;AACjE,SAAO;AAAA,IACL,IAAI,GAAG,KAAK,IAAI,SAAS,KAAK,UAAU;AAAA,IACxC,MAAM,KAAK;AAAA,IACX,iBAAiB;AAAA,MACf,QAAQ,KAAK;AAAA,MACb,WAAW;AAAA,MACX,YAAY,OAAO,KAAK,UAAU;AAAA,IACpC;AAAA,EACF;AACF;AAEA,eAAsB,yBACpB,QACqB;AACrB,QAAM,YAAwB,CAAC;AAC/B,mBAAiB,QAAQ,QAAQ;AAC/B,cAAU,KAAK,kBAAkB,IAAI,CAAC;AAAA,EACxC;AACA,SAAO;AACT;AAKA,SAAS,GAAM,OAA2B;AACxC,SAAO,EAAE,IAAI,MAAM,MAAM;AAC3B;AAEA,SAAS,IAAI,OAAqC;AAChD,SAAO,EAAE,IAAI,OAAO,MAAM;AAC5B;AAEA,eAAe,WAAW,SAAoC;AAC5D,UAAQ,MAAM,KAAK,SAAS,EAAE,UAAU,WAAW,OAAO,GAAG,WAAW,KAAK,CAAC,GAAG;AAAA,IAC/E,CAAC,MAAM,UAAU,KAAK,cAAc,KAAK;AAAA,EAC3C;AACF;AAEA,eAAe,cACb,MACA,MAC+D;AAC/D,MAAI,CAAC,KAAK,IAAI;AACZ,WAAO;AAAA,EACT;AACA,MAAI,SAAS,UAAU;AACrB,WAAO;AAAA,EACT;AACA,MAAI;AACF,UAAM,OAAO,MAAM,eAAe,KAAK,KAAK;AAC5C,WAAO,SAAS,SAAS,GAAG,IAAI,IAAI,GAAG,EAAE,MAAM,KAAK,MAAM,MAAM,KAAK,CAAC;AAAA,EACxE,SAAS,OAAO;AACd,WAAO,IAAI,KAAK;AAAA,EAClB;AACF;AAEA,eAAe,aACb,MACA,MACgG;AAChG,MAAI,CAAC,KAAK,IAAI;AACZ,WAAO,CAAC,IAAI;AAAA,EACd;AACA,MAAI,SAAS,UAAU;AACrB,WAAO,CAAC,IAAI;AAAA,EACd;AACA,MAAI;AACF,UAAM,QAAQ,MAAM,aAAa,KAAK,KAAK;AAC3C,QAAI,SAAS,QAAQ;AACnB,aAAO,CAAC,GAAG,MAAM,IAAI,CAAC,SAAS,KAAK,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;AAAA,IACrD;AACA,QAAI,SAAS,gBAAgB;AAC3B,aAAO,CAAC,GAAG,EAAE,MAAM,KAAK,MAAM,MAAM,MAAM,MAAM,IAAI,CAAC,SAAS,KAAK,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC;AAAA,IACtF;AACA,QAAI,SAAS,kBAAkB;AAC7B,aAAO,MAAM,IAAI,CAAC,SAAS,GAAG,EAAE,MAAM,KAAK,MAAM,MAAM,GAAG,KAAK,CAAC,CAAC;AAAA,IACnE;AACA,WAAO,MAAM,IAAI,EAAE;AAAA,EACrB,SAAS,OAAO;AACd,WAAO,CAAC,IAAI,KAAK,CAAC;AAAA,EACpB;AACF;AAEA,eAAe,eAAe,QAAqC;AACjE,MAAI,WAAW,QAAQ;AACrB,WAAO,IAAI,YAAY,EAAE,OAAO,OAAO,KAAK;AAAA,EAC9C;AACA,SAAO,SAAS,OAAO,MAAM,MAAM;AACrC;AAEA,eAAe,aAAa,QAAuC;AACjE,QAAM,QAAQ,MAAM,OAAO,iCAAiC;AAC5D,QAAM,QAAQ,WAAW,SAAS,OAAO,QAAQ,aAAa,MAAM,SAAS,OAAO,IAAI,CAAC;AACzF,QAAM,cAAc,MAAM,YAAY,EAAE,MAAM,MAAM,CAAC;AACrD,QAAM,WAAW,MAAM,YAAY;AACnC,QAAM,QAAmB,CAAC;AAC1B,MAAI;AACF,aAAS,QAAQ,GAAG,SAAS,SAAS,UAAU,SAAS,GAAG;AAC1D,YAAM,OAAO,MAAM,SAAS,QAAQ,KAAK;AACzC,YAAM,UAAU,MAAM,KAAK,eAAe;AAC1C,YAAM,OAAO,QAAQ,MAClB;AAAA,QAAQ,CAAC,SACR,OAAO,SAAS,YAAY,SAAS,QAAQ,SAAS,OAAO,CAAC,OAAO,KAAK,GAAG,CAAC,IAAI,CAAC;AAAA,MACrF,EACC,KAAK,IAAI;AACZ,YAAM,KAAK,EAAE,YAAY,QAAQ,GAAG,MAAM,KAAK,SAAS,IAAI,GAAG,IAAI;AAAA,IAAO,GAAG,CAAC;AAAA,IAChF;AAAA,EACF,UAAE;AACA,UAAM,YAAY,QAAQ;AAAA,EAC5B;AACA,SAAO;AACT;AAEA,SAAS,aAAa,OAA6C;AACjE,MAAI,iBAAiB,YAAY;AAC/B,WAAO,IAAI;AAAA,MACT,MAAM,OAAO,MAAM,MAAM,YAAY,MAAM,aAAa,MAAM,UAAU;AAAA,IAC1E;AAAA,EACF;AACA,SAAO,IAAI,WAAW,KAAK;AAC7B;","names":[]}

package/dist/mcp/index.d.ts

@@ -1,10 +1,10 @@
-import { a as McpConnection, c as McpServer, e as McpStdioOptions, b as McpHttpOptions, d as McpSseOptions } from '../types-DpFoVF_g.js';
-export { M as McpClient, f as McpToolCallContent, g as McpToolCallResult, h as McpToolDefinition } from '../types-DpFoVF_g.js';
+import { a as McpConnection, c as McpServer, e as McpStdioOptions, b as McpHttpOptions, d as McpSseOptions } from '../types-fLi8uM5R.js';
+export { M as McpClient, f as McpToolCallContent, g as McpToolCallResult, h as McpToolDefinition } from '../types-fLi8uM5R.js';
 import '@modelcontextprotocol/sdk/client/sse.js';
 import '@modelcontextprotocol/sdk/client/stdio.js';
 import '@modelcontextprotocol/sdk/client/streamableHttp.js';
 import '../types-BM8-Y8Hy.js';
-import '../tool-FEHplpd9.js';
+import '../tool-ClZYES-Z.js';
 
 declare function connectMcp(connection: McpConnection): Promise<McpServer>;
 

package/dist/{middleware-CKsY_zZE.d.ts → middleware-2FR89muX.d.ts}

@@ -1,8 +1,8 @@
 import { z } from 'zod';
-import { q as ToolDefinition } from './types-BM8-Y8Hy.js';
+import { q as ToolDefinition, u as ToolResultContent } from './types-BM8-Y8Hy.js';
 import { VectorMetadata, EmbeddingModel, EmbeddedDocument } from './embeddings/index.js';
 import { VectorSearchIndex } from './vector-store/index.js';
-import { A as AnyTool, d as ToolCallContext } from './tool-FEHplpd9.js';
+import { A as AnyTool, d as ToolCallContext, N as NormalizedToolOutput } from './tool-ClZYES-Z.js';
 
 type ZodSchema<T = unknown> = z.ZodType<T>;
 
@@ -16,7 +16,7 @@ declare class ToolSet {
     get(toolName: string): AnyTool | undefined;
     values(): AnyTool[];
     getToolDefinitions(prompt?: string): Promise<ToolDefinition[]>;
-    call(toolName: string, args: string, context?: ToolCallContext): Promise<string>;
+    call(toolName: string, args: string, context?: ToolCallContext): Promise<NormalizedToolOutput>;
 }
 
 type ToolSearchDocument<Metadata extends VectorMetadata = VectorMetadata> = {
@@ -42,6 +42,8 @@ type ToolResultMiddlewareArgs = {
     args: string;
     result: string;
     originalResult: string;
+    structuredResult?: ToolResultContent[] | undefined;
+    originalStructuredResult?: ToolResultContent[] | undefined;
     turn: number;
     toolCallId?: string | undefined;
     internalCallId: string;

package/dist/observability/index.d.ts

@@ -1,5 +1,5 @@
-import { M as Message, d as CompletionRequest, J as JsonObject, c as CompletionModelCapabilities, e as CompletionResponse, n as ToolCall, q as ToolDefinition, U as Usage } from '../types-BM8-Y8Hy.js';
-import { e as ToolCallStreamEvent } from '../tool-FEHplpd9.js';
+import { M as Message, d as CompletionRequest, J as JsonObject, c as CompletionModelCapabilities, e as CompletionResponse, n as ToolCall, q as ToolDefinition, u as ToolResultContent, U as Usage } from '../types-BM8-Y8Hy.js';
+import { e as ToolCallStreamEvent } from '../tool-ClZYES-Z.js';
 
 type AgentTraceInfo = {
     traceId?: string | undefined;
@@ -65,6 +65,7 @@ type AgentToolStartArgs = {
 };
 type AgentToolEndArgs = AgentToolStartArgs & {
     result: string;
+    structuredResult?: ToolResultContent[] | undefined;
     skipped: boolean;
 };
 type AgentToolErrorArgs = AgentToolStartArgs & {

package/dist/observability/index.js

@@ -1,6 +1,6 @@
 import {
   createObserver
-} from "../chunk-OD6TY44B.js";
+} from "../chunk-ZSYIQYTV.js";
 export {
   createObserver
 };

package/dist/pipeline/index.d.ts

@@ -1,10 +1,10 @@
 import { J as JsonObject, b as CompletionModel } from '../types-BM8-Y8Hy.js';
-import { A as Agent } from '../agent-ufaeGoaz.js';
+import { A as Agent } from '../agent-D_jXqPAZ.js';
 import { Extractor } from '../extractor/index.js';
 import '../memory/index.js';
 import '../observability/index.js';
-import '../tool-FEHplpd9.js';
-import '../middleware-CKsY_zZE.js';
+import '../tool-ClZYES-Z.js';
+import '../middleware-2FR89muX.js';
 import 'zod';
 import '../embeddings/index.js';
 import '../vector-store/index.js';

package/dist/skills/index.d.ts

@@ -1,6 +1,6 @@
-import { a as SkillLoader, b as SkillSet } from '../types-CW3K1MJm.js';
-export { S as Skill, c as SkillValidationError, d as SkillValidationIssue } from '../types-CW3K1MJm.js';
-import '../tool-FEHplpd9.js';
+import { a as SkillLoader, b as SkillSet } from '../types-B0iXLkqi.js';
+export { S as Skill, c as SkillValidationError, d as SkillValidationIssue } from '../types-B0iXLkqi.js';
+import '../tool-ClZYES-Z.js';
 import '../types-BM8-Y8Hy.js';
 
 declare function loadSkills(loaders: SkillLoader | SkillLoader[]): Promise<SkillSet>;

package/dist/skills/index.js

@@ -2,12 +2,10 @@ import {
   SkillValidationError,
   loadSkills,
   skill
-} from "../chunk-2VCWYCPV.js";
-import "../chunk-V5EETE3U.js";
-import "../chunk-4F7RVGFR.js";
+} from "../chunk-TILE6Z2N.js";
 import "../chunk-YK4WAAS4.js";
-import "../chunk-3H7FVGHU.js";
-import "../chunk-445TT5Q5.js";
+import "../chunk-I2HOMD3R.js";
+import "../chunk-65QV627O.js";
 import "../chunk-A7VDIZQN.js";
 export {
   SkillValidationError,

package/dist/tool/index.d.ts

@@ -1,22 +1,24 @@
 import { z } from 'zod';
-import { Z as ZodSchema } from '../middleware-CKsY_zZE.js';
-export { D as DynamicToolIndex, E as EmbedToolsOptions, T as ToolMiddleware, a as ToolResultMiddlewareArgs, b as ToolSearchDocument, c as ToolSet, d as createToolIndex, e as createToolMiddleware, f as embedTools, i as isDynamicToolIndex } from '../middleware-CKsY_zZE.js';
-import { b as ToolApprovalPolicy, d as ToolCallContext, T as Tool } from '../tool-FEHplpd9.js';
-export { A as AnyTool, a as ToolApprovalContext, c as ToolApprovalRunContext, e as ToolCallStreamEvent, p as parseToolArgs, s as serializeToolOutput } from '../tool-FEHplpd9.js';
+import { Z as ZodSchema } from '../middleware-2FR89muX.js';
+export { D as DynamicToolIndex, E as EmbedToolsOptions, T as ToolMiddleware, a as ToolResultMiddlewareArgs, b as ToolSearchDocument, c as ToolSet, d as createToolIndex, e as createToolMiddleware, f as embedTools, i as isDynamicToolIndex } from '../middleware-2FR89muX.js';
+import { b as ToolApprovalPolicy, d as ToolCallContext, T as Tool } from '../tool-ClZYES-Z.js';
+export { A as AnyTool, N as NormalizedToolOutput, a as ToolApprovalContext, c as ToolApprovalRunContext, e as ToolCallStreamEvent, f as ToolOutput, i as isToolResultContentArray, n as normalizeToolResultOutput, p as parseToolArgs, s as serializeToolOutput, t as toolResultContentToText } from '../tool-ClZYES-Z.js';
 import '../types-BM8-Y8Hy.js';
 import '../embeddings/index.js';
 import '../vector-store/index.js';
 
-type CreateToolOptions<InputSchema extends ZodSchema, OutputSchema extends ZodSchema | undefined = undefined> = {
+type CreateToolOptions<InputSchema extends ZodSchema, OutputSchema extends ZodSchema | undefined = undefined, Output = unknown> = {
     name: string;
     description: string;
     input: InputSchema;
     output?: OutputSchema;
     approval?: ToolApprovalPolicy<z.output<InputSchema>>;
-    execute(args: z.output<InputSchema>, context: ToolCallContext): OutputSchema extends ZodSchema ? z.input<OutputSchema> | Promise<z.input<OutputSchema>> : unknown | Promise<unknown>;
+    execute(args: z.output<InputSchema>, context: ToolCallContext): OutputSchema extends ZodSchema ? z.input<OutputSchema> | Promise<z.input<OutputSchema>> : Output | Promise<Output>;
 };
-type ToolOutput<OutputSchema extends ZodSchema | undefined> = OutputSchema extends ZodSchema ? z.output<OutputSchema> : unknown;
-declare function createTool<InputSchema extends ZodSchema, OutputSchema extends ZodSchema | undefined = undefined>(options: CreateToolOptions<InputSchema, OutputSchema>): Tool<z.output<InputSchema>, ToolOutput<OutputSchema>>;
+declare function createTool<InputSchema extends ZodSchema, Output = unknown>(options: CreateToolOptions<InputSchema, undefined, Output> & {
+    output?: undefined;
+}): Tool<z.output<InputSchema>, Output>;
+declare function createTool<InputSchema extends ZodSchema, OutputSchema extends ZodSchema>(options: CreateToolOptions<InputSchema, OutputSchema>): Tool<z.output<InputSchema>, z.output<OutputSchema>>;
 
 declare class ToolCallError extends Error {
     readonly cause?: unknown | undefined;

package/dist/tool/index.js

@@ -1,27 +1,29 @@
-import {
-  createThinkTool,
-  createToolIndex,
-  createToolMiddleware,
-  embedTools,
-  isDynamicToolIndex
-} from "../chunk-V5EETE3U.js";
-import "../chunk-4F7RVGFR.js";
 import {
   ToolCallError,
   ToolJsonError,
   ToolNotFoundError,
+  ToolOutput,
   ToolSet,
+  createThinkTool,
+  createToolIndex,
+  createToolMiddleware,
+  embedTools,
+  isDynamicToolIndex,
+  isToolResultContentArray,
+  normalizeToolResultOutput,
   parseToolArgs,
-  serializeToolOutput
-} from "../chunk-3H7FVGHU.js";
+  serializeToolOutput,
+  toolResultContentToText
+} from "../chunk-I2HOMD3R.js";
 import {
   createTool
-} from "../chunk-445TT5Q5.js";
+} from "../chunk-65QV627O.js";
 import "../chunk-A7VDIZQN.js";
 export {
   ToolCallError,
   ToolJsonError,
   ToolNotFoundError,
+  ToolOutput,
   ToolSet,
   createThinkTool,
   createTool,
@@ -29,7 +31,10 @@ export {
   createToolMiddleware,
   embedTools,
   isDynamicToolIndex,
+  isToolResultContentArray,
+  normalizeToolResultOutput,
   parseToolArgs,
-  serializeToolOutput
+  serializeToolOutput,
+  toolResultContentToText
 };
 //# sourceMappingURL=index.js.map

package/dist/{tool-FEHplpd9.d.ts → tool-ClZYES-Z.d.ts}

@@ -1,4 +1,4 @@
-import { J as JsonObject, q as ToolDefinition, j as JsonValue } from './types-BM8-Y8Hy.js';
+import { J as JsonObject, q as ToolDefinition, u as ToolResultContent, j as JsonValue } from './types-BM8-Y8Hy.js';
 
 type ToolApprovalRunContext = {
     agentId: string;
@@ -37,7 +37,14 @@ interface Tool<Args = unknown, Output = unknown> {
 type AnyTool = Omit<Tool<unknown, unknown>, "approval"> & {
     readonly approval?: unknown;
 };
+type NormalizedToolOutput = string | ToolResultContent[];
+declare const ToolOutput: {
+    content(content: ToolResultContent[]): ToolResultContent[];
+};
 declare function serializeToolOutput(output: unknown): string;
+declare function isToolResultContentArray(value: unknown): value is ToolResultContent[];
+declare function normalizeToolResultOutput(output: unknown): NormalizedToolOutput;
+declare function toolResultContentToText(content: ToolResultContent[]): string;
 declare function parseToolArgs(args: string): JsonValue;
 
-export { type AnyTool as A, type Tool as T, type ToolApprovalContext as a, type ToolApprovalPolicy as b, type ToolApprovalRunContext as c, type ToolCallContext as d, type ToolCallStreamEvent as e, parseToolArgs as p, serializeToolOutput as s };
+export { type AnyTool as A, type NormalizedToolOutput as N, type Tool as T, type ToolApprovalContext as a, type ToolApprovalPolicy as b, type ToolApprovalRunContext as c, type ToolCallContext as d, type ToolCallStreamEvent as e, ToolOutput as f, isToolResultContentArray as i, normalizeToolResultOutput as n, parseToolArgs as p, serializeToolOutput as s, toolResultContentToText as t };

package/dist/{types-CW3K1MJm.d.ts → types-B0iXLkqi.d.ts}

@@ -1,4 +1,4 @@
-import { A as AnyTool } from './tool-FEHplpd9.js';
+import { A as AnyTool } from './tool-ClZYES-Z.js';
 
 type Skill = {
     readonly name: string;

package/dist/{types-DpFoVF_g.d.ts → types-fLi8uM5R.d.ts}

@@ -2,7 +2,7 @@ import { SSEClientTransportOptions } from '@modelcontextprotocol/sdk/client/sse.
 import { StdioServerParameters } from '@modelcontextprotocol/sdk/client/stdio.js';
 import { StreamableHTTPClientTransportOptions } from '@modelcontextprotocol/sdk/client/streamableHttp.js';
 import { J as JsonObject } from './types-BM8-Y8Hy.js';
-import { A as AnyTool } from './tool-FEHplpd9.js';
+import { A as AnyTool } from './tool-ClZYES-Z.js';
 
 type McpToolDefinition = {
     name: string;

package/dist/vector-store/index.d.ts

@@ -1,5 +1,5 @@
 import { VectorMetadataValue, VectorMetadata, EmbeddingModel, EmbeddedDocument, Embedding } from '../embeddings/index.js';
-import { T as Tool } from '../tool-FEHplpd9.js';
+import { T as Tool } from '../tool-ClZYES-Z.js';
 import '../types-BM8-Y8Hy.js';
 
 type VectorFilter = {

package/dist/vector-store/index.js

@@ -3,8 +3,7 @@ import {
   InMemoryVectorStore,
   createVectorSearchTool,
   vectorFilter
-} from "../chunk-4F7RVGFR.js";
-import "../chunk-445TT5Q5.js";
+} from "../chunk-65QV627O.js";
 import "../chunk-A7VDIZQN.js";
 export {
   InMemoryVectorIndex,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@anvia/core",
-  "version": "0.2.4",
+  "version": "0.3.1",
   "description": "Core runtime primitives for context-aware Anvia agents.",
   "author": "anvia",
   "maintainer": "Indra Zulfi",
@@ -99,8 +99,8 @@
   },
   "dependencies": {
     "@modelcontextprotocol/sdk": "^1.29.0",
-    "pdfjs-dist": "^5.7.284",
-    "tinyglobby": "^0.2.16",
+    "pdfjs-dist": "^6.0.227",
+    "tinyglobby": "^0.2.17",
     "yaml": "^2.9.0",
     "zod": "^4.4.3"
   },

package/dist/chunk-3H7FVGHU.js

@@ -1,113 +0,0 @@
-// src/tool/errors.ts
-var ToolCallError = class extends Error {
-  constructor(message, cause) {
-    super(message);
-    this.cause = cause;
-    this.name = "ToolCallError";
-  }
-  cause;
-};
-var ToolNotFoundError = class extends Error {
-  constructor(toolName) {
-    super(`Tool not found: ${toolName}`);
-    this.toolName = toolName;
-    this.name = "ToolNotFoundError";
-  }
-  toolName;
-};
-var ToolJsonError = class extends Error {
-  constructor(message, cause) {
-    super(message);
-    this.cause = cause;
-    this.name = "ToolJsonError";
-  }
-  cause;
-};
-
-// src/tool/tool.ts
-function serializeToolOutput(output) {
-  if (typeof output === "string") {
-    return output;
-  }
-  const serialized = JSON.stringify(output);
-  return serialized === void 0 ? String(output) : serialized;
-}
-function parseToolArgs(args) {
-  if (args.trim() === "") {
-    return {};
-  }
-  return JSON.parse(args);
-}
-
-// src/tool/tool-set.ts
-var ToolSet = class _ToolSet {
-  tools = /* @__PURE__ */ new Map();
-  static fromTools(tools) {
-    const toolSet = new _ToolSet();
-    for (const tool of tools) {
-      toolSet.addTool(tool);
-    }
-    return toolSet;
-  }
-  addTool(tool) {
-    this.tools.set(tool.name, tool);
-    return this;
-  }
-  addTools(tools) {
-    const values = Array.isArray(tools) ? tools : tools.values();
-    for (const tool of values) {
-      this.addTool(tool);
-    }
-    return this;
-  }
-  deleteTool(toolName) {
-    return this.tools.delete(toolName);
-  }
-  contains(toolName) {
-    return this.tools.has(toolName);
-  }
-  get(toolName) {
-    return this.tools.get(toolName);
-  }
-  values() {
-    return [...this.tools.values()];
-  }
-  async getToolDefinitions(prompt = "") {
-    const defs = [];
-    for (const tool of this.tools.values()) {
-      defs.push(await tool.definition(prompt));
-    }
-    return defs;
-  }
-  async call(toolName, args, context) {
-    const tool = this.tools.get(toolName);
-    if (tool === void 0) {
-      throw new ToolNotFoundError(toolName);
-    }
-    let parsedArgs;
-    try {
-      parsedArgs = parseToolArgs(args);
-    } catch (error) {
-      throw new ToolJsonError(`Invalid JSON arguments for tool ${toolName}`, error);
-    }
-    try {
-      const output = await tool.call(parsedArgs, context);
-      return serializeToolOutput(output);
-    } catch (error) {
-      if (error instanceof Error) {
-        throw new ToolCallError(error.message, error);
-      }
-      throw new ToolCallError(`Tool ${toolName} failed`, error);
-    }
-  }
-};
-
-export {
-  ToolCallError,
-  ToolNotFoundError,
-  ToolJsonError,
-  serializeToolOutput,
-  parseToolArgs,
-  ToolSet
-};
-//# sourceMappingURL=chunk-3H7FVGHU.js.map

package/dist/chunk-3H7FVGHU.js.map

@@ -1 +0,0 @@
-{"version":3,"sources":["../src/tool/errors.ts","../src/tool/tool.ts","../src/tool/tool-set.ts"],"sourcesContent":["export class ToolCallError extends Error {\n  constructor(\n    message: string,\n    readonly cause?: unknown,\n  ) {\n    super(message);\n    this.name = \"ToolCallError\";\n  }\n}\n\nexport class ToolNotFoundError extends Error {\n  constructor(readonly toolName: string) {\n    super(`Tool not found: ${toolName}`);\n    this.name = \"ToolNotFoundError\";\n  }\n}\n\nexport class ToolJsonError extends Error {\n  constructor(\n    message: string,\n    readonly cause?: unknown,\n  ) {\n    super(message);\n    this.name = \"ToolJsonError\";\n  }\n}\n","import type { JsonObject, JsonValue, ToolDefinition } from \"../completion/types\";\n\nexport type ToolApprovalRunContext = {\n  agentId: string;\n  runId: string;\n  sessionId?: string;\n  metadata?: JsonObject;\n};\n\nexport type ToolApprovalContext<Args = unknown> = {\n  toolName: string;\n  args: Args;\n  rawArgs: string;\n  toolCallId?: string;\n  internalCallId: string;\n  run: ToolApprovalRunContext;\n};\n\nexport type ToolApprovalPolicy<Args = unknown> = {\n  when(ctx: ToolApprovalContext<Args>): boolean | Promise<boolean>;\n  reason?: string | ((ctx: ToolApprovalContext<Args>) => string | Promise<string>);\n  rejectMessage?: string | ((ctx: ToolApprovalContext<Args>) => string | Promise<string>);\n};\n\nexport type ToolCallStreamEvent = {\n  agentId: string;\n  agentName?: string | undefined;\n  event: unknown;\n};\n\nexport type ToolCallContext = {\n  emitStreamEvent?(event: ToolCallStreamEvent): void | Promise<void>;\n};\n\nexport interface Tool<Args = unknown, Output = unknown> {\n  readonly name: string;\n  readonly approval?: ToolApprovalPolicy<Args>;\n  definition(prompt: string): ToolDefinition | Promise<ToolDefinition>;\n  call(args: Args, context?: ToolCallContext): Output | Promise<Output>;\n  parseApprovalArgs?(args: unknown): Args;\n}\n\nexport type AnyTool = Omit<Tool<unknown, unknown>, \"approval\"> & {\n  readonly approval?: unknown;\n};\n\nexport function serializeToolOutput(output: unknown): string {\n  if (typeof output === \"string\") {\n    return output;\n  }\n\n  const serialized = JSON.stringify(output);\n  return serialized === undefined ? String(output) : serialized;\n}\n\nexport function parseToolArgs(args: string): JsonValue {\n  if (args.trim() === \"\") {\n    return {};\n  }\n\n  return JSON.parse(args) as JsonValue;\n}\n","import type { ToolDefinition } from \"../completion/types\";\nimport { ToolCallError, ToolJsonError, ToolNotFoundError } from \"./errors\";\nimport { type AnyTool, parseToolArgs, serializeToolOutput, type ToolCallContext } from \"./tool\";\n\nexport class ToolSet {\n  private readonly tools = new Map<string, AnyTool>();\n\n  static fromTools(tools: AnyTool[]): ToolSet {\n    const toolSet = new ToolSet();\n    for (const tool of tools) {\n      toolSet.addTool(tool);\n    }\n    return toolSet;\n  }\n\n  addTool(tool: AnyTool): this {\n    this.tools.set(tool.name, tool);\n    return this;\n  }\n\n  addTools(tools: AnyTool[] | ToolSet): this {\n    const values = Array.isArray(tools) ? tools : tools.values();\n    for (const tool of values) {\n      this.addTool(tool);\n    }\n    return this;\n  }\n\n  deleteTool(toolName: string): boolean {\n    return this.tools.delete(toolName);\n  }\n\n  contains(toolName: string): boolean {\n    return this.tools.has(toolName);\n  }\n\n  get(toolName: string): AnyTool | undefined {\n    return this.tools.get(toolName);\n  }\n\n  values(): AnyTool[] {\n    return [...this.tools.values()];\n  }\n\n  async getToolDefinitions(prompt = \"\"): Promise<ToolDefinition[]> {\n    const defs: ToolDefinition[] = [];\n    for (const tool of this.tools.values()) {\n      defs.push(await tool.definition(prompt));\n    }\n    return defs;\n  }\n\n  async call(toolName: string, args: string, context?: ToolCallContext): Promise<string> {\n    const tool = this.tools.get(toolName);\n    if (tool === undefined) {\n      throw new ToolNotFoundError(toolName);\n    }\n\n    let parsedArgs: unknown;\n    try {\n      parsedArgs = parseToolArgs(args);\n    } catch (error) {\n      throw new ToolJsonError(`Invalid JSON arguments for tool ${toolName}`, error);\n    }\n\n    try {\n      const output = await tool.call(parsedArgs, context);\n      return serializeToolOutput(output);\n    } catch (error) {\n      if (error instanceof Error) {\n        throw new ToolCallError(error.message, error);\n      }\n      throw new ToolCallError(`Tool ${toolName} failed`, error);\n    }\n  }\n}\n"],"mappings":";AAAO,IAAM,gBAAN,cAA4B,MAAM;AAAA,EACvC,YACE,SACS,OACT;AACA,UAAM,OAAO;AAFJ;AAGT,SAAK,OAAO;AAAA,EACd;AAAA,EAJW;AAKb;AAEO,IAAM,oBAAN,cAAgC,MAAM;AAAA,EAC3C,YAAqB,UAAkB;AACrC,UAAM,mBAAmB,QAAQ,EAAE;AADhB;AAEnB,SAAK,OAAO;AAAA,EACd;AAAA,EAHqB;AAIvB;AAEO,IAAM,gBAAN,cAA4B,MAAM;AAAA,EACvC,YACE,SACS,OACT;AACA,UAAM,OAAO;AAFJ;AAGT,SAAK,OAAO;AAAA,EACd;AAAA,EAJW;AAKb;;;ACqBO,SAAS,oBAAoB,QAAyB;AAC3D,MAAI,OAAO,WAAW,UAAU;AAC9B,WAAO;AAAA,EACT;AAEA,QAAM,aAAa,KAAK,UAAU,MAAM;AACxC,SAAO,eAAe,SAAY,OAAO,MAAM,IAAI;AACrD;AAEO,SAAS,cAAc,MAAyB;AACrD,MAAI,KAAK,KAAK,MAAM,IAAI;AACtB,WAAO,CAAC;AAAA,EACV;AAEA,SAAO,KAAK,MAAM,IAAI;AACxB;;;ACzDO,IAAM,UAAN,MAAM,SAAQ;AAAA,EACF,QAAQ,oBAAI,IAAqB;AAAA,EAElD,OAAO,UAAU,OAA2B;AAC1C,UAAM,UAAU,IAAI,SAAQ;AAC5B,eAAW,QAAQ,OAAO;AACxB,cAAQ,QAAQ,IAAI;AAAA,IACtB;AACA,WAAO;AAAA,EACT;AAAA,EAEA,QAAQ,MAAqB;AAC3B,SAAK,MAAM,IAAI,KAAK,MAAM,IAAI;AAC9B,WAAO;AAAA,EACT;AAAA,EAEA,SAAS,OAAkC;AACzC,UAAM,SAAS,MAAM,QAAQ,KAAK,IAAI,QAAQ,MAAM,OAAO;AAC3D,eAAW,QAAQ,QAAQ;AACzB,WAAK,QAAQ,IAAI;AAAA,IACnB;AACA,WAAO;AAAA,EACT;AAAA,EAEA,WAAW,UAA2B;AACpC,WAAO,KAAK,MAAM,OAAO,QAAQ;AAAA,EACnC;AAAA,EAEA,SAAS,UAA2B;AAClC,WAAO,KAAK,MAAM,IAAI,QAAQ;AAAA,EAChC;AAAA,EAEA,IAAI,UAAuC;AACzC,WAAO,KAAK,MAAM,IAAI,QAAQ;AAAA,EAChC;AAAA,EAEA,SAAoB;AAClB,WAAO,CAAC,GAAG,KAAK,MAAM,OAAO,CAAC;AAAA,EAChC;AAAA,EAEA,MAAM,mBAAmB,SAAS,IAA+B;AAC/D,UAAM,OAAyB,CAAC;AAChC,eAAW,QAAQ,KAAK,MAAM,OAAO,GAAG;AACtC,WAAK,KAAK,MAAM,KAAK,WAAW,MAAM,CAAC;AAAA,IACzC;AACA,WAAO;AAAA,EACT;AAAA,EAEA,MAAM,KAAK,UAAkB,MAAc,SAA4C;AACrF,UAAM,OAAO,KAAK,MAAM,IAAI,QAAQ;AACpC,QAAI,SAAS,QAAW;AACtB,YAAM,IAAI,kBAAkB,QAAQ;AAAA,IACtC;AAEA,QAAI;AACJ,QAAI;AACF,mBAAa,cAAc,IAAI;AAAA,IACjC,SAAS,OAAO;AACd,YAAM,IAAI,cAAc,mCAAmC,QAAQ,IAAI,KAAK;AAAA,IAC9E;AAEA,QAAI;AACF,YAAM,SAAS,MAAM,KAAK,KAAK,YAAY,OAAO;AAClD,aAAO,oBAAoB,MAAM;AAAA,IACnC,SAAS,OAAO;AACd,UAAI,iBAAiB,OAAO;AAC1B,cAAM,IAAI,cAAc,MAAM,SAAS,KAAK;AAAA,MAC9C;AACA,YAAM,IAAI,cAAc,QAAQ,QAAQ,WAAW,KAAK;AAAA,IAC1D;AAAA,EACF;AACF;","names":[]}

package/dist/chunk-445TT5Q5.js

@@ -1,37 +0,0 @@
-// src/schema/zod-schema.ts
-import { z } from "zod";
-function toProviderJsonSchema(schema) {
-  const jsonSchema = z.toJSONSchema(schema);
-  const { $schema: _schema, ...providerSchema } = jsonSchema;
-  return providerSchema;
-}
-
-// src/tool/create-tool.ts
-function createTool(options) {
-  const parameters = toProviderJsonSchema(options.input);
-  return {
-    name: options.name,
-    ...options.approval === void 0 ? {} : { approval: options.approval },
-    definition() {
-      return {
-        name: options.name,
-        description: options.description,
-        parameters
-      };
-    },
-    async call(args, context = {}) {
-      const parsedArgs = options.input.parse(args);
-      const result = await options.execute(parsedArgs, context);
-      return options.output === void 0 ? result : options.output.parse(result);
-    },
-    parseApprovalArgs(args) {
-      return options.input.parse(args);
-    }
-  };
-}
-
-export {
-  toProviderJsonSchema,
-  createTool
-};
-//# sourceMappingURL=chunk-445TT5Q5.js.map

package/dist/chunk-445TT5Q5.js.map

@@ -1 +0,0 @@
-{"version":3,"sources":["../src/schema/zod-schema.ts","../src/tool/create-tool.ts"],"sourcesContent":["import { z } from \"zod\";\nimport type { JsonObject } from \"../completion/index\";\n\nexport type ZodSchema<T = unknown> = z.ZodType<T>;\n\nexport function toProviderJsonSchema(schema: z.ZodType): JsonObject {\n  const jsonSchema = z.toJSONSchema(schema) as JsonObject;\n  const { $schema: _schema, ...providerSchema } = jsonSchema;\n  return providerSchema;\n}\n","import type { z } from \"zod\";\nimport { toProviderJsonSchema, type ZodSchema } from \"../schema/zod-schema\";\nimport type { Tool, ToolApprovalPolicy, ToolCallContext } from \"./tool\";\n\nexport type CreateToolOptions<\n  InputSchema extends ZodSchema,\n  OutputSchema extends ZodSchema | undefined = undefined,\n> = {\n  name: string;\n  description: string;\n  input: InputSchema;\n  output?: OutputSchema;\n  approval?: ToolApprovalPolicy<z.output<InputSchema>>;\n  execute(\n    args: z.output<InputSchema>,\n    context: ToolCallContext,\n  ): OutputSchema extends ZodSchema\n    ? z.input<OutputSchema> | Promise<z.input<OutputSchema>>\n    : unknown | Promise<unknown>;\n};\n\ntype ToolOutput<OutputSchema extends ZodSchema | undefined> = OutputSchema extends ZodSchema\n  ? z.output<OutputSchema>\n  : unknown;\n\nexport function createTool<\n  InputSchema extends ZodSchema,\n  OutputSchema extends ZodSchema | undefined = undefined,\n>(\n  options: CreateToolOptions<InputSchema, OutputSchema>,\n): Tool<z.output<InputSchema>, ToolOutput<OutputSchema>> {\n  const parameters = toProviderJsonSchema(options.input);\n\n  return {\n    name: options.name,\n    ...(options.approval === undefined ? {} : { approval: options.approval }),\n    definition() {\n      return {\n        name: options.name,\n        description: options.description,\n        parameters,\n      };\n    },\n    async call(args, context = {}): Promise<ToolOutput<OutputSchema>> {\n      const parsedArgs = options.input.parse(args);\n      const result = await options.execute(parsedArgs, context);\n      return (\n        options.output === undefined ? result : options.output.parse(result)\n      ) as ToolOutput<OutputSchema>;\n    },\n    parseApprovalArgs(args): z.output<InputSchema> {\n      return options.input.parse(args);\n    },\n  };\n}\n"],"mappings":";AAAA,SAAS,SAAS;AAKX,SAAS,qBAAqB,QAA+B;AAClE,QAAM,aAAa,EAAE,aAAa,MAAM;AACxC,QAAM,EAAE,SAAS,SAAS,GAAG,eAAe,IAAI;AAChD,SAAO;AACT;;;ACgBO,SAAS,WAId,SACuD;AACvD,QAAM,aAAa,qBAAqB,QAAQ,KAAK;AAErD,SAAO;AAAA,IACL,MAAM,QAAQ;AAAA,IACd,GAAI,QAAQ,aAAa,SAAY,CAAC,IAAI,EAAE,UAAU,QAAQ,SAAS;AAAA,IACvE,aAAa;AACX,aAAO;AAAA,QACL,MAAM,QAAQ;AAAA,QACd,aAAa,QAAQ;AAAA,QACrB;AAAA,MACF;AAAA,IACF;AAAA,IACA,MAAM,KAAK,MAAM,UAAU,CAAC,GAAsC;AAChE,YAAM,aAAa,QAAQ,MAAM,MAAM,IAAI;AAC3C,YAAM,SAAS,MAAM,QAAQ,QAAQ,YAAY,OAAO;AACxD,aACE,QAAQ,WAAW,SAAY,SAAS,QAAQ,OAAO,MAAM,MAAM;AAAA,IAEvE;AAAA,IACA,kBAAkB,MAA6B;AAC7C,aAAO,QAAQ,MAAM,MAAM,IAAI;AAAA,IACjC;AAAA,EACF;AACF;","names":[]}

package/dist/chunk-4F7RVGFR.js.map

@@ -1 +0,0 @@
-{"version":3,"sources":["../src/vector-store/index.ts","../src/vector-store/filter.ts","../src/vector-store/lsh.ts"],"sourcesContent":["import { z } from \"zod\";\nimport {\n  cosineSimilarity,\n  type EmbeddedDocument,\n  type Embedding,\n  type EmbeddingModel,\n  embedText,\n  type VectorMetadata,\n} from \"../embeddings\";\nimport { createTool } from \"../tool/create-tool\";\nimport type { Tool } from \"../tool/tool\";\nimport { matchesVectorFilter, type VectorFilter } from \"./filter\";\nimport { LshIndex, type LshOptions } from \"./lsh\";\n\nexport { type VectorFilter, vectorFilter } from \"./filter\";\n\nexport type IndexStrategy = { type: \"bruteForce\" } | LshOptions;\n\nexport type VectorSearchRequest = {\n  query: string;\n  topK: number;\n  threshold?: number | undefined;\n  filter?: VectorFilter | undefined;\n};\n\nexport type VectorSearchResult<T = unknown, Metadata extends VectorMetadata = VectorMetadata> = {\n  score: number;\n  id: string;\n  document: T;\n  metadata?: Metadata | undefined;\n};\n\nexport type VectorInspectRequest = {\n  limit: number;\n  cursor?: string | undefined;\n  filter?: VectorFilter | undefined;\n};\n\nexport type VectorInspectItem<T = unknown, Metadata extends VectorMetadata = VectorMetadata> = {\n  id: string;\n  document: T;\n  metadata?: Metadata | undefined;\n};\n\nexport type VectorInspectPage<T = unknown, Metadata extends VectorMetadata = VectorMetadata> = {\n  items: Array<VectorInspectItem<T, Metadata>>;\n  nextCursor?: string | undefined;\n  totalCount?: number | undefined;\n};\n\nexport interface VectorSearchIndex<T = unknown, Metadata extends VectorMetadata = VectorMetadata> {\n  search(request: VectorSearchRequest): Promise<Array<VectorSearchResult<T, Metadata>>>;\n  searchIds(request: VectorSearchRequest): Promise<Array<{ score: number; id: string }>>;\n  asTool(options: VectorSearchToolOptions): Tool<{ query: string; topK?: number }, unknown>;\n  inspect?(request: VectorInspectRequest): Promise<VectorInspectPage<T, Metadata>>;\n}\n\nexport type VectorSearchToolOptions = {\n  name: string;\n  description?: string | undefined;\n  topK?: number | undefined;\n  threshold?: number | undefined;\n  filter?: VectorFilter | undefined;\n};\n\ntype StoredDocument<T, Metadata extends VectorMetadata> = EmbeddedDocument<T, Metadata>;\n\nexport class InMemoryVectorStore<T, Metadata extends VectorMetadata = VectorMetadata> {\n  private readonly documents = new Map<string, StoredDocument<T, Metadata>>();\n  private indexStrategy: IndexStrategy;\n  private lshIndex: LshIndex | undefined;\n\n  constructor(options: { index?: IndexStrategy } = {}) {\n    this.indexStrategy = options.index ?? { type: \"bruteForce\" };\n  }\n\n  static fromDocuments<T, Metadata extends VectorMetadata = VectorMetadata>(\n    documents: Array<EmbeddedDocument<T, Metadata>>,\n    options: { index?: IndexStrategy } = {},\n  ): InMemoryVectorStore<T, Metadata> {\n    const store = new InMemoryVectorStore<T, Metadata>(options);\n    store.addDocuments(documents);\n    return store;\n  }\n\n  addDocuments(documents: Array<EmbeddedDocument<T, Metadata>>): this {\n    for (const document of documents) {\n      this.documents.set(document.id, document);\n    }\n    this.rebuildLshIndex();\n    return this;\n  }\n\n  get(id: string): StoredDocument<T, Metadata> | undefined {\n    return this.documents.get(id);\n  }\n\n  values(): Array<StoredDocument<T, Metadata>> {\n    return [...this.documents.values()];\n  }\n\n  len(): number {\n    return this.documents.size;\n  }\n\n  isEmpty(): boolean {\n    return this.documents.size === 0;\n  }\n\n  index(model: EmbeddingModel): InMemoryVectorIndex<T, Metadata> {\n    return new InMemoryVectorIndex(model, this);\n  }\n\n  candidates(queryEmbedding: Embedding): Array<StoredDocument<T, Metadata>> {\n    if (this.indexStrategy.type !== \"lsh\" || this.lshIndex === undefined) {\n      return this.values();\n    }\n\n    const candidateIds = this.lshIndex.query(queryEmbedding.vector);\n    if (candidateIds.size === 0) {\n      return this.values();\n    }\n\n    return [...candidateIds].flatMap((id) => {\n      const document = this.documents.get(id);\n      return document === undefined ? [] : [document];\n    });\n  }\n\n  private rebuildLshIndex(): void {\n    if (this.indexStrategy.type !== \"lsh\") {\n      this.lshIndex = undefined;\n      return;\n    }\n\n    const firstEmbedding = this.values().flatMap((document) => document.embeddings)[0];\n    if (firstEmbedding === undefined) {\n      this.lshIndex = undefined;\n      return;\n    }\n\n    const index = new LshIndex(firstEmbedding.vector.length, this.indexStrategy);\n    for (const document of this.documents.values()) {\n      for (const embedding of document.embeddings) {\n        index.insert(document.id, embedding.vector);\n      }\n    }\n    this.lshIndex = index;\n  }\n}\n\nexport class InMemoryVectorIndex<T, Metadata extends VectorMetadata = VectorMetadata>\n  implements VectorSearchIndex<T, Metadata>\n{\n  constructor(\n    private readonly model: EmbeddingModel,\n    private readonly store: InMemoryVectorStore<T, Metadata>,\n  ) {}\n\n  async search(request: VectorSearchRequest): Promise<Array<VectorSearchResult<T, Metadata>>> {\n    const queryEmbedding = await embedText(this.model, request.query);\n    return this.store\n      .candidates(queryEmbedding)\n      .filter((document) => matchesVectorFilter(document.metadata, request.filter))\n      .flatMap((document) => {\n        const score = bestScore(queryEmbedding, document.embeddings);\n        if (score === undefined) {\n          return [];\n        }\n        if (request.threshold !== undefined && score < request.threshold) {\n          return [];\n        }\n        return [\n          {\n            score,\n            id: document.id,\n            document: document.document,\n            ...(document.metadata === undefined ? {} : { metadata: document.metadata }),\n          },\n        ];\n      })\n      .sort((left, right) => right.score - left.score)\n      .slice(0, Math.max(0, Math.trunc(request.topK)));\n  }\n\n  async searchIds(request: VectorSearchRequest): Promise<Array<{ score: number; id: string }>> {\n    return (await this.search(request)).map(({ score, id }) => ({ score, id }));\n  }\n\n  async inspect(request: VectorInspectRequest): Promise<VectorInspectPage<T, Metadata>> {\n    const limit = Math.max(0, Math.trunc(request.limit));\n    const start = Math.max(0, Math.trunc(Number(request.cursor ?? \"0\")));\n    const documents = this.store\n      .values()\n      .filter((document) => matchesVectorFilter(document.metadata, request.filter));\n    const page = documents.slice(start, start + limit);\n    const nextOffset = start + page.length;\n    return {\n      items: page.map((document) => ({\n        id: document.id,\n        document: document.document,\n        ...(document.metadata === undefined ? {} : { metadata: document.metadata }),\n      })),\n      ...(nextOffset < documents.length ? { nextCursor: String(nextOffset) } : {}),\n      totalCount: documents.length,\n    };\n  }\n\n  asTool(options: VectorSearchToolOptions): Tool<{ query: string; topK?: number }, unknown> {\n    return createVectorSearchTool(this, options);\n  }\n}\n\nexport function createVectorSearchTool<T, Metadata extends VectorMetadata>(\n  index: VectorSearchIndex<T, Metadata>,\n  options: VectorSearchToolOptions,\n): Tool<{ query: string; topK?: number }, Array<VectorSearchResult<T, Metadata>>> {\n  return createTool({\n    name: options.name,\n    description:\n      options.description ?? \"Search a vector store for documents relevant to the provided query.\",\n    input: z.object({\n      query: z.string().describe(\"The query string to search for relevant documents.\"),\n      topK: z.number().int().positive().optional().describe(\"The maximum number of results.\"),\n    }),\n    output: z.array(\n      z.object({\n        score: z.number(),\n        id: z.string(),\n        document: z.any(),\n        metadata: z\n          .record(z.string(), z.union([z.string(), z.number(), z.boolean(), z.null()]))\n          .optional(),\n      }),\n    ),\n    execute: ({ query, topK }) =>\n      index.search({\n        query,\n        topK: topK ?? options.topK ?? 5,\n        threshold: options.threshold,\n        filter: options.filter,\n      }),\n  }) as Tool<{ query: string; topK?: number }, Array<VectorSearchResult<T, Metadata>>>;\n}\n\nfunction bestScore(queryEmbedding: Embedding, embeddings: Embedding[]): number | undefined {\n  let best: number | undefined;\n  for (const embedding of embeddings) {\n    const score = cosineSimilarity(queryEmbedding.vector, embedding.vector);\n    best = best === undefined ? score : Math.max(best, score);\n  }\n  return best;\n}\n","import type { VectorMetadata, VectorMetadataValue } from \"../embeddings\";\n\nexport type VectorFilter =\n  | { type: \"eq\"; key: string; value: VectorMetadataValue }\n  | { type: \"gt\"; key: string; value: VectorMetadataValue }\n  | { type: \"lt\"; key: string; value: VectorMetadataValue }\n  | { type: \"and\"; filters: [VectorFilter, VectorFilter] }\n  | { type: \"or\"; filters: [VectorFilter, VectorFilter] };\n\nexport const vectorFilter = {\n  eq(key: string, value: VectorMetadataValue): VectorFilter {\n    return { type: \"eq\", key, value };\n  },\n  gt(key: string, value: VectorMetadataValue): VectorFilter {\n    return { type: \"gt\", key, value };\n  },\n  lt(key: string, value: VectorMetadataValue): VectorFilter {\n    return { type: \"lt\", key, value };\n  },\n  and(left: VectorFilter, right: VectorFilter): VectorFilter {\n    return { type: \"and\", filters: [left, right] };\n  },\n  or(left: VectorFilter, right: VectorFilter): VectorFilter {\n    return { type: \"or\", filters: [left, right] };\n  },\n};\n\nexport function matchesVectorFilter(\n  metadata: VectorMetadata | undefined,\n  filter: VectorFilter | undefined,\n): boolean {\n  if (filter === undefined) {\n    return true;\n  }\n  if (metadata === undefined) {\n    return false;\n  }\n\n  switch (filter.type) {\n    case \"eq\":\n      return metadata[filter.key] === filter.value;\n    case \"gt\":\n      return compare(metadata[filter.key], filter.value) > 0;\n    case \"lt\":\n      return compare(metadata[filter.key], filter.value) < 0;\n    case \"and\":\n      return (\n        matchesVectorFilter(metadata, filter.filters[0]) &&\n        matchesVectorFilter(metadata, filter.filters[1])\n      );\n    case \"or\":\n      return (\n        matchesVectorFilter(metadata, filter.filters[0]) ||\n        matchesVectorFilter(metadata, filter.filters[1])\n      );\n  }\n}\n\nfunction compare(left: VectorMetadataValue | undefined, right: VectorMetadataValue): number {\n  if (typeof left === \"number\" && typeof right === \"number\") {\n    return left - right;\n  }\n  if (typeof left === \"string\" && typeof right === \"string\") {\n    return left.localeCompare(right);\n  }\n  if (typeof left === \"boolean\" && typeof right === \"boolean\") {\n    return Number(left) - Number(right);\n  }\n  return 0;\n}\n","export type LshOptions = {\n  type: \"lsh\";\n  numTables: number;\n  numHyperplanes: number;\n  seed?: number | undefined;\n};\n\nexport class LshIndex {\n  private readonly hyperplanes: number[][];\n  private readonly tables: Array<Map<string, Set<string>>>;\n\n  constructor(\n    dimensions: number,\n    private readonly options: LshOptions,\n  ) {\n    const rng = seededRandom(options.seed ?? 42);\n    this.hyperplanes = [];\n    for (let index = 0; index < options.numTables * options.numHyperplanes; index += 1) {\n      const plane = Array.from({ length: dimensions }, () => rng() * 2 - 1);\n      const norm = Math.sqrt(plane.reduce((sum, value) => sum + value ** 2, 0));\n      this.hyperplanes.push(norm === 0 ? plane : plane.map((value) => value / norm));\n    }\n    this.tables = Array.from({ length: options.numTables }, () => new Map());\n  }\n\n  insert(id: string, vector: number[]): void {\n    for (let table = 0; table < this.options.numTables; table += 1) {\n      const hash = this.hash(vector, table);\n      const bucket = this.tables[table]?.get(hash) ?? new Set<string>();\n      bucket.add(id);\n      this.tables[table]?.set(hash, bucket);\n    }\n  }\n\n  query(vector: number[]): Set<string> {\n    const candidates = new Set<string>();\n    for (let table = 0; table < this.options.numTables; table += 1) {\n      const hash = this.hash(vector, table);\n      for (const id of this.tables[table]?.get(hash) ?? []) {\n        candidates.add(id);\n      }\n    }\n    return candidates;\n  }\n\n  private hash(vector: number[], table: number): string {\n    let hash = \"\";\n    const start = table * this.options.numHyperplanes;\n    for (let offset = 0; offset < this.options.numHyperplanes; offset += 1) {\n      const plane = this.hyperplanes[start + offset] as number[];\n      const dot = vector.reduce((sum, value, index) => sum + value * (plane[index] ?? 0), 0);\n      hash += dot >= 0 ? \"1\" : \"0\";\n    }\n    return hash;\n  }\n}\n\nfunction seededRandom(seed: number): () => number {\n  let state = seed >>> 0;\n  return () => {\n    state = (state * 1664525 + 1013904223) >>> 0;\n    return state / 0x100000000;\n  };\n}\n"],"mappings":";;;;;;;;;AAAA,SAAS,SAAS;;;ACSX,IAAM,eAAe;AAAA,EAC1B,GAAG,KAAa,OAA0C;AACxD,WAAO,EAAE,MAAM,MAAM,KAAK,MAAM;AAAA,EAClC;AAAA,EACA,GAAG,KAAa,OAA0C;AACxD,WAAO,EAAE,MAAM,MAAM,KAAK,MAAM;AAAA,EAClC;AAAA,EACA,GAAG,KAAa,OAA0C;AACxD,WAAO,EAAE,MAAM,MAAM,KAAK,MAAM;AAAA,EAClC;AAAA,EACA,IAAI,MAAoB,OAAmC;AACzD,WAAO,EAAE,MAAM,OAAO,SAAS,CAAC,MAAM,KAAK,EAAE;AAAA,EAC/C;AAAA,EACA,GAAG,MAAoB,OAAmC;AACxD,WAAO,EAAE,MAAM,MAAM,SAAS,CAAC,MAAM,KAAK,EAAE;AAAA,EAC9C;AACF;AAEO,SAAS,oBACd,UACA,QACS;AACT,MAAI,WAAW,QAAW;AACxB,WAAO;AAAA,EACT;AACA,MAAI,aAAa,QAAW;AAC1B,WAAO;AAAA,EACT;AAEA,UAAQ,OAAO,MAAM;AAAA,IACnB,KAAK;AACH,aAAO,SAAS,OAAO,GAAG,MAAM,OAAO;AAAA,IACzC,KAAK;AACH,aAAO,QAAQ,SAAS,OAAO,GAAG,GAAG,OAAO,KAAK,IAAI;AAAA,IACvD,KAAK;AACH,aAAO,QAAQ,SAAS,OAAO,GAAG,GAAG,OAAO,KAAK,IAAI;AAAA,IACvD,KAAK;AACH,aACE,oBAAoB,UAAU,OAAO,QAAQ,CAAC,CAAC,KAC/C,oBAAoB,UAAU,OAAO,QAAQ,CAAC,CAAC;AAAA,IAEnD,KAAK;AACH,aACE,oBAAoB,UAAU,OAAO,QAAQ,CAAC,CAAC,KAC/C,oBAAoB,UAAU,OAAO,QAAQ,CAAC,CAAC;AAAA,EAErD;AACF;AAEA,SAAS,QAAQ,MAAuC,OAAoC;AAC1F,MAAI,OAAO,SAAS,YAAY,OAAO,UAAU,UAAU;AACzD,WAAO,OAAO;AAAA,EAChB;AACA,MAAI,OAAO,SAAS,YAAY,OAAO,UAAU,UAAU;AACzD,WAAO,KAAK,cAAc,KAAK;AAAA,EACjC;AACA,MAAI,OAAO,SAAS,aAAa,OAAO,UAAU,WAAW;AAC3D,WAAO,OAAO,IAAI,IAAI,OAAO,KAAK;AAAA,EACpC;AACA,SAAO;AACT;;;AC9DO,IAAM,WAAN,MAAe;AAAA,EAIpB,YACE,YACiB,SACjB;AADiB;AAEjB,UAAM,MAAM,aAAa,QAAQ,QAAQ,EAAE;AAC3C,SAAK,cAAc,CAAC;AACpB,aAAS,QAAQ,GAAG,QAAQ,QAAQ,YAAY,QAAQ,gBAAgB,SAAS,GAAG;AAClF,YAAM,QAAQ,MAAM,KAAK,EAAE,QAAQ,WAAW,GAAG,MAAM,IAAI,IAAI,IAAI,CAAC;AACpE,YAAM,OAAO,KAAK,KAAK,MAAM,OAAO,CAAC,KAAK,UAAU,MAAM,SAAS,GAAG,CAAC,CAAC;AACxE,WAAK,YAAY,KAAK,SAAS,IAAI,QAAQ,MAAM,IAAI,CAAC,UAAU,QAAQ,IAAI,CAAC;AAAA,IAC/E;AACA,SAAK,SAAS,MAAM,KAAK,EAAE,QAAQ,QAAQ,UAAU,GAAG,MAAM,oBAAI,IAAI,CAAC;AAAA,EACzE;AAAA,EAVmB;AAAA,EALF;AAAA,EACA;AAAA,EAgBjB,OAAO,IAAY,QAAwB;AACzC,aAAS,QAAQ,GAAG,QAAQ,KAAK,QAAQ,WAAW,SAAS,GAAG;AAC9D,YAAM,OAAO,KAAK,KAAK,QAAQ,KAAK;AACpC,YAAM,SAAS,KAAK,OAAO,KAAK,GAAG,IAAI,IAAI,KAAK,oBAAI,IAAY;AAChE,aAAO,IAAI,EAAE;AACb,WAAK,OAAO,KAAK,GAAG,IAAI,MAAM,MAAM;AAAA,IACtC;AAAA,EACF;AAAA,EAEA,MAAM,QAA+B;AACnC,UAAM,aAAa,oBAAI,IAAY;AACnC,aAAS,QAAQ,GAAG,QAAQ,KAAK,QAAQ,WAAW,SAAS,GAAG;AAC9D,YAAM,OAAO,KAAK,KAAK,QAAQ,KAAK;AACpC,iBAAW,MAAM,KAAK,OAAO,KAAK,GAAG,IAAI,IAAI,KAAK,CAAC,GAAG;AACpD,mBAAW,IAAI,EAAE;AAAA,MACnB;AAAA,IACF;AACA,WAAO;AAAA,EACT;AAAA,EAEQ,KAAK,QAAkB,OAAuB;AACpD,QAAI,OAAO;AACX,UAAM,QAAQ,QAAQ,KAAK,QAAQ;AACnC,aAAS,SAAS,GAAG,SAAS,KAAK,QAAQ,gBAAgB,UAAU,GAAG;AACtE,YAAM,QAAQ,KAAK,YAAY,QAAQ,MAAM;AAC7C,YAAM,MAAM,OAAO,OAAO,CAAC,KAAK,OAAO,UAAU,MAAM,SAAS,MAAM,KAAK,KAAK,IAAI,CAAC;AACrF,cAAQ,OAAO,IAAI,MAAM;AAAA,IAC3B;AACA,WAAO;AAAA,EACT;AACF;AAEA,SAAS,aAAa,MAA4B;AAChD,MAAI,QAAQ,SAAS;AACrB,SAAO,MAAM;AACX,YAAS,QAAQ,UAAU,eAAgB;AAC3C,WAAO,QAAQ;AAAA,EACjB;AACF;;;AFIO,IAAM,sBAAN,MAAM,qBAAyE;AAAA,EACnE,YAAY,oBAAI,IAAyC;AAAA,EAClE;AAAA,EACA;AAAA,EAER,YAAY,UAAqC,CAAC,GAAG;AACnD,SAAK,gBAAgB,QAAQ,SAAS,EAAE,MAAM,aAAa;AAAA,EAC7D;AAAA,EAEA,OAAO,cACL,WACA,UAAqC,CAAC,GACJ;AAClC,UAAM,QAAQ,IAAI,qBAAiC,OAAO;AAC1D,UAAM,aAAa,SAAS;AAC5B,WAAO;AAAA,EACT;AAAA,EAEA,aAAa,WAAuD;AAClE,eAAW,YAAY,WAAW;AAChC,WAAK,UAAU,IAAI,SAAS,IAAI,QAAQ;AAAA,IAC1C;AACA,SAAK,gBAAgB;AACrB,WAAO;AAAA,EACT;AAAA,EAEA,IAAI,IAAqD;AACvD,WAAO,KAAK,UAAU,IAAI,EAAE;AAAA,EAC9B;AAAA,EAEA,SAA6C;AAC3C,WAAO,CAAC,GAAG,KAAK,UAAU,OAAO,CAAC;AAAA,EACpC;AAAA,EAEA,MAAc;AACZ,WAAO,KAAK,UAAU;AAAA,EACxB;AAAA,EAEA,UAAmB;AACjB,WAAO,KAAK,UAAU,SAAS;AAAA,EACjC;AAAA,EAEA,MAAM,OAAyD;AAC7D,WAAO,IAAI,oBAAoB,OAAO,IAAI;AAAA,EAC5C;AAAA,EAEA,WAAW,gBAA+D;AACxE,QAAI,KAAK,cAAc,SAAS,SAAS,KAAK,aAAa,QAAW;AACpE,aAAO,KAAK,OAAO;AAAA,IACrB;AAEA,UAAM,eAAe,KAAK,SAAS,MAAM,eAAe,MAAM;AAC9D,QAAI,aAAa,SAAS,GAAG;AAC3B,aAAO,KAAK,OAAO;AAAA,IACrB;AAEA,WAAO,CAAC,GAAG,YAAY,EAAE,QAAQ,CAAC,OAAO;AACvC,YAAM,WAAW,KAAK,UAAU,IAAI,EAAE;AACtC,aAAO,aAAa,SAAY,CAAC,IAAI,CAAC,QAAQ;AAAA,IAChD,CAAC;AAAA,EACH;AAAA,EAEQ,kBAAwB;AAC9B,QAAI,KAAK,cAAc,SAAS,OAAO;AACrC,WAAK,WAAW;AAChB;AAAA,IACF;AAEA,UAAM,iBAAiB,KAAK,OAAO,EAAE,QAAQ,CAAC,aAAa,SAAS,UAAU,EAAE,CAAC;AACjF,QAAI,mBAAmB,QAAW;AAChC,WAAK,WAAW;AAChB;AAAA,IACF;AAEA,UAAM,QAAQ,IAAI,SAAS,eAAe,OAAO,QAAQ,KAAK,aAAa;AAC3E,eAAW,YAAY,KAAK,UAAU,OAAO,GAAG;AAC9C,iBAAW,aAAa,SAAS,YAAY;AAC3C,cAAM,OAAO,SAAS,IAAI,UAAU,MAAM;AAAA,MAC5C;AAAA,IACF;AACA,SAAK,WAAW;AAAA,EAClB;AACF;AAEO,IAAM,sBAAN,MAEP;AAAA,EACE,YACmB,OACA,OACjB;AAFiB;AACA;AAAA,EAChB;AAAA,EAFgB;AAAA,EACA;AAAA,EAGnB,MAAM,OAAO,SAA+E;AAC1F,UAAM,iBAAiB,MAAM,UAAU,KAAK,OAAO,QAAQ,KAAK;AAChE,WAAO,KAAK,MACT,WAAW,cAAc,EACzB,OAAO,CAAC,aAAa,oBAAoB,SAAS,UAAU,QAAQ,MAAM,CAAC,EAC3E,QAAQ,CAAC,aAAa;AACrB,YAAM,QAAQ,UAAU,gBAAgB,SAAS,UAAU;AAC3D,UAAI,UAAU,QAAW;AACvB,eAAO,CAAC;AAAA,MACV;AACA,UAAI,QAAQ,cAAc,UAAa,QAAQ,QAAQ,WAAW;AAChE,eAAO,CAAC;AAAA,MACV;AACA,aAAO;AAAA,QACL;AAAA,UACE;AAAA,UACA,IAAI,SAAS;AAAA,UACb,UAAU,SAAS;AAAA,UACnB,GAAI,SAAS,aAAa,SAAY,CAAC,IAAI,EAAE,UAAU,SAAS,SAAS;AAAA,QAC3E;AAAA,MACF;AAAA,IACF,CAAC,EACA,KAAK,CAAC,MAAM,UAAU,MAAM,QAAQ,KAAK,KAAK,EAC9C,MAAM,GAAG,KAAK,IAAI,GAAG,KAAK,MAAM,QAAQ,IAAI,CAAC,CAAC;AAAA,EACnD;AAAA,EAEA,MAAM,UAAU,SAA6E;AAC3F,YAAQ,MAAM,KAAK,OAAO,OAAO,GAAG,IAAI,CAAC,EAAE,OAAO,GAAG,OAAO,EAAE,OAAO,GAAG,EAAE;AAAA,EAC5E;AAAA,EAEA,MAAM,QAAQ,SAAwE;AACpF,UAAM,QAAQ,KAAK,IAAI,GAAG,KAAK,MAAM,QAAQ,KAAK,CAAC;AACnD,UAAM,QAAQ,KAAK,IAAI,GAAG,KAAK,MAAM,OAAO,QAAQ,UAAU,GAAG,CAAC,CAAC;AACnE,UAAM,YAAY,KAAK,MACpB,OAAO,EACP,OAAO,CAAC,aAAa,oBAAoB,SAAS,UAAU,QAAQ,MAAM,CAAC;AAC9E,UAAM,OAAO,UAAU,MAAM,OAAO,QAAQ,KAAK;AACjD,UAAM,aAAa,QAAQ,KAAK;AAChC,WAAO;AAAA,MACL,OAAO,KAAK,IAAI,CAAC,cAAc;AAAA,QAC7B,IAAI,SAAS;AAAA,QACb,UAAU,SAAS;AAAA,QACnB,GAAI,SAAS,aAAa,SAAY,CAAC,IAAI,EAAE,UAAU,SAAS,SAAS;AAAA,MAC3E,EAAE;AAAA,MACF,GAAI,aAAa,UAAU,SAAS,EAAE,YAAY,OAAO,UAAU,EAAE,IAAI,CAAC;AAAA,MAC1E,YAAY,UAAU;AAAA,IACxB;AAAA,EACF;AAAA,EAEA,OAAO,SAAmF;AACxF,WAAO,uBAAuB,MAAM,OAAO;AAAA,EAC7C;AACF;AAEO,SAAS,uBACd,OACA,SACgF;AAChF,SAAO,WAAW;AAAA,IAChB,MAAM,QAAQ;AAAA,IACd,aACE,QAAQ,eAAe;AAAA,IACzB,OAAO,EAAE,OAAO;AAAA,MACd,OAAO,EAAE,OAAO,EAAE,SAAS,oDAAoD;AAAA,MAC/E,MAAM,EAAE,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,SAAS,EAAE,SAAS,gCAAgC;AAAA,IACxF,CAAC;AAAA,IACD,QAAQ,EAAE;AAAA,MACR,EAAE,OAAO;AAAA,QACP,OAAO,EAAE,OAAO;AAAA,QAChB,IAAI,EAAE,OAAO;AAAA,QACb,UAAU,EAAE,IAAI;AAAA,QAChB,UAAU,EACP,OAAO,EAAE,OAAO,GAAG,EAAE,MAAM,CAAC,EAAE,OAAO,GAAG,EAAE,OAAO,GAAG,EAAE,QAAQ,GAAG,EAAE,KAAK,CAAC,CAAC,CAAC,EAC3E,SAAS;AAAA,MACd,CAAC;AAAA,IACH;AAAA,IACA,SAAS,CAAC,EAAE,OAAO,KAAK,MACtB,MAAM,OAAO;AAAA,MACX;AAAA,MACA,MAAM,QAAQ,QAAQ,QAAQ;AAAA,MAC9B,WAAW,QAAQ;AAAA,MACnB,QAAQ,QAAQ;AAAA,IAClB,CAAC;AAAA,EACL,CAAC;AACH;AAEA,SAAS,UAAU,gBAA2B,YAA6C;AACzF,MAAI;AACJ,aAAW,aAAa,YAAY;AAClC,UAAM,QAAQ,iBAAiB,eAAe,QAAQ,UAAU,MAAM;AACtE,WAAO,SAAS,SAAY,QAAQ,KAAK,IAAI,MAAM,KAAK;AAAA,EAC1D;AACA,SAAO;AACT;","names":[]}