@anvia/core 0.3.0 → 0.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/loaders/index.js
CHANGED
|
@@ -249,7 +249,8 @@ async function readFileSource(source) {
|
|
|
249
249
|
async function readPdfPages(source) {
|
|
250
250
|
const pdfjs = await import("pdfjs-dist/legacy/build/pdf.mjs");
|
|
251
251
|
const bytes = "bytes" in source ? source.bytes : toUint8Array(await readFile(source.path));
|
|
252
|
-
const
|
|
252
|
+
const loadingTask = pdfjs.getDocument({ data: bytes });
|
|
253
|
+
const document = await loadingTask.promise;
|
|
253
254
|
const pages = [];
|
|
254
255
|
try {
|
|
255
256
|
for (let index = 1; index <= document.numPages; index += 1) {
|
|
@@ -262,7 +263,7 @@ async function readPdfPages(source) {
|
|
|
262
263
|
` : "" });
|
|
263
264
|
}
|
|
264
265
|
} finally {
|
|
265
|
-
await
|
|
266
|
+
await loadingTask.destroy();
|
|
266
267
|
}
|
|
267
268
|
return pages;
|
|
268
269
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["../../src/loaders/index.ts"],"sourcesContent":["import { readdir, readFile } from \"node:fs/promises\";\nimport { isAbsolute, join } from \"node:path\";\nimport { glob } from \"tinyglobby\";\nimport type { Document } from \"../completion\";\n\nexport type LoaderResult<T> = { ok: true; value: T } | { ok: false; error: unknown };\n\nexport type FileSource = { path: string } | { path: \"<memory>\"; bytes: Uint8Array };\nexport type FileReadWithPath = { path: string; text: string };\n\nexport type PdfSource = { path: string } | { path: \"<memory>\"; bytes: Uint8Array };\nexport type PdfReadWithPath = { path: string; text: string };\nexport type PdfPage = { pageNumber: number; text: string };\nexport type PdfPageWithPath = { path: string; pageNumber: number; text: string };\n\ntype FileMode = \"source\" | \"read\" | \"readWithPath\";\ntype PdfMode = \"source\" | \"read\" | \"readWithPath\" | \"byPage\" | \"byPageWithPath\";\n\nexport class FileLoader<T = LoaderResult<FileSource>> implements AsyncIterable<T> {\n private constructor(\n private readonly source: () => AsyncIterable<LoaderResult<FileSource>>,\n private readonly mode: FileMode,\n private readonly skipErrors: boolean,\n ) {}\n\n static withGlob(pattern: string): FileLoader<LoaderResult<FileSource>> {\n return new FileLoader(\n async function* () {\n for (const path of await sortedGlob(pattern)) {\n yield ok({ path });\n }\n },\n \"source\",\n false,\n );\n }\n\n static withDir(directory: string): FileLoader<LoaderResult<FileSource>> {\n return new FileLoader(\n async function* () {\n const entries = await readdir(directory, { withFileTypes: true });\n const files = entries\n .filter((entry) => entry.isFile())\n .map((entry) => join(directory, entry.name))\n .sort((left, right) => left.localeCompare(right));\n for (const path of files) {\n yield ok({ path });\n }\n },\n \"source\",\n false,\n );\n }\n\n static fromBytes(bytes: Uint8Array | ArrayBuffer): FileLoader<LoaderResult<FileSource>> {\n return FileLoader.fromBytesMany([bytes]);\n }\n\n static fromBytesMany(\n bytes: Array<Uint8Array | ArrayBuffer>,\n ): FileLoader<LoaderResult<FileSource>> {\n return new FileLoader(\n async function* () {\n for (const value of bytes) {\n yield ok({ path: \"<memory>\", bytes: toUint8Array(value) });\n }\n },\n \"source\",\n false,\n );\n }\n\n read(): FileLoader<LoaderResult<string>> {\n return new FileLoader(this.source, \"read\", this.skipErrors);\n }\n\n readWithPath(): FileLoader<LoaderResult<FileReadWithPath>> {\n return new FileLoader(this.source, \"readWithPath\", this.skipErrors);\n }\n\n ignoreErrors(): FileLoader<UnwrapLoaderResult<T>> {\n return new FileLoader(this.source, this.mode, true) as FileLoader<UnwrapLoaderResult<T>>;\n }\n\n async *[Symbol.asyncIterator](): AsyncIterator<T> {\n for await (const item of this.source()) {\n const result = await mapFileResult(item, this.mode);\n if (!result.ok && this.skipErrors) {\n continue;\n }\n yield (this.skipErrors && result.ok ? result.value : result) as T;\n }\n }\n}\n\nexport class PdfFileLoader<T = LoaderResult<PdfSource>> implements AsyncIterable<T> {\n private constructor(\n private readonly source: () => AsyncIterable<LoaderResult<PdfSource>>,\n private readonly mode: PdfMode,\n private readonly skipErrors: boolean,\n ) {}\n\n static withGlob(pattern: string): PdfFileLoader<LoaderResult<PdfSource>> {\n return new PdfFileLoader(\n async function* () {\n for (const path of await sortedGlob(pattern)) {\n yield ok({ path });\n }\n },\n \"source\",\n false,\n );\n }\n\n static withDir(directory: string): PdfFileLoader<LoaderResult<PdfSource>> {\n return new PdfFileLoader(\n async function* () {\n const entries = await readdir(directory, { withFileTypes: true });\n const files = entries\n .filter((entry) => entry.isFile())\n .map((entry) => join(directory, entry.name))\n .sort((left, right) => left.localeCompare(right));\n for (const path of files) {\n yield ok({ path });\n }\n },\n \"source\",\n false,\n );\n }\n\n static fromBytes(bytes: Uint8Array | ArrayBuffer): PdfFileLoader<LoaderResult<PdfSource>> {\n return PdfFileLoader.fromBytesMany([bytes]);\n }\n\n static fromBytesMany(\n bytes: Array<Uint8Array | ArrayBuffer>,\n ): PdfFileLoader<LoaderResult<PdfSource>> {\n return new PdfFileLoader(\n async function* () {\n for (const value of bytes) {\n yield ok({ path: \"<memory>\", bytes: toUint8Array(value) });\n }\n },\n \"source\",\n false,\n );\n }\n\n read(): PdfFileLoader<LoaderResult<string>> {\n return new PdfFileLoader(this.source, \"read\", this.skipErrors);\n }\n\n readWithPath(): PdfFileLoader<LoaderResult<PdfReadWithPath>> {\n return new PdfFileLoader(this.source, \"readWithPath\", this.skipErrors);\n }\n\n byPage(\n this: PdfFileLoader<LoaderResult<PdfReadWithPath>>,\n ): PdfFileLoader<LoaderResult<PdfPageWithPath>>;\n byPage(this: PdfFileLoader<PdfReadWithPath>): PdfFileLoader<PdfPageWithPath>;\n byPage(): PdfFileLoader<LoaderResult<PdfPage>>;\n byPage(): PdfFileLoader<LoaderResult<PdfPage> | LoaderResult<PdfPageWithPath> | PdfPageWithPath> {\n const mode = this.mode === \"readWithPath\" ? \"byPageWithPath\" : \"byPage\";\n return new PdfFileLoader(this.source, mode, this.skipErrors);\n }\n\n ignoreErrors(): PdfFileLoader<UnwrapLoaderResult<T>> {\n return new PdfFileLoader(this.source, this.mode, true) as PdfFileLoader<UnwrapLoaderResult<T>>;\n }\n\n async *[Symbol.asyncIterator](): AsyncIterator<T> {\n for await (const item of this.source()) {\n const results = await mapPdfResult(item, this.mode);\n for (const result of results) {\n if (!result.ok && this.skipErrors) {\n continue;\n }\n yield (this.skipErrors && result.ok ? result.value : result) as T;\n }\n }\n }\n}\n\nexport function fileToDocument(file: FileReadWithPath): Document {\n return {\n id: file.path,\n text: file.text,\n additionalProps: {\n source: file.path,\n mediaType: \"text/plain\",\n },\n };\n}\n\nexport async function fileLoaderToDocuments(\n loader: AsyncIterable<FileReadWithPath>,\n): Promise<Document[]> {\n const documents: Document[] = [];\n for await (const file of loader) {\n documents.push(fileToDocument(file));\n }\n return documents;\n}\n\nexport function pdfToDocument(pdf: PdfReadWithPath): Document {\n return {\n id: pdf.path,\n text: pdf.text,\n additionalProps: {\n source: pdf.path,\n mediaType: \"application/pdf\",\n },\n };\n}\n\nexport async function pdfLoaderToDocuments(\n loader: AsyncIterable<PdfReadWithPath>,\n): Promise<Document[]> {\n const documents: Document[] = [];\n for await (const pdf of loader) {\n documents.push(pdfToDocument(pdf));\n }\n return documents;\n}\n\nexport function pdfPageToDocument(page: PdfPageWithPath): Document {\n return {\n id: `${page.path}#page=${page.pageNumber}`,\n text: page.text,\n additionalProps: {\n source: page.path,\n mediaType: \"application/pdf\",\n pageNumber: String(page.pageNumber),\n },\n };\n}\n\nexport async function pdfPageLoaderToDocuments(\n loader: AsyncIterable<PdfPageWithPath>,\n): Promise<Document[]> {\n const documents: Document[] = [];\n for await (const page of loader) {\n documents.push(pdfPageToDocument(page));\n }\n return documents;\n}\n\ntype LoaderValue<T> = T extends { ok: true; value: infer Value } ? Value : never;\ntype UnwrapLoaderResult<T> = [LoaderValue<T>] extends [never] ? T : LoaderValue<T>;\n\nfunction ok<T>(value: T): LoaderResult<T> {\n return { ok: true, value };\n}\n\nfunction err(error: unknown): LoaderResult<never> {\n return { ok: false, error };\n}\n\nasync function sortedGlob(pattern: string): Promise<string[]> {\n return (await glob(pattern, { absolute: isAbsolute(pattern), onlyFiles: true })).sort(\n (left, right) => left.localeCompare(right),\n );\n}\n\nasync function mapFileResult(\n item: LoaderResult<FileSource>,\n mode: FileMode,\n): Promise<LoaderResult<FileSource | string | FileReadWithPath>> {\n if (!item.ok) {\n return item;\n }\n if (mode === \"source\") {\n return item;\n }\n try {\n const text = await readFileSource(item.value);\n return mode === \"read\" ? ok(text) : ok({ path: item.value.path, text });\n } catch (error) {\n return err(error);\n }\n}\n\nasync function mapPdfResult(\n item: LoaderResult<PdfSource>,\n mode: PdfMode,\n): Promise<Array<LoaderResult<PdfSource | string | PdfReadWithPath | PdfPage | PdfPageWithPath>>> {\n if (!item.ok) {\n return [item];\n }\n if (mode === \"source\") {\n return [item];\n }\n try {\n const pages = await readPdfPages(item.value);\n if (mode === \"read\") {\n return [ok(pages.map((page) => page.text).join(\"\"))];\n }\n if (mode === \"readWithPath\") {\n return [ok({ path: item.value.path, text: pages.map((page) => page.text).join(\"\") })];\n }\n if (mode === \"byPageWithPath\") {\n return pages.map((page) => ok({ path: item.value.path, ...page }));\n }\n return pages.map(ok);\n } catch (error) {\n return [err(error)];\n }\n}\n\nasync function readFileSource(source: FileSource): Promise<string> {\n if (\"bytes\" in source) {\n return new TextDecoder().decode(source.bytes);\n }\n return readFile(source.path, \"utf8\");\n}\n\nasync function readPdfPages(source: PdfSource): Promise<PdfPage[]> {\n const pdfjs = await import(\"pdfjs-dist/legacy/build/pdf.mjs\");\n const bytes = \"bytes\" in source ? source.bytes : toUint8Array(await readFile(source.path));\n const document = await pdfjs.getDocument({ data: bytes }).promise;\n const pages: PdfPage[] = [];\n try {\n for (let index = 1; index <= document.numPages; index += 1) {\n const page = await document.getPage(index);\n const content = await page.getTextContent();\n const text = content.items\n .flatMap((item) =>\n typeof item === \"object\" && item !== null && \"str\" in item ? [String(item.str)] : [],\n )\n .join(\"\\n\");\n pages.push({ pageNumber: index - 1, text: text.length > 0 ? `${text}\\n` : \"\" });\n }\n } finally {\n await document.destroy();\n }\n return pages;\n}\n\nfunction toUint8Array(bytes: Uint8Array | ArrayBuffer): Uint8Array {\n if (bytes instanceof Uint8Array) {\n return new Uint8Array(\n bytes.buffer.slice(bytes.byteOffset, bytes.byteOffset + bytes.byteLength),\n );\n }\n return new Uint8Array(bytes);\n}\n"],"mappings":";AAAA,SAAS,SAAS,gBAAgB;AAClC,SAAS,YAAY,YAAY;AACjC,SAAS,YAAY;AAgBd,IAAM,aAAN,MAAM,YAAqE;AAAA,EACxE,YACW,QACA,MACA,YACjB;AAHiB;AACA;AACA;AAAA,EAChB;AAAA,EAHgB;AAAA,EACA;AAAA,EACA;AAAA,EAGnB,OAAO,SAAS,SAAuD;AACrE,WAAO,IAAI;AAAA,MACT,mBAAmB;AACjB,mBAAW,QAAQ,MAAM,WAAW,OAAO,GAAG;AAC5C,gBAAM,GAAG,EAAE,KAAK,CAAC;AAAA,QACnB;AAAA,MACF;AAAA,MACA;AAAA,MACA;AAAA,IACF;AAAA,EACF;AAAA,EAEA,OAAO,QAAQ,WAAyD;AACtE,WAAO,IAAI;AAAA,MACT,mBAAmB;AACjB,cAAM,UAAU,MAAM,QAAQ,WAAW,EAAE,eAAe,KAAK,CAAC;AAChE,cAAM,QAAQ,QACX,OAAO,CAAC,UAAU,MAAM,OAAO,CAAC,EAChC,IAAI,CAAC,UAAU,KAAK,WAAW,MAAM,IAAI,CAAC,EAC1C,KAAK,CAAC,MAAM,UAAU,KAAK,cAAc,KAAK,CAAC;AAClD,mBAAW,QAAQ,OAAO;AACxB,gBAAM,GAAG,EAAE,KAAK,CAAC;AAAA,QACnB;AAAA,MACF;AAAA,MACA;AAAA,MACA;AAAA,IACF;AAAA,EACF;AAAA,EAEA,OAAO,UAAU,OAAuE;AACtF,WAAO,YAAW,cAAc,CAAC,KAAK,CAAC;AAAA,EACzC;AAAA,EAEA,OAAO,cACL,OACsC;AACtC,WAAO,IAAI;AAAA,MACT,mBAAmB;AACjB,mBAAW,SAAS,OAAO;AACzB,gBAAM,GAAG,EAAE,MAAM,YAAY,OAAO,aAAa,KAAK,EAAE,CAAC;AAAA,QAC3D;AAAA,MACF;AAAA,MACA;AAAA,MACA;AAAA,IACF;AAAA,EACF;AAAA,EAEA,OAAyC;AACvC,WAAO,IAAI,YAAW,KAAK,QAAQ,QAAQ,KAAK,UAAU;AAAA,EAC5D;AAAA,EAEA,eAA2D;AACzD,WAAO,IAAI,YAAW,KAAK,QAAQ,gBAAgB,KAAK,UAAU;AAAA,EACpE;AAAA,EAEA,eAAkD;AAChD,WAAO,IAAI,YAAW,KAAK,QAAQ,KAAK,MAAM,IAAI;AAAA,EACpD;AAAA,EAEA,QAAQ,OAAO,aAAa,IAAsB;AAChD,qBAAiB,QAAQ,KAAK,OAAO,GAAG;AACtC,YAAM,SAAS,MAAM,cAAc,MAAM,KAAK,IAAI;AAClD,UAAI,CAAC,OAAO,MAAM,KAAK,YAAY;AACjC;AAAA,MACF;AACA,YAAO,KAAK,cAAc,OAAO,KAAK,OAAO,QAAQ;AAAA,IACvD;AAAA,EACF;AACF;AAEO,IAAM,gBAAN,MAAM,eAAuE;AAAA,EAC1E,YACW,QACA,MACA,YACjB;AAHiB;AACA;AACA;AAAA,EAChB;AAAA,EAHgB;AAAA,EACA;AAAA,EACA;AAAA,EAGnB,OAAO,SAAS,SAAyD;AACvE,WAAO,IAAI;AAAA,MACT,mBAAmB;AACjB,mBAAW,QAAQ,MAAM,WAAW,OAAO,GAAG;AAC5C,gBAAM,GAAG,EAAE,KAAK,CAAC;AAAA,QACnB;AAAA,MACF;AAAA,MACA;AAAA,MACA;AAAA,IACF;AAAA,EACF;AAAA,EAEA,OAAO,QAAQ,WAA2D;AACxE,WAAO,IAAI;AAAA,MACT,mBAAmB;AACjB,cAAM,UAAU,MAAM,QAAQ,WAAW,EAAE,eAAe,KAAK,CAAC;AAChE,cAAM,QAAQ,QACX,OAAO,CAAC,UAAU,MAAM,OAAO,CAAC,EAChC,IAAI,CAAC,UAAU,KAAK,WAAW,MAAM,IAAI,CAAC,EAC1C,KAAK,CAAC,MAAM,UAAU,KAAK,cAAc,KAAK,CAAC;AAClD,mBAAW,QAAQ,OAAO;AACxB,gBAAM,GAAG,EAAE,KAAK,CAAC;AAAA,QACnB;AAAA,MACF;AAAA,MACA;AAAA,MACA;AAAA,IACF;AAAA,EACF;AAAA,EAEA,OAAO,UAAU,OAAyE;AACxF,WAAO,eAAc,cAAc,CAAC,KAAK,CAAC;AAAA,EAC5C;AAAA,EAEA,OAAO,cACL,OACwC;AACxC,WAAO,IAAI;AAAA,MACT,mBAAmB;AACjB,mBAAW,SAAS,OAAO;AACzB,gBAAM,GAAG,EAAE,MAAM,YAAY,OAAO,aAAa,KAAK,EAAE,CAAC;AAAA,QAC3D;AAAA,MACF;AAAA,MACA;AAAA,MACA;AAAA,IACF;AAAA,EACF;AAAA,EAEA,OAA4C;AAC1C,WAAO,IAAI,eAAc,KAAK,QAAQ,QAAQ,KAAK,UAAU;AAAA,EAC/D;AAAA,EAEA,eAA6D;AAC3D,WAAO,IAAI,eAAc,KAAK,QAAQ,gBAAgB,KAAK,UAAU;AAAA,EACvE;AAAA,EAOA,SAAiG;AAC/F,UAAM,OAAO,KAAK,SAAS,iBAAiB,mBAAmB;AAC/D,WAAO,IAAI,eAAc,KAAK,QAAQ,MAAM,KAAK,UAAU;AAAA,EAC7D;AAAA,EAEA,eAAqD;AACnD,WAAO,IAAI,eAAc,KAAK,QAAQ,KAAK,MAAM,IAAI;AAAA,EACvD;AAAA,EAEA,QAAQ,OAAO,aAAa,IAAsB;AAChD,qBAAiB,QAAQ,KAAK,OAAO,GAAG;AACtC,YAAM,UAAU,MAAM,aAAa,MAAM,KAAK,IAAI;AAClD,iBAAW,UAAU,SAAS;AAC5B,YAAI,CAAC,OAAO,MAAM,KAAK,YAAY;AACjC;AAAA,QACF;AACA,cAAO,KAAK,cAAc,OAAO,KAAK,OAAO,QAAQ;AAAA,MACvD;AAAA,IACF;AAAA,EACF;AACF;AAEO,SAAS,eAAe,MAAkC;AAC/D,SAAO;AAAA,IACL,IAAI,KAAK;AAAA,IACT,MAAM,KAAK;AAAA,IACX,iBAAiB;AAAA,MACf,QAAQ,KAAK;AAAA,MACb,WAAW;AAAA,IACb;AAAA,EACF;AACF;AAEA,eAAsB,sBACpB,QACqB;AACrB,QAAM,YAAwB,CAAC;AAC/B,mBAAiB,QAAQ,QAAQ;AAC/B,cAAU,KAAK,eAAe,IAAI,CAAC;AAAA,EACrC;AACA,SAAO;AACT;AAEO,SAAS,cAAc,KAAgC;AAC5D,SAAO;AAAA,IACL,IAAI,IAAI;AAAA,IACR,MAAM,IAAI;AAAA,IACV,iBAAiB;AAAA,MACf,QAAQ,IAAI;AAAA,MACZ,WAAW;AAAA,IACb;AAAA,EACF;AACF;AAEA,eAAsB,qBACpB,QACqB;AACrB,QAAM,YAAwB,CAAC;AAC/B,mBAAiB,OAAO,QAAQ;AAC9B,cAAU,KAAK,cAAc,GAAG,CAAC;AAAA,EACnC;AACA,SAAO;AACT;AAEO,SAAS,kBAAkB,MAAiC;AACjE,SAAO;AAAA,IACL,IAAI,GAAG,KAAK,IAAI,SAAS,KAAK,UAAU;AAAA,IACxC,MAAM,KAAK;AAAA,IACX,iBAAiB;AAAA,MACf,QAAQ,KAAK;AAAA,MACb,WAAW;AAAA,MACX,YAAY,OAAO,KAAK,UAAU;AAAA,IACpC;AAAA,EACF;AACF;AAEA,eAAsB,yBACpB,QACqB;AACrB,QAAM,YAAwB,CAAC;AAC/B,mBAAiB,QAAQ,QAAQ;AAC/B,cAAU,KAAK,kBAAkB,IAAI,CAAC;AAAA,EACxC;AACA,SAAO;AACT;AAKA,SAAS,GAAM,OAA2B;AACxC,SAAO,EAAE,IAAI,MAAM,MAAM;AAC3B;AAEA,SAAS,IAAI,OAAqC;AAChD,SAAO,EAAE,IAAI,OAAO,MAAM;AAC5B;AAEA,eAAe,WAAW,SAAoC;AAC5D,UAAQ,MAAM,KAAK,SAAS,EAAE,UAAU,WAAW,OAAO,GAAG,WAAW,KAAK,CAAC,GAAG;AAAA,IAC/E,CAAC,MAAM,UAAU,KAAK,cAAc,KAAK;AAAA,EAC3C;AACF;AAEA,eAAe,cACb,MACA,MAC+D;AAC/D,MAAI,CAAC,KAAK,IAAI;AACZ,WAAO;AAAA,EACT;AACA,MAAI,SAAS,UAAU;AACrB,WAAO;AAAA,EACT;AACA,MAAI;AACF,UAAM,OAAO,MAAM,eAAe,KAAK,KAAK;AAC5C,WAAO,SAAS,SAAS,GAAG,IAAI,IAAI,GAAG,EAAE,MAAM,KAAK,MAAM,MAAM,KAAK,CAAC;AAAA,EACxE,SAAS,OAAO;AACd,WAAO,IAAI,KAAK;AAAA,EAClB;AACF;AAEA,eAAe,aACb,MACA,MACgG;AAChG,MAAI,CAAC,KAAK,IAAI;AACZ,WAAO,CAAC,IAAI;AAAA,EACd;AACA,MAAI,SAAS,UAAU;AACrB,WAAO,CAAC,IAAI;AAAA,EACd;AACA,MAAI;AACF,UAAM,QAAQ,MAAM,aAAa,KAAK,KAAK;AAC3C,QAAI,SAAS,QAAQ;AACnB,aAAO,CAAC,GAAG,MAAM,IAAI,CAAC,SAAS,KAAK,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;AAAA,IACrD;AACA,QAAI,SAAS,gBAAgB;AAC3B,aAAO,CAAC,GAAG,EAAE,MAAM,KAAK,MAAM,MAAM,MAAM,MAAM,IAAI,CAAC,SAAS,KAAK,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC;AAAA,IACtF;AACA,QAAI,SAAS,kBAAkB;AAC7B,aAAO,MAAM,IAAI,CAAC,SAAS,GAAG,EAAE,MAAM,KAAK,MAAM,MAAM,GAAG,KAAK,CAAC,CAAC;AAAA,IACnE;AACA,WAAO,MAAM,IAAI,EAAE;AAAA,EACrB,SAAS,OAAO;AACd,WAAO,CAAC,IAAI,KAAK,CAAC;AAAA,EACpB;AACF;AAEA,eAAe,eAAe,QAAqC;AACjE,MAAI,WAAW,QAAQ;AACrB,WAAO,IAAI,YAAY,EAAE,OAAO,OAAO,KAAK;AAAA,EAC9C;AACA,SAAO,SAAS,OAAO,MAAM,MAAM;AACrC;AAEA,eAAe,aAAa,QAAuC;AACjE,QAAM,QAAQ,MAAM,OAAO,iCAAiC;AAC5D,QAAM,QAAQ,WAAW,SAAS,OAAO,QAAQ,aAAa,MAAM,SAAS,OAAO,IAAI,CAAC;AACzF,QAAM,WAAW,MAAM,MAAM,YAAY,EAAE,MAAM,MAAM,CAAC,EAAE;AAC1D,QAAM,QAAmB,CAAC;AAC1B,MAAI;AACF,aAAS,QAAQ,GAAG,SAAS,SAAS,UAAU,SAAS,GAAG;AAC1D,YAAM,OAAO,MAAM,SAAS,QAAQ,KAAK;AACzC,YAAM,UAAU,MAAM,KAAK,eAAe;AAC1C,YAAM,OAAO,QAAQ,MAClB;AAAA,QAAQ,CAAC,SACR,OAAO,SAAS,YAAY,SAAS,QAAQ,SAAS,OAAO,CAAC,OAAO,KAAK,GAAG,CAAC,IAAI,CAAC;AAAA,MACrF,EACC,KAAK,IAAI;AACZ,YAAM,KAAK,EAAE,YAAY,QAAQ,GAAG,MAAM,KAAK,SAAS,IAAI,GAAG,IAAI;AAAA,IAAO,GAAG,CAAC;AAAA,IAChF;AAAA,EACF,UAAE;AACA,UAAM,SAAS,QAAQ;AAAA,EACzB;AACA,SAAO;AACT;AAEA,SAAS,aAAa,OAA6C;AACjE,MAAI,iBAAiB,YAAY;AAC/B,WAAO,IAAI;AAAA,MACT,MAAM,OAAO,MAAM,MAAM,YAAY,MAAM,aAAa,MAAM,UAAU;AAAA,IAC1E;AAAA,EACF;AACA,SAAO,IAAI,WAAW,KAAK;AAC7B;","names":[]}
|
|
1
|
+
{"version":3,"sources":["../../src/loaders/index.ts"],"sourcesContent":["import { readdir, readFile } from \"node:fs/promises\";\nimport { isAbsolute, join } from \"node:path\";\nimport { glob } from \"tinyglobby\";\nimport type { Document } from \"../completion\";\n\nexport type LoaderResult<T> = { ok: true; value: T } | { ok: false; error: unknown };\n\nexport type FileSource = { path: string } | { path: \"<memory>\"; bytes: Uint8Array };\nexport type FileReadWithPath = { path: string; text: string };\n\nexport type PdfSource = { path: string } | { path: \"<memory>\"; bytes: Uint8Array };\nexport type PdfReadWithPath = { path: string; text: string };\nexport type PdfPage = { pageNumber: number; text: string };\nexport type PdfPageWithPath = { path: string; pageNumber: number; text: string };\n\ntype FileMode = \"source\" | \"read\" | \"readWithPath\";\ntype PdfMode = \"source\" | \"read\" | \"readWithPath\" | \"byPage\" | \"byPageWithPath\";\n\nexport class FileLoader<T = LoaderResult<FileSource>> implements AsyncIterable<T> {\n private constructor(\n private readonly source: () => AsyncIterable<LoaderResult<FileSource>>,\n private readonly mode: FileMode,\n private readonly skipErrors: boolean,\n ) {}\n\n static withGlob(pattern: string): FileLoader<LoaderResult<FileSource>> {\n return new FileLoader(\n async function* () {\n for (const path of await sortedGlob(pattern)) {\n yield ok({ path });\n }\n },\n \"source\",\n false,\n );\n }\n\n static withDir(directory: string): FileLoader<LoaderResult<FileSource>> {\n return new FileLoader(\n async function* () {\n const entries = await readdir(directory, { withFileTypes: true });\n const files = entries\n .filter((entry) => entry.isFile())\n .map((entry) => join(directory, entry.name))\n .sort((left, right) => left.localeCompare(right));\n for (const path of files) {\n yield ok({ path });\n }\n },\n \"source\",\n false,\n );\n }\n\n static fromBytes(bytes: Uint8Array | ArrayBuffer): FileLoader<LoaderResult<FileSource>> {\n return FileLoader.fromBytesMany([bytes]);\n }\n\n static fromBytesMany(\n bytes: Array<Uint8Array | ArrayBuffer>,\n ): FileLoader<LoaderResult<FileSource>> {\n return new FileLoader(\n async function* () {\n for (const value of bytes) {\n yield ok({ path: \"<memory>\", bytes: toUint8Array(value) });\n }\n },\n \"source\",\n false,\n );\n }\n\n read(): FileLoader<LoaderResult<string>> {\n return new FileLoader(this.source, \"read\", this.skipErrors);\n }\n\n readWithPath(): FileLoader<LoaderResult<FileReadWithPath>> {\n return new FileLoader(this.source, \"readWithPath\", this.skipErrors);\n }\n\n ignoreErrors(): FileLoader<UnwrapLoaderResult<T>> {\n return new FileLoader(this.source, this.mode, true) as FileLoader<UnwrapLoaderResult<T>>;\n }\n\n async *[Symbol.asyncIterator](): AsyncIterator<T> {\n for await (const item of this.source()) {\n const result = await mapFileResult(item, this.mode);\n if (!result.ok && this.skipErrors) {\n continue;\n }\n yield (this.skipErrors && result.ok ? result.value : result) as T;\n }\n }\n}\n\nexport class PdfFileLoader<T = LoaderResult<PdfSource>> implements AsyncIterable<T> {\n private constructor(\n private readonly source: () => AsyncIterable<LoaderResult<PdfSource>>,\n private readonly mode: PdfMode,\n private readonly skipErrors: boolean,\n ) {}\n\n static withGlob(pattern: string): PdfFileLoader<LoaderResult<PdfSource>> {\n return new PdfFileLoader(\n async function* () {\n for (const path of await sortedGlob(pattern)) {\n yield ok({ path });\n }\n },\n \"source\",\n false,\n );\n }\n\n static withDir(directory: string): PdfFileLoader<LoaderResult<PdfSource>> {\n return new PdfFileLoader(\n async function* () {\n const entries = await readdir(directory, { withFileTypes: true });\n const files = entries\n .filter((entry) => entry.isFile())\n .map((entry) => join(directory, entry.name))\n .sort((left, right) => left.localeCompare(right));\n for (const path of files) {\n yield ok({ path });\n }\n },\n \"source\",\n false,\n );\n }\n\n static fromBytes(bytes: Uint8Array | ArrayBuffer): PdfFileLoader<LoaderResult<PdfSource>> {\n return PdfFileLoader.fromBytesMany([bytes]);\n }\n\n static fromBytesMany(\n bytes: Array<Uint8Array | ArrayBuffer>,\n ): PdfFileLoader<LoaderResult<PdfSource>> {\n return new PdfFileLoader(\n async function* () {\n for (const value of bytes) {\n yield ok({ path: \"<memory>\", bytes: toUint8Array(value) });\n }\n },\n \"source\",\n false,\n );\n }\n\n read(): PdfFileLoader<LoaderResult<string>> {\n return new PdfFileLoader(this.source, \"read\", this.skipErrors);\n }\n\n readWithPath(): PdfFileLoader<LoaderResult<PdfReadWithPath>> {\n return new PdfFileLoader(this.source, \"readWithPath\", this.skipErrors);\n }\n\n byPage(\n this: PdfFileLoader<LoaderResult<PdfReadWithPath>>,\n ): PdfFileLoader<LoaderResult<PdfPageWithPath>>;\n byPage(this: PdfFileLoader<PdfReadWithPath>): PdfFileLoader<PdfPageWithPath>;\n byPage(): PdfFileLoader<LoaderResult<PdfPage>>;\n byPage(): PdfFileLoader<LoaderResult<PdfPage> | LoaderResult<PdfPageWithPath> | PdfPageWithPath> {\n const mode = this.mode === \"readWithPath\" ? \"byPageWithPath\" : \"byPage\";\n return new PdfFileLoader(this.source, mode, this.skipErrors);\n }\n\n ignoreErrors(): PdfFileLoader<UnwrapLoaderResult<T>> {\n return new PdfFileLoader(this.source, this.mode, true) as PdfFileLoader<UnwrapLoaderResult<T>>;\n }\n\n async *[Symbol.asyncIterator](): AsyncIterator<T> {\n for await (const item of this.source()) {\n const results = await mapPdfResult(item, this.mode);\n for (const result of results) {\n if (!result.ok && this.skipErrors) {\n continue;\n }\n yield (this.skipErrors && result.ok ? result.value : result) as T;\n }\n }\n }\n}\n\nexport function fileToDocument(file: FileReadWithPath): Document {\n return {\n id: file.path,\n text: file.text,\n additionalProps: {\n source: file.path,\n mediaType: \"text/plain\",\n },\n };\n}\n\nexport async function fileLoaderToDocuments(\n loader: AsyncIterable<FileReadWithPath>,\n): Promise<Document[]> {\n const documents: Document[] = [];\n for await (const file of loader) {\n documents.push(fileToDocument(file));\n }\n return documents;\n}\n\nexport function pdfToDocument(pdf: PdfReadWithPath): Document {\n return {\n id: pdf.path,\n text: pdf.text,\n additionalProps: {\n source: pdf.path,\n mediaType: \"application/pdf\",\n },\n };\n}\n\nexport async function pdfLoaderToDocuments(\n loader: AsyncIterable<PdfReadWithPath>,\n): Promise<Document[]> {\n const documents: Document[] = [];\n for await (const pdf of loader) {\n documents.push(pdfToDocument(pdf));\n }\n return documents;\n}\n\nexport function pdfPageToDocument(page: PdfPageWithPath): Document {\n return {\n id: `${page.path}#page=${page.pageNumber}`,\n text: page.text,\n additionalProps: {\n source: page.path,\n mediaType: \"application/pdf\",\n pageNumber: String(page.pageNumber),\n },\n };\n}\n\nexport async function pdfPageLoaderToDocuments(\n loader: AsyncIterable<PdfPageWithPath>,\n): Promise<Document[]> {\n const documents: Document[] = [];\n for await (const page of loader) {\n documents.push(pdfPageToDocument(page));\n }\n return documents;\n}\n\ntype LoaderValue<T> = T extends { ok: true; value: infer Value } ? Value : never;\ntype UnwrapLoaderResult<T> = [LoaderValue<T>] extends [never] ? T : LoaderValue<T>;\n\nfunction ok<T>(value: T): LoaderResult<T> {\n return { ok: true, value };\n}\n\nfunction err(error: unknown): LoaderResult<never> {\n return { ok: false, error };\n}\n\nasync function sortedGlob(pattern: string): Promise<string[]> {\n return (await glob(pattern, { absolute: isAbsolute(pattern), onlyFiles: true })).sort(\n (left, right) => left.localeCompare(right),\n );\n}\n\nasync function mapFileResult(\n item: LoaderResult<FileSource>,\n mode: FileMode,\n): Promise<LoaderResult<FileSource | string | FileReadWithPath>> {\n if (!item.ok) {\n return item;\n }\n if (mode === \"source\") {\n return item;\n }\n try {\n const text = await readFileSource(item.value);\n return mode === \"read\" ? ok(text) : ok({ path: item.value.path, text });\n } catch (error) {\n return err(error);\n }\n}\n\nasync function mapPdfResult(\n item: LoaderResult<PdfSource>,\n mode: PdfMode,\n): Promise<Array<LoaderResult<PdfSource | string | PdfReadWithPath | PdfPage | PdfPageWithPath>>> {\n if (!item.ok) {\n return [item];\n }\n if (mode === \"source\") {\n return [item];\n }\n try {\n const pages = await readPdfPages(item.value);\n if (mode === \"read\") {\n return [ok(pages.map((page) => page.text).join(\"\"))];\n }\n if (mode === \"readWithPath\") {\n return [ok({ path: item.value.path, text: pages.map((page) => page.text).join(\"\") })];\n }\n if (mode === \"byPageWithPath\") {\n return pages.map((page) => ok({ path: item.value.path, ...page }));\n }\n return pages.map(ok);\n } catch (error) {\n return [err(error)];\n }\n}\n\nasync function readFileSource(source: FileSource): Promise<string> {\n if (\"bytes\" in source) {\n return new TextDecoder().decode(source.bytes);\n }\n return readFile(source.path, \"utf8\");\n}\n\nasync function readPdfPages(source: PdfSource): Promise<PdfPage[]> {\n const pdfjs = await import(\"pdfjs-dist/legacy/build/pdf.mjs\");\n const bytes = \"bytes\" in source ? source.bytes : toUint8Array(await readFile(source.path));\n const loadingTask = pdfjs.getDocument({ data: bytes });\n const document = await loadingTask.promise;\n const pages: PdfPage[] = [];\n try {\n for (let index = 1; index <= document.numPages; index += 1) {\n const page = await document.getPage(index);\n const content = await page.getTextContent();\n const text = content.items\n .flatMap((item) =>\n typeof item === \"object\" && item !== null && \"str\" in item ? [String(item.str)] : [],\n )\n .join(\"\\n\");\n pages.push({ pageNumber: index - 1, text: text.length > 0 ? `${text}\\n` : \"\" });\n }\n } finally {\n await loadingTask.destroy();\n }\n return pages;\n}\n\nfunction toUint8Array(bytes: Uint8Array | ArrayBuffer): Uint8Array {\n if (bytes instanceof Uint8Array) {\n return new Uint8Array(\n bytes.buffer.slice(bytes.byteOffset, bytes.byteOffset + bytes.byteLength),\n );\n }\n return new Uint8Array(bytes);\n}\n"],"mappings":";AAAA,SAAS,SAAS,gBAAgB;AAClC,SAAS,YAAY,YAAY;AACjC,SAAS,YAAY;AAgBd,IAAM,aAAN,MAAM,YAAqE;AAAA,EACxE,YACW,QACA,MACA,YACjB;AAHiB;AACA;AACA;AAAA,EAChB;AAAA,EAHgB;AAAA,EACA;AAAA,EACA;AAAA,EAGnB,OAAO,SAAS,SAAuD;AACrE,WAAO,IAAI;AAAA,MACT,mBAAmB;AACjB,mBAAW,QAAQ,MAAM,WAAW,OAAO,GAAG;AAC5C,gBAAM,GAAG,EAAE,KAAK,CAAC;AAAA,QACnB;AAAA,MACF;AAAA,MACA;AAAA,MACA;AAAA,IACF;AAAA,EACF;AAAA,EAEA,OAAO,QAAQ,WAAyD;AACtE,WAAO,IAAI;AAAA,MACT,mBAAmB;AACjB,cAAM,UAAU,MAAM,QAAQ,WAAW,EAAE,eAAe,KAAK,CAAC;AAChE,cAAM,QAAQ,QACX,OAAO,CAAC,UAAU,MAAM,OAAO,CAAC,EAChC,IAAI,CAAC,UAAU,KAAK,WAAW,MAAM,IAAI,CAAC,EAC1C,KAAK,CAAC,MAAM,UAAU,KAAK,cAAc,KAAK,CAAC;AAClD,mBAAW,QAAQ,OAAO;AACxB,gBAAM,GAAG,EAAE,KAAK,CAAC;AAAA,QACnB;AAAA,MACF;AAAA,MACA;AAAA,MACA;AAAA,IACF;AAAA,EACF;AAAA,EAEA,OAAO,UAAU,OAAuE;AACtF,WAAO,YAAW,cAAc,CAAC,KAAK,CAAC;AAAA,EACzC;AAAA,EAEA,OAAO,cACL,OACsC;AACtC,WAAO,IAAI;AAAA,MACT,mBAAmB;AACjB,mBAAW,SAAS,OAAO;AACzB,gBAAM,GAAG,EAAE,MAAM,YAAY,OAAO,aAAa,KAAK,EAAE,CAAC;AAAA,QAC3D;AAAA,MACF;AAAA,MACA;AAAA,MACA;AAAA,IACF;AAAA,EACF;AAAA,EAEA,OAAyC;AACvC,WAAO,IAAI,YAAW,KAAK,QAAQ,QAAQ,KAAK,UAAU;AAAA,EAC5D;AAAA,EAEA,eAA2D;AACzD,WAAO,IAAI,YAAW,KAAK,QAAQ,gBAAgB,KAAK,UAAU;AAAA,EACpE;AAAA,EAEA,eAAkD;AAChD,WAAO,IAAI,YAAW,KAAK,QAAQ,KAAK,MAAM,IAAI;AAAA,EACpD;AAAA,EAEA,QAAQ,OAAO,aAAa,IAAsB;AAChD,qBAAiB,QAAQ,KAAK,OAAO,GAAG;AACtC,YAAM,SAAS,MAAM,cAAc,MAAM,KAAK,IAAI;AAClD,UAAI,CAAC,OAAO,MAAM,KAAK,YAAY;AACjC;AAAA,MACF;AACA,YAAO,KAAK,cAAc,OAAO,KAAK,OAAO,QAAQ;AAAA,IACvD;AAAA,EACF;AACF;AAEO,IAAM,gBAAN,MAAM,eAAuE;AAAA,EAC1E,YACW,QACA,MACA,YACjB;AAHiB;AACA;AACA;AAAA,EAChB;AAAA,EAHgB;AAAA,EACA;AAAA,EACA;AAAA,EAGnB,OAAO,SAAS,SAAyD;AACvE,WAAO,IAAI;AAAA,MACT,mBAAmB;AACjB,mBAAW,QAAQ,MAAM,WAAW,OAAO,GAAG;AAC5C,gBAAM,GAAG,EAAE,KAAK,CAAC;AAAA,QACnB;AAAA,MACF;AAAA,MACA;AAAA,MACA;AAAA,IACF;AAAA,EACF;AAAA,EAEA,OAAO,QAAQ,WAA2D;AACxE,WAAO,IAAI;AAAA,MACT,mBAAmB;AACjB,cAAM,UAAU,MAAM,QAAQ,WAAW,EAAE,eAAe,KAAK,CAAC;AAChE,cAAM,QAAQ,QACX,OAAO,CAAC,UAAU,MAAM,OAAO,CAAC,EAChC,IAAI,CAAC,UAAU,KAAK,WAAW,MAAM,IAAI,CAAC,EAC1C,KAAK,CAAC,MAAM,UAAU,KAAK,cAAc,KAAK,CAAC;AAClD,mBAAW,QAAQ,OAAO;AACxB,gBAAM,GAAG,EAAE,KAAK,CAAC;AAAA,QACnB;AAAA,MACF;AAAA,MACA;AAAA,MACA;AAAA,IACF;AAAA,EACF;AAAA,EAEA,OAAO,UAAU,OAAyE;AACxF,WAAO,eAAc,cAAc,CAAC,KAAK,CAAC;AAAA,EAC5C;AAAA,EAEA,OAAO,cACL,OACwC;AACxC,WAAO,IAAI;AAAA,MACT,mBAAmB;AACjB,mBAAW,SAAS,OAAO;AACzB,gBAAM,GAAG,EAAE,MAAM,YAAY,OAAO,aAAa,KAAK,EAAE,CAAC;AAAA,QAC3D;AAAA,MACF;AAAA,MACA;AAAA,MACA;AAAA,IACF;AAAA,EACF;AAAA,EAEA,OAA4C;AAC1C,WAAO,IAAI,eAAc,KAAK,QAAQ,QAAQ,KAAK,UAAU;AAAA,EAC/D;AAAA,EAEA,eAA6D;AAC3D,WAAO,IAAI,eAAc,KAAK,QAAQ,gBAAgB,KAAK,UAAU;AAAA,EACvE;AAAA,EAOA,SAAiG;AAC/F,UAAM,OAAO,KAAK,SAAS,iBAAiB,mBAAmB;AAC/D,WAAO,IAAI,eAAc,KAAK,QAAQ,MAAM,KAAK,UAAU;AAAA,EAC7D;AAAA,EAEA,eAAqD;AACnD,WAAO,IAAI,eAAc,KAAK,QAAQ,KAAK,MAAM,IAAI;AAAA,EACvD;AAAA,EAEA,QAAQ,OAAO,aAAa,IAAsB;AAChD,qBAAiB,QAAQ,KAAK,OAAO,GAAG;AACtC,YAAM,UAAU,MAAM,aAAa,MAAM,KAAK,IAAI;AAClD,iBAAW,UAAU,SAAS;AAC5B,YAAI,CAAC,OAAO,MAAM,KAAK,YAAY;AACjC;AAAA,QACF;AACA,cAAO,KAAK,cAAc,OAAO,KAAK,OAAO,QAAQ;AAAA,MACvD;AAAA,IACF;AAAA,EACF;AACF;AAEO,SAAS,eAAe,MAAkC;AAC/D,SAAO;AAAA,IACL,IAAI,KAAK;AAAA,IACT,MAAM,KAAK;AAAA,IACX,iBAAiB;AAAA,MACf,QAAQ,KAAK;AAAA,MACb,WAAW;AAAA,IACb;AAAA,EACF;AACF;AAEA,eAAsB,sBACpB,QACqB;AACrB,QAAM,YAAwB,CAAC;AAC/B,mBAAiB,QAAQ,QAAQ;AAC/B,cAAU,KAAK,eAAe,IAAI,CAAC;AAAA,EACrC;AACA,SAAO;AACT;AAEO,SAAS,cAAc,KAAgC;AAC5D,SAAO;AAAA,IACL,IAAI,IAAI;AAAA,IACR,MAAM,IAAI;AAAA,IACV,iBAAiB;AAAA,MACf,QAAQ,IAAI;AAAA,MACZ,WAAW;AAAA,IACb;AAAA,EACF;AACF;AAEA,eAAsB,qBACpB,QACqB;AACrB,QAAM,YAAwB,CAAC;AAC/B,mBAAiB,OAAO,QAAQ;AAC9B,cAAU,KAAK,cAAc,GAAG,CAAC;AAAA,EACnC;AACA,SAAO;AACT;AAEO,SAAS,kBAAkB,MAAiC;AACjE,SAAO;AAAA,IACL,IAAI,GAAG,KAAK,IAAI,SAAS,KAAK,UAAU;AAAA,IACxC,MAAM,KAAK;AAAA,IACX,iBAAiB;AAAA,MACf,QAAQ,KAAK;AAAA,MACb,WAAW;AAAA,MACX,YAAY,OAAO,KAAK,UAAU;AAAA,IACpC;AAAA,EACF;AACF;AAEA,eAAsB,yBACpB,QACqB;AACrB,QAAM,YAAwB,CAAC;AAC/B,mBAAiB,QAAQ,QAAQ;AAC/B,cAAU,KAAK,kBAAkB,IAAI,CAAC;AAAA,EACxC;AACA,SAAO;AACT;AAKA,SAAS,GAAM,OAA2B;AACxC,SAAO,EAAE,IAAI,MAAM,MAAM;AAC3B;AAEA,SAAS,IAAI,OAAqC;AAChD,SAAO,EAAE,IAAI,OAAO,MAAM;AAC5B;AAEA,eAAe,WAAW,SAAoC;AAC5D,UAAQ,MAAM,KAAK,SAAS,EAAE,UAAU,WAAW,OAAO,GAAG,WAAW,KAAK,CAAC,GAAG;AAAA,IAC/E,CAAC,MAAM,UAAU,KAAK,cAAc,KAAK;AAAA,EAC3C;AACF;AAEA,eAAe,cACb,MACA,MAC+D;AAC/D,MAAI,CAAC,KAAK,IAAI;AACZ,WAAO;AAAA,EACT;AACA,MAAI,SAAS,UAAU;AACrB,WAAO;AAAA,EACT;AACA,MAAI;AACF,UAAM,OAAO,MAAM,eAAe,KAAK,KAAK;AAC5C,WAAO,SAAS,SAAS,GAAG,IAAI,IAAI,GAAG,EAAE,MAAM,KAAK,MAAM,MAAM,KAAK,CAAC;AAAA,EACxE,SAAS,OAAO;AACd,WAAO,IAAI,KAAK;AAAA,EAClB;AACF;AAEA,eAAe,aACb,MACA,MACgG;AAChG,MAAI,CAAC,KAAK,IAAI;AACZ,WAAO,CAAC,IAAI;AAAA,EACd;AACA,MAAI,SAAS,UAAU;AACrB,WAAO,CAAC,IAAI;AAAA,EACd;AACA,MAAI;AACF,UAAM,QAAQ,MAAM,aAAa,KAAK,KAAK;AAC3C,QAAI,SAAS,QAAQ;AACnB,aAAO,CAAC,GAAG,MAAM,IAAI,CAAC,SAAS,KAAK,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;AAAA,IACrD;AACA,QAAI,SAAS,gBAAgB;AAC3B,aAAO,CAAC,GAAG,EAAE,MAAM,KAAK,MAAM,MAAM,MAAM,MAAM,IAAI,CAAC,SAAS,KAAK,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC;AAAA,IACtF;AACA,QAAI,SAAS,kBAAkB;AAC7B,aAAO,MAAM,IAAI,CAAC,SAAS,GAAG,EAAE,MAAM,KAAK,MAAM,MAAM,GAAG,KAAK,CAAC,CAAC;AAAA,IACnE;AACA,WAAO,MAAM,IAAI,EAAE;AAAA,EACrB,SAAS,OAAO;AACd,WAAO,CAAC,IAAI,KAAK,CAAC;AAAA,EACpB;AACF;AAEA,eAAe,eAAe,QAAqC;AACjE,MAAI,WAAW,QAAQ;AACrB,WAAO,IAAI,YAAY,EAAE,OAAO,OAAO,KAAK;AAAA,EAC9C;AACA,SAAO,SAAS,OAAO,MAAM,MAAM;AACrC;AAEA,eAAe,aAAa,QAAuC;AACjE,QAAM,QAAQ,MAAM,OAAO,iCAAiC;AAC5D,QAAM,QAAQ,WAAW,SAAS,OAAO,QAAQ,aAAa,MAAM,SAAS,OAAO,IAAI,CAAC;AACzF,QAAM,cAAc,MAAM,YAAY,EAAE,MAAM,MAAM,CAAC;AACrD,QAAM,WAAW,MAAM,YAAY;AACnC,QAAM,QAAmB,CAAC;AAC1B,MAAI;AACF,aAAS,QAAQ,GAAG,SAAS,SAAS,UAAU,SAAS,GAAG;AAC1D,YAAM,OAAO,MAAM,SAAS,QAAQ,KAAK;AACzC,YAAM,UAAU,MAAM,KAAK,eAAe;AAC1C,YAAM,OAAO,QAAQ,MAClB;AAAA,QAAQ,CAAC,SACR,OAAO,SAAS,YAAY,SAAS,QAAQ,SAAS,OAAO,CAAC,OAAO,KAAK,GAAG,CAAC,IAAI,CAAC;AAAA,MACrF,EACC,KAAK,IAAI;AACZ,YAAM,KAAK,EAAE,YAAY,QAAQ,GAAG,MAAM,KAAK,SAAS,IAAI,GAAG,IAAI;AAAA,IAAO,GAAG,CAAC;AAAA,IAChF;AAAA,EACF,UAAE;AACA,UAAM,YAAY,QAAQ;AAAA,EAC5B;AACA,SAAO;AACT;AAEA,SAAS,aAAa,OAA6C;AACjE,MAAI,iBAAiB,YAAY;AAC/B,WAAO,IAAI;AAAA,MACT,MAAM,OAAO,MAAM,MAAM,YAAY,MAAM,aAAa,MAAM,UAAU;AAAA,IAC1E;AAAA,EACF;AACA,SAAO,IAAI,WAAW,KAAK;AAC7B;","names":[]}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@anvia/core",
|
|
3
|
-
"version": "0.3.
|
|
3
|
+
"version": "0.3.1",
|
|
4
4
|
"description": "Core runtime primitives for context-aware Anvia agents.",
|
|
5
5
|
"author": "anvia",
|
|
6
6
|
"maintainer": "Indra Zulfi",
|
|
@@ -99,8 +99,8 @@
|
|
|
99
99
|
},
|
|
100
100
|
"dependencies": {
|
|
101
101
|
"@modelcontextprotocol/sdk": "^1.29.0",
|
|
102
|
-
"pdfjs-dist": "^
|
|
103
|
-
"tinyglobby": "^0.2.
|
|
102
|
+
"pdfjs-dist": "^6.0.227",
|
|
103
|
+
"tinyglobby": "^0.2.17",
|
|
104
104
|
"yaml": "^2.9.0",
|
|
105
105
|
"zod": "^4.4.3"
|
|
106
106
|
},
|