@opengeni/documents 0.8.6 → 0.8.14-canary.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@opengeni/documents",
-  "version": "0.8.6",
+  "version": "0.8.14-canary.0",
   "license": "Apache-2.0",
   "repository": {
     "type": "git",
@@ -39,11 +39,11 @@
     "prepublishOnly": "bash ../../scripts/prepublish-guard"
   },
   "dependencies": {
-    "@llamaindex/liteparse": "^1.5.3",
-    "@opengeni/config": "^0.22.0",
-    "@opengeni/contracts": "^2.7.0",
-    "@opengeni/db": "^3.6.0",
-    "@opengeni/storage": "^0.2.110",
+    "@llamaindex/liteparse": "2.14.2",
+    "@opengeni/config": "^0.22.5-canary.1",
+    "@opengeni/contracts": "^2.9.2-canary.1",
+    "@opengeni/db": "^3.7.4-canary.0",
+    "@opengeni/storage": "^0.2.115-canary.1",
     "drizzle-orm": "^0.45.2",
     "openai": "6.47.0"
   }

package/src/index.ts

@@ -331,8 +331,8 @@ export class LiteParseDocumentParser implements DocumentParser {
   private async parseWithLiteParse(bytes: Uint8Array): Promise<string> {
     return await this.enqueueParse(async () => {
       const { LiteParse } = await import("@llamaindex/liteparse");
-      const parser = new LiteParse({ ocrEnabled: true, numWorkers: 1 });
-      const result = await parser.parse(Buffer.from(bytes), true);
+      const parser = new LiteParse({ ocrEnabled: true, numWorkers: 1, quiet: true });
+      const result = await parser.parse(Buffer.from(bytes));
       const text = typeof result?.text === "string" ? result.text : "";
       return text.replace(/\0/g, " ").trim();
     });
@@ -4168,13 +4168,17 @@ function validateEmbedding(values: number[], dimensions: number, model: string):
 }
 
 function isTextLike(file: FileAsset): boolean {
-  const contentType = file.contentType.toLowerCase();
+  const contentType = file.contentType.split(";", 1)[0]?.trim().toLowerCase() ?? "";
   const filename = file.filename.toLowerCase();
   return (
     contentType.startsWith("text/") ||
     contentType === "application/json" ||
     contentType === "application/xml" ||
     contentType === "application/x-yaml" ||
+    contentType === "application/yaml" ||
+    contentType === "message/rfc822" ||
+    filename.endsWith(".txt") ||
+    filename.endsWith(".log") ||
     filename.endsWith(".md") ||
     filename.endsWith(".markdown") ||
     filename.endsWith(".json") ||
@@ -4182,7 +4186,10 @@ function isTextLike(file: FileAsset): boolean {
     filename.endsWith(".yml") ||
     filename.endsWith(".csv") ||
     filename.endsWith(".tsv") ||
-    filename.endsWith(".xml")
+    filename.endsWith(".xml") ||
+    filename.endsWith(".html") ||
+    filename.endsWith(".htm") ||
+    filename.endsWith(".eml")
   );
 }