@opengeni/documents 0.8.6 → 0.8.14-canary.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.js +4 -4
- package/dist/index.js.map +1 -1
- package/package.json +6 -6
- package/src/index.ts +11 -4
package/dist/index.js
CHANGED
|
@@ -248,8 +248,8 @@ var LiteParseDocumentParser = class {
|
|
|
248
248
|
async parseWithLiteParse(bytes) {
|
|
249
249
|
return await this.enqueueParse(async () => {
|
|
250
250
|
const { LiteParse } = await import("@llamaindex/liteparse");
|
|
251
|
-
const parser = new LiteParse({ ocrEnabled: true, numWorkers: 1 });
|
|
252
|
-
const result = await parser.parse(Buffer.from(bytes)
|
|
251
|
+
const parser = new LiteParse({ ocrEnabled: true, numWorkers: 1, quiet: true });
|
|
252
|
+
const result = await parser.parse(Buffer.from(bytes));
|
|
253
253
|
const text = typeof result?.text === "string" ? result.text : "";
|
|
254
254
|
return text.replace(/\0/g, " ").trim();
|
|
255
255
|
});
|
|
@@ -2933,9 +2933,9 @@ function validateEmbedding(values, dimensions, model) {
|
|
|
2933
2933
|
return values;
|
|
2934
2934
|
}
|
|
2935
2935
|
function isTextLike(file) {
|
|
2936
|
-
const contentType = file.contentType.toLowerCase();
|
|
2936
|
+
const contentType = file.contentType.split(";", 1)[0]?.trim().toLowerCase() ?? "";
|
|
2937
2937
|
const filename = file.filename.toLowerCase();
|
|
2938
|
-
return contentType.startsWith("text/") || contentType === "application/json" || contentType === "application/xml" || contentType === "application/x-yaml" || filename.endsWith(".md") || filename.endsWith(".markdown") || filename.endsWith(".json") || filename.endsWith(".yaml") || filename.endsWith(".yml") || filename.endsWith(".csv") || filename.endsWith(".tsv") || filename.endsWith(".xml");
|
|
2938
|
+
return contentType.startsWith("text/") || contentType === "application/json" || contentType === "application/xml" || contentType === "application/x-yaml" || contentType === "application/yaml" || contentType === "message/rfc822" || filename.endsWith(".txt") || filename.endsWith(".log") || filename.endsWith(".md") || filename.endsWith(".markdown") || filename.endsWith(".json") || filename.endsWith(".yaml") || filename.endsWith(".yml") || filename.endsWith(".csv") || filename.endsWith(".tsv") || filename.endsWith(".xml") || filename.endsWith(".html") || filename.endsWith(".htm") || filename.endsWith(".eml");
|
|
2939
2939
|
}
|
|
2940
2940
|
function splitOversizedText(text, maxChars) {
|
|
2941
2941
|
if (text.length <= maxChars) return [text];
|