@veryfront/ext-document-kreuzberg 0.1.990 → 0.1.992

package/esm/extensions/ext-document-kreuzberg/src/extraction-config.d.ts

@@ -0,0 +1,2 @@
+export declare function extractionConfigForMimeType(mimeType: string): Record<string, unknown> | undefined;
+//# sourceMappingURL=extraction-config.d.ts.map

package/esm/extensions/ext-document-kreuzberg/src/extraction-config.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"extraction-config.d.ts","sourceRoot":"","sources":["../../../../src/extensions/ext-document-kreuzberg/src/extraction-config.ts"],"names":[],"mappings":"AAcA,wBAAgB,2BAA2B,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,SAAS,CAIjG"}

package/esm/extensions/ext-document-kreuzberg/src/extraction-config.js

@@ -0,0 +1,17 @@
+const PDF_TEXT_ONLY_EXTRACTION_CONFIG = {
+    images: { extractImages: false },
+    pdfOptions: {
+        extractImages: false,
+        extractMetadata: false,
+        extractAnnotations: false,
+        hierarchy: { enabled: false, includeBbox: false },
+    },
+};
+function normalizeMimeType(mimeType) {
+    return mimeType.toLowerCase().split(";")[0]?.trim() ?? "";
+}
+export function extractionConfigForMimeType(mimeType) {
+    return normalizeMimeType(mimeType) === "application/pdf"
+        ? PDF_TEXT_ONLY_EXTRACTION_CONFIG
+        : undefined;
+}

package/esm/extensions/ext-document-kreuzberg/src/index.d.ts

@@ -9,10 +9,10 @@
 import "../../../_dnt.polyfills.js";
 import type { ExtensionFactory, ExtensionLogger } from "veryfront/extensions";
 import type { DocumentExtractionOptions, DocumentExtractor, KreuzbergExtractor } from "veryfront/extensions/compat";
-/** Maximum time to wait for fallback worker extraction before aborting. */
-export declare const EXTRACTION_TIMEOUT_MS = 120000;
 export declare const NATIVE_PROGRESS_IDLE_TIMEOUT_MS = 120000;
 export declare const NATIVE_PROGRESS_HARD_TIMEOUT_MS: number;
+/** Maximum time to wait for fallback worker extraction before aborting. */
+export declare const EXTRACTION_TIMEOUT_MS: number;
 declare function extractInWorkerDeno(buffer: ArrayBuffer, mimeType: string): Promise<string>;
 export interface KreuzbergDocumentExtractorDeps {
     isDenoRuntime?: boolean;

package/esm/extensions/ext-document-kreuzberg/src/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/extensions/ext-document-kreuzberg/src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AACH,OAAO,4BAA4B,CAAC;AAKpC,OAAO,KAAK,EAAE,gBAAgB,EAAE,eAAe,EAAE,MAAM,kCAAkC,CAAC;AAC1F,OAAO,KAAK,EACV,yBAAyB,EAEzB,iBAAiB,EACjB,kBAAkB,EACnB,MAAM,yCAAyC,CAAC;AAIjD,2EAA2E;AAC3E,eAAO,MAAM,qBAAqB,SAAU,CAAC;AAC7C,eAAO,MAAM,+BAA+B,SAAU,CAAC;AACvD,eAAO,MAAM,+BAA+B,QAAc,CAAC;AAE3D,iBAAS,mBAAmB,CAC1B,MAAM,EAAE,WAAW,EACnB,QAAQ,EAAE,MAAM,GACf,OAAO,CAAC,MAAM,CAAC,CA0CjB;AAED,MAAM,WAAW,8BAA8B;IAC7C,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,mBAAmB,CAAC,EAAE,MAAM,OAAO,CAAC,kBAAkB,CAAC,CAAC;IACxD,mBAAmB,CAAC,EAAE,OAAO,mBAAmB,CAAC;IACjD,6BAA6B,CAAC,EAAE,OAAO,6BAA6B,CAAC;IACrE,MAAM,CAAC,EAAE,IAAI,CAAC,eAAe,EAAE,MAAM,CAAC,CAAC;CACxC;AAuCD,iBAAS,6BAA6B,CACpC,MAAM,EAAE,WAAW,EACnB,QAAQ,EAAE,MAAM,EAChB,OAAO,EAAE,yBAAyB,GACjC,OAAO,CAAC,MAAM,CAAC,CAgFjB;AAED,qBAAa,0BAA2B,YAAW,iBAAiB;IACtD,OAAO,CAAC,QAAQ,CAAC,IAAI;gBAAJ,IAAI,GAAE,8BAAmC;IAEtE,eAAe,IAAI,OAAO,CAAC,kBAAkB,CAAC;IAIxC,eAAe,CACnB,MAAM,EAAE,WAAW,EACnB,QAAQ,EAAE,MAAM,EAChB,OAAO,GAAE,yBAA8B,GACtC,OAAO,CAAC,MAAM,CAAC;CAgDnB;AAED,QAAA,MAAM,oBAAoB,EAAE,gBAiB3B,CAAC;AAEF,eAAe,oBAAoB,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/extensions/ext-document-kreuzberg/src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AACH,OAAO,4BAA4B,CAAC;AAKpC,OAAO,KAAK,EAAE,gBAAgB,EAAE,eAAe,EAAE,MAAM,kCAAkC,CAAC;AAC1F,OAAO,KAAK,EACV,yBAAyB,EAEzB,iBAAiB,EACjB,kBAAkB,EACnB,MAAM,yCAAyC,CAAC;AAKjD,eAAO,MAAM,+BAA+B,SAAU,CAAC;AACvD,eAAO,MAAM,+BAA+B,QAAc,CAAC;AAC3D,2EAA2E;AAC3E,eAAO,MAAM,qBAAqB,QAAkC,CAAC;AAErE,iBAAS,mBAAmB,CAC1B,MAAM,EAAE,WAAW,EACnB,QAAQ,EAAE,MAAM,GACf,OAAO,CAAC,MAAM,CAAC,CA0CjB;AAED,MAAM,WAAW,8BAA8B;IAC7C,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,mBAAmB,CAAC,EAAE,MAAM,OAAO,CAAC,kBAAkB,CAAC,CAAC;IACxD,mBAAmB,CAAC,EAAE,OAAO,mBAAmB,CAAC;IACjD,6BAA6B,CAAC,EAAE,OAAO,6BAA6B,CAAC;IACrE,MAAM,CAAC,EAAE,IAAI,CAAC,eAAe,EAAE,MAAM,CAAC,CAAC;CACxC;AA2CD,iBAAS,6BAA6B,CACpC,MAAM,EAAE,WAAW,EACnB,QAAQ,EAAE,MAAM,EAChB,OAAO,EAAE,yBAAyB,GACjC,OAAO,CAAC,MAAM,CAAC,CAgFjB;AAED,qBAAa,0BAA2B,YAAW,iBAAiB;IACtD,OAAO,CAAC,QAAQ,CAAC,IAAI;gBAAJ,IAAI,GAAE,8BAAmC;IAEtE,eAAe,IAAI,OAAO,CAAC,kBAAkB,CAAC;IAIxC,eAAe,CACnB,MAAM,EAAE,WAAW,EACnB,QAAQ,EAAE,MAAM,EAChB,OAAO,GAAE,yBAA8B,GACtC,OAAO,CAAC,MAAM,CAAC;CAoDnB;AAED,QAAA,MAAM,oBAAoB,EAAE,gBAiB3B,CAAC;AAEF,eAAe,oBAAoB,CAAC"}

package/esm/extensions/ext-document-kreuzberg/src/index.js

@@ -9,11 +9,12 @@
 import "../../../_dnt.polyfills.js";
 import * as dntShim from "../../../_dnt.shims.js";
 import { isMissingPackageError, loadKreuzberg, loadKreuzbergNative } from "./kreuzberg.js";
+import { extractionConfigForMimeType } from "./extraction-config.js";
 import { isDeno } from "./runtime.js";
-/** Maximum time to wait for fallback worker extraction before aborting. */
-export const EXTRACTION_TIMEOUT_MS = 120_000;
 export const NATIVE_PROGRESS_IDLE_TIMEOUT_MS = 120_000;
 export const NATIVE_PROGRESS_HARD_TIMEOUT_MS = 10 * 60_000;
+/** Maximum time to wait for fallback worker extraction before aborting. */
+export const EXTRACTION_TIMEOUT_MS = NATIVE_PROGRESS_HARD_TIMEOUT_MS;
 function extractInWorkerDeno(buffer, mimeType) {
     return new Promise((resolve, reject) => {
         // The worker ships as raw TypeScript in the compiled binary and from source
@@ -62,7 +63,7 @@ function isNativeProgressMimeType(mimeType) {
 }
 async function extractWithNativeKreuzberg(buffer, mimeType, loadNative) {
     const { extractBytes } = await loadNative();
-    const result = await extractBytes(new Uint8Array(buffer), mimeType);
+    const result = await extractBytes(new Uint8Array(buffer), mimeType, extractionConfigForMimeType(mimeType));
     return result.content;
 }
 function warningDetails(mimeType, error) {
@@ -159,7 +160,7 @@ export class KreuzbergDocumentExtractor {
         // PDF path can hang on valid large manuals.
         if (!isDenoRuntime) {
             const { extractBytes } = await loadKreuzberg();
-            const result = await extractBytes(new Uint8Array(buffer), mimeType);
+            const result = await extractBytes(new Uint8Array(buffer), mimeType, extractionConfigForMimeType(mimeType));
             return result.content;
         }
         if (options.onProgress && isNativeProgressMimeType(mimeType)) {

package/esm/extensions/ext-document-kreuzberg/src/kreuzberg.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"kreuzberg.d.ts","sourceRoot":"","sources":["../../../../src/extensions/ext-document-kreuzberg/src/kreuzberg.ts"],"names":[],"mappings":"AAaA,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,yCAAyC,CAAC;AAWlF,wBAAsB,mBAAmB,IAAI,OAAO,CAAC,kBAAkB,CAAC,CAWvE;AAED,wBAAsB,aAAa,IAAI,OAAO,CAAC,kBAAkB,CAAC,CA2BjE;AAeD,wBAAgB,qBAAqB,CAAC,KAAK,EAAE,OAAO,GAAG,OAAO,CAqB7D"}
+{"version":3,"file":"kreuzberg.d.ts","sourceRoot":"","sources":["../../../../src/extensions/ext-document-kreuzberg/src/kreuzberg.ts"],"names":[],"mappings":"AAaA,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,yCAAyC,CAAC;AAOlF,wBAAsB,mBAAmB,IAAI,OAAO,CAAC,kBAAkB,CAAC,CAWvE;AAED,wBAAsB,aAAa,IAAI,OAAO,CAAC,kBAAkB,CAAC,CA2BjE;AAeD,wBAAgB,qBAAqB,CAAC,KAAK,EAAE,OAAO,GAAG,OAAO,CAqB7D"}

package/esm/extensions/ext-document-kreuzberg/src/native-progress-extraction-worker.js

@@ -1,5 +1,6 @@
 import { PDFDocument } from "pdf-lib";
 import JSZip from "jszip";
+import { extractionConfigForMimeType } from "./extraction-config.js";
 import { loadKreuzbergNative } from "./kreuzberg.js";
 function postProgress(event) {
   self.postMessage({ type: "progress", event });
@@ -93,12 +94,17 @@ async function extractPdfByPage(buffer) {
   const source = await PDFDocument.load(buffer, { ignoreEncryption: true });
   const total = source.getPageCount();
   const pages = [];
+  const pdfConfig = extractionConfigForMimeType("application/pdf");
   for (let index = 0; index < total; index += 1) {
     const singlePage = await PDFDocument.create();
     const [page] = await singlePage.copyPages(source, [index]);
     singlePage.addPage(page);
     const bytes = await singlePage.save({ useObjectStreams: false });
-    const result = await extractBytes(new Uint8Array(bytes), "application/pdf");
+    const result = await extractBytes(
+      new Uint8Array(bytes),
+      "application/pdf",
+      pdfConfig
+    );
     const content = result.content.trim();
     pages.push(content);
     postProgress({

package/esm/extensions/ext-document-kreuzberg/src/upload-extraction-worker.js

@@ -1,4 +1,5 @@
 import { loadKreuzberg } from "./kreuzberg.js";
+import { extractionConfigForMimeType } from "./extraction-config.js";
 self.onmessage = async (event) => {
   if (event.origin && event.origin !== self.location.origin) {
     self.postMessage(
@@ -11,7 +12,11 @@ self.onmessage = async (event) => {
   try {
     const { buffer, mimeType } = event.data;
     const { extractBytes } = await loadKreuzberg();
-    const result = await extractBytes(new Uint8Array(buffer), mimeType);
+    const result = await extractBytes(
+      new Uint8Array(buffer),
+      mimeType,
+      extractionConfigForMimeType(mimeType)
+    );
     self.postMessage({ content: result.content });
   } catch (err) {
     const message = err instanceof Error ? err.message : String(err);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@veryfront/ext-document-kreuzberg",
-  "version": "0.1.990",
+  "version": "0.1.992",
   "description": "Veryfront first-party extension package for ext-document-kreuzberg",
   "keywords": [
     "veryfront",
@@ -55,7 +55,7 @@
     "pdf-lib": "1.17.1"
   },
   "peerDependencies": {
-    "veryfront": "^0.1.990"
+    "veryfront": "^0.1.992"
   },
   "type": "module",
   "types": "./esm/extensions/ext-document-kreuzberg/src/index.d.ts",