@lumifai/harness-tool-pack-file-processing 0.1.0 → 0.2.0

package/README.md

@@ -1,6 +1,6 @@
 # @lumifai/harness-tool-pack-file-processing
 
-Extract text from workspace PDF files and write results under `/processed/`.
+Extract workspace PDF files into Markdown or text files, with optional document metadata.
 
 ```ts
 import { createFileProcessingToolPack } from '@lumifai/harness-tool-pack-file-processing';
@@ -8,6 +8,12 @@ import { createFileProcessingToolPack } from '@lumifai/harness-tool-pack-file-pr
 createFileProcessingToolPack();
 ```
 
-`process_pdf` reads a workspace PDF, runs OCR-aware extraction via LiteParse, and returns an `output_path` reference instead of inline content. Bundled agent skills ship in this package and are merged into `workspace.skills` when the pack is enabled.
+`process_pdf` reads a workspace PDF, runs OCR-aware extraction via LiteParse, and returns an `output_path` reference instead of inline content. Markdown is the default format and preserves more document structure, including tables and links. Bundled agent skills ship in this package and are merged into `workspace.skills` when the pack is enabled.
+
+The tool applies consistent OCR-aware parsing and returns metadata, annotations, form fields, and page complexity by default. The small set of meaningful optional inputs is:
+
+- `format`: `markdown` (default) or `text`.
+- `include_images`: extracted image metadata and workspace paths to image files.
+- `include_complexity`: per-page OCR and layout complexity signals (default `true`).
 
 Tool pack id: `file-processing`. Disabled by default (`enabledByDefault: false`). Default policy: `process_pdf` → `ask`.

package/dist/file-processing.js

@@ -42,30 +42,133 @@ const utils_1 = require("./utils");
 const workspaceFilePathSchema = zod_1.z
     .string()
     .min(1)
-    .describe('Absolute workspace path to a file, e.g. /documents/report.pdf');
+    .describe('Workspace-relative path to a file, e.g. documents/report.pdf');
+const processPdfInputSchema = zod_1.z.object({
+    path: workspaceFilePathSchema,
+    format: zod_1.z
+        .enum(['markdown', 'text'])
+        .default('markdown')
+        .describe('Output format. Markdown preserves document structure and improves table extraction.'),
+    include_images: zod_1.z
+        .boolean()
+        .default(false)
+        .describe('Extract embedded images and write them beside the processed output.'),
+    include_complexity: zod_1.z
+        .boolean()
+        .default(true)
+        .describe('Include per-page layout and OCR complexity signals.'),
+});
+function safeImageFileName(name, id, index) {
+    const extension = name.split('.').pop() || 'bin';
+    const safeId = id.replace(/[^a-zA-Z0-9_-]/g, '_') || `image-${index + 1}`;
+    return `${safeId}.${extension.replace(/[^a-zA-Z0-9]/g, '') || 'bin'}`;
+}
+function serializeComplexity(pageNum, complexity) {
+    if (!complexity || typeof complexity !== 'object') {
+        return { page_number: pageNum };
+    }
+    const value = complexity;
+    return {
+        page_number: pageNum,
+        text_length: value.textLength,
+        text_coverage: value.textCoverage,
+        has_substantial_images: value.hasSubstantialImages,
+        image_block_count: value.imageBlockCount,
+        image_coverage: value.imageCoverage,
+        largest_image_coverage: value.largestImageCoverage,
+        full_page_image: value.fullPageImage,
+        uncovered_vector_area: value.uncoveredVectorArea,
+        is_garbled: value.isGarbled,
+        page_area: value.pageArea,
+        needs_ocr: value.needsOcr,
+        reasons: value.reasons,
+        layout: value.layout,
+    };
+}
 function createFileProcessingTools() {
     const processPdf = (0, tools_1.createTool)({
         id: 'process_pdf',
-        description: 'Extract text content from a PDF file and write it to /processed/ as plain text. Returns a reference path, not the content itself.',
-        inputSchema: zod_1.z.object({
-            path: workspaceFilePathSchema,
-        }),
+        description: 'Extract a PDF into a workspace Markdown or text file. Markdown preserves structure and tables. The result includes document metadata, annotations, form fields, and page complexity signals; image extraction is optional because it can be expensive.',
+        inputSchema: processPdfInputSchema,
         execute: async (inputData, context) => {
             const { LiteParse } = await Promise.resolve().then(() => __importStar(require('@llamaindex/liteparse')));
             const { normalizedPath, buffer } = await (0, utils_1.readWorkspaceBinaryFile)(context, inputData.path);
-            const parser = new LiteParse({ ocrEnabled: true, quiet: true });
+            const parser = new LiteParse({
+                ocrEnabled: true,
+                quiet: true,
+                outputFormat: inputData.format,
+                extractLinks: inputData.format === 'markdown',
+                imageMode: inputData.format === 'markdown' ? 'placeholder' : 'off',
+                extractImages: inputData.include_images,
+                extractAnnotations: true,
+                extractFormFields: true,
+                extractDocumentMetadata: true,
+                includeComplexity: inputData.include_complexity,
+            });
             const result = await parser.parse(buffer);
-            const outputContent = result.pages
-                .map((page) => `--- Page ${page.pageNum} ---\n${page.text}`)
+            const outputPath = (0, utils_1.deriveOutputPath)(normalizedPath, inputData.format === 'markdown' ? 'md' : 'txt');
+            const imageDirectory = `${outputPath}.images`;
+            const imageOutputs = [];
+            let outputContent = result.pages
+                .map((page) => {
+                const pageContent = inputData.format === 'markdown' ? page.markdown : page.text;
+                return `--- Page ${page.pageNum} ---\n${pageContent}`;
+            })
                 .join('\n\n');
-            const outputPath = (0, utils_1.deriveOutputPath)(normalizedPath, 'txt');
+            if (inputData.include_images) {
+                const relativeImageDirectory = imageDirectory.substring(imageDirectory.lastIndexOf('/') + 1);
+                for (const [index, image] of result.images.entries()) {
+                    const imageName = safeImageFileName(image.name, image.id, index);
+                    const imagePath = `${imageDirectory}/${imageName}`;
+                    await (0, utils_1.writeProcessedBinary)(context, imagePath, image.bytes);
+                    imageOutputs.push({
+                        id: image.id,
+                        name: image.name,
+                        output_path: imagePath,
+                        page: image.page,
+                        bbox: image.bbox,
+                        width: image.width,
+                        height: image.height,
+                        rotation: image.rotation,
+                        format: image.format,
+                        duplicate_of: image.duplicateOf,
+                    });
+                    if (inputData.format === 'markdown') {
+                        const relativeImagePath = `./${relativeImageDirectory}/${imageName}`;
+                        if (image.name)
+                            outputContent = outputContent.replaceAll(image.name, relativeImagePath);
+                        if (image.id && image.id !== image.name)
+                            outputContent = outputContent.replaceAll(image.id, relativeImagePath);
+                    }
+                }
+            }
             await (0, utils_1.writeProcessedText)(context, outputPath, outputContent);
-            return {
+            const toolResult = {
                 source_path: normalizedPath,
                 output_path: outputPath,
                 page_count: result.pages.length,
                 char_count: outputContent.length,
             };
+            toolResult.document_metadata = result.docMeta ?? null;
+            toolResult.creator = result.creator ?? null;
+            toolResult.producer = result.producer ?? null;
+            toolResult.annotations = result.pages.flatMap((page) => (page.annotations ?? []).map((annotation) => ({
+                page_number: page.pageNum,
+                ...annotation,
+            })));
+            toolResult.form_fields = result.pages.flatMap((page) => (page.formFields ?? []).map((field) => ({
+                page_number: page.pageNum,
+                ...field,
+            })));
+            toolResult.form_type = result.formType ?? null;
+            if (inputData.include_complexity) {
+                toolResult.complexity = result.pages.map((page) => serializeComplexity(page.pageNum, page.complexity));
+            }
+            if (inputData.include_images) {
+                toolResult.images = imageOutputs;
+                toolResult.image_error_count = result.imageErrorCount;
+            }
+            return toolResult;
         },
     });
     return {

package/dist/file-processing.js.map

@@ -1 +1 @@
-{"version":3,"file":"file-processing.js","sourceRoot":"","sources":["../src/file-processing.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAaA,8DAkCC;AAED,oEAUC;AA1DD,8CAAgD;AAChD,6BAAwB;AAExB,oEAA6D;AAE7D,mCAAwF;AAExF,MAAM,uBAAuB,GAAG,OAAC;KAC9B,MAAM,EAAE;KACR,GAAG,CAAC,CAAC,CAAC;KACN,QAAQ,CAAC,+DAA+D,CAAC,CAAC;AAE7E,SAAgB,yBAAyB;IACvC,MAAM,UAAU,GAAG,IAAA,kBAAU,EAAC;QAC5B,EAAE,EAAE,aAAa;QACjB,WAAW,EACT,mIAAmI;QACrI,WAAW,EAAE,OAAC,CAAC,MAAM,CAAC;YACpB,IAAI,EAAE,uBAAuB;SAC9B,CAAC;QACF,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,OAAO,EAAE,EAAE;YACpC,MAAM,EAAE,SAAS,EAAE,GAAG,wDAAa,uBAAuB,GAAC,CAAC;YAC5D,MAAM,EAAE,cAAc,EAAE,MAAM,EAAE,GAAG,MAAM,IAAA,+BAAuB,EAAC,OAAO,EAAE,SAAS,CAAC,IAAI,CAAC,CAAC;YAE1F,MAAM,MAAM,GAAG,IAAI,SAAS,CAAC,EAAE,UAAU,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;YAChE,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;YAE1C,MAAM,aAAa,GAAG,MAAM,CAAC,KAAK;iBAC/B,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,YAAY,IAAI,CAAC,OAAO,SAAS,IAAI,CAAC,IAAI,EAAE,CAAC;iBAC3D,IAAI,CAAC,MAAM,CAAC,CAAC;YAEhB,MAAM,UAAU,GAAG,IAAA,wBAAgB,EAAC,cAAc,EAAE,KAAK,CAAC,CAAC;YAC3D,MAAM,IAAA,0BAAkB,EAAC,OAAO,EAAE,UAAU,EAAE,aAAa,CAAC,CAAC;YAE7D,OAAO;gBACL,WAAW,EAAE,cAAc;gBAC3B,WAAW,EAAE,UAAU;gBACvB,UAAU,EAAE,MAAM,CAAC,KAAK,CAAC,MAAM;gBAC/B,UAAU,EAAE,aAAa,CAAC,MAAM;aACjC,CAAC;QACJ,CAAC;KACF,CAAC,CAAC;IAEH,OAAO;QACL,UAAU;KACX,CAAC;AACJ,CAAC;AAED,SAAgB,4BAA4B;IAC1C,OAAO,IAAA,mCAAc,EAAC;QACpB,EAAE,EAAE,iBAAiB;QACrB,WAAW,EAAE,8BAA8B;QAC3C,gBAAgB,EAAE,KAAK;QACvB,KAAK,EAAE,yBAAyB,EAAE;QAClC,kBAAkB,EAAE;YAClB,WAAW,EAAE,KAAK;SACnB;KACF,CAAC,CAAC;AACL,CAAC"}
+{"version":3,"file":"file-processing.js","sourceRoot":"","sources":["../src/file-processing.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAmEA,8DAiHC;AAED,oEAUC;AA9LD,8CAAgD;AAChD,6BAAwB;AAExB,oEAA6D;AAE7D,mCAKiB;AAEjB,MAAM,uBAAuB,GAAG,OAAC;KAC9B,MAAM,EAAE;KACR,GAAG,CAAC,CAAC,CAAC;KACN,QAAQ,CAAC,8DAA8D,CAAC,CAAC;AAE5E,MAAM,qBAAqB,GAAG,OAAC,CAAC,MAAM,CAAC;IACrC,IAAI,EAAE,uBAAuB;IAC7B,MAAM,EAAE,OAAC;SACN,IAAI,CAAC,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC;SAC1B,OAAO,CAAC,UAAU,CAAC;SACnB,QAAQ,CACP,qFAAqF,CACtF;IACH,cAAc,EAAE,OAAC;SACd,OAAO,EAAE;SACT,OAAO,CAAC,KAAK,CAAC;SACd,QAAQ,CAAC,qEAAqE,CAAC;IAClF,kBAAkB,EAAE,OAAC;SAClB,OAAO,EAAE;SACT,OAAO,CAAC,IAAI,CAAC;SACb,QAAQ,CAAC,qDAAqD,CAAC;CACnE,CAAC,CAAC;AAEH,SAAS,iBAAiB,CAAC,IAAY,EAAE,EAAU,EAAE,KAAa;IAChE,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,IAAI,KAAK,CAAC;IACjD,MAAM,MAAM,GAAG,EAAE,CAAC,OAAO,CAAC,iBAAiB,EAAE,GAAG,CAAC,IAAI,SAAS,KAAK,GAAG,CAAC,EAAE,CAAC;IAC1E,OAAO,GAAG,MAAM,IAAI,SAAS,CAAC,OAAO,CAAC,eAAe,EAAE,EAAE,CAAC,IAAI,KAAK,EAAE,CAAC;AACxE,CAAC;AAED,SAAS,mBAAmB,CAAC,OAAe,EAAE,UAA2C;IACvF,IAAI,CAAC,UAAU,IAAI,OAAO,UAAU,KAAK,QAAQ,EAAE,CAAC;QAClD,OAAO,EAAE,WAAW,EAAE,OAAO,EAAE,CAAC;IAClC,CAAC;IAED,MAAM,KAAK,GAAG,UAAgD,CAAC;IAC/D,OAAO;QACL,WAAW,EAAE,OAAO;QACpB,WAAW,EAAE,KAAK,CAAC,UAAU;QAC7B,aAAa,EAAE,KAAK,CAAC,YAAY;QACjC,sBAAsB,EAAE,KAAK,CAAC,oBAAoB;QAClD,iBAAiB,EAAE,KAAK,CAAC,eAAe;QACxC,cAAc,EAAE,KAAK,CAAC,aAAa;QACnC,sBAAsB,EAAE,KAAK,CAAC,oBAAoB;QAClD,eAAe,EAAE,KAAK,CAAC,aAAa;QACpC,qBAAqB,EAAE,KAAK,CAAC,mBAAmB;QAChD,UAAU,EAAE,KAAK,CAAC,SAAS;QAC3B,SAAS,EAAE,KAAK,CAAC,QAAQ;QACzB,SAAS,EAAE,KAAK,CAAC,QAAQ;QACzB,OAAO,EAAE,KAAK,CAAC,OAAO;QACtB,MAAM,EAAE,KAAK,CAAC,MAAM;KACrB,CAAC;AACJ,CAAC;AAED,SAAgB,yBAAyB;IACvC,MAAM,UAAU,GAAG,IAAA,kBAAU,EAAC;QAC5B,EAAE,EAAE,aAAa;QACjB,WAAW,EACT,wPAAwP;QAC1P,WAAW,EAAE,qBAAqB;QAClC,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,OAAO,EAAE,EAAE;YACpC,MAAM,EAAE,SAAS,EAAE,GAAG,wDAAa,uBAAuB,GAAC,CAAC;YAC5D,MAAM,EAAE,cAAc,EAAE,MAAM,EAAE,GAAG,MAAM,IAAA,+BAAuB,EAAC,OAAO,EAAE,SAAS,CAAC,IAAI,CAAC,CAAC;YAE1F,MAAM,MAAM,GAAG,IAAI,SAAS,CAAC;gBAC3B,UAAU,EAAE,IAAI;gBAChB,KAAK,EAAE,IAAI;gBACX,YAAY,EAAE,SAAS,CAAC,MAAM;gBAC9B,YAAY,EAAE,SAAS,CAAC,MAAM,KAAK,UAAU;gBAC7C,SAAS,EAAE,SAAS,CAAC,MAAM,KAAK,UAAU,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,KAAK;gBAClE,aAAa,EAAE,SAAS,CAAC,cAAc;gBACvC,kBAAkB,EAAE,IAAI;gBACxB,iBAAiB,EAAE,IAAI;gBACvB,uBAAuB,EAAE,IAAI;gBAC7B,iBAAiB,EAAE,SAAS,CAAC,kBAAkB;aAChD,CAAC,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;YAE1C,MAAM,UAAU,GAAG,IAAA,wBAAgB,EACjC,cAAc,EACd,SAAS,CAAC,MAAM,KAAK,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,CAC/C,CAAC;YACF,MAAM,cAAc,GAAG,GAAG,UAAU,SAAS,CAAC;YAC9C,MAAM,YAAY,GAAG,EAAE,CAAC;YAExB,IAAI,aAAa,GAAG,MAAM,CAAC,KAAK;iBAC7B,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;gBACZ,MAAM,WAAW,GAAG,SAAS,CAAC,MAAM,KAAK,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC;gBAChF,OAAO,YAAY,IAAI,CAAC,OAAO,SAAS,WAAW,EAAE,CAAC;YACxD,CAAC,CAAC;iBACD,IAAI,CAAC,MAAM,CAAC,CAAC;YAEhB,IAAI,SAAS,CAAC,cAAc,EAAE,CAAC;gBAC7B,MAAM,sBAAsB,GAAG,cAAc,CAAC,SAAS,CACrD,cAAc,CAAC,WAAW,CAAC,GAAG,CAAC,GAAG,CAAC,CACpC,CAAC;gBAEF,KAAK,MAAM,CAAC,KAAK,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,MAAM,CAAC,OAAO,EAAE,EAAE,CAAC;oBACrD,MAAM,SAAS,GAAG,iBAAiB,CAAC,KAAK,CAAC,IAAI,EAAE,KAAK,CAAC,EAAE,EAAE,KAAK,CAAC,CAAC;oBACjE,MAAM,SAAS,GAAG,GAAG,cAAc,IAAI,SAAS,EAAE,CAAC;oBACnD,MAAM,IAAA,4BAAoB,EAAC,OAAO,EAAE,SAAS,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC;oBAC5D,YAAY,CAAC,IAAI,CAAC;wBAChB,EAAE,EAAE,KAAK,CAAC,EAAE;wBACZ,IAAI,EAAE,KAAK,CAAC,IAAI;wBAChB,WAAW,EAAE,SAAS;wBACtB,IAAI,EAAE,KAAK,CAAC,IAAI;wBAChB,IAAI,EAAE,KAAK,CAAC,IAAI;wBAChB,KAAK,EAAE,KAAK,CAAC,KAAK;wBAClB,MAAM,EAAE,KAAK,CAAC,MAAM;wBACpB,QAAQ,EAAE,KAAK,CAAC,QAAQ;wBACxB,MAAM,EAAE,KAAK,CAAC,MAAM;wBACpB,YAAY,EAAE,KAAK,CAAC,WAAW;qBAChC,CAAC,CAAC;oBAEH,IAAI,SAAS,CAAC,MAAM,KAAK,UAAU,EAAE,CAAC;wBACpC,MAAM,iBAAiB,GAAG,KAAK,sBAAsB,IAAI,SAAS,EAAE,CAAC;wBACrE,IAAI,KAAK,CAAC,IAAI;4BAAE,aAAa,GAAG,aAAa,CAAC,UAAU,CAAC,KAAK,CAAC,IAAI,EAAE,iBAAiB,CAAC,CAAC;wBACxF,IAAI,KAAK,CAAC,EAAE,IAAI,KAAK,CAAC,EAAE,KAAK,KAAK,CAAC,IAAI;4BACrC,aAAa,GAAG,aAAa,CAAC,UAAU,CAAC,KAAK,CAAC,EAAE,EAAE,iBAAiB,CAAC,CAAC;oBAC1E,CAAC;gBACH,CAAC;YACH,CAAC;YAED,MAAM,IAAA,0BAAkB,EAAC,OAAO,EAAE,UAAU,EAAE,aAAa,CAAC,CAAC;YAE7D,MAAM,UAAU,GAA4B;gBAC1C,WAAW,EAAE,cAAc;gBAC3B,WAAW,EAAE,UAAU;gBACvB,UAAU,EAAE,MAAM,CAAC,KAAK,CAAC,MAAM;gBAC/B,UAAU,EAAE,aAAa,CAAC,MAAM;aACjC,CAAC;YAEF,UAAU,CAAC,iBAAiB,GAAG,MAAM,CAAC,OAAO,IAAI,IAAI,CAAC;YACtD,UAAU,CAAC,OAAO,GAAG,MAAM,CAAC,OAAO,IAAI,IAAI,CAAC;YAC5C,UAAU,CAAC,QAAQ,GAAG,MAAM,CAAC,QAAQ,IAAI,IAAI,CAAC;YAC9C,UAAU,CAAC,WAAW,GAAG,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE,CACrD,CAAC,IAAI,CAAC,WAAW,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC;gBAC5C,WAAW,EAAE,IAAI,CAAC,OAAO;gBACzB,GAAG,UAAU;aACd,CAAC,CAAC,CACJ,CAAC;YACF,UAAU,CAAC,WAAW,GAAG,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE,CACrD,CAAC,IAAI,CAAC,UAAU,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;gBACtC,WAAW,EAAE,IAAI,CAAC,OAAO;gBACzB,GAAG,KAAK;aACT,CAAC,CAAC,CACJ,CAAC;YACF,UAAU,CAAC,SAAS,GAAG,MAAM,CAAC,QAAQ,IAAI,IAAI,CAAC;YAE/C,IAAI,SAAS,CAAC,kBAAkB,EAAE,CAAC;gBACjC,UAAU,CAAC,UAAU,GAAG,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAChD,mBAAmB,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,UAAU,CAAC,CACnD,CAAC;YACJ,CAAC;YAED,IAAI,SAAS,CAAC,cAAc,EAAE,CAAC;gBAC7B,UAAU,CAAC,MAAM,GAAG,YAAY,CAAC;gBACjC,UAAU,CAAC,iBAAiB,GAAG,MAAM,CAAC,eAAe,CAAC;YACxD,CAAC;YAED,OAAO,UAAU,CAAC;QACpB,CAAC;KACF,CAAC,CAAC;IAEH,OAAO;QACL,UAAU;KACX,CAAC;AACJ,CAAC;AAED,SAAgB,4BAA4B;IAC1C,OAAO,IAAA,mCAAc,EAAC;QACpB,EAAE,EAAE,iBAAiB;QACrB,WAAW,EAAE,8BAA8B;QAC3C,gBAAgB,EAAE,KAAK;QACvB,KAAK,EAAE,yBAAyB,EAAE;QAClC,kBAAkB,EAAE;YAClB,WAAW,EAAE,KAAK;SACnB;KACF,CAAC,CAAC;AACL,CAAC"}

package/dist/utils.d.ts

@@ -1,8 +1,11 @@
 import type { ToolExecutionContext } from '@mastra/core/tools';
+/** Hard cap for files loaded into memory for PDF OCR / LiteParse. */
+export declare const DEFAULT_MAX_FILE_BYTES: number;
 export declare function normalizeWorkspaceFilePath(path: string): string;
-export declare function readWorkspaceBinaryFile(context: ToolExecutionContext, path: string): Promise<{
+export declare function readWorkspaceBinaryFile(context: ToolExecutionContext, path: string, maxBytes?: number): Promise<{
     normalizedPath: string;
     buffer: Buffer;
 }>;
 export declare function writeProcessedText(context: ToolExecutionContext, outputPath: string, content: string): Promise<void>;
+export declare function writeProcessedBinary(context: ToolExecutionContext, outputPath: string, content: Uint8Array): Promise<void>;
 export declare function deriveOutputPath(sourcePath: string, suffix: string): string;

package/dist/utils.js

@@ -1,40 +1,32 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
+exports.DEFAULT_MAX_FILE_BYTES = void 0;
 exports.normalizeWorkspaceFilePath = normalizeWorkspaceFilePath;
 exports.readWorkspaceBinaryFile = readWorkspaceBinaryFile;
 exports.writeProcessedText = writeProcessedText;
+exports.writeProcessedBinary = writeProcessedBinary;
 exports.deriveOutputPath = deriveOutputPath;
+const harness_tool_packs_1 = require("@lumifai/harness-tool-packs");
+/** Hard cap for files loaded into memory for PDF OCR / LiteParse. */
+exports.DEFAULT_MAX_FILE_BYTES = 64 * 1024 * 1024;
+const workspaceFiles = (0, harness_tool_packs_1.createWorkspaceFiles)({ maxFileBytes: exports.DEFAULT_MAX_FILE_BYTES });
 function normalizeWorkspaceFilePath(path) {
-    const trimmed = path.trim();
-    if (!trimmed.startsWith('/')) {
-        throw new Error(`Path must be absolute within the workspace: ${path}`);
-    }
-    return trimmed;
+    return (0, harness_tool_packs_1.normalizeWorkspacePath)(path);
 }
-async function readWorkspaceBinaryFile(context, path) {
-    const normalizedPath = normalizeWorkspaceFilePath(path);
-    const workspace = context.workspace;
-    if (!workspace?.filesystem?.readFile) {
-        throw new Error(`File processing tools require a workspace filesystem to read ${normalizedPath}`);
-    }
-    const fileContent = await workspace.filesystem.readFile(normalizedPath);
-    const buffer = Buffer.isBuffer(fileContent) ? fileContent : Buffer.from(fileContent);
-    return { normalizedPath, buffer };
+async function readWorkspaceBinaryFile(context, path, maxBytes = exports.DEFAULT_MAX_FILE_BYTES) {
+    const file = await workspaceFiles.read(context, path, { maxBytes });
+    return { normalizedPath: file.path, buffer: file.bytes };
 }
 async function writeProcessedText(context, outputPath, content) {
-    const workspace = context.workspace;
-    if (!workspace?.filesystem?.writeFile) {
-        throw new Error(`File processing tools require a workspace filesystem to write ${outputPath}`);
-    }
-    const dir = outputPath.substring(0, outputPath.lastIndexOf('/'));
-    if (dir && dir !== '/') {
-        await workspace.filesystem.mkdir(dir, { recursive: true });
-    }
-    await workspace.filesystem.writeFile(outputPath, content);
+    await workspaceFiles.write(context, outputPath, content, { overwrite: true });
+}
+async function writeProcessedBinary(context, outputPath, content) {
+    await workspaceFiles.write(context, outputPath, Buffer.from(content), { overwrite: true });
 }
 function deriveOutputPath(sourcePath, suffix) {
-    const parts = sourcePath.split('/');
-    const filename = parts[parts.length - 1] ?? sourcePath;
+    const normalized = (0, harness_tool_packs_1.normalizeWorkspacePath)(sourcePath);
+    const parts = normalized.split('/');
+    const filename = parts[parts.length - 1] ?? normalized;
     parts[parts.length - 1] = `${filename}.${suffix}`;
     return parts.join('/');
 }

package/dist/utils.js.map

@@ -1 +1 @@
-{"version":3,"file":"utils.js","sourceRoot":"","sources":["../src/utils.ts"],"names":[],"mappings":";;AAEA,gEAMC;AAED,0DAeC;AAED,gDAgBC;AAED,4CAKC;AAhDD,SAAgB,0BAA0B,CAAC,IAAY;IACrD,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;IAC5B,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;QAC7B,MAAM,IAAI,KAAK,CAAC,+CAA+C,IAAI,EAAE,CAAC,CAAC;IACzE,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC;AAEM,KAAK,UAAU,uBAAuB,CAC3C,OAA6B,EAC7B,IAAY;IAEZ,MAAM,cAAc,GAAG,0BAA0B,CAAC,IAAI,CAAC,CAAC;IACxD,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,CAAC;IACpC,IAAI,CAAC,SAAS,EAAE,UAAU,EAAE,QAAQ,EAAE,CAAC;QACrC,MAAM,IAAI,KAAK,CACb,gEAAgE,cAAc,EAAE,CACjF,CAAC;IACJ,CAAC;IAED,MAAM,WAAW,GAAG,MAAM,SAAS,CAAC,UAAU,CAAC,QAAQ,CAAC,cAAc,CAAC,CAAC;IACxE,MAAM,MAAM,GAAG,MAAM,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;IACrF,OAAO,EAAE,cAAc,EAAE,MAAM,EAAE,CAAC;AACpC,CAAC;AAEM,KAAK,UAAU,kBAAkB,CACtC,OAA6B,EAC7B,UAAkB,EAClB,OAAe;IAEf,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,CAAC;IACpC,IAAI,CAAC,SAAS,EAAE,UAAU,EAAE,SAAS,EAAE,CAAC;QACtC,MAAM,IAAI,KAAK,CAAC,iEAAiE,UAAU,EAAE,CAAC,CAAC;IACjG,CAAC;IAED,MAAM,GAAG,GAAG,UAAU,CAAC,SAAS,CAAC,CAAC,EAAE,UAAU,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC;IACjE,IAAI,GAAG,IAAI,GAAG,KAAK,GAAG,EAAE,CAAC;QACvB,MAAM,SAAS,CAAC,UAAU,CAAC,KAAK,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAC7D,CAAC;IAED,MAAM,SAAS,CAAC,UAAU,CAAC,SAAS,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;AAC5D,CAAC;AAED,SAAgB,gBAAgB,CAAC,UAAkB,EAAE,MAAc;IACjE,MAAM,KAAK,GAAG,UAAU,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IACpC,MAAM,QAAQ,GAAG,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,IAAI,UAAU,CAAC;IACvD,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,GAAG,GAAG,QAAQ,IAAI,MAAM,EAAE,CAAC;IAClD,OAAO,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AACzB,CAAC"}
+{"version":3,"file":"utils.js","sourceRoot":"","sources":["../src/utils.ts"],"names":[],"mappings":";;;AAQA,gEAEC;AAED,0DAOC;AAED,gDAMC;AAED,oDAMC;AAED,4CAMC;AA1CD,oEAA2F;AAE3F,qEAAqE;AACxD,QAAA,sBAAsB,GAAG,EAAE,GAAG,IAAI,GAAG,IAAI,CAAC;AAEvD,MAAM,cAAc,GAAG,IAAA,yCAAoB,EAAC,EAAE,YAAY,EAAE,8BAAsB,EAAE,CAAC,CAAC;AAEtF,SAAgB,0BAA0B,CAAC,IAAY;IACrD,OAAO,IAAA,2CAAsB,EAAC,IAAI,CAAC,CAAC;AACtC,CAAC;AAEM,KAAK,UAAU,uBAAuB,CAC3C,OAA6B,EAC7B,IAAY,EACZ,WAAmB,8BAAsB;IAEzC,MAAM,IAAI,GAAG,MAAM,cAAc,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,EAAE,EAAE,QAAQ,EAAE,CAAC,CAAC;IACpE,OAAO,EAAE,cAAc,EAAE,IAAI,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,CAAC,KAAK,EAAE,CAAC;AAC3D,CAAC;AAEM,KAAK,UAAU,kBAAkB,CACtC,OAA6B,EAC7B,UAAkB,EAClB,OAAe;IAEf,MAAM,cAAc,CAAC,KAAK,CAAC,OAAO,EAAE,UAAU,EAAE,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;AAChF,CAAC;AAEM,KAAK,UAAU,oBAAoB,CACxC,OAA6B,EAC7B,UAAkB,EAClB,OAAmB;IAEnB,MAAM,cAAc,CAAC,KAAK,CAAC,OAAO,EAAE,UAAU,EAAE,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;AAC7F,CAAC;AAED,SAAgB,gBAAgB,CAAC,UAAkB,EAAE,MAAc;IACjE,MAAM,UAAU,GAAG,IAAA,2CAAsB,EAAC,UAAU,CAAC,CAAC;IACtD,MAAM,KAAK,GAAG,UAAU,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IACpC,MAAM,QAAQ,GAAG,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,IAAI,UAAU,CAAC;IACvD,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,GAAG,GAAG,QAAQ,IAAI,MAAM,EAAE,CAAC;IAClD,OAAO,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AACzB,CAAC"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lumifai/harness-tool-pack-file-processing",
-  "version": "0.1.0",
+  "version": "0.2.0",
   "license": "UNLICENSED",
   "repository": {
     "type": "git",
@@ -27,9 +27,9 @@
     "skills"
   ],
   "dependencies": {
-    "@llamaindex/liteparse": "^2.0.8",
-    "@lumifai/harness-tool-packs": "0.1.0",
-    "@mastra/core": "1.50.1",
+    "@llamaindex/liteparse": "^2.11.0",
+    "@lumifai/harness-tool-packs": "0.2.1",
+    "@mastra/core": "1.67.0",
     "zod": "^4.4.3"
   },
   "publishConfig": {

package/skills/file-processing/SKILL.md

@@ -1,7 +1,7 @@
 ---
 name: file-processing
-description: Extract text from PDF files for LLM consumption
-version: 1.1.0
+description: Extract structured Markdown or text from PDF files for LLM consumption
+version: 1.2.0
 tags:
   - file-processing
   - pdf
@@ -9,21 +9,21 @@ tags:
 
 # File Processing Tools
 
-Use the file-processing tool pack to extract text from PDF documents. Extracted content is written to the workspace filesystem as plain text files, keeping your context window clean.
+Use the file-processing tool pack to extract structured content from PDF documents. Extracted content is written to the workspace filesystem, keeping your context window clean.
 
 ## Available Tools
 
-| Tool          | Format | Output                 |
-| ------------- | ------ | ---------------------- |
-| `process_pdf` | PDF    | Text with page markers |
+| Tool          | Format | Output                             |
+| ------------- | ------ | ---------------------------------- |
+| `process_pdf` | PDF    | Markdown or text with page markers |
 
 ## Workflow
 
-1. Call `process_pdf` with the absolute workspace path to the file.
-2. The tool writes extracted text to `/processed/<filename>.txt`.
+1. Call `process_pdf` with the workspace-relative path to the file.
+2. The tool writes Markdown to `<source>.md` by default. Pass `format: "text"` for the legacy plain-text format.
 3. Use the returned `output_path` to reference the extracted content in subsequent operations.
 
 ## Tips
 
-- All tools return metadata only (paths, counts) — the extracted text stays in the filesystem.
-- Processed files are written to `/processed/` to keep them organized and separate from source files.
+- All tools return metadata only — the extracted document body stays in the filesystem.
+- Metadata, annotations, form fields, and complexity signals are included by default. Pass `include_images: true` only when embedded image files are needed; image extraction can be expensive.