officeparser 6.1.1 → 7.0.1

package/dist/generators/BaseGenerator.js

@@ -0,0 +1,107 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.BaseGenerator = void 0;
+const configUtils_js_1 = require("../utils/configUtils.js");
+const errorUtils_js_1 = require("../utils/errorUtils.js");
+const styleMapper_js_1 = require("../utils/styleMapper.js");
+/**
+ * Base class for all document generators.
+ * Provides common traversal logic and configuration handling.
+ */
+class BaseGenerator {
+    destination;
+    config;
+    ast;
+    messages = [];
+    styleMapper;
+    constructor(destination, ast, config) {
+        this.destination = destination;
+        this.config = (0, configUtils_js_1.resolveGeneratorConfig)(destination, ast.config, config);
+        this.ast = ast;
+        this.styleMapper = new styleMapper_js_1.StyleMapper(this.config.styleMap, this.config.ignoreDefaultStyleMap);
+    }
+    /**
+     * Retrieves the semantic mapping for a node, respecting the includeFormatting flag.
+     * Per design requirements: Style mapping is bypassed if formatting is disabled.
+     */
+    getSemanticMapping(node) {
+        if (this.config.includeFormatting === false) {
+            return undefined;
+        }
+        return this.styleMapper.getMapping(node);
+    }
+    /**
+     * Centralized logic for handling the onNode callback.
+     * Evaluates the callback and returns a result that tells the generator how to proceed.
+     *
+     * @returns
+     * - `string`: Use this as the node's output, skip default processing.
+     * - `false`: Skip this node and its subtree.
+     * - `void`: Proceed with default processing.
+     */
+    async handleOnNode(node) {
+        const result = await this.config.onNode(node);
+        if (result === false)
+            return false;
+        if (typeof result === 'string')
+            return result;
+    }
+    /**
+     * Recursively processes nodes and builds output.
+     *
+     * @param node - The current node being processed
+     * @param processor - A function that takes a node and its children's output and returns the node's output string.
+     * @returns The generated string for this node and its subtree.
+     */
+    async processNodeRecursive(node, processor) {
+        const override = await this.handleOnNode(node);
+        if (override === false)
+            return '';
+        if (typeof override === 'string')
+            return override;
+        let childrenOutput = '';
+        if (node.children) {
+            for (const child of node.children) {
+                childrenOutput += await this.processNodeRecursive(child, processor);
+            }
+        }
+        return await processor(node, childrenOutput);
+    }
+    /**
+     * Helper to generate a unique ID (slug) from text.
+     */
+    slugify(text) {
+        return text
+            .toLowerCase()
+            .replace(/[^\w\s-]/g, '')
+            .replace(/[\s_-]+/g, '-')
+            .replace(/^-+|-+$/g, '');
+    }
+    /**
+     * Recursively extracts plain text from a node and its children.
+     */
+    getNodeText(node) {
+        if (node.text)
+            return node.text;
+        if (node.children) {
+            return node.children.map(c => this.getNodeText(c)).join('');
+        }
+        return '';
+    }
+    /**
+     * Reports a warning to the user and collects it for the final result.
+     */
+    warn(type, info, node) {
+        const message = (0, errorUtils_js_1.getWarningMessage)(type, info);
+        const issue = {
+            type: 'warning',
+            code: type,
+            message,
+            node,
+            details: info
+        };
+        this.messages.push(issue);
+        this.config.onWarning(issue);
+    }
+}
+exports.BaseGenerator = BaseGenerator;

package/dist/generators/ChunkingGenerator.d.ts

@@ -0,0 +1,81 @@
+import { ConversionResult, GeneratorConfig, OfficeParserAST } from '../types.js';
+import { BaseGenerator } from './BaseGenerator.js';
+/**
+ * Generates a list of OfficeChunk objects from an AST for use in RAG pipelines.
+ * Supports three strategies: 'fixed-size', 'document-structure', and 'semantic'.
+ */
+export declare class ChunkingGenerator extends BaseGenerator<'chunks'> {
+    /** The resolved chunking config (with defaults applied). */
+    private chunkConfig;
+    /** Whether the user provided an explicit sentence boundary regex. */
+    private isCustomRegex;
+    constructor(ast: OfficeParserAST, config?: GeneratorConfig<'chunks'>);
+    /**
+     * Merges the user's chunking config with the appropriate defaults for the chosen strategy.
+     */
+    private resolveChunkingConfig;
+    /**
+     * Main entry point. Routes to the correct strategy implementation.
+     * Note: ConversionResult.value is a JSON string of OfficeChunk[] for the 'chunks' destination.
+     */
+    generate(): Promise<ConversionResult<'chunks'>>;
+    /**
+     * Splits the full document text into fixed-size chunks with optional overlap.
+     * Attempts to split on natural separators before hard-cutting.
+     */
+    private generateFixedSize;
+    /**
+     * Recursively tries separators to split text into chunks of at most `chunkSize`,
+     * with `chunkOverlap` characters of overlap between consecutive chunks.
+     */
+    private splitTextRecursively;
+    /**
+     * Walks the AST and splits at the designated structural boundaries (slide, page, heading, paragraph).
+     */
+    private generateDocumentStructure;
+    private processNodeForStructure;
+    private isStructuralBoundary;
+    /**
+     * Handles table chunking with the configured tableSplitStrategy.
+     * 'row': keeps header row attached to every chunk.
+     * 'flatten': converts table to text and splits normally.
+     */
+    private processTableNode;
+    /**
+     * Renders a list of row nodes as a pipe-separated text string.
+     */
+    private renderRowsAsText;
+    /**
+     * Splits document into semantically coherent chunks using cosine similarity
+     * between sentence embeddings. A new chunk begins when similarity drops
+     * below `similarityThreshold`.
+     */
+    private generateSemantic;
+    /**
+     * Extracts all text sentences from the AST with their contextual metadata.
+     */
+    private extractSentences;
+    /**
+     * Builds a flat text string from the entire document and a map of
+     * character offsets to AST node metadata for position-based metadata lookups.
+     */
+    private buildFlatTextWithPositions;
+    /**
+     * Finds the closest AST metadata for a given character position.
+     */
+    private enrichMetadataFromPosition;
+    /**
+     * Applies final post-processing: strips whitespace, sets sourceType.
+     */
+    private finalizeChunks;
+    /**
+     * Helper to process embeddings in sequential batches to avoid API rate limits and memory issues.
+     */
+    private batchEmbeddings;
+    private cosineSimilarity;
+    private averageEmbeddings;
+    /**
+     * Robustly splits text into sentences, respecting abbreviations and non-Western punctuation.
+     */
+    private splitIntoSentences;
+}