@exulu/backend 1.48.2 → 1.49.0

package/ee/documents/processing/chunk_markdown.py

@@ -1,263 +0,0 @@
-#!/usr/bin/env python3
-"""
-Markdown Chunker using Docling HybridChunker
-Converts markdown files into chunks using Docling's hybrid chunking approach.
-
-Usage:
-    chunk_markdown.py <markdown_file_path> [-o OUTPUT_PATH] [--max-tokens MAX_TOKENS]
-"""
-
-import sys
-import os
-import warnings
-import argparse
-import json
-from pathlib import Path
-
-# Suppress warnings
-warnings.filterwarnings('ignore')
-os.environ['PYTHONWARNINGS'] = 'ignore'
-
-from docling.document_converter import DocumentConverter
-from docling.chunking import HybridChunker
-
-
-def find_page_for_chunk(chunk_text: str, pages_data: list) -> int:
-    """
-    Find which page a chunk belongs to by matching text content.
-
-    Args:
-        chunk_text: The text content of the chunk
-        pages_data: List of page objects from index_validated.json
-
-    Returns:
-        Page number (1-indexed) or None if not found
-    """
-    # Take first 100 characters for matching (remove extra whitespace)
-    search_text = ' '.join(chunk_text[:100].split())
-
-    # Search through all pages
-    for page in pages_data:
-        # Check both content and vlm_corrected_text
-        content_sources = [page.get('content', '')]
-        if page.get('vlm_corrected_text'):
-            content_sources.append(page['vlm_corrected_text'])
-
-        for content in content_sources:
-            # Normalize whitespace in content for comparison
-            normalized_content = ' '.join(content.split())
-
-            if search_text in normalized_content:
-                return page.get('page')
-
-    return None
-
-
-def chunk_markdown_file(markdown_path: str, max_tokens: int = 512, output_path: str = None, use_markdown_tables: bool = True, index_json_path: str = None) -> dict:
-    """
-    Chunk a markdown file using Docling's HybridChunker.
-
-    Args:
-        markdown_path: Path to the markdown file
-        max_tokens: Maximum number of tokens per chunk (default: 512)
-        output_path: Optional output path for JSON file
-        use_markdown_tables: Use markdown table format instead of triplets (default: True)
-        index_json_path: Optional path to index_validated.json for page mapping
-
-    Returns:
-        Dictionary containing chunks and metadata
-    """
-    # Convert the markdown document
-    print(f"Converting markdown document: {markdown_path}", file=sys.stderr)
-    converter = DocumentConverter()
-    result = converter.convert(source=markdown_path)
-    doc = result.document
-
-    # Initialize the chunker with specified max_tokens and markdown tables enabled
-    print(f"Initializing HybridChunker with max_tokens={max_tokens}, use_markdown_tables={use_markdown_tables}", file=sys.stderr)
-    chunker = HybridChunker(
-        max_tokens=max_tokens,
-        use_markdown_tables=use_markdown_tables
-    )
-
-    # Load page data for page mapping if provided
-    pages_data = None
-    if index_json_path and Path(index_json_path).exists():
-        print(f"Loading page data from: {index_json_path}", file=sys.stderr)
-        with open(index_json_path, 'r', encoding='utf-8') as f:
-            pages_data = json.load(f)
-
-    # Chunk the document
-    print(f"Chunking document...", file=sys.stderr)
-    chunk_iter = chunker.chunk(dl_doc=doc)
-
-    # Process chunks and collect results
-    chunks = []
-    for i, chunk in enumerate(chunk_iter):
-        # Get the context-enriched text (recommended for embeddings)
-        enriched_text = chunker.contextualize(chunk=chunk)
-
-        # Extract heading hierarchy from chunk metadata
-        heading_hierarchy = []
-        if hasattr(chunk.meta, 'headings') and chunk.meta.headings:
-            heading_hierarchy = chunk.meta.headings if isinstance(chunk.meta.headings, list) else [chunk.meta.headings]
-
-        # Find page number by matching chunk text with pages data
-        page_number = None
-        if pages_data:
-            page_number = find_page_for_chunk(chunk.text, pages_data)
-
-        # Debug for first few chunks
-        if i < 3:
-            print(f"\nDEBUG Chunk {i}:", file=sys.stderr)
-            print(f"  Text preview: {chunk.text[:100]}...", file=sys.stderr)
-            print(f"  Heading hierarchy: {heading_hierarchy}", file=sys.stderr)
-            print(f"  Page number: {page_number}", file=sys.stderr)
-
-        chunks.append({
-            "chunk_id": i,
-            "text": chunk.text,
-            "enriched_text": enriched_text,
-            "metadata": {
-                "page": page_number,
-                "path": getattr(chunk, 'path', None),
-                "headings": heading_hierarchy
-            }
-        })
-
-    # Return the results
-    return {
-        "source": markdown_path,
-        "total_chunks": len(chunks),
-        "max_tokens": max_tokens,
-        "chunks": chunks
-    }
-
-
-def main():
-    """Main entry point for the script."""
-    # Set up argument parser
-    parser = argparse.ArgumentParser(
-        description='Chunk markdown file using Docling HybridChunker',
-        formatter_class=argparse.RawDescriptionHelpFormatter
-    )
-
-    parser.add_argument(
-        'markdown_path',
-        type=str,
-        help='Path to the markdown file to chunk'
-    )
-
-    parser.add_argument(
-        '-o', '--output',
-        type=str,
-        dest='output_path',
-        help='Output path for the JSON file (default: same name as markdown with _chunks.json suffix)'
-    )
-
-    parser.add_argument(
-        '--max-tokens',
-        type=int,
-        dest='max_tokens',
-        default=512,
-        help='Maximum number of tokens per chunk (default: 512)'
-    )
-
-    parser.add_argument(
-        '--no-markdown-tables',
-        action='store_true',
-        dest='no_markdown_tables',
-        help='Disable markdown table format (use triplets instead)'
-    )
-
-    parser.add_argument(
-        '--index-json',
-        type=str,
-        dest='index_json_path',
-        help='Path to processed.json for page number mapping'
-    )
-
-    # Parse arguments
-    args = parser.parse_args()
-
-    markdown_path = args.markdown_path
-    output_path = args.output_path
-    max_tokens = args.max_tokens
-    use_markdown_tables = not args.no_markdown_tables
-    index_json_path = args.index_json_path
-
-    # Auto-detect index_validated.json if not provided
-    if not index_json_path:
-        # Try to find index_validated.json in the same directory as the markdown file
-        markdown_dir = Path(markdown_path).parent
-        potential_index = markdown_dir / 'processed.json'
-        if potential_index.exists():
-            index_json_path = str(potential_index)
-            print(f"Auto-detected index file: {index_json_path}", file=sys.stderr)
-
-    # Validate the file exists
-    if not Path(markdown_path).exists():
-        print(f"Error: File not found: {markdown_path}", file=sys.stderr)
-        sys.exit(1)
-
-    # Default: same name as markdown but with _chunks.json suffix
-    if not output_path:
-        markdown_file = Path(markdown_path)
-        output_path = str(markdown_file.parent / f"chunks.json")
-    else:
-        # If output_path is a directory, append default filename
-        output_path_obj = Path(output_path)
-        if output_path_obj.is_dir():
-            markdown_file = Path(markdown_path)
-            output_path = str(output_path_obj / f"chunks.json")
-        elif not output_path_obj.suffix:
-            # If no extension provided, treat as directory
-            output_path_obj.mkdir(exist_ok=True)
-            markdown_file = Path(markdown_path)
-            output_path = str(output_path_obj / f"chunks.json")
-
-    try:
-        # Chunk the markdown file
-        print(f"Processing markdown: {markdown_path}", file=sys.stderr)
-        print(f"Max tokens per chunk: {max_tokens}", file=sys.stderr)
-        print(f"Use markdown tables: {use_markdown_tables}", file=sys.stderr)
-        result = chunk_markdown_file(
-            markdown_path,
-            max_tokens=max_tokens,
-            output_path=output_path,
-            use_markdown_tables=use_markdown_tables,
-            index_json_path=index_json_path
-        )
-
-        # Save to JSON file
-        with open(output_path, 'w', encoding='utf-8') as f:
-            json.dump(result, f, indent=2, ensure_ascii=False)
-            f.flush()
-
-        print(f"\nSuccessfully saved {result['total_chunks']} chunks to: {output_path}", file=sys.stderr)
-
-        # Print stats
-        total_chars = sum(len(chunk['text']) for chunk in result['chunks'])
-        avg_chars = total_chars / len(result['chunks']) if result['chunks'] else 0
-
-        print(f"\nChunking stats:", file=sys.stderr)
-        print(f"  Total chunks: {result['total_chunks']}", file=sys.stderr)
-        print(f"  Total characters: {total_chars}", file=sys.stderr)
-        print(f"  Average characters per chunk: {avg_chars:.0f}", file=sys.stderr)
-
-        sys.stderr.flush()
-        sys.stdout.flush()
-
-        # Exit cleanly
-        os._exit(0)
-
-    except Exception as e:
-        print(f"Error processing markdown: {str(e)}", file=sys.stderr)
-        import traceback
-        traceback.print_exc(file=sys.stderr)
-        sys.stderr.flush()
-        os._exit(1)
-
-
-if __name__ == "__main__":
-    main()

package/ee/documents/processing/pdf_processor.spec

@@ -1,115 +0,0 @@
-# -*- mode: python ; coding: utf-8 -*-
-from PyInstaller.utils.hooks import copy_metadata, collect_data_files
-import os
-
-block_cipher = None
-
-# Collect package metadata for packages that need it
-datas = []
-datas += copy_metadata('docling')
-datas += copy_metadata('docling-core')
-datas += copy_metadata('docling-parse')
-datas += copy_metadata('docling-ibm-models')
-datas += copy_metadata('transformers')
-datas += copy_metadata('torch')
-datas += copy_metadata('tokenizers')
-datas += copy_metadata('huggingface-hub')
-datas += copy_metadata('pydantic')
-datas += copy_metadata('pydantic-core')
-
-# Collect data files from docling packages
-datas += collect_data_files('docling_parse')
-datas += collect_data_files('docling')
-datas += collect_data_files('docling_core')
-datas += collect_data_files('docling_ibm_models')
-datas += collect_data_files('transformers')
-
-# Collect all data files from docling and transformers packages
-a = Analysis(
-    ['pdf_processor.py'],
-    pathex=[],
-    binaries=[],
-    datas=datas,
-    hiddenimports=[
-        'docling',
-        'docling.document_converter',
-        'docling.chunking',
-        'docling.models',
-        'docling.models.plugins',
-        'docling.models.plugins.defaults',
-        'docling.backend',
-        'docling.backend.docling_parse_backend',
-        'docling.backend.asciidoc_backend',
-        'docling.backend.html_backend',
-        'docling.backend.md_backend',
-        'docling.backend.msexcel_backend',
-        'docling.backend.mspowerpoint_backend',
-        'docling.backend.msword_backend',
-        'docling.datamodel',
-        'docling.datamodel.document',
-        'docling_core',
-        'docling_core.transforms.chunker',
-        'docling_core.transforms.chunker.tokenizer',
-        'docling_core.transforms.chunker.tokenizer.huggingface',
-        'transformers',
-        'transformers.models',
-        'transformers.models.auto',
-        'torch',
-        'numpy',
-        'PIL',
-        'pdfplumber',
-        'pypdf',
-        'pikepdf',
-        'lxml',
-        'bs4',
-        'tiktoken',
-        'tokenizers',
-        'sentencepiece',
-        'safetensors',
-        'huggingface_hub',
-        'tqdm',
-        'regex',
-        'requests',
-        'urllib3',
-        'certifi',
-        'charset_normalizer',
-        'idna',
-        'packaging',
-        'filelock',
-        'pyyaml',
-        'jinja2',
-        'markupsafe',
-    ],
-    hookspath=[],
-    hooksconfig={},
-    runtime_hooks=[],
-    excludes=[],
-    win_no_prefer_redirects=False,
-    win_private_assemblies=False,
-    cipher=block_cipher,
-    noarchive=False,
-)
-
-pyz = PYZ(a.pure, a.zipped_data, cipher=block_cipher)
-
-exe = EXE(
-    pyz,
-    a.scripts,
-    a.binaries,
-    a.zipfiles,
-    a.datas,
-    [],
-    name='pdf_processor',
-    debug=False,
-    bootloader_ignore_signals=False,
-    strip=False,
-    upx=True,
-    upx_exclude=[],
-    runtime_tmpdir=None,
-    console=True,
-    disable_windowed_traceback=False,
-    argv_emulation=False,
-    target_arch=None,
-    codesign_identity=None,
-    entitlements_file=None,
-)

package/eslint.config.js

@@ -1,88 +0,0 @@
-import tseslint from '@typescript-eslint/eslint-plugin';
-import tsparser from '@typescript-eslint/parser';
-import js from '@eslint/js';
-
-export default [
-  {
-    ignores: ['dist/**', 'node_modules/**', '*.config.js', '*.config.cjs', '*.config.ts']
-  },
-  js.configs.recommended,
-  {
-    files: ['**/*.ts', '**/*.tsx'],
-    languageOptions: {
-      parser: tsparser,
-      parserOptions: {
-        ecmaVersion: 'latest',
-        sourceType: 'module',
-        project: './tsconfig.json'
-      },
-      globals: {
-        console: 'readonly',
-        process: 'readonly',
-        Buffer: 'readonly',
-        setTimeout: 'readonly',
-        clearTimeout: 'readonly',
-        setInterval: 'readonly',
-        clearInterval: 'readonly',
-        setImmediate: 'readonly',
-        clearImmediate: 'readonly',
-        __dirname: 'readonly',
-        __filename: 'readonly',
-        exports: 'writable',
-        module: 'writable',
-        require: 'readonly',
-        global: 'readonly',
-        performance: 'readonly',
-        TextDecoder: 'readonly',
-        TextEncoder: 'readonly'
-      }
-    },
-    plugins: {
-      '@typescript-eslint': tseslint
-    },
-    rules: {
-      ...tseslint.configs.recommended.rules,
-      ...tseslint.configs['recommended-requiring-type-checking'].rules,
-      'no-console': 'warn',
-      'no-extra-boolean-cast': 'off',
-      'no-async-promise-executor': 'off',
-      'no-useless-escape': 'off',
-      '@typescript-eslint/no-unused-vars': [
-        'error',
-        {
-          argsIgnorePattern: '^_',
-          varsIgnorePattern: '^_',
-          args: 'none'
-        }
-      ],
-      '@typescript-eslint/explicit-function-return-type': [
-        'warn',
-        {
-          allowExpressions: true,
-          allowTypedFunctionExpressions: true
-        }
-      ],
-      '@typescript-eslint/no-redundant-type-constituents': 'off',
-      '@typescript-eslint/no-explicit-any': 'off',
-      '@typescript-eslint/no-floating-promises': 'off',
-      '@typescript-eslint/await-thenable': 'off',
-      '@typescript-eslint/no-misused-promises': 'off',
-      '@typescript-eslint/unbound-method': 'off',
-      '@typescript-eslint/no-unnecessary-type-assertion': 'warn',
-      '@typescript-eslint/prefer-nullish-coalescing': 'warn',
-      '@typescript-eslint/prefer-optional-chain': 'warn',
-      'no-unused-expressions': 'off',
-      '@typescript-eslint/no-unused-expressions': 'error',
-      'require-await': 'off',
-      'no-case-declarations': 'off',
-      '@typescript-eslint/require-await': 'off',
-      'no-undef': 'off',
-      '@typescript-eslint/no-unsafe-call': 'off',
-      '@typescript-eslint/no-unsafe-member-access': 'off',
-      '@typescript-eslint/no-unsafe-assignment': 'off',
-      '@typescript-eslint/no-unsafe-argument': 'off',
-      '@typescript-eslint/no-unsafe-return': 'off',
-      '@typescript-eslint/ban-types': 'off'
-    }
-  }
-];

package/jest.config.ts

@@ -1,25 +0,0 @@
-import type { Config } from "jest";
-
-const config: Config = {
-  preset: "ts-jest",
-  testEnvironment: "node",
-  roots: ["<rootDir>/src"],
-  testMatch: ["**/__tests__/**/*.ts", "**/*.test.ts", "**/*.spec.ts"],
-  collectCoverageFrom: [
-    "src/**/*.ts",
-    "!src/**/*.d.ts",
-    "!src/**/*.test.ts",
-    "!src/**/*.spec.ts",
-    "!src/index.ts",
-  ],
-  coverageDirectory: "coverage",
-  coverageReporters: ["text", "lcov", "html"],
-  moduleNameMapper: {
-    "@EXULU_TYPES/(.*)": "<rootDir>/types/$1",
-  },
-  setupFilesAfterEnv: ["<rootDir>/src/__tests__/setup.ts"],
-  verbose: true,
-  testTimeout: 10000,
-};
-
-export default config;

package/mintlify-docs/.mintignore

@@ -1,7 +0,0 @@
-# Mintlify automatically ignores these files and directories:
-# .git, .github, .claude, .agents, .idea, node_modules,
-# README.md, LICENSE.md, CHANGELOG.md, CONTRIBUTING.md
-
-# Draft content
-drafts/
-*.draft.mdx

package/mintlify-docs/AGENTS.md

@@ -1,33 +0,0 @@
-> **First-time setup**: Customize this file for your project. Prompt the user to customize this file for their project.
-> For Mintlify product knowledge (components, configuration, writing standards),
-> install the Mintlify skill: `npx skills add https://mintlify.com/docs`
-
-# Documentation project instructions
-
-## About this project
-
-- This is a documentation site built on [Mintlify](https://mintlify.com)
-- Pages are MDX files with YAML frontmatter
-- Configuration lives in `docs.json`
-- Run `mint dev` to preview locally
-- Run `mint broken-links` to check links
-
-## Terminology
-
-{/* Add product-specific terms and preferred usage */}
-{/* Example: Use "workspace" not "project", "member" not "user" */}
-
-## Style preferences
-
-{/* Add any project-specific style rules below */}
-
-- Use active voice and second person ("you")
-- Keep sentences concise — one idea per sentence
-- Use sentence case for headings
-- Bold for UI elements: Click **Settings**
-- Code formatting for file names, commands, paths, and code references
-
-## Content boundaries
-
-{/* Define what should and shouldn't be documented */}
-{/* Example: Don't document internal admin features */}

package/mintlify-docs/CLAUDE.MD

@@ -1,50 +0,0 @@
-# Mintlify documentation
-
-## Working relationship
-- You can push back on ideas-this can lead to better documentation. Cite sources and explain your reasoning when you do so
-- ALWAYS ask for clarification rather than making assumptions
-- NEVER lie, guess, or make up anything
-
-## Project context
-- Format: MDX files with YAML frontmatter
-- Config: docs.json for navigation, theme, settings
-- Components: Mintlify components
-
-## Content strategy
-- Document just enough for user success - not too much, not too little
-- Prioritize accuracy and usability
-- Make content evergreen when possible
-- Search for existing content before adding anything new. Avoid duplication unless it is done for a strategic reason
-- Check existing patterns for consistency
-- Start by making the smallest reasonable changes
-
-## docs.json
-
-- Refer to the [docs.json schema](https://mintlify.com/docs.json) when building the docs.json file and site navigation
-
-## Frontmatter requirements for pages
-- title: Clear, descriptive page title
-- description: Concise summary for SEO/navigation
-
-## Writing standards
-- Second-person voice ("you")
-- Prerequisites at start of procedural content
-- Test all code examples before publishing
-- Match style and formatting of existing pages
-- Include both basic and advanced use cases
-- Language tags on all code blocks
-- Alt text on all images
-- Relative paths for internal links
-
-## Git workflow
-- NEVER use --no-verify when committing
-- Ask how to handle uncommitted changes before starting
-- Create a new branch when no clear branch exists for changes
-- Commit frequently throughout development
-- NEVER skip or disable pre-commit hooks
-
-## Do not
-- Skip frontmatter on any MDX file
-- Use absolute URLs for internal links
-- Include untested code examples
-- Make assumptions - always ask for clarification

package/mintlify-docs/CONTRIBUTING.md

@@ -1,32 +0,0 @@
-# Contribute to the documentation
-
-Thank you for your interest in contributing to our documentation! This guide will help you get started.
-
-## How to contribute
-
-### Option 1: Edit directly on GitHub
-
-1. Navigate to the page you want to edit
-2. Click the "Edit this file" button (the pencil icon)
-3. Make your changes and submit a pull request
-
-### Option 2: Local development
-
-1. Fork and clone this repository
-2. Install the Mintlify CLI: `npm i -g mint`
-3. Create a branch for your changes
-4. Make changes
-5. Navigate to the docs directory (mintlify-docs) and run `mint dev`
-6. Preview your changes at `http://localhost:3000`
-7. Commit your changes and submit a pull request
-
-For more details on local development, see our [development guide](development.mdx).
-
-## Writing guidelines
-
-- **Use active voice**: "Run the command" not "The command should be run"
-- **Address the reader directly**: Use "you" instead of "the user"
-- **Keep sentences concise**: Aim for one idea per sentence
-- **Lead with the goal**: Start instructions with what the user wants to accomplish
-- **Use consistent terminology**: Don't alternate between synonyms for the same concept
-- **Include examples**: Show, don't just tell

package/mintlify-docs/LICENSE

@@ -1,21 +0,0 @@
-MIT License
-
-Copyright (c) 2023 Mintlify
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all
-copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-SOFTWARE.