parsekit-bin 0.1.2

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 0a6447155b1ecdb5426c2051e680a12a6b0091a43539d46c711443a5bc98f4bd
+  data.tar.gz: 1dc5570e92dbc491b50669d8dcc68aec57ca9249dd6f31fd82c5767c1109beea
+SHA512:
+  metadata.gz: 681e2892319f1f8ccf51982836224fae0b38946c54dd44eaf184fca8deefc1b21875ce9359523663d09610eb2cfeec1efe6425d897584d6d8bbc9fc3ca51196b
+  data.tar.gz: 891180dcb47736714c2f33c42bb13032bf8f6fddcf09e663915edcb525c8715d137ac9309b1821702c7c8b8918f3aa552b291e9a340bf169e69da6e5b257ac5d

data/CHANGELOG.md

@@ -0,0 +1,53 @@
+# Changelog
+
+All notable changes to this project will be documented in this file.
+
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
+and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+## [Unreleased]
+
+### Added
+- Nothing yet
+
+### Changed
+- Nothing yet
+
+### Deprecated
+- Nothing yet
+
+### Removed
+- Nothing yet
+
+### Fixed
+- Nothing yet
+
+### Security
+- Nothing yet
+
+## [0.1.0] - 2024-08-09
+
+### Added
+- Initial release of parsekit
+- Basic parser functionality with Ruby bindings via Magnus
+- Support for parsing strings and files
+- Configurable parser with options (strict_mode, max_depth, encoding)
+- Parser class with instance methods
+- Module-level convenience methods
+- Error handling with custom error classes
+- Thread-safe parsing operations
+- Cross-platform support (Linux, macOS, Windows)
+- Ruby 3.0+ support
+- Comprehensive test suite with RSpec
+- CI/CD with GitHub Actions
+- Documentation and examples
+- Integration with ruby-nlp ecosystem
+
+### Technical Details
+- Built with Magnus 0.7 for Ruby-Rust bindings
+- Uses rb_sys 0.9 for build system integration
+- Rust edition 2021
+- Cross-compilation support for multiple platforms
+
+[Unreleased]: https://github.com/scientist-labs/parsekit/compare/v0.1.0...HEAD
+[0.1.0]: https://github.com/scientist-labs/parsekit/releases/tag/v0.1.0

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2024 Your Name
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,195 @@
+<img src="/docs/assets/parsekit-wide.png" alt="parsekit" height="80px">
+
+[![Gem Version](https://badge.fury.io/rb/parsekit-bin.svg)](https://badge.fury.io/rb/parsekit-bin)
+[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
+
+Native Ruby bindings for the [parser-core](https://crates.io/crates/parser-core) Rust crate, providing high-performance document parsing and text extraction capabilities through Magnus. This gem wraps parser-core to extract text from PDFs, Office documents (DOCX, XLSX), images (with OCR), and more. Part of the ruby-nlp ecosystem.
+
+## Features
+
+- 📄 **Document Parsing**: Extract text from PDFs, Office documents (DOCX, XLSX)
+- 🖼️ **OCR Support**: Extract text from images using Tesseract OCR
+- 🚀 **High Performance**: Native Rust performance with Ruby convenience
+- 🔧 **Unified API**: Single interface for multiple document formats
+- 📦 **Cross-Platform**: Works on Linux, macOS, and Windows
+- 🧪 **Well Tested**: Comprehensive test suite with RSpec
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'parsekit-bin'
+```
+
+And then execute:
+
+    $ bundle install
+
+Or install it yourself as:
+
+```bash
+gem install parsekit-bin
+```
+
+### Requirements
+
+- Ruby >= 3.0.0
+- Rust toolchain (stable)
+- C compiler (for linking)
+
+That's it! ParseKit bundles all necessary libraries including Tesseract for OCR, so you don't need to install any system dependencies.
+
+## Usage
+
+### Basic Usage
+
+```ruby
+require 'parsekit'
+
+# Parse a PDF file
+text = ParseKit.parse_file("document.pdf")
+puts text  # Extracted text from the PDF
+
+# Parse an Excel file
+text = ParseKit.parse_file("spreadsheet.xlsx")
+puts text  # Extracted text from all sheets
+
+# Parse binary data directly
+file_data = File.binread("document.pdf")
+text = ParseKit.parse_bytes(file_data)
+puts text
+
+# Parse with a Parser instance
+parser = ParseKit::Parser.new
+text = parser.parse_file("report.docx")
+puts text
+```
+
+### Module-Level Convenience Methods
+
+```ruby
+# Parse files directly
+content = ParseKit.parse_file('document.pdf')
+
+# Parse bytes
+data = File.read('document.pdf', mode: 'rb')
+content = ParseKit.parse_bytes(data.bytes)
+
+# Check supported formats
+formats = ParseKit.supported_formats
+# => ["txt", "json", "xml", "html", "docx", "xlsx", "xls", "csv", "pdf", "png", "jpg", "jpeg", "tiff", "bmp"]
+
+# Check if a file is supported
+ParseKit.supports_file?('document.pdf')  # => true
+```
+
+### Configuration Options
+
+```ruby
+# Create parser with options
+parser = ParseKit::Parser.new(
+  strict_mode: true,
+  max_size: 50 * 1024 * 1024,  # 50MB limit
+  encoding: 'UTF-8'
+)
+
+# Or use the strict convenience method
+parser = ParseKit::Parser.strict
+```
+
+### Format-Specific Parsing
+
+```ruby
+parser = ParseKit::Parser.new
+
+# Direct access to format-specific parsers
+pdf_data = File.read('document.pdf', mode: 'rb').bytes
+pdf_text = parser.parse_pdf(pdf_data)
+
+image_data = File.read('image.png', mode: 'rb').bytes
+ocr_text = parser.ocr_image(image_data)
+
+excel_data = File.read('data.xlsx', mode: 'rb').bytes
+excel_text = parser.parse_xlsx(excel_data)
+```
+
+## Supported Formats
+
+| Format | Extensions | Method | Notes |
+|--------|------------|--------|-------|
+| PDF | .pdf | `parse_pdf` | Text extraction via MuPDF |
+| Word | .docx | `parse_docx` | Office Open XML format |
+| Excel | .xlsx, .xls | `parse_xlsx` | Both modern and legacy formats |
+| PowerPoint | .pptx | `parse_pptx` | Text extraction from slides and notes |
+| Images | .png, .jpg, .jpeg, .tiff, .bmp | `ocr_image` | OCR via bundled Tesseract |
+| JSON | .json | `parse_json` | Pretty-printed output |
+| XML/HTML | .xml, .html | `parse_xml` | Extracts text content |
+| Text | .txt, .csv, .md | `parse_text` | With encoding detection |
+
+## Performance
+
+ParseKit is built with performance in mind:
+
+- Native Rust implementation for speed
+- Statically linked C libraries (MuPDF, Tesseract) compiled with optimizations
+- Efficient memory usage with streaming where possible
+- Configurable size limits to prevent memory issues
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests.
+
+To compile the Rust extension:
+
+```bash
+rake compile
+```
+
+To run tests with coverage:
+
+```bash
+rake dev:coverage
+```
+
+### OCR Mode Configuration
+
+By default, ParseKit bundles Tesseract for zero-dependency OCR support. Advanced users who already have Tesseract installed system-wide and want faster gem installation can use system mode:
+
+**Using system Tesseract during installation:**
+```bash
+gem install parsekit-bin -- --no-default-features
+```
+
+**For development with system Tesseract:**
+```bash
+rake compile CARGO_FEATURES=""  # Disables bundled-tesseract feature
+```
+
+**System Tesseract requirements:**
+- **macOS**: `brew install tesseract`
+- **Ubuntu/Debian**: `sudo apt-get install libtesseract-dev`
+- **Fedora/RHEL**: `sudo dnf install tesseract-devel`
+
+The bundled mode adds ~1-3 minutes to initial gem installation but provides a completely self-contained experience with no external dependencies.
+
+## Architecture
+
+ParseKit uses a hybrid Ruby/Rust architecture:
+
+- **Ruby Layer**: Provides convenient API and format detection
+- **Rust Layer**: Implements high-performance parsing using:
+  - MuPDF for PDF text extraction (statically linked)
+  - tesseract-rs for OCR (with bundled Tesseract by default)
+  - Pure Rust libraries for DOCX/XLSX parsing
+  - Magnus for Ruby-Rust FFI bindings
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/scientist-labs/parsekit.
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).
+
+Note: This gem includes statically linked versions of MuPDF (AGPL/Commercial) and Tesseract (Apache 2.0). Please review their respective licenses for compliance with your use case.

data/ext/parsekit/Cargo.toml

@@ -0,0 +1,36 @@
+[package]
+name = "parsekit"
+version = "0.1.0"
+edition = "2021"
+authors = ["Your Name <your.email@example.com>"]
+license = "MIT"
+publish = false
+
+[lib]
+crate-type = ["cdylib"]
+name = "parsekit"
+
+[dependencies]
+magnus = { version = "0.8", features = ["rb-sys"] }
+# Document parsing - testing embedded C libraries
+# MuPDF builds from source and statically links
+mupdf = { version = "0.5", default-features = false, features = [] }
+# OCR - Using tesseract-rs for both system and bundled modes
+tesseract-rs = "0.1"  # Tesseract with optional bundling
+image = "0.25"  # Image processing library (match rusty-tesseract's version)
+calamine = "0.31"  # Excel parsing
+docx-rs = "0.4"  # Word document parsing
+quick-xml = "0.38"  # XML parsing
+zip = "5.0"  # ZIP archive handling for PPTX
+serde_json = "1.0"  # JSON parsing
+regex = "1.10"  # Text parsing
+encoding_rs = "0.8"  # Encoding detection
+
+[features]
+default = ["bundled-tesseract"]
+bundled-tesseract = []
+
+[profile.release]
+opt-level = 3
+lto = true
+codegen-units = 1

data/ext/parsekit/extconf.rb

@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+
+require "mkmf"
+require "rb_sys/mkmf"
+
+create_rust_makefile("parsekit/parsekit")

data/ext/parsekit/src/error.rs

@@ -0,0 +1,45 @@
+use magnus::{Error, RModule, Ruby, Module};
+
+/// Custom error types for ParseKit
+#[derive(Debug)]
+#[allow(dead_code)]
+pub enum ParserError {
+    ParseError(String),
+    ConfigError(String),
+    IoError(String),
+}
+
+impl ParserError {
+    /// Convert to Magnus Error
+    #[allow(dead_code)]
+    pub fn to_error(&self) -> Error {
+        match self {
+            ParserError::ParseError(msg) => {
+                Error::new(Ruby::get().unwrap().exception_runtime_error(), msg.clone())
+            }
+            ParserError::ConfigError(msg) => {
+                Error::new(Ruby::get().unwrap().exception_arg_error(), msg.clone())
+            }
+            ParserError::IoError(msg) => {
+                Error::new(Ruby::get().unwrap().exception_io_error(), msg.clone())
+            }
+        }
+    }
+}
+
+/// Initialize error classes
+/// For simplicity, we'll just create Ruby classes that inherit from Object,
+/// and document that they should be treated as exceptions
+pub fn init(_ruby: &Ruby, module: RModule) -> Result<(), Error> {
+    // For now, just create placeholder classes
+    // In a real implementation, you'd want to properly set up exception classes
+    // but Magnus 0.7's API for this is complex
+    
+    // Define error classes as regular Ruby classes
+    // Users can still rescue them by name in Ruby code
+    let _error = module.define_class("Error", Ruby::get().unwrap().class_object())?;
+    let _parse_error = module.define_class("ParseError", Ruby::get().unwrap().class_object())?;
+    let _config_error = module.define_class("ConfigError", Ruby::get().unwrap().class_object())?;
+    
+    Ok(())
+}

data/ext/parsekit/src/format_detector.rs

@@ -0,0 +1,233 @@
+use std::path::Path;
+
+/// Represents a detected file format
+#[derive(Debug, Clone, PartialEq)]
+pub enum FileFormat {
+    Pdf,
+    Docx,
+    Xlsx,
+    Xls,
+    Pptx,
+    Png,
+    Jpeg,
+    Tiff,
+    Bmp,
+    Json,
+    Xml,
+    Html,
+    Text,
+    Unknown,
+}
+
+impl FileFormat {
+    /// Convert to Ruby symbol representation
+    pub fn to_symbol(&self) -> &'static str {
+        match self {
+            FileFormat::Pdf => "pdf",
+            FileFormat::Docx => "docx",
+            FileFormat::Xlsx => "xlsx",
+            FileFormat::Xls => "xls",
+            FileFormat::Pptx => "pptx",
+            FileFormat::Png => "png",
+            FileFormat::Jpeg => "jpeg",
+            FileFormat::Tiff => "tiff",
+            FileFormat::Bmp => "bmp",
+            FileFormat::Json => "json",
+            FileFormat::Xml => "xml",
+            FileFormat::Html => "xml", // HTML is treated as XML in Ruby
+            FileFormat::Text => "text",
+            FileFormat::Unknown => "unknown",
+        }
+    }
+}
+
+/// Central format detection logic
+pub struct FormatDetector;
+
+impl FormatDetector {
+    /// Detect format from filename and content
+    /// Prioritizes content detection over extension when both are available
+    pub fn detect(filename: Option<&str>, content: Option<&[u8]>) -> FileFormat {
+        // First try content-based detection if content is provided
+        if let Some(data) = content {
+            let format = Self::detect_from_content(data);
+            // If we got a definitive format from content, use it
+            if !matches!(format, FileFormat::Text | FileFormat::Unknown) {
+                return format;
+            }
+        }
+        
+        // Fall back to extension-based detection
+        if let Some(name) = filename {
+            let ext_format = Self::detect_from_extension(name);
+            if ext_format != FileFormat::Unknown {
+                return ext_format;
+            }
+        }
+        
+        // If content detection returned Text and no extension match, return Text
+        if let Some(data) = content {
+            let format = Self::detect_from_content(data);
+            if format == FileFormat::Text {
+                return FileFormat::Text;
+            }
+        }
+        
+        FileFormat::Unknown
+    }
+    
+    /// Detect format from file extension
+    pub fn detect_from_extension(filename: &str) -> FileFormat {
+        let path = Path::new(filename);
+        let ext = match path.extension().and_then(|s| s.to_str()) {
+            Some(e) => e.to_lowercase(),
+            None => return FileFormat::Unknown,
+        };
+        
+        match ext.as_str() {
+            "pdf" => FileFormat::Pdf,
+            "docx" => FileFormat::Docx,
+            "xlsx" => FileFormat::Xlsx,
+            "xls" => FileFormat::Xls,
+            "pptx" => FileFormat::Pptx,
+            "png" => FileFormat::Png,
+            "jpg" | "jpeg" => FileFormat::Jpeg,
+            "tiff" | "tif" => FileFormat::Tiff,
+            "bmp" => FileFormat::Bmp,
+            "json" => FileFormat::Json,
+            "xml" => FileFormat::Xml,
+            "html" | "htm" => FileFormat::Html,
+            "txt" | "text" | "md" | "markdown" | "csv" => FileFormat::Text,
+            _ => FileFormat::Unknown,
+        }
+    }
+    
+    /// Detect format from file content (magic bytes)
+    pub fn detect_from_content(data: &[u8]) -> FileFormat {
+        if data.is_empty() {
+            return FileFormat::Text; // Empty files are treated as text
+        }
+        
+        // PDF
+        if data.len() >= 4 && data.starts_with(b"%PDF") {
+            return FileFormat::Pdf;
+        }
+        
+        // PNG
+        if data.len() >= 8 && data.starts_with(&[0x89, 0x50, 0x4E, 0x47, 0x0D, 0x0A, 0x1A, 0x0A]) {
+            return FileFormat::Png;
+        }
+        
+        // JPEG
+        if data.len() >= 3 && data.starts_with(&[0xFF, 0xD8, 0xFF]) {
+            return FileFormat::Jpeg;
+        }
+        
+        // BMP
+        if data.len() >= 2 && data.starts_with(b"BM") {
+            return FileFormat::Bmp;
+        }
+        
+        // TIFF (little-endian or big-endian)
+        if data.len() >= 4 {
+            if data.starts_with(b"II\x2A\x00") || data.starts_with(b"MM\x00\x2A") {
+                return FileFormat::Tiff;
+            }
+        }
+        
+        // OLE Compound Document (old Excel/Word)
+        if data.len() >= 4 && data.starts_with(&[0xD0, 0xCF, 0x11, 0xE0]) {
+            return FileFormat::Xls; // Old Office format, usually Excel
+        }
+        
+        // ZIP archive (could be DOCX, XLSX, PPTX)
+        if data.len() >= 2 && data.starts_with(b"PK") {
+            return Self::detect_office_format(data);
+        }
+        
+        // XML
+        if data.len() >= 5 {
+            let start = String::from_utf8_lossy(&data[0..5.min(data.len())]);
+            if start.starts_with("<?xml") || start.starts_with("<!") {
+                return FileFormat::Xml;
+            }
+        }
+        
+        // HTML
+        if data.len() >= 14 {
+            let start = String::from_utf8_lossy(&data[0..14.min(data.len())]).to_lowercase();
+            if start.contains("<!doctype") || start.contains("<html") {
+                return FileFormat::Html;
+            }
+        }
+        
+        // JSON
+        if let Some(&first_non_ws) = data.iter().find(|&&b| !b" \t\n\r".contains(&b)) {
+            if first_non_ws == b'{' || first_non_ws == b'[' {
+                return FileFormat::Json;
+            }
+        }
+        
+        // Default to text for unrecognized formats
+        FileFormat::Text
+    }
+    
+    /// Detect specific Office format from ZIP data
+    fn detect_office_format(data: &[u8]) -> FileFormat {
+        // Look for Office-specific directory names in first 2KB of ZIP
+        let check_len = 2000.min(data.len());
+        let content = String::from_utf8_lossy(&data[0..check_len]);
+        
+        // Check for format-specific markers
+        if content.contains("word/") || content.contains("word/_rels") {
+            FileFormat::Docx
+        } else if content.contains("xl/") || content.contains("xl/_rels") {
+            FileFormat::Xlsx
+        } else if content.contains("ppt/") || content.contains("ppt/_rels") {
+            FileFormat::Pptx
+        } else {
+            // Default to XLSX for generic ZIP (most common Office format)
+            FileFormat::Xlsx
+        }
+    }
+    
+    
+    /// Get all supported extensions
+    pub fn supported_extensions() -> Vec<&'static str> {
+        vec![
+            "pdf", "docx", "xlsx", "xls", "pptx",
+            "png", "jpg", "jpeg", "tiff", "tif", "bmp",
+            "json", "xml", "html", "htm",
+            "txt", "text", "md", "markdown", "csv"
+        ]
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    
+    #[test]
+    fn test_detect_pdf() {
+        let pdf_data = b"%PDF-1.5\n";
+        assert_eq!(FormatDetector::detect_from_content(pdf_data), FileFormat::Pdf);
+    }
+    
+    #[test]
+    fn test_detect_png() {
+        let png_data = &[0x89, 0x50, 0x4E, 0x47, 0x0D, 0x0A, 0x1A, 0x0A];
+        assert_eq!(FormatDetector::detect_from_content(png_data), FileFormat::Png);
+    }
+    
+    #[test]
+    fn test_detect_from_extension() {
+        assert_eq!(FormatDetector::detect_from_extension("document.pdf"), FileFormat::Pdf);
+        assert_eq!(FormatDetector::detect_from_extension("Document.PDF"), FileFormat::Pdf);
+        assert_eq!(FormatDetector::detect_from_extension("data.xlsx"), FileFormat::Xlsx);
+    }
+    
+    #[test]
+    fn test_empty_data() {
+        assert_eq!(FormatDetector::detect_from_content(&[]), FileFormat::Text);
+    }
+}

data/ext/parsekit/src/lib.rs

@@ -0,0 +1,25 @@
+use magnus::{function, prelude::*, Error, Ruby};
+
+mod parser;
+mod error;
+mod format_detector;
+
+/// Initialize the ParseKit module and its submodules
+#[magnus::init]
+fn init(ruby: &Ruby) -> Result<(), Error> {
+    let module = ruby.define_module("ParseKit")?;
+    
+    // Initialize submodules
+    parser::init(ruby, module)?;
+    error::init(ruby, module)?;
+    
+    // Add module-level methods
+    module.define_singleton_method("version", function!(version, 0))?;
+    
+    Ok(())
+}
+
+/// Return the version of the parsekit gem
+fn version() -> String {
+    env!("CARGO_PKG_VERSION").to_string()
+}