universal_document_processor 1.0.1 → 1.0.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c33ef9db6830ddb62e98966d0bf3dd85106833e260303651df297bb3d46b9529
-  data.tar.gz: 280fa75bf3d842fc1af11dd95ca49e0526ed22512b5732aed0a7c25d4a57a7d9
+  metadata.gz: 8ec66decfe8626354f9fe05b757dbdc11921b21fa6b5dccfdb4d8ce5deba2c3f
+  data.tar.gz: 19c2802d337d0517ab91cfe71bdb2b051213e17f0e1a76605c5bce895429eed4
 SHA512:
-  metadata.gz: 01d6c5129dc7ec1a7911a77ba69b8dbafd32bd9397e0374175a4cf09cf24f8c06225a0bc927b1e3bba8250e3d1e93d40bbbc1dbf3aa1a6ff0fbc92cb7f5f24a4
-  data.tar.gz: b445b8d773e2865e6e2c5593c123cc7d0da580d4d16822e4b9e45c851568f077f4c949d921f2f602c4ed3ff941ccf8a1d002eb14401c554b0f8355dc293bff62
+  metadata.gz: f2cbb1944e533a4a75d6248dd6df279219e4a4c7b77dac3b0e4d474b5b4375203d188bff5d388af30716b3dc5487fcd293955c8504565c4a1b56d552a8484993
+  data.tar.gz: 2d95c2f173de302d14cdfda6d3357b5d7d9a5cf82cabc2e5622bdb8f6d7e60c56bab75eccf85a312045be5f8e66b743b4344420847192a3b0268cd4a70c5f414

data/CHANGELOG.md

@@ -7,6 +7,26 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [1.2.0] - 2024-01-15
+### Added
+- **TSV (Tab-Separated Values) File Support**: Complete built-in TSV processing capabilities
+  - Native TSV parsing using Ruby CSV library with tab delimiter
+  - Text extraction with proper formatting
+  - Comprehensive metadata detection (format, delimiter, encoding)
+  - Table structure analysis and header detection
+  - Statistical analysis and data validation
+  - Format conversions: TSV ↔ CSV, TSV → JSON
+  - Cross-format compatibility with existing CSV and Excel features
+  - New `to_tsv()` method for converting other formats to TSV
+  - Enhanced file detector with TSV MIME type mapping
+  - Full integration with existing Document class API
+
+### Enhanced
+- **ExcelProcessor**: Extended to handle TSV files alongside CSV and Excel formats
+- **File Detection**: Added TSV MIME type support (`text/tab-separated-values`)
+- **Document Class**: Added `to_tsv()` method and TSV format support
+- **Supported Formats**: Updated to include TSV in format list
+
 ## [1.0.1] - 2025-06-23
 
 ### Fixed

data/README.md

@@ -16,7 +16,7 @@ A comprehensive Ruby gem that provides unified document processing capabilities
 
 ### **Supported File Formats**
 - **📄 Documents**: PDF, DOC, DOCX, RTF
-- **📊 Spreadsheets**: XLS, XLSX, CSV
+- **📊 Spreadsheets**: XLS, XLSX, CSV, TSV
 - **📺 Presentations**: PPT, PPTX
 - **🖼️ Images**: JPG, PNG, GIF, BMP, TIFF
 - **📁 Archives**: ZIP, RAR, 7Z
@@ -236,6 +236,58 @@ tables.each_with_index do |table, index|
 end
 ```
 
+### Processing TSV (Tab-Separated Values) Files
+
+```ruby
+# Process TSV files with built-in support
+result = UniversalDocumentProcessor.process('data.tsv')
+
+# TSV-specific metadata
+metadata = result[:metadata]
+puts "Format: #{metadata[:format]}"        # => "tsv"
+puts "Delimiter: #{metadata[:delimiter]}"  # => "tab"
+puts "Rows: #{metadata[:total_rows]}"
+puts "Columns: #{metadata[:total_columns]}"
+puts "Has headers: #{metadata[:has_headers]}"
+
+# Extract structured data
+tables = result[:tables]
+table = tables.first
+puts "Headers: #{table[:headers].join(', ')}"
+puts "Sample row: #{table[:data][1].join(' | ')}"
+
+# Format conversions
+document = UniversalDocumentProcessor::Document.new('data.tsv')
+
+# Convert TSV to CSV
+csv_output = document.to_csv
+puts "CSV conversion: #{csv_output.length} characters"
+
+# Convert TSV to JSON
+json_output = document.to_json
+puts "JSON conversion: #{json_output.length} characters"
+
+# Convert CSV to TSV
+csv_document = UniversalDocumentProcessor::Document.new('data.csv')
+tsv_output = csv_document.to_tsv
+puts "TSV conversion: #{tsv_output.length} characters"
+
+# Statistical analysis
+stats = document.extract_statistics
+sheet_stats = stats['Sheet1']
+puts "Total cells: #{sheet_stats[:total_cells]}"
+puts "Numeric cells: #{sheet_stats[:numeric_cells]}"
+puts "Text cells: #{sheet_stats[:text_cells]}"
+puts "Average value: #{sheet_stats[:average_value]}"
+
+# Data validation
+validation = document.validate_data
+sheet_validation = validation['Sheet1']
+puts "Data quality score: #{sheet_validation[:data_quality_score]}%"
+puts "Empty rows: #{sheet_validation[:empty_rows]}"
+puts "Duplicate rows: #{sheet_validation[:duplicate_rows]}"
+```
+
 ### Processing Word Documents
 
 ```ruby

data/Rakefile

@@ -1,12 +1,105 @@
-require "bundler/gem_tasks"
-require "rspec/core/rake_task"
-require "rubocop/rake_task"
+require 'rake/testtask'
+require 'bundler/gem_tasks'
 
-RSpec::Core::RakeTask.new(:spec)
-RuboCop::RakeTask.new
+# Default task
+task default: :test
 
-desc "Run tests and linting"
-task default: %i[spec rubocop]
+# Test task
+Rake::TestTask.new(:test) do |t|
+  t.libs << 'test'
+  t.libs << 'lib'
+  t.test_files = FileList['test/test_*.rb']
+  t.verbose = true
+end
+
+# Individual test tasks
+Rake::TestTask.new(:test_core) do |t|
+  t.libs << 'test'
+  t.libs << 'lib'
+  t.test_files = FileList['test/test_universal_document_processor.rb']
+  t.verbose = true
+end
+
+Rake::TestTask.new(:test_ai) do |t|
+  t.libs << 'test'
+  t.libs << 'lib'
+  t.test_files = FileList['test/test_ai_agent.rb']
+  t.verbose = true
+end
+
+Rake::TestTask.new(:test_processors) do |t|
+  t.libs << 'test'
+  t.libs << 'lib'
+  t.test_files = FileList['test/test_processors.rb']
+  t.verbose = true
+end
+
+# Coverage task (if simplecov is available)
+desc "Run tests with coverage"
+task :coverage do
+  ENV['COVERAGE'] = 'true'
+  Rake::Task[:test].invoke
+end
+
+# Lint task (if rubocop is available)
+desc "Run RuboCop"
+task :lint do
+  begin
+    require 'rubocop/rake_task'
+    RuboCop::RakeTask.new
+  rescue LoadError
+    puts "RuboCop not available. Install it with: gem install rubocop"
+  end
+end
+
+# Documentation task
+desc "Generate documentation"
+task :doc do
+  system "yard doc"
+end
+
+# Clean task
+desc "Clean up generated files"
+task :clean do
+  FileUtils.rm_rf('coverage')
+  FileUtils.rm_rf('doc')
+  FileUtils.rm_rf('pkg')
+  FileUtils.rm_f('Gemfile.lock')
+end
+
+# Install dependencies
+desc "Install dependencies"
+task :install do
+  system "bundle install"
+end
+
+# Quality check task
+desc "Run all quality checks"
+task quality: [:test, :lint]
+
+# CI task
+desc "Run CI tasks"
+task ci: [:install, :test]
+
+# Development setup
+desc "Setup development environment"
+task :setup do
+  puts "Setting up development environment..."
+  Rake::Task[:install].invoke
+  puts "Development environment ready!"
+  puts ""
+  puts "Available tasks:"
+  puts "  rake test        - Run all tests"
+  puts "  rake test_core   - Run core functionality tests"
+  puts "  rake test_ai     - Run AI agent tests"
+  puts "  rake test_processors - Run processor tests"
+  puts "  rake coverage    - Run tests with coverage"
+  puts "  rake lint        - Run RuboCop linting"
+  puts "  rake doc         - Generate documentation"
+  puts "  rake clean       - Clean up generated files"
+  puts ""
+  puts "To run tests with AI features, set OPENAI_API_KEY environment variable"
+end
 
 desc "Build the gem"
 task :build do
@@ -23,14 +116,4 @@ desc "Install the gem locally"
 task :install do
   system "gem build universal_document_processor.gemspec"
   system "gem install universal_document_processor-*.gem"
-end
-
-desc "Clean build artifacts"
-task :clean do
-  system "rm -f *.gem"
-end
-
-desc "Generate documentation"
-task :doc do
-  system "yard doc"
 end 

data/lib/universal_document_processor/ai_agent.rb

@@ -4,7 +4,7 @@ require 'uri'
 
 module UniversalDocumentProcessor
   class AIAgent
-    attr_reader :api_key, :model, :base_url, :conversation_history
+    attr_reader :api_key, :model, :base_url, :conversation_history, :ai_enabled
 
     def initialize(options = {})
       @api_key = options[:api_key] || ENV['OPENAI_API_KEY']
@@ -13,12 +13,15 @@ module UniversalDocumentProcessor
       @conversation_history = []
       @max_history = options[:max_history] || 10
       @temperature = options[:temperature] || 0.7
+      @ai_enabled = false
       
       validate_configuration
     end
 
     # Main document analysis with AI
     def analyze_document(document_result, query = nil)
+      ensure_ai_available!
+      
       context = build_document_context(document_result)
       
       if query
@@ -63,6 +66,8 @@ Please provide:
 
     # Ask specific questions about a document
     def ask_document_question(document_result, question)
+      ensure_ai_available!
+      
       context = build_document_context(document_result)
       
       prompt = build_question_prompt(context, question)
@@ -74,6 +79,8 @@ Please provide:
 
     # Summarize document content
     def summarize_document(document_result, length: :medium)
+      ensure_ai_available!
+      
       context = build_document_context(document_result)
       
       length_instruction = case length
@@ -92,6 +99,8 @@ Please provide:
 
     # Extract key information from document
     def extract_key_information(document_result, categories = nil)
+      ensure_ai_available!
+      
       context = build_document_context(document_result)
       categories ||= ['key_facts', 'important_dates', 'names', 'locations', 'numbers']
       
@@ -104,6 +113,8 @@ Please provide:
 
     # Translate document content
     def translate_document(document_result, target_language)
+      ensure_ai_available!
+      
       context = build_document_context(document_result)
       
       prompt = build_translation_prompt(context, target_language)
@@ -115,6 +126,8 @@ Please provide:
 
     # Generate document insights and recommendations
     def generate_insights(document_result)
+      ensure_ai_available!
+      
       context = build_document_context(document_result)
       
       prompt = build_insights_prompt(context)
@@ -126,6 +139,8 @@ Please provide:
 
     # Compare multiple documents
     def compare_documents(document_results, comparison_type = :content)
+      ensure_ai_available!
+      
       contexts = document_results.map { |doc| build_document_context(doc) }
       
       prompt = build_comparison_prompt(contexts, comparison_type)
@@ -137,6 +152,8 @@ Please provide:
 
     # Classify document type and purpose
     def classify_document(document_result)
+      ensure_ai_available!
+      
       context = build_document_context(document_result)
       
       prompt = build_classification_prompt(context)
@@ -148,6 +165,8 @@ Please provide:
 
     # Generate action items from document
     def extract_action_items(document_result)
+      ensure_ai_available!
+      
       context = build_document_context(document_result)
       
       prompt = build_action_items_prompt(context)
@@ -159,6 +178,8 @@ Please provide:
 
     # Chat about the document
     def chat(message, document_result = nil)
+      ensure_ai_available!
+      
       if document_result
         context = build_document_context(document_result)
         prompt = build_chat_prompt(context, message)
@@ -180,6 +201,10 @@ Please provide:
     def conversation_summary
       return "No conversation history" if @conversation_history.empty?
       
+      unless @ai_enabled
+        return "AI features are disabled. Cannot generate conversation summary."
+      end
+      
       history_text = @conversation_history.map do |entry|
         "Q: #{entry[:question]}\nA: #{entry[:answer]}"
       end.join("\n\n")
@@ -188,11 +213,27 @@ Please provide:
       call_openai_api(prompt)
     end
 
+    # Check if AI features are available
+    def ai_available?
+      @ai_enabled
+    end
+
     private
 
     def validate_configuration
-      raise ArgumentError, "OpenAI API key is required" unless @api_key
-      raise ArgumentError, "OpenAI API key cannot be empty" if @api_key.empty?
+      if @api_key && !@api_key.empty?
+        @ai_enabled = true
+      else
+        @ai_enabled = false
+        warn "Warning: OpenAI API key not provided. AI features will be disabled. Set OPENAI_API_KEY environment variable or pass api_key option to enable AI features."
+      end
+    end
+
+    # Ensure AI is available before making API calls
+    def ensure_ai_available!
+      unless @ai_enabled
+        raise DependencyMissingError, "AI features are not available. Please provide an OpenAI API key to use AI functionality."
+      end
     end
 
     def build_document_context(document_result)

data/lib/universal_document_processor/document.rb

@@ -48,6 +48,42 @@ module UniversalDocumentProcessor
       []
     end
 
+    def extract_statistics
+      processor.respond_to?(:extract_statistics) ? processor.extract_statistics : {}
+    rescue => e
+      {}
+    end
+
+    def validate_data
+      processor.respond_to?(:validate_data) ? processor.validate_data : {}
+    rescue => e
+      {}
+    end
+
+    def extract_formulas
+      processor.respond_to?(:extract_formulas) ? processor.extract_formulas : []
+    rescue => e
+      []
+    end
+
+    def to_json
+      processor.respond_to?(:to_json) ? processor.to_json : process.to_json
+    rescue => e
+      process.to_json
+    end
+
+    def to_csv(sheet_name = nil)
+      processor.respond_to?(:to_csv) ? processor.to_csv(sheet_name) : ""
+    rescue => e
+      ""
+    end
+
+    def to_tsv(sheet_name = nil)
+      processor.respond_to?(:to_tsv) ? processor.to_tsv(sheet_name) : ""
+    rescue => e
+      ""
+    end
+
     def convert_to(target_format)
       case target_format.to_sym
       when :pdf
@@ -64,7 +100,7 @@ module UniversalDocumentProcessor
     end
 
     def supported_formats
-      %w[pdf docx doc xlsx xls pptx ppt txt rtf html xml csv jpg jpeg png gif bmp tiff zip rar 7z]
+      %w[pdf docx doc xlsx xls pptx ppt txt rtf html xml csv tsv jpg jpeg png gif bmp tiff zip rar 7z]
     end
 
     def supported?
@@ -139,11 +175,11 @@ module UniversalDocumentProcessor
       case @content_type
       when /pdf/
         Processors::PdfProcessor.new(@file_path, @options)
-      when /word/, /document/
+      when /wordprocessingml/, /msword/
         Processors::WordProcessor.new(@file_path, @options)
-      when /excel/, /spreadsheet/
+      when /spreadsheetml/, /ms-excel/, /csv/, /tab-separated/
         Processors::ExcelProcessor.new(@file_path, @options)
-      when /powerpoint/, /presentation/
+      when /presentationml/, /ms-powerpoint/
         Processors::PowerpointProcessor.new(@file_path, @options)
       when /image/
         Processors::ImageProcessor.new(@file_path, @options)