universal_document_processor 1.0.5 → 1.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4b4c918d869d7ecc4420b740c032d07eb9d5344fc5049f2522c2de92ac5ced17
-  data.tar.gz: acc85eb5cf922ce1e29384fc5624e1095df40a444bc5ee39fff23ce875f8b5a4
+  metadata.gz: 8444e9dc03cd125a0a9e62df6370b7dbba4adf4777d89478d1d51c60f5c83d70
+  data.tar.gz: 3a5fc000774c34683c7b0d95c0ca9a034838cd5e69be51c43c98792070b278aa
 SHA512:
-  metadata.gz: 9a072e0dda668c534edbcc118591807fe55d8acca8257c2d339d709ca5892f3b6b9eca53a4467763f87977c30016546f6c0fbcb2c81c61c96fd2d9c427905c0f
-  data.tar.gz: c5567a97e9630cd89822afaa151ac4aff39ca6195be4fefe7b67bf72686f29d665c8727bd86d29897c8e7c587c85da4f8abd8495c4c9a4ef48d2b8c22537fd33
+  metadata.gz: be66ce6b411fcfa52eaf6353ef1f37785cbe996c8c977bb671c971988c43a500108004741b157ca7636d9e2f20991c43b44884134c3ffd4fde2f6b4a90d27380
+  data.tar.gz: 94e90e17615093e529db4100d674944150d0cbbcd527df3341b8c025ada3fe389dc8dbc64099b1cfa87473105c57038ca1fb5106281242b8658e583042cc52c9

data/CHANGELOG.md

@@ -7,6 +7,19 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [1.1.0] - 2025-01-XX
+### Added
+- **ZIP File Creation**: New functionality to create ZIP archives programmatically
+  - `ArchiveProcessor.create_zip()` class method for creating ZIP files
+  - Support for creating archives from individual files or entire directories
+  - Recursive directory archiving with proper path structure preservation
+  - Comprehensive test coverage with error handling
+  - Integration with existing archive processing capabilities
+
+### Enhanced
+- **ArchiveProcessor**: Extended with ZIP creation capabilities alongside existing extraction features
+- **Archive Support**: Now supports both reading/extracting and creating ZIP archives
+
 ## [1.2.0] - 2024-01-15
 ### Added
 - **TSV (Tab-Separated Values) File Support**: Complete built-in TSV processing capabilities

data/README.md

@@ -29,6 +29,7 @@ A comprehensive Ruby gem that provides unified document processing capabilities
 - **Table Detection**: Structured data extraction
 - **Character Validation**: Invalid character detection and cleaning
 - **Multi-language Support**: Full Unicode support including Japanese (日本語)
+- **Archive Creation**: Create ZIP files from individual files or directories
 
 ### **Character & Encoding Support**
 - **Smart encoding detection** (UTF-8, Shift_JIS, EUC-JP, ISO-8859-1)
@@ -110,7 +111,7 @@ result = UniversalDocumentProcessor.process('document.pdf')
   },
   images: [...],
   tables: [...],
-  processed_at: 2024-01-15 10:30:00 UTC
+  processed_at: 2025-07-06 10:30:00 UTC
 }
 ```
 
@@ -421,6 +422,119 @@ agent = UniversalDocumentProcessor.create_ai_agent(
 )
 ```
 
+## 📦 Archive Processing (ZIP Creation & Extraction)
+
+The gem provides comprehensive archive processing capabilities, including both extracting from existing archives and creating new ZIP files.
+
+### Extracting from Archives
+
+```ruby
+# Extract text and metadata from ZIP archives
+result = UniversalDocumentProcessor.process('archive.zip')
+
+# Access archive-specific metadata
+metadata = result[:metadata]
+puts "Archive type: #{metadata[:archive_type]}"           # => "zip"
+puts "Total files: #{metadata[:total_files]}"             # => 15
+puts "Uncompressed size: #{metadata[:total_uncompressed_size]} bytes"
+puts "Compression ratio: #{metadata[:compression_ratio]}%" # => 75%
+puts "Directory structure: #{metadata[:directory_structure]}"
+
+# Check for specific file types
+puts "File types: #{metadata[:file_types]}"               # => {"txt"=>5, "pdf"=>3, "jpg"=>7}
+puts "Has executables: #{metadata[:has_executable_files]}" # => false
+puts "Largest file: #{metadata[:largest_file][:path]} (#{metadata[:largest_file][:size]} bytes)"
+
+# Extract text from text files within the archive
+text_content = result[:text_content]
+puts "Combined text from archive: #{text_content.length} characters"
+```
+
+### Creating ZIP Archives
+
+```ruby
+# Create ZIP from individual files
+files_to_zip = ['document1.pdf', 'document2.txt', 'image.jpg']
+output_zip = 'my_archive.zip'
+
+zip_path = UniversalDocumentProcessor::Processors::ArchiveProcessor.create_zip(
+  output_zip, 
+  files_to_zip
+)
+puts "ZIP created: #{zip_path}"
+
+# Create ZIP from entire directory (preserves folder structure)
+directory_to_zip = '/path/to/documents'
+archive_path = UniversalDocumentProcessor::Processors::ArchiveProcessor.create_zip(
+  'directory_backup.zip',
+  directory_to_zip
+)
+puts "Directory archived: #{archive_path}"
+
+# Working with temporary directories
+require 'tmpdir'
+
+Dir.mktmpdir do |tmpdir|
+  # Create some test files
+  File.write(File.join(tmpdir, 'file1.txt'), 'Hello from file 1')
+  File.write(File.join(tmpdir, 'file2.txt'), 'Hello from file 2')
+  
+  # Create subdirectory with files
+  subdir = File.join(tmpdir, 'subfolder')
+  Dir.mkdir(subdir)
+  File.write(File.join(subdir, 'file3.txt'), 'Hello from subfolder')
+  
+  # Archive the entire directory structure
+  zip_file = File.join(tmpdir, 'complete_backup.zip')
+  UniversalDocumentProcessor::Processors::ArchiveProcessor.create_zip(zip_file, tmpdir)
+  
+  puts "Archive size: #{File.size(zip_file)} bytes"
+  
+  # Verify archive contents by processing it
+  archive_result = UniversalDocumentProcessor.process(zip_file)
+  puts "Files in archive: #{archive_result[:metadata][:total_files]}"
+end
+
+# Error handling for ZIP creation
+begin
+  UniversalDocumentProcessor::Processors::ArchiveProcessor.create_zip(
+    '/invalid/path/archive.zip',
+    ['file1.txt', 'file2.txt']
+  )
+rescue => e
+  puts "Error creating ZIP: #{e.message}"
+end
+
+# Validate input before creating ZIP
+files = ['doc1.pdf', 'doc2.txt']
+files.each do |file|
+  unless File.exist?(file)
+    puts "Warning: #{file} does not exist"
+  end
+end
+```
+
+### Archive Analysis
+
+```ruby
+# Analyze archive security and structure
+result = UniversalDocumentProcessor.process('suspicious_archive.zip')
+metadata = result[:metadata]
+
+# Security analysis
+if metadata[:has_executable_files]
+  puts "⚠️  Archive contains executable files"
+end
+
+# Directory structure analysis
+structure = metadata[:directory_structure]
+puts "Top-level directories: #{structure.keys.join(', ')}"
+
+# File type distribution
+file_types = metadata[:file_types]
+puts "Most common file type: #{file_types.max_by{|k,v| v}}"
+```
+
 ## 🎌 Japanese Filename Support
 
 The gem provides comprehensive support for Japanese filenames across all operating systems:

data/lib/universal_document_processor/processors/archive_processor.rb

@@ -91,6 +91,32 @@ module UniversalDocumentProcessor
         super + [:list_files, :extract_file, :extract_all, :analyze_security]
       end
 
+      # Class method to create a zip file from a list of files or a directory
+      def self.create_zip(output_zip_path, files_or_directory)
+        require 'zip'
+        files = []
+        if files_or_directory.is_a?(String) && File.directory?(files_or_directory)
+          # Recursively collect all files in the directory
+          Dir[File.join(files_or_directory, '**', '**')].each do |file|
+            files << file unless File.directory?(file)
+          end
+          base_dir = files_or_directory
+        elsif files_or_directory.is_a?(Array)
+          files = files_or_directory
+          base_dir = nil
+        else
+          raise ArgumentError, 'files_or_directory must be a directory path or an array of file paths'
+        end
+
+        Zip::File.open(output_zip_path, Zip::File::CREATE) do |zipfile|
+          files.each do |file|
+            entry_name = base_dir ? file.sub(/^#{Regexp.escape(base_dir)}\/?/, '') : File.basename(file)
+            zipfile.add(entry_name, file)
+          end
+        end
+        output_zip_path
+      end
+
       private
 
       def detect_archive_type

data/lib/universal_document_processor/version.rb

@@ -1,3 +1,3 @@
 module UniversalDocumentProcessor
-  VERSION = "1.0.5"
+  VERSION = "1.1.0"
 end 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: universal_document_processor
 version: !ruby/object:Gem::Version
-  version: 1.0.5
+  version: 1.1.0
 platform: ruby
 authors:
 - Vikas Patil
@@ -201,7 +201,6 @@ files:
 - README.md
 - Rakefile
 - USER_GUIDE.md
-- debug_test.rb
 - lib/universal_document_processor.rb
 - lib/universal_document_processor/ai_agent.rb
 - lib/universal_document_processor/document.rb
@@ -217,10 +216,6 @@ files:
 - lib/universal_document_processor/utils/file_detector.rb
 - lib/universal_document_processor/utils/japanese_filename_handler.rb
 - lib/universal_document_processor/version.rb
-- test_ai_dependency.rb
-- test_core_functionality.rb
-- test_performance_memory.rb
-- test_published_gem.rb
 homepage: https://github.com/vpatil160/universal_document_processor
 licenses:
 - MIT

data/debug_test.rb

@@ -1,35 +0,0 @@
-#!/usr/bin/env ruby
-
-# Add lib directory to load path
-$LOAD_PATH.unshift File.expand_path('lib', __dir__)
-
-# Load the gem
-require 'universal_document_processor'
-require 'tempfile'
-
-# Create a simple text file
-txt_file = Tempfile.new(['test', '.txt'])
-txt_file.write("This is a sample text file.\nIt has multiple lines.\nUsed for testing.")
-txt_file.close
-
-puts "Testing text file: #{txt_file.path}"
-
-begin
-  puts "Processing file..."
-  result = UniversalDocumentProcessor.process(txt_file.path)
-  
-  puts "Result keys: #{result.keys}"
-  puts "Result type: #{result.class}"
-  
-  if result.is_a?(Hash)
-    result.each do |key, value|
-      puts "#{key}: #{value.class} - #{value.to_s[0..100]}..."
-    end
-  end
-  
-rescue => e
-  puts "Error: #{e.class} - #{e.message}"
-  puts e.backtrace.first(5)
-end
-
-txt_file.unlink 

data/test_ai_dependency.rb

@@ -1,80 +0,0 @@
-#!/usr/bin/env ruby
-
-# Add lib directory to load path
-$LOAD_PATH.unshift File.expand_path('lib', __dir__)
-
-# Load the gem
-require 'universal_document_processor'
-
-puts "Testing AI Dependency Handling"
-puts "=" * 50
-
-# Test 1: Check AI availability without API key
-puts "\n1. Testing AI availability without API key:"
-ai_available = UniversalDocumentProcessor.ai_available?
-puts "   AI Available: #{ai_available}"
-
-# Test 2: Create AI agent without API key
-puts "\n2. Creating AI agent without API key:"
-agent = UniversalDocumentProcessor.create_ai_agent
-puts "   Agent created: #{agent.class}"
-puts "   AI enabled: #{agent.ai_enabled}"
-puts "   AI available: #{agent.ai_available?}"
-
-# Test 3: Try to use AI methods without API key
-puts "\n3. Testing AI methods without API key:"
-
-# Create a sample text file
-require 'tempfile'
-sample_file = Tempfile.new(['test', '.txt'])
-sample_file.write("This is a test document for AI processing.")
-sample_file.close
-
-begin
-  result = UniversalDocumentProcessor.ai_analyze(sample_file.path)
-  puts "   ERROR: Should have raised an exception!"
-rescue UniversalDocumentProcessor::DependencyMissingError => e
-  puts "   ✓ Correctly raised DependencyMissingError: #{e.message}"
-rescue => e
-  puts "   ✗ Unexpected error: #{e.class} - #{e.message}"
-end
-
-# Test 4: Check available features
-puts "\n4. Available features:"
-features = UniversalDocumentProcessor.available_features
-puts "   Features: #{features.join(', ')}"
-puts "   AI processing included: #{features.include?(:ai_processing)}"
-
-# Test 5: Check optional dependencies
-puts "\n5. Optional dependencies:"
-optional_deps = UniversalDocumentProcessor.optional_dependencies
-puts "   Optional dependencies: #{optional_deps.keys.join(', ')}"
-
-missing_deps = UniversalDocumentProcessor.missing_dependencies
-puts "   Missing dependencies: #{missing_deps.join(', ')}"
-
-# Test 6: Installation instructions
-puts "\n6. Installation instructions:"
-instructions = UniversalDocumentProcessor.installation_instructions
-puts instructions
-
-# Test 7: Test with API key if provided
-if ENV['OPENAI_API_KEY'] && !ENV['OPENAI_API_KEY'].empty?
-  puts "\n7. Testing with API key:"
-  ai_available_with_key = UniversalDocumentProcessor.ai_available?
-  puts "   AI Available with key: #{ai_available_with_key}"
-  
-  agent_with_key = UniversalDocumentProcessor.create_ai_agent
-  puts "   Agent AI enabled: #{agent_with_key.ai_enabled}"
-else
-  puts "\n7. Skipping API key test (OPENAI_API_KEY not set)"
-end
-
-# Clean up
-sample_file.unlink
-
-puts "\n" + "=" * 50
-puts "AI Dependency Test Complete!"
-puts "✓ AI features are properly optional"
-puts "✓ Clear error messages when dependencies missing"
-puts "✓ Graceful degradation when features unavailable" 

data/test_core_functionality.rb

@@ -1,280 +0,0 @@
-#!/usr/bin/env ruby
-
-# Add lib directory to load path
-$LOAD_PATH.unshift File.expand_path('lib', __dir__)
-
-# Load the gem
-require 'universal_document_processor'
-require 'tempfile'
-
-puts "Testing Core Functionality"
-puts "=" * 50
-
-test_count = 0
-passed_count = 0
-
-def test(description)
-  global_test_count = caller_locations.first.lineno
-  print "#{global_test_count}. #{description}... "
-  
-  begin
-    yield
-    puts "✓ PASS"
-    return true
-  rescue => e
-    puts "✗ FAIL: #{e.message}"
-    puts "   #{e.backtrace.first}" if ENV['DEBUG']
-    return false
-  end
-end
-
-# Create sample files for testing
-puts "\nCreating sample files..."
-
-# Text file
-txt_file = Tempfile.new(['test', '.txt'])
-txt_file.write("This is a sample text file.\nIt has multiple lines.\nUsed for testing.")
-txt_file.close
-
-# CSV file
-csv_file = Tempfile.new(['test', '.csv'])
-csv_file.write("Name,Age,City\nJohn,25,New York\nJane,30,Los Angeles\nBob,35,Chicago")
-csv_file.close
-
-# TSV file
-tsv_file = Tempfile.new(['test', '.tsv'])
-tsv_file.write("Name\tAge\tCity\nJohn\t25\tNew York\nJane\t30\tLos Angeles\nBob\t35\tChicago")
-tsv_file.close
-
-# JSON file
-json_file = Tempfile.new(['test', '.json'])
-json_file.write('{"name": "Test Document", "type": "sample", "data": [1, 2, 3, 4, 5]}')
-json_file.close
-
-# XML file
-xml_file = Tempfile.new(['test', '.xml'])
-xml_file.write(<<~XML)
-  <?xml version="1.0" encoding="UTF-8"?>
-  <document>
-    <title>Sample XML Document</title>
-    <content>This is a sample XML file for testing.</content>
-  </document>
-XML
-xml_file.close
-
-puts "Sample files created successfully!"
-
-# Run tests
-puts "\nRunning Core Tests:"
-puts "-" * 30
-
-# Test 1: Version number
-test_count += 1
-passed = test("Version number is defined") do
-  version = UniversalDocumentProcessor::VERSION
-  raise "Version is nil" if version.nil?
-  raise "Version format invalid" unless version.match?(/\d+\.\d+\.\d+/)
-end
-passed_count += 1 if passed
-
-# Test 2: Text file processing
-test_count += 1
-passed = test("Text file processing") do
-  result = UniversalDocumentProcessor.process(txt_file.path)
-  raise "Result is not a hash" unless result.is_a?(Hash)
-  raise "Missing text key" unless result.has_key?(:text)
-  raise "Missing metadata key" unless result.has_key?(:metadata)
-  raise "Text content incorrect" unless result[:text].include?("sample text file")
-  raise "Format incorrect" unless result[:metadata][:format] == "txt"
-end
-passed_count += 1 if passed
-
-# Test 3: Text extraction
-test_count += 1
-passed = test("Text extraction method") do
-  text = UniversalDocumentProcessor.extract_text(txt_file.path)
-  raise "Text is not a string" unless text.is_a?(String)
-  raise "Text content missing" unless text.include?("sample text file")
-end
-passed_count += 1 if passed
-
-# Test 4: Metadata extraction
-test_count += 1
-passed = test("Metadata extraction") do
-  metadata = UniversalDocumentProcessor.get_metadata(txt_file.path)
-  raise "Metadata is not a hash" unless metadata.is_a?(Hash)
-  raise "Format missing" unless metadata[:format] == "txt"
-  raise "File size missing" unless metadata[:file_size] > 0
-end
-passed_count += 1 if passed
-
-# Test 5: CSV processing
-test_count += 1
-passed = test("CSV file processing") do
-  result = UniversalDocumentProcessor.process(csv_file.path)
-  raise "Result is not a hash" unless result.is_a?(Hash)
-  raise "Missing tables key" unless result.has_key?(:tables)
-  raise "Format incorrect" unless result[:metadata][:format] == "csv"
-  raise "Delimiter incorrect" unless result[:metadata][:delimiter] == "comma"
-  raise "No tables found" unless result[:tables].length > 0
-end
-passed_count += 1 if passed
-
-# Test 6: TSV processing
-test_count += 1
-passed = test("TSV file processing") do
-  result = UniversalDocumentProcessor.process(tsv_file.path)
-  raise "Result is not a hash" unless result.is_a?(Hash)
-  raise "Missing tables key" unless result.has_key?(:tables)
-  raise "Format incorrect" unless result[:metadata][:format] == "tsv"
-  raise "Delimiter incorrect" unless result[:metadata][:delimiter] == "tab"
-  raise "No tables found" unless result[:tables].length > 0
-end
-passed_count += 1 if passed
-
-# Test 7: JSON processing
-test_count += 1
-passed = test("JSON file processing") do
-  result = UniversalDocumentProcessor.process(json_file.path)
-  raise "Result is not a hash" unless result.is_a?(Hash)
-  raise "Format incorrect" unless result[:metadata][:format] == "json"
-  raise "Text missing" unless result[:text].include?("Test Document")
-end
-passed_count += 1 if passed
-
-# Test 8: XML processing
-test_count += 1
-passed = test("XML file processing") do
-  result = UniversalDocumentProcessor.process(xml_file.path)
-  raise "Result is not a hash" unless result.is_a?(Hash)
-  raise "Format incorrect" unless result[:metadata][:format] == "xml"
-  raise "Text missing" unless result[:text].include?("Sample XML Document")
-end
-passed_count += 1 if passed
-
-# Test 9: Batch processing
-test_count += 1
-passed = test("Batch processing") do
-  files = [txt_file.path, csv_file.path, json_file.path]
-  results = UniversalDocumentProcessor.batch_process(files)
-  raise "Results not array" unless results.is_a?(Array)
-  raise "Wrong number of results" unless results.length == 3
-  results.each do |result|
-    raise "Missing text or error key" unless result.has_key?(:text) || result.has_key?(:error)
-  end
-end
-passed_count += 1 if passed
-
-# Test 10: Available features
-test_count += 1
-passed = test("Available features check") do
-  features = UniversalDocumentProcessor.available_features
-  raise "Features not array" unless features.is_a?(Array)
-  raise "Missing text processing" unless features.include?(:text_processing)
-  raise "Missing CSV processing" unless features.include?(:csv_processing)
-  raise "Missing TSV processing" unless features.include?(:tsv_processing)
-end
-passed_count += 1 if passed
-
-# Test 11: Dependency checking
-test_count += 1
-passed = test("Dependency availability check") do
-  # These may or may not be available, just test the method works
-  pdf_available = UniversalDocumentProcessor.dependency_available?(:pdf_reader)
-  raise "Dependency check failed" unless [true, false].include?(pdf_available)
-end
-passed_count += 1 if passed
-
-# Test 12: Text quality analysis
-test_count += 1
-passed = test("Text quality analysis") do
-  analysis = UniversalDocumentProcessor.analyze_text_quality("Clean text")
-  raise "Analysis not hash" unless analysis.is_a?(Hash)
-  raise "Missing valid_characters" unless analysis.has_key?(:valid_characters)
-  raise "Missing invalid_characters" unless analysis.has_key?(:invalid_characters)
-end
-passed_count += 1 if passed
-
-# Test 13: Text cleaning
-test_count += 1
-passed = test("Text cleaning") do
-  dirty_text = "Clean\x00text"
-  clean_text = UniversalDocumentProcessor.clean_text(dirty_text)
-  raise "Cleaning failed" if clean_text.include?("\x00")
-end
-passed_count += 1 if passed
-
-# Test 14: Japanese text detection
-test_count += 1
-passed = test("Japanese text detection") do
-  english = "This is English"
-  japanese = "これは日本語"
-  raise "English detected as Japanese" if UniversalDocumentProcessor.japanese_text?(english)
-  raise "Japanese not detected" unless UniversalDocumentProcessor.japanese_text?(japanese)
-end
-passed_count += 1 if passed
-
-# Test 15: Optional dependencies info
-test_count += 1
-passed = test("Optional dependencies information") do
-  optional_deps = UniversalDocumentProcessor.optional_dependencies
-  raise "Optional deps not hash" unless optional_deps.is_a?(Hash)
-  raise "Missing pdf-reader" unless optional_deps.has_key?('pdf-reader')
-  
-  missing_deps = UniversalDocumentProcessor.missing_dependencies
-  raise "Missing deps not array" unless missing_deps.is_a?(Array)
-  
-  instructions = UniversalDocumentProcessor.installation_instructions
-  raise "Instructions not string" unless instructions.is_a?(String)
-end
-passed_count += 1 if passed
-
-# Test 16: AI availability check (should be false without API key)
-test_count += 1
-passed = test("AI availability check") do
-  ai_available = UniversalDocumentProcessor.ai_available?
-  raise "AI should not be available without key" if ai_available
-end
-passed_count += 1 if passed
-
-# Test 17: Error handling for unsupported format
-test_count += 1
-passed = test("Error handling for unsupported format") do
-  unsupported_file = Tempfile.new(['test', '.unknown'])
-  unsupported_file.write("test content")
-  unsupported_file.close
-  
-  begin
-    UniversalDocumentProcessor.process(unsupported_file.path)
-    raise "Should have raised UnsupportedFormatError"
-  rescue UniversalDocumentProcessor::UnsupportedFormatError
-    # Expected error
-  rescue => e
-    raise "Wrong error type: #{e.class}"
-  ensure
-    unsupported_file.unlink
-  end
-end
-passed_count += 1 if passed
-
-# Clean up
-puts "\nCleaning up temporary files..."
-[txt_file, csv_file, tsv_file, json_file, xml_file].each do |file|
-  file.unlink if File.exist?(file.path)
-end
-
-# Results
-puts "\n" + "=" * 50
-puts "Test Results:"
-puts "  Total tests: #{test_count}"
-puts "  Passed: #{passed_count}"
-puts "  Failed: #{test_count - passed_count}"
-puts "  Success rate: #{((passed_count.to_f / test_count) * 100).round(1)}%"
-
-if passed_count == test_count
-  puts "\n🎉 All tests passed! Core functionality is working correctly."
-  exit 0
-else
-  puts "\n❌ Some tests failed. Please check the issues above."
-  exit 1
-end 

data/test_performance_memory.rb

@@ -1,271 +0,0 @@
-#!/usr/bin/env ruby
-
-# Performance and Memory Usage Analysis for Universal Document Processor
-# This test checks if we need to add performance guidelines and memory usage documentation
-
-puts "🚀 Performance & Memory Analysis - Universal Document Processor"
-puts "=" * 70
-
-$LOAD_PATH.unshift File.expand_path('lib', __dir__)
-require 'universal_document_processor'
-require 'tempfile'
-require 'benchmark'
-
-# Helper to get memory usage (Windows-specific)
-def get_memory_usage
-  begin
-    result = `tasklist /FI "PID eq #{Process.pid}" /FO CSV 2>nul`
-    if result && !result.empty?
-      lines = result.split("\n")
-      if lines.length > 1
-        memory_str = lines[1].split(",")[4].gsub('"', '').gsub(',', '')
-        return memory_str.to_i # KB
-      end
-    end
-  rescue
-    # Fallback for non-Windows or error cases
-  end
-  return 0
-end
-
-def format_memory(kb)
-  if kb > 1024
-    "#{(kb / 1024.0).round(1)} MB"
-  else
-    "#{kb} KB"
-  end
-end
-
-def create_test_file(size_description, content_generator)
-  file = Tempfile.new(['perf_test', '.txt'])
-  content = content_generator.call
-  file.write(content)
-  file.close
-  
-  actual_size = File.size(file.path)
-  puts "  📁 Created #{size_description}: #{format_memory(actual_size / 1024)} (#{file.path})"
-  
-  return file, actual_size
-end
-
-issues_found = []
-performance_concerns = []
-
-puts "\n📊 PERFORMANCE TESTING"
-puts "-" * 50
-
-# Test 1: Small file performance (baseline)
-puts "\n1️⃣ Small File Performance (Baseline)"
-small_file, small_size = create_test_file("small file", -> { "Hello World!\n" * 100 })
-
-start_memory = get_memory_usage
-time_taken = Benchmark.realtime do
-  result = UniversalDocumentProcessor.process(small_file.path)
-end
-end_memory = get_memory_usage
-
-puts "  ⏱️  Processing time: #{(time_taken * 1000).round(2)} ms"
-puts "  🧠 Memory change: #{format_memory(end_memory - start_memory)}"
-
-small_file.unlink
-baseline_time = time_taken
-
-# Test 2: Medium file performance
-puts "\n2️⃣ Medium File Performance (1MB)"
-medium_file, medium_size = create_test_file("medium file", -> { "This is a test line with some content.\n" * 25000 })
-
-start_memory = get_memory_usage
-time_taken = Benchmark.realtime do
-  result = UniversalDocumentProcessor.process(medium_file.path)
-end
-end_memory = get_memory_usage
-
-puts "  ⏱️  Processing time: #{(time_taken * 1000).round(2)} ms"
-puts "  🧠 Memory change: #{format_memory(end_memory - start_memory)}"
-puts "  📈 Speed ratio: #{(time_taken / baseline_time).round(1)}x slower than baseline"
-
-if time_taken > 2.0
-  performance_concerns << "Medium files (1MB) take #{time_taken.round(2)} seconds to process"
-end
-
-medium_file.unlink
-
-# Test 3: Large file performance
-puts "\n3️⃣ Large File Performance (5MB)"
-large_file, large_size = create_test_file("large file", -> { "This is a longer test line with more content to simulate real documents.\n" * 75000 })
-
-start_memory = get_memory_usage
-time_taken = Benchmark.realtime do
-  result = UniversalDocumentProcessor.process(large_file.path)
-end
-end_memory = get_memory_usage
-
-puts "  ⏱️  Processing time: #{(time_taken * 1000).round(2)} ms"
-puts "  🧠 Memory change: #{format_memory(end_memory - start_memory)}"
-puts "  📈 Speed ratio: #{(time_taken / baseline_time).round(1)}x slower than baseline"
-
-if time_taken > 10.0
-  performance_concerns << "Large files (5MB) take #{time_taken.round(2)} seconds to process"
-end
-
-if (end_memory - start_memory) > 100000  # 100MB
-  performance_concerns << "Large files use #{format_memory(end_memory - start_memory)} of memory"
-end
-
-large_file.unlink
-
-puts "\n💾 MEMORY USAGE TESTING"
-puts "-" * 50
-
-# Test 4: Memory usage with multiple files
-puts "\n4️⃣ Batch Processing Memory Test"
-files = []
-file_sizes = []
-
-5.times do |i|
-  file, size = create_test_file("batch file #{i+1}", -> { "Batch processing test content line #{i}.\n" * 5000 })
-  files << file.path
-  file_sizes << size
-end
-
-total_file_size = file_sizes.sum
-puts "  📦 Total file size: #{format_memory(total_file_size / 1024)}"
-
-start_memory = get_memory_usage
-time_taken = Benchmark.realtime do
-  results = UniversalDocumentProcessor.batch_process(files)
-end
-end_memory = get_memory_usage
-
-memory_used = end_memory - start_memory
-puts "  ⏱️  Batch processing time: #{(time_taken * 1000).round(2)} ms"
-puts "  🧠 Memory used: #{format_memory(memory_used)}"
-puts "  📊 Memory efficiency: #{(memory_used.to_f / (total_file_size / 1024)).round(2)}x file size"
-
-if memory_used > (total_file_size / 1024) * 3  # More than 3x file size
-  performance_concerns << "Batch processing uses #{(memory_used.to_f / (total_file_size / 1024)).round(1)}x the file size in memory"
-end
-
-# Cleanup
-files.each { |f| File.delete(f) if File.exist?(f) }
-
-# Test 5: CSV/TSV processing performance
-puts "\n5️⃣ Structured Data Processing Performance"
-
-# Large CSV test
-csv_content = "Name,Age,Email,Department,Salary,Location,Phone\n"
-csv_content += 10000.times.map { |i| "User#{i},#{20+i%50},user#{i}@example.com,Dept#{i%10},#{30000+i*10},City#{i%100},555-#{i.to_s.rjust(4, '0')}" }.join("\n")
-
-csv_file = Tempfile.new(['large', '.csv'])
-csv_file.write(csv_content)
-csv_file.close
-
-csv_size = File.size(csv_file.path)
-puts "  📊 Large CSV size: #{format_memory(csv_size / 1024)}"
-
-start_memory = get_memory_usage
-time_taken = Benchmark.realtime do
-  result = UniversalDocumentProcessor.process(csv_file.path)
-end
-end_memory = get_memory_usage
-
-puts "  ⏱️  CSV processing time: #{(time_taken * 1000).round(2)} ms"
-puts "  🧠 Memory change: #{format_memory(end_memory - start_memory)}"
-
-if time_taken > 5.0
-  performance_concerns << "Large CSV files (#{format_memory(csv_size / 1024)}) take #{time_taken.round(2)} seconds"
-end
-
-csv_file.unlink
-
-# Test 6: Unicode content performance
-puts "\n6️⃣ Unicode Content Performance"
-unicode_content = "これは日本語のテストです。🌟 This includes emoji and special characters: áéíóú, ñ, ç, ü\n" * 5000
-
-unicode_file = Tempfile.new(['unicode', '.txt'])
-unicode_file.write(unicode_content)
-unicode_file.close
-
-start_memory = get_memory_usage
-time_taken = Benchmark.realtime do
-  result = UniversalDocumentProcessor.process(unicode_file.path)
-end
-end_memory = get_memory_usage
-
-puts "  ⏱️  Unicode processing time: #{(time_taken * 1000).round(2)} ms"
-puts "  🧠 Memory change: #{format_memory(end_memory - start_memory)}"
-
-unicode_file.unlink
-
-puts "\n" + "=" * 70
-puts "🎯 PERFORMANCE & MEMORY ANALYSIS RESULTS"
-puts "=" * 70
-
-puts "\n📈 PERFORMANCE CONCERNS FOUND:"
-if performance_concerns.empty?
-  puts "✅ No significant performance issues detected!"
-  puts "   The gem performs well within reasonable limits."
-else
-  performance_concerns.each_with_index do |concern, i|
-    puts "⚠️  #{i + 1}. #{concern}"
-  end
-end
-
-puts "\n📚 DOCUMENTATION RECOMMENDATIONS:"
-
-puts "\n4️⃣ Performance Guidelines Needed:"
-guidelines_needed = []
-
-if performance_concerns.any? { |c| c.include?("seconds") }
-  guidelines_needed << "Processing time expectations for different file sizes"
-  guidelines_needed << "Recommended file size limits for real-time processing"
-end
-
-if performance_concerns.any? { |c| c.include?("memory") }
-  guidelines_needed << "Memory usage patterns and optimization tips"
-  guidelines_needed << "Best practices for batch processing large files"
-end
-
-guidelines_needed << "Performance comparison between different file formats"
-guidelines_needed << "Optimization tips for production environments"
-
-if guidelines_needed.any?
-  puts "📋 Suggested documentation additions:"
-  guidelines_needed.each_with_index do |guideline, i|
-    puts "   #{i + 1}. #{guideline}"
-  end
-else
-  puts "✅ Current performance is good - minimal documentation needed"
-end
-
-puts "\n5️⃣ Memory Usage Documentation Needed:"
-memory_docs_needed = []
-
-memory_docs_needed << "Expected memory usage patterns (typically 2-3x file size)"
-memory_docs_needed << "Memory-efficient processing tips for large files"
-memory_docs_needed << "Batch processing memory considerations"
-memory_docs_needed << "When to process files individually vs. in batches"
-
-puts "📋 Suggested memory usage documentation:"
-memory_docs_needed.each_with_index do |doc, i|
-  puts "   #{i + 1}. #{doc}"
-end
-
-puts "\n💡 SPECIFIC RECOMMENDATIONS:"
-puts "1. Add a PERFORMANCE.md file with benchmarks and guidelines"
-puts "2. Include memory usage examples in README"
-puts "3. Add performance tips to method documentation"
-puts "4. Consider adding a performance_info method to the gem"
-puts "5. Document recommended file size limits for different use cases"
-
-puts "\n🎯 CONCLUSION:"
-if performance_concerns.length > 2
-  puts "❌ Performance documentation is NEEDED - several concerns found"
-  exit 1
-elsif performance_concerns.length > 0
-  puts "⚠️  Performance documentation would be HELPFUL - some concerns found"
-  exit 2
-else
-  puts "✅ Performance is good, but documentation would still be valuable for users"
-  exit 0
-end 

data/test_published_gem.rb

@@ -1,349 +0,0 @@
-#!/usr/bin/env ruby
-
-# Test script to check for potential issues with the published gem
-# This simulates real-world usage scenarios
-
-puts "🔍 Testing Universal Document Processor v1.0.3 for Potential Issues"
-puts "=" * 70
-
-# Add lib directory to load path for local testing
-$LOAD_PATH.unshift File.expand_path('lib', __dir__)
-
-require 'universal_document_processor'
-require 'tempfile'
-
-test_count = 0
-issue_count = 0
-warnings = []
-
-def test_issue(description)
-  global_test_count = caller_locations.first.lineno
-  print "#{global_test_count}. #{description}... "
-  
-  begin
-    yield
-    puts "✅ OK"
-    return false # No issue
-  rescue => e
-    puts "❌ ISSUE: #{e.message}"
-    puts "   #{e.backtrace.first}" if ENV['DEBUG']
-    return true # Issue found
-  end
-end
-
-def check_warning(description)
-  print "⚠️  #{description}... "
-  begin
-    result = yield
-    if result
-      puts "FOUND"
-      return result
-    else
-      puts "OK"
-      return nil
-    end
-  rescue => e
-    puts "ERROR: #{e.message}"
-    return e.message
-  end
-end
-
-puts "\n🧪 Testing Core Functionality Issues"
-puts "-" * 40
-
-# Test 1: Basic gem loading
-test_count += 1
-issue_found = test_issue("Gem loads without errors") do
-  # Just loading the gem should work
-  raise "VERSION not defined" unless defined?(UniversalDocumentProcessor::VERSION)
-  raise "Main module not available" unless defined?(UniversalDocumentProcessor)
-end
-issue_count += 1 if issue_found
-
-# Test 2: AI agent without API key (should not crash)
-test_count += 1
-issue_found = test_issue("AI agent creation without API key") do
-  agent = UniversalDocumentProcessor.create_ai_agent
-  raise "Agent not created" unless agent.is_a?(UniversalDocumentProcessor::AIAgent)
-  raise "AI should not be available" if agent.ai_available?
-end
-issue_count += 1 if issue_found
-
-# Test 3: Text file processing
-test_count += 1
-issue_found = test_issue("Basic text file processing") do
-  txt_file = Tempfile.new(['test', '.txt'])
-  txt_file.write("Sample text content")
-  txt_file.close
-  
-  result = UniversalDocumentProcessor.process(txt_file.path)
-  raise "No text_content key" unless result.has_key?(:text_content)
-  raise "No metadata key" unless result.has_key?(:metadata)
-  
-  txt_file.unlink
-end
-issue_count += 1 if issue_found
-
-# Test 4: CSV processing
-test_count += 1
-issue_found = test_issue("CSV file processing") do
-  csv_file = Tempfile.new(['test', '.csv'])
-  csv_file.write("Name,Age\nJohn,25\nJane,30")
-  csv_file.close
-  
-  result = UniversalDocumentProcessor.process(csv_file.path)
-  raise "Wrong format detected" unless result[:metadata][:format] == "csv"
-  raise "No tables extracted" unless result[:tables].length > 0
-  
-  csv_file.unlink
-end
-issue_count += 1 if issue_found
-
-# Test 5: TSV processing (our new feature)
-test_count += 1
-issue_found = test_issue("TSV file processing") do
-  tsv_file = Tempfile.new(['test', '.tsv'])
-  tsv_file.write("Name\tAge\nJohn\t25\nJane\t30")
-  tsv_file.close
-  
-  result = UniversalDocumentProcessor.process(tsv_file.path)
-  raise "Wrong format detected" unless result[:metadata][:format] == "tsv"
-  raise "Wrong delimiter" unless result[:metadata][:delimiter] == "tab"
-  raise "No tables extracted" unless result[:tables].length > 0
-  
-  tsv_file.unlink
-end
-issue_count += 1 if issue_found
-
-puts "\n🔒 Testing Dependency Issues"
-puts "-" * 40
-
-# Test 6: Optional dependency checking
-test_count += 1
-issue_found = test_issue("Optional dependency information") do
-  deps = UniversalDocumentProcessor.optional_dependencies
-  raise "No optional deps info" if deps.empty?
-  
-  missing = UniversalDocumentProcessor.missing_dependencies
-  raise "Missing deps not array" unless missing.is_a?(Array)
-  
-  instructions = UniversalDocumentProcessor.installation_instructions
-  raise "No installation instructions" if instructions.empty?
-end
-issue_count += 1 if issue_found
-
-# Test 7: PDF processing without pdf-reader gem
-test_count += 1
-issue_found = test_issue("PDF processing dependency handling") do
-  # Create a fake PDF file (just for testing error handling)
-  pdf_file = Tempfile.new(['test', '.pdf'])
-  pdf_file.write("%PDF-1.4\nFake PDF content")
-  pdf_file.close
-  
-  begin
-    result = UniversalDocumentProcessor.process(pdf_file.path)
-    # Should either work (if pdf-reader available) or give graceful error
-  rescue UniversalDocumentProcessor::DependencyMissingError => e
-    # This is expected and good
-    raise "Wrong error message" unless e.message.include?("pdf-reader")
-  end
-  
-  pdf_file.unlink
-end
-issue_count += 1 if issue_found
-
-puts "\n⚠️  Testing Edge Cases & Potential Warnings"
-puts "-" * 40
-
-# Warning 1: Large file handling
-warning = check_warning("Large file memory usage") do
-  # Create a moderately large text file
-  large_file = Tempfile.new(['large_test', '.txt'])
-  content = "This is a test line.\n" * 10000  # ~200KB
-  large_file.write(content)
-  large_file.close
-  
-  start_time = Time.now
-  result = UniversalDocumentProcessor.process(large_file.path)
-  end_time = Time.now
-  
-  large_file.unlink
-  
-  processing_time = end_time - start_time
-  if processing_time > 5.0
-    "Large file processing took #{processing_time.round(2)} seconds"
-  else
-    false
-  end
-end
-warnings << warning if warning
-
-# Warning 2: Unicode/Japanese filename handling
-warning = check_warning("Unicode filename handling") do
-  begin
-    japanese_content = "これは日本語のテストです。"
-    unicode_file = Tempfile.new(['テスト', '.txt'])
-    unicode_file.write(japanese_content)
-    unicode_file.close
-    
-    result = UniversalDocumentProcessor.process(unicode_file.path)
-    unicode_file.unlink
-    false
-  rescue => e
-    "Unicode filename issue: #{e.message}"
-  end
-end
-warnings << warning if warning
-
-# Warning 3: Empty file handling
-warning = check_warning("Empty file handling") do
-  empty_file = Tempfile.new(['empty', '.txt'])
-  empty_file.close
-  
-  begin
-    result = UniversalDocumentProcessor.process(empty_file.path)
-    empty_file.unlink
-    
-    if result[:text_content].nil? || result[:text_content].empty?
-      false # This is expected
-    else
-      false # Also fine
-    end
-  rescue => e
-    empty_file.unlink
-    "Empty file processing issue: #{e.message}"
-  end
-end
-warnings << warning if warning
-
-# Warning 4: Invalid file extension handling
-warning = check_warning("Invalid file extension handling") do
-  invalid_file = Tempfile.new(['test', '.xyz'])
-  invalid_file.write("Test content")
-  invalid_file.close
-  
-  begin
-    result = UniversalDocumentProcessor.process(invalid_file.path)
-    invalid_file.unlink
-    false # Processed successfully
-  rescue UniversalDocumentProcessor::UnsupportedFormatError
-    invalid_file.unlink
-    false # Expected error, good
-  rescue => e
-    invalid_file.unlink
-    "Unexpected error for unsupported format: #{e.message}"
-  end
-end
-warnings << warning if warning
-
-# Warning 5: Memory usage with multiple files
-warning = check_warning("Memory usage with batch processing") do
-  files = []
-  5.times do |i|
-    file = Tempfile.new(["batch_#{i}", '.txt'])
-    file.write("Batch test content #{i}\n" * 1000)
-    file.close
-    files << file.path
-  end
-  
-  begin
-    start_memory = `tasklist /FI "PID eq #{Process.pid}" /FO CSV`.split("\n")[1].split(",")[4].gsub('"', '').gsub(',', '').to_i rescue 0
-    
-    results = UniversalDocumentProcessor.batch_process(files)
-    
-    end_memory = `tasklist /FI "PID eq #{Process.pid}" /FO CSV`.split("\n")[1].split(",")[4].gsub('"', '').gsub(',', '').to_i rescue 0
-    
-    files.each { |f| File.delete(f) if File.exist?(f) }
-    
-    memory_increase = end_memory - start_memory
-    if memory_increase > 50000  # 50MB increase
-      "High memory usage: #{memory_increase}KB increase"
-    else
-      false
-    end
-  rescue => e
-    files.each { |f| File.delete(f) if File.exist?(f) }
-    "Batch processing memory test failed: #{e.message}"
-  end
-end
-warnings << warning if warning
-
-puts "\n🔍 Testing AI Features (Without API Key)"
-puts "-" * 40
-
-# Test 8: AI methods should fail gracefully
-test_count += 1
-issue_found = test_issue("AI methods fail gracefully without API key") do
-  txt_file = Tempfile.new(['ai_test', '.txt'])
-  txt_file.write("Test content for AI")
-  txt_file.close
-  
-  begin
-    UniversalDocumentProcessor.ai_analyze(txt_file.path)
-    raise "Should have raised DependencyMissingError"
-  rescue UniversalDocumentProcessor::DependencyMissingError => e
-    # Expected - this is good
-    raise "Wrong error message" unless e.message.include?("OpenAI API key")
-  end
-  
-  txt_file.unlink
-end
-issue_count += 1 if issue_found
-
-puts "\n📊 Testing Available Features"
-puts "-" * 40
-
-# Test 9: Feature detection
-test_count += 1
-issue_found = test_issue("Feature detection works correctly") do
-  features = UniversalDocumentProcessor.available_features
-  raise "No features detected" if features.empty?
-  raise "Missing basic features" unless features.include?(:text_processing)
-  raise "Missing TSV support" unless features.include?(:tsv_processing)
-  
-  # AI should not be available without API key
-  raise "AI should not be available" if features.include?(:ai_processing)
-end
-issue_count += 1 if issue_found
-
-puts "\n" + "=" * 70
-puts "🎯 ISSUE ANALYSIS COMPLETE"
-puts "=" * 70
-
-puts "\n📈 SUMMARY:"
-puts "  Total tests run: #{test_count}"
-puts "  Issues found: #{issue_count}"
-puts "  Warnings: #{warnings.compact.length}"
-
-if issue_count == 0
-  puts "\n✅ NO CRITICAL ISSUES FOUND!"
-  puts "The gem appears to be working correctly for basic usage."
-else
-  puts "\n❌ CRITICAL ISSUES DETECTED!"
-  puts "The gem has #{issue_count} critical issues that need attention."
-end
-
-if warnings.compact.length > 0
-  puts "\n⚠️  WARNINGS TO CONSIDER:"
-  warnings.compact.each_with_index do |warning, i|
-    puts "  #{i + 1}. #{warning}"
-  end
-else
-  puts "\n✅ No significant warnings detected."
-end
-
-puts "\n🔮 POTENTIAL USER ISSUES TO WATCH FOR:"
-puts "1. Users trying to use AI features without setting OPENAI_API_KEY"
-puts "2. Users expecting PDF/Word processing without installing optional gems"
-puts "3. Large file processing performance"
-puts "4. Unicode filename handling on different systems"
-puts "5. Memory usage with batch processing of many files"
-
-puts "\n💡 RECOMMENDATIONS:"
-puts "1. ✅ AI dependency handling is working correctly"
-puts "2. ✅ TSV processing is functional"
-puts "3. ✅ Error messages are helpful"
-puts "4. 📚 Consider adding performance guidelines to documentation"
-puts "5. 📚 Consider adding memory usage notes for large files"
-
-exit issue_count