kreuzberg 4.0.0.pre.rc.29 → 4.0.0.rc1

data/spec/binding/keywords_extraction_spec.rb

@@ -1,600 +0,0 @@
-# frozen_string_literal: true
-
-RSpec.describe 'Keyword Extraction' do
-  describe 'basic keyword extraction' do
-    it 'extracts keywords from text' do
-      config = Kreuzberg::Config::Extraction.new(
-        keywords: Kreuzberg::Config::Keywords.new(
-          algorithm: 'yake',
-          max_keywords: 10
-        )
-      )
-
-      text = 'Machine learning and artificial intelligence are transforming technology. Neural networks and deep learning are key areas of AI research. These technologies enable predictions and data analysis.'
-      result = Kreuzberg.extract_bytes_sync(data: text, mime_type: 'text/plain', config: config)
-
-      expect(result).not_to be_nil
-      expect(result.content).not_to be_nil
-      expect(result.content).to include('Machine learning')
-    end
-
-    it 'returns keywords in metadata' do
-      config = Kreuzberg::Config::Extraction.new(
-        keywords: Kreuzberg::Config::Keywords.new(
-          algorithm: 'yake',
-          max_keywords: 5
-        )
-      )
-
-      text = 'Artificial intelligence transforms technology development. Machine learning algorithms improve with training data.'
-      result = Kreuzberg.extract_bytes_sync(data: text, mime_type: 'text/plain', config: config)
-
-      expect(result).not_to be_nil
-      expect(result.metadata).to be_a(Hash)
-    end
-
-    it 'respects max_keywords parameter' do
-      max_keywords_values = [1, 5, 10]
-
-      max_keywords_values.each do |max_kw|
-        config = Kreuzberg::Config::Extraction.new(
-          keywords: Kreuzberg::Config::Keywords.new(
-            algorithm: 'yake',
-            max_keywords: max_kw
-          )
-        )
-
-        text = 'Machine learning and artificial intelligence are transforming technology. Neural networks and deep learning are key research areas. Data science enables predictions. Algorithms process information efficiently.'
-        result = Kreuzberg.extract_bytes_sync(data: text, mime_type: 'text/plain', config: config)
-
-        expect(result).not_to be_nil
-        expect(result.content).not_to be_nil
-      end
-    end
-
-    it 'returns content when keywords enabled' do
-      config = Kreuzberg::Config::Extraction.new(
-        keywords: Kreuzberg::Config::Keywords.new(
-          algorithm: 'yake',
-          max_keywords: 5
-        )
-      )
-
-      text = 'Artificial intelligence is transforming the world.'
-      result = Kreuzberg.extract_bytes_sync(data: text, mime_type: 'text/plain', config: config)
-
-      expect(result).not_to be_nil
-      expect(result.content).not_to be_nil
-      expect(result.content).to include('Artificial intelligence')
-    end
-  end
-
-  describe 'multilingual keyword extraction' do
-    it 'extracts keywords from English text' do
-      config = Kreuzberg::Config::Extraction.new(
-        keywords: Kreuzberg::Config::Keywords.new(
-          algorithm: 'yake',
-          language: 'en',
-          max_keywords: 5
-        )
-      )
-
-      text = 'Machine learning and artificial intelligence are transforming technology development globally.'
-      result = Kreuzberg.extract_bytes_sync(data: text, mime_type: 'text/plain', config: config)
-
-      expect(result).not_to be_nil
-      expect(result.content).not_to be_nil
-    end
-
-    it 'extracts keywords from German text' do
-      config = Kreuzberg::Config::Extraction.new(
-        keywords: Kreuzberg::Config::Keywords.new(
-          algorithm: 'yake',
-          language: 'de',
-          max_keywords: 5
-        )
-      )
-
-      text = 'Maschinelles Lernen und künstliche Intelligenz transformieren die Technologieentwicklung.'
-      result = Kreuzberg.extract_bytes_sync(data: text, mime_type: 'text/plain', config: config)
-
-      expect(result).not_to be_nil
-      expect(result.content).not_to be_nil
-    end
-
-    it 'extracts keywords from French text' do
-      config = Kreuzberg::Config::Extraction.new(
-        keywords: Kreuzberg::Config::Keywords.new(
-          algorithm: 'yake',
-          language: 'fr',
-          max_keywords: 5
-        )
-      )
-
-      text = "L'apprentissage automatique et l'intelligence artificielle transforment le développement technologique."
-      result = Kreuzberg.extract_bytes_sync(data: text, mime_type: 'text/plain', config: config)
-
-      expect(result).not_to be_nil
-      expect(result.content).not_to be_nil
-    end
-
-    it 'handles language parameter correctly' do
-      languages = %w[en de fr es it pt nl]
-
-      languages.each do |lang|
-        config = Kreuzberg::Config::Extraction.new(
-          keywords: Kreuzberg::Config::Keywords.new(
-            algorithm: 'yake',
-            language: lang,
-            max_keywords: 3
-          )
-        )
-
-        text = 'Machine learning and artificial intelligence.'
-        result = Kreuzberg.extract_bytes_sync(data: text, mime_type: 'text/plain', config: config)
-
-        expect(result).not_to be_nil
-      end
-    end
-  end
-
-  describe 'min_score filtering' do
-    it 'filters keywords with different score thresholds' do
-      text = 'Machine learning and artificial intelligence are transforming technology. Neural networks and deep learning are key research areas.'
-
-      results_by_threshold = {}
-      thresholds = [0.1, 0.5, 0.9]
-
-      thresholds.each do |threshold|
-        config = Kreuzberg::Config::Extraction.new(
-          keywords: Kreuzberg::Config::Keywords.new(
-            algorithm: 'yake',
-            max_keywords: 100,
-            min_score: threshold
-          )
-        )
-
-        result = Kreuzberg.extract_bytes_sync(data: text, mime_type: 'text/plain', config: config)
-        results_by_threshold[threshold] = result
-
-        expect(result).not_to be_nil
-        expect(result.content).not_to be_nil
-      end
-
-      # Lower thresholds should produce more or equal results
-      low_score_content = results_by_threshold[0.1]
-      high_score_content = results_by_threshold[0.9]
-      expect(low_score_content.content.length).to be >= high_score_content.content.length
-    end
-
-    it 'produces consistent results with same score threshold' do
-      config = Kreuzberg::Config::Extraction.new(
-        keywords: Kreuzberg::Config::Keywords.new(
-          algorithm: 'yake',
-          max_keywords: 50,
-          min_score: 0.3
-        )
-      )
-
-      text = 'Artificial intelligence is transforming data science and machine learning research globally with neural networks.'
-      result1 = Kreuzberg.extract_bytes_sync(data: text, mime_type: 'text/plain', config: config)
-      result2 = Kreuzberg.extract_bytes_sync(data: text, mime_type: 'text/plain', config: config)
-      result3 = Kreuzberg.extract_bytes_sync(data: text, mime_type: 'text/plain', config: config)
-
-      # Results should be identical across runs
-      expect(result1.content).to eq(result2.content)
-      expect(result2.content).to eq(result3.content)
-    end
-  end
-
-  describe 'ngram_range variations' do
-    it 'extracts keywords with different ngram configurations' do
-      text = 'Machine learning and artificial intelligence are transforming data science and technology development globally.'
-
-      configs = {
-        'single_words' => [1, 1],
-        'bigrams' => [2, 2],
-        'unigram_bigram' => [1, 2],
-        'unigram_trigram' => [1, 3]
-      }
-
-      results = {}
-      configs.each do |label, range|
-        config = Kreuzberg::Config::Extraction.new(
-          keywords: Kreuzberg::Config::Keywords.new(
-            algorithm: 'yake',
-            max_keywords: 15,
-            ngram_range: range
-          )
-        )
-
-        result = Kreuzberg.extract_bytes_sync(data: text, mime_type: 'text/plain', config: config)
-        results[label] = result
-
-        expect(result).not_to be_nil
-        expect(result.content).not_to be_nil
-        expect(result.content.length).to be > 0
-      end
-
-      # Wider n-gram ranges typically produce more content due to phrase inclusion
-      expect(results['unigram_trigram'].content.length).to be >= results['single_words'].content.length
-    end
-
-    it 'produces consistent results with same ngram_range' do
-      config = Kreuzberg::Config::Extraction.new(
-        keywords: Kreuzberg::Config::Keywords.new(
-          algorithm: 'yake',
-          max_keywords: 10,
-          ngram_range: [1, 2]
-        )
-      )
-
-      text = 'Machine learning and artificial intelligence are transforming technology development across industry.'
-      result1 = Kreuzberg.extract_bytes_sync(data: text, mime_type: 'text/plain', config: config)
-      result2 = Kreuzberg.extract_bytes_sync(data: text, mime_type: 'text/plain', config: config)
-
-      # Results should be identical
-      expect(result1.content).to eq(result2.content)
-    end
-  end
-
-  describe 'algorithm selection' do
-    it 'both YAKE and RAKE algorithms produce results' do
-      text = 'Machine learning and artificial intelligence are transforming technology and neural networks enable deep learning.'
-
-      yake_config = Kreuzberg::Config::Extraction.new(
-        keywords: Kreuzberg::Config::Keywords.new(
-          algorithm: 'yake',
-          max_keywords: 10
-        )
-      )
-
-      rake_config = Kreuzberg::Config::Extraction.new(
-        keywords: Kreuzberg::Config::Keywords.new(
-          algorithm: 'rake',
-          max_keywords: 10
-        )
-      )
-
-      yake_result = Kreuzberg.extract_bytes_sync(data: text, mime_type: 'text/plain', config: yake_config)
-      rake_result = Kreuzberg.extract_bytes_sync(data: text, mime_type: 'text/plain', config: rake_config)
-
-      expect(yake_result).not_to be_nil
-      expect(yake_result.content).not_to be_nil
-      expect(yake_result.content.length).to be > 0
-
-      expect(rake_result).not_to be_nil
-      expect(rake_result.content).not_to be_nil
-      expect(rake_result.content.length).to be > 0
-    end
-
-    it 'algorithm-specific parameters affect extraction' do
-      text = 'Machine learning and artificial intelligence are transforming technology development and research.'
-
-      # YAKE with different window sizes
-      yake_config_small = Kreuzberg::Config::Extraction.new(
-        keywords: Kreuzberg::Config::Keywords.new(
-          algorithm: 'yake',
-          max_keywords: 10,
-          yake_params: Kreuzberg::Config::KeywordYakeParams.new(window_size: 2)
-        )
-      )
-
-      yake_config_large = Kreuzberg::Config::Extraction.new(
-        keywords: Kreuzberg::Config::Keywords.new(
-          algorithm: 'yake',
-          max_keywords: 10,
-          yake_params: Kreuzberg::Config::KeywordYakeParams.new(window_size: 4)
-        )
-      )
-
-      result_small = Kreuzberg.extract_bytes_sync(data: text, mime_type: 'text/plain', config: yake_config_small)
-      result_large = Kreuzberg.extract_bytes_sync(data: text, mime_type: 'text/plain', config: yake_config_large)
-
-      expect(result_small).not_to be_nil
-      expect(result_large).not_to be_nil
-      # Both should produce valid results
-      expect(result_small.content.length).to be > 0
-      expect(result_large.content.length).to be > 0
-    end
-
-    it 'RAKE with min_word_length parameter works' do
-      config = Kreuzberg::Config::Extraction.new(
-        keywords: Kreuzberg::Config::Keywords.new(
-          algorithm: 'rake',
-          max_keywords: 10,
-          rake_params: Kreuzberg::Config::KeywordRakeParams.new(
-            min_word_length: 2,
-            max_words_per_phrase: 4
-          )
-        )
-      )
-
-      text = 'Machine learning and artificial intelligence are transforming technology.'
-      result = Kreuzberg.extract_bytes_sync(data: text, mime_type: 'text/plain', config: config)
-
-      expect(result).not_to be_nil
-      expect(result.content).not_to be_nil
-      expect(result.content.length).to be > 0
-    end
-  end
-
-  describe 'batch keyword extraction' do
-    it 'processes multiple texts with same configuration' do
-      config = Kreuzberg::Config::Extraction.new(
-        keywords: Kreuzberg::Config::Keywords.new(
-          algorithm: 'yake',
-          max_keywords: 5
-        )
-      )
-
-      texts = [
-        'Machine learning and artificial intelligence are transforming technology.',
-        'Deep learning neural networks enable advanced data science applications.',
-        'Artificial intelligence enables predictions and automation globally.'
-      ]
-
-      results = texts.map { |text| Kreuzberg.extract_bytes_sync(text, 'text/plain', config: config) }
-
-      expect(results.length).to eq(3)
-      results.each do |result|
-        expect(result).not_to be_nil
-        expect(result.content).not_to be_nil
-      end
-    end
-
-    it 'maintains consistency across batch processing' do
-      config = Kreuzberg::Config::Extraction.new(
-        keywords: Kreuzberg::Config::Keywords.new(
-          algorithm: 'yake',
-          max_keywords: 10
-        )
-      )
-
-      text = 'Machine learning and artificial intelligence are transforming technology development globally.'
-
-      result1 = Kreuzberg.extract_bytes_sync(data: text, mime_type: 'text/plain', config: config)
-      result2 = Kreuzberg.extract_bytes_sync(data: text, mime_type: 'text/plain', config: config)
-
-      expect(result1).not_to be_nil
-      expect(result2).not_to be_nil
-      expect(result1.content).to eq(result2.content)
-    end
-
-    it 'handles different configurations per batch item' do
-      text = 'Machine learning and artificial intelligence are transforming technology.'
-
-      configs = [
-        Kreuzberg::Config::Extraction.new(
-          keywords: Kreuzberg::Config::Keywords.new(algorithm: 'yake', max_keywords: 5)
-        ),
-        Kreuzberg::Config::Extraction.new(
-          keywords: Kreuzberg::Config::Keywords.new(algorithm: 'rake', max_keywords: 5)
-        ),
-        Kreuzberg::Config::Extraction.new(
-          keywords: Kreuzberg::Config::Keywords.new(algorithm: 'yake', max_keywords: 10)
-        )
-      ]
-
-      results = configs.map { |cfg| Kreuzberg.extract_bytes_sync(text, 'text/plain', config: cfg) }
-
-      expect(results.length).to eq(3)
-      results.each do |result|
-        expect(result).not_to be_nil
-      end
-    end
-  end
-
-  describe 'score normalization validation' do
-    it 'returns normalized scores between 0 and 1' do
-      config = Kreuzberg::Config::Extraction.new(
-        keywords: Kreuzberg::Config::Keywords.new(
-          algorithm: 'yake',
-          max_keywords: 20
-        )
-      )
-
-      text = 'Machine learning and artificial intelligence are transforming technology. Neural networks and deep learning enable data science applications.'
-      result = Kreuzberg.extract_bytes_sync(data: text, mime_type: 'text/plain', config: config)
-
-      expect(result).not_to be_nil
-      expect(result.content).not_to be_nil
-    end
-
-    it 'validates score values are realistic' do
-      config = Kreuzberg::Config::Extraction.new(
-        keywords: Kreuzberg::Config::Keywords.new(
-          algorithm: 'yake',
-          max_keywords: 50
-        )
-      )
-
-      text = 'Artificial intelligence machine learning data science neural networks deep learning transforming technology.'
-      result = Kreuzberg.extract_bytes_sync(data: text, mime_type: 'text/plain', config: config)
-
-      expect(result).not_to be_nil
-      expect(result.content).not_to be_nil
-    end
-
-    it 'handles score filtering with normalized ranges' do
-      scores_to_test = [0.0, 0.25, 0.5, 0.75, 1.0]
-
-      scores_to_test.each do |score_threshold|
-        config = Kreuzberg::Config::Extraction.new(
-          keywords: Kreuzberg::Config::Keywords.new(
-            algorithm: 'yake',
-            max_keywords: 100,
-            min_score: score_threshold
-          )
-        )
-
-        text = 'Machine learning artificial intelligence data science neural networks deep learning technology.'
-        result = Kreuzberg.extract_bytes_sync(data: text, mime_type: 'text/plain', config: config)
-
-        expect(result).not_to be_nil
-      end
-    end
-
-    it 'ensures score consistency for same text' do
-      config = Kreuzberg::Config::Extraction.new(
-        keywords: Kreuzberg::Config::Keywords.new(
-          algorithm: 'yake',
-          max_keywords: 10
-        )
-      )
-
-      text = 'Machine learning and artificial intelligence transform technology.'
-      result1 = Kreuzberg.extract_bytes_sync(data: text, mime_type: 'text/plain', config: config)
-      result2 = Kreuzberg.extract_bytes_sync(data: text, mime_type: 'text/plain', config: config)
-
-      expect(result1).not_to be_nil
-      expect(result2).not_to be_nil
-    end
-  end
-
-  describe 'empty and edge cases' do
-    it 'handles very short text gracefully' do
-      config = Kreuzberg::Config::Extraction.new(
-        keywords: Kreuzberg::Config::Keywords.new(
-          algorithm: 'yake',
-          max_keywords: 5
-        )
-      )
-
-      text = 'AI'
-      result = Kreuzberg.extract_bytes_sync(data: text, mime_type: 'text/plain', config: config)
-
-      expect(result).not_to be_nil
-      expect(result.content).not_to be_nil
-    end
-
-    it 'handles text with no obvious keywords' do
-      config = Kreuzberg::Config::Extraction.new(
-        keywords: Kreuzberg::Config::Keywords.new(
-          algorithm: 'yake',
-          max_keywords: 5
-        )
-      )
-
-      text = 'a b c d e'
-      result = Kreuzberg.extract_bytes_sync(data: text, mime_type: 'text/plain', config: config)
-
-      expect(result).not_to be_nil
-    end
-
-    it 'handles text with repeated keywords' do
-      config = Kreuzberg::Config::Extraction.new(
-        keywords: Kreuzberg::Config::Keywords.new(
-          algorithm: 'yake',
-          max_keywords: 5
-        )
-      )
-
-      text = 'Machine machine machine learning learning learning artificial artificial artificial intelligence intelligence.'
-      result = Kreuzberg.extract_bytes_sync(data: text, mime_type: 'text/plain', config: config)
-
-      expect(result).not_to be_nil
-      expect(result.content).not_to be_nil
-    end
-
-    it 'handles max_keywords of 0' do
-      config = Kreuzberg::Config::Extraction.new(
-        keywords: Kreuzberg::Config::Keywords.new(
-          algorithm: 'yake',
-          max_keywords: 0
-        )
-      )
-
-      text = 'Machine learning and artificial intelligence.'
-      result = Kreuzberg.extract_bytes_sync(data: text, mime_type: 'text/plain', config: config)
-
-      expect(result).not_to be_nil
-    end
-
-    it 'handles large max_keywords value' do
-      config = Kreuzberg::Config::Extraction.new(
-        keywords: Kreuzberg::Config::Keywords.new(
-          algorithm: 'yake',
-          max_keywords: 1000
-        )
-      )
-
-      text = 'Machine learning and artificial intelligence are transforming technology.'
-      result = Kreuzberg.extract_bytes_sync(data: text, mime_type: 'text/plain', config: config)
-
-      expect(result).not_to be_nil
-    end
-
-    it 'handles disabled keyword extraction' do
-      config = Kreuzberg::Config::Extraction.new
-      text = 'Machine learning and artificial intelligence.'
-      result = Kreuzberg.extract_bytes_sync(data: text, mime_type: 'text/plain', config: config)
-
-      expect(result).not_to be_nil
-      expect(result.content).not_to be_nil
-    end
-
-    it 'handles keywords config with nil values' do
-      config = Kreuzberg::Config::Extraction.new(
-        keywords: Kreuzberg::Config::Keywords.new(
-          algorithm: nil,
-          max_keywords: nil
-        )
-      )
-
-      text = 'Machine learning and artificial intelligence.'
-      result = Kreuzberg.extract_bytes_sync(data: text, mime_type: 'text/plain', config: config)
-
-      expect(result).not_to be_nil
-    end
-  end
-
-  describe 'integration with Extraction config' do
-    it 'accepts Keywords config in Extraction' do
-      keywords = Kreuzberg::Config::Keywords.new(
-        algorithm: 'yake',
-        max_keywords: 10
-      )
-      config = Kreuzberg::Config::Extraction.new(keywords: keywords)
-
-      expect(config.keywords).to be_a(Kreuzberg::Config::Keywords)
-      expect(config.keywords.algorithm).to eq('yake')
-    end
-
-    it 'accepts keywords config as hash in Extraction' do
-      config = Kreuzberg::Config::Extraction.new(
-        keywords: {
-          algorithm: 'rake',
-          max_keywords: 15,
-          min_score: 0.3
-        }
-      )
-
-      expect(config.keywords).to be_a(Kreuzberg::Config::Keywords)
-      expect(config.keywords.algorithm).to eq('rake')
-      expect(config.keywords.max_keywords).to eq(15)
-    end
-
-    it 'includes keywords config in to_h' do
-      keywords = Kreuzberg::Config::Keywords.new(
-        algorithm: 'yake',
-        max_keywords: 10
-      )
-      config = Kreuzberg::Config::Extraction.new(keywords: keywords)
-
-      hash = config.to_h
-
-      expect(hash).to include(:keywords)
-      expect(hash[:keywords]).to be_a(Hash)
-      expect(hash[:keywords][:algorithm]).to eq('yake')
-    end
-
-    it 'handles nil keywords config' do
-      config = Kreuzberg::Config::Extraction.new(keywords: nil)
-
-      expect(config.keywords).to be_nil
-      hash = config.to_h
-      expect(hash[:keywords]).to be_nil
-    end
-  end
-end