keyword-ruby 0.1.1 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 433764acb546b2b35c3e05f56095ec7ba4642a8c67591d9466fb1a6a8ae47bdf
-  data.tar.gz: e83e1a3e1b454b584efccf44a592c7868e788ff756e946e4c7631e696982cc09
+  metadata.gz: ae61b0d85e3b754d30502e323aac159abbe570b4b30341b8b8c1d86f0e3f65c2
+  data.tar.gz: 729dc165b59cb53568032f884ccc9eae34c80958cb5d87698bcceff0eb0f6061
 SHA512:
-  metadata.gz: b5dbf9ec6be9a3b3b657c03b6397df41de8152a02ab8539f5b62c435de8dd51ba80ca6d16645f50971d5bb48e9c2e1351a9e63081d61f89251c2d257f23cef54
-  data.tar.gz: 0d51a0fe597f68dff3231881ac131d69b85ee210bf638cdb95d8e8e8d1b2cdfab4f4e50c8f15d4c6271e394d647e5f4778e8a104270ea17a851c70d88cafc942
+  metadata.gz: a8bf751b18fff5031917192ebc67cd30da67705d72ebc30d2b31c4e03b488223f4572f99c36261e8dd81c66491f212e9db2ce3e456e8df89efb3bb642d3e7790
+  data.tar.gz: 1a1fa33626b2946b913c93eb984dc9854cb65c65b0b208d5959e4aac9dcb255f80dfbf668b1219cafa803acb227bed075b26031af305fcf15fb263bc5637a748

data/README.md

@@ -1,11 +1,11 @@
 # keyword-ruby
 
-Pure Ruby keyword and keyphrase extraction using RAKE, YAKE, and TF-IDF algorithms.
+Keyword extraction for Ruby using RAKE, YAKE, and TF-IDF algorithms. Extract the most relevant terms from any text.
 
 ## Installation
 
 ```ruby
-gem "keyword-ruby", "~> 0.1"
+gem "keyword-ruby"
 ```
 
 ## Usage
@@ -13,22 +13,31 @@ gem "keyword-ruby", "~> 0.1"
 ```ruby
 require "keyword_ruby"
 
-# RAKE (default)
-keywords = KeywordRuby.extract("Your text here...")
-keywords.each { |kw| puts "#{kw.phrase}: #{kw.score}" }
+text = "Ruby is a dynamic programming language focused on simplicity and productivity."
 
-# YAKE
-keywords = KeywordRuby.extract(text, algorithm: :yake)
+# RAKE (Rapid Automatic Keyword Extraction)
+keywords = KeywordRuby.extract(text, algorithm: :rake, top_n: 5)
 
-# TF-IDF (with corpus)
+# YAKE (Yet Another Keyword Extractor)
+keywords = KeywordRuby.extract(text, algorithm: :yake, top_n: 5)
+
+# TF-IDF
 extractor = KeywordRuby::Extractors::Tfidf.new
-extractor.fit(corpus_documents)
-keywords = extractor.extract(text)
+extractor.fit(corpus)  # optional: fit on a corpus
+keywords = extractor.extract(text, top_n: 5)
 
-# Batch extraction
-results = KeywordRuby.extract_batch(documents, algorithm: :rake, top_n: 5)
+keywords.each { |kw| puts "#{kw.text}: #{kw.score}" }
 ```
 
+## Features
+
+- RAKE with proper co-occurrence degree calculation
+- YAKE with stop word handling in multi-word phrases
+- TF-IDF with optional corpus fitting (falls back to TF-only)
+- Score normalization to 0.0-1.0 range
+- English contraction expansion (don't → do not)
+- Input validation and language support
+
 ## License
 
 MIT

data/lib/keyword_ruby/configuration.rb

@@ -3,7 +3,7 @@
 module KeywordRuby
   class Configuration
     attr_accessor :default_algorithm, :default_language, :default_top_n,
-                  :max_phrase_length, :min_word_length
+                  :max_phrase_length, :min_word_length, :custom_stop_words
 
     def initialize
       @default_algorithm = :rake
@@ -11,6 +11,7 @@ module KeywordRuby
       @default_top_n = 10
       @max_phrase_length = 4
       @min_word_length = 2
+      @custom_stop_words = []
     end
   end
 end

data/lib/keyword_ruby/extractors/base.rb

@@ -3,7 +3,7 @@
 module KeywordRuby
   module Extractors
     class Base
-      SUPPORTED_LANGUAGES = %i[en id].freeze
+      SUPPORTED_LANGUAGES = %i[en id ms nl fr de es pt ar ja].freeze
 
       def initialize(language: nil, top_n: nil, max_length: nil, min_word_length: nil, normalize: true)
         config = KeywordRuby.configuration

data/lib/keyword_ruby/extractors/textrank.rb

@@ -0,0 +1,107 @@
+# frozen_string_literal: true
+
+module KeywordRuby
+  module Extractors
+    class TextRank < Base
+      DEFAULT_DAMPING = 0.85
+      DEFAULT_ITERATIONS = 30
+      DEFAULT_CONVERGENCE = 0.0001
+
+      def initialize(damping: DEFAULT_DAMPING, iterations: DEFAULT_ITERATIONS, **opts)
+        super(**opts)
+        @damping = damping
+        @iterations = iterations
+      end
+
+      def extract(text)
+        validate_text!(text)
+        return [] if text.nil? || text.strip.empty?
+
+        words = TextProcessing::Tokenizer.tokenize(text)
+          .reject { |w| stop_word?(w) || w.length < @min_word_length }
+        return [] if words.empty?
+
+        # Build co-occurrence graph (window size = 4)
+        graph = build_graph(words, window: 4)
+        return [] if graph.empty?
+
+        # Run PageRank
+        scores = pagerank(graph)
+
+        # Generate multi-word candidates
+        all_words = TextProcessing::Tokenizer.tokenize(text)
+        candidates = generate_phrases(all_words, scores)
+
+        results = candidates.sort.first(@top_n)
+        normalize_scores(results)
+      end
+
+      private
+
+      def build_graph(words, window: 4)
+        graph = Hash.new { |h, k| h[k] = Hash.new(0.0) }
+
+        words.each_cons(window) do |group|
+          group.uniq.combination(2) do |a, b|
+            graph[a][b] += 1.0
+            graph[b][a] += 1.0
+          end
+        end
+
+        graph
+      end
+
+      def pagerank(graph)
+        nodes = graph.keys
+        n = nodes.size.to_f
+        scores = nodes.map { |node| [node, 1.0 / n] }.to_h
+
+        @iterations.times do
+          new_scores = {}
+          max_diff = 0.0
+
+          nodes.each do |node|
+            rank = (1.0 - @damping) / n
+            neighbors = graph[node]
+
+            neighbors.each do |neighbor, weight|
+              out_weight = graph[neighbor].values.sum
+              rank += @damping * (scores[neighbor] || 0.0) * weight / out_weight if out_weight > 0
+            end
+
+            new_scores[node] = rank
+            max_diff = [max_diff, (rank - (scores[node] || 0.0)).abs].max
+          end
+
+          scores = new_scores
+          break if max_diff < DEFAULT_CONVERGENCE
+        end
+
+        scores
+      end
+
+      def generate_phrases(words, word_scores)
+        phrases = {}
+
+        # Single words
+        word_scores.each do |word, score|
+          phrases[word] = score
+        end
+
+        # Multi-word phrases (2-4 words)
+        (2..@max_length).each do |n|
+          words.each_cons(n) do |gram|
+            next if stop_word?(gram.first) || stop_word?(gram.last)
+            next if gram.first.length < @min_word_length || gram.last.length < @min_word_length
+
+            phrase = gram.join(" ")
+            score = gram.sum { |w| word_scores[w] || 0.0 }
+            phrases[phrase] = [phrases[phrase] || 0.0, score].max
+          end
+        end
+
+        phrases.map { |phrase, score| Keyword.new(phrase: phrase, score: score) }
+      end
+    end
+  end
+end

data/lib/keyword_ruby/stop_words/ar.txt

@@ -0,0 +1,29 @@
+في
+من
+على
+إلى
+أن
+هذا
+هذه
+التي
+الذي
+عن
+مع
+كان
+لا
+ما
+هو
+هي
+قد
+بين
+أو
+عند
+بعد
+كل
+ذلك
+تلك
+حتى
+إذا
+ثم
+لم
+لن

data/lib/keyword_ruby/stop_words/de.txt

@@ -0,0 +1,39 @@
+der
+die
+das
+und
+ist
+ein
+eine
+in
+den
+von
+zu
+mit
+auf
+für
+nicht
+sich
+des
+dem
+als
+auch
+es
+an
+er
+so
+dass
+aus
+bei
+nach
+wie
+über
+hat
+oder
+noch
+aber
+um
+wenn
+kann
+nur
+werden

data/lib/keyword_ruby/stop_words/es.txt

@@ -0,0 +1,37 @@
+el
+la
+los
+las
+de
+del
+en
+un
+una
+que
+es
+por
+con
+no
+para
+se
+al
+lo
+su
+como
+más
+pero
+sus
+le
+ya
+fue
+este
+ha
+sí
+porque
+esta
+entre
+cuando
+muy
+sin
+sobre
+también

data/lib/keyword_ruby/stop_words/fr.txt

@@ -0,0 +1,44 @@
+le
+la
+les
+de
+des
+du
+un
+une
+et
+est
+en
+que
+qui
+dans
+ce
+il
+ne
+sur
+se
+pas
+plus
+par
+je
+avec
+tout
+faire
+son
+au
+mais
+nous
+ont
+cette
+ou
+été
+aussi
+leur
+bien
+peut
+même
+ces
+quand
+entre
+notre
+après

data/lib/keyword_ruby/stop_words/ja.txt

@@ -0,0 +1,40 @@
+の
+に
+は
+を
+た
+が
+で
+て
+と
+し
+れ
+さ
+ある
+いる
+も
+する
+から
+な
+こと
+として
+い
+や
+れる
+など
+なっ
+ない
+この
+ため
+その
+あっ
+よう
+また
+もの
+という
+あり
+まで
+られ
+なる
+へ
+か

data/lib/keyword_ruby/stop_words/ms.txt

@@ -0,0 +1,29 @@
+yang
+dan
+di
+ini
+itu
+dengan
+untuk
+dari
+adalah
+pada
+tidak
+dalam
+akan
+telah
+ke
+oleh
+ada
+juga
+saya
+mereka
+sudah
+boleh
+kami
+kita
+semua
+antara
+lebih
+atas
+bagi

data/lib/keyword_ruby/stop_words/nl.txt

@@ -0,0 +1,29 @@
+de
+het
+een
+van
+in
+en
+is
+dat
+op
+te
+voor
+met
+zijn
+er
+niet
+ook
+maar
+door
+als
+aan
+bij
+of
+om
+nog
+uit
+dan
+naar
+over
+tot

data/lib/keyword_ruby/stop_words/pt.txt

@@ -0,0 +1,36 @@
+de
+que
+não
+do
+da
+em
+um
+para
+com
+uma
+os
+no
+se
+na
+por
+mais
+as
+dos
+como
+mas
+ao
+ele
+das
+tem
+seu
+sua
+ou
+quando
+muito
+nos
+já
+também
+só
+pelo
+pela
+até

data/lib/keyword_ruby/text_processing/stop_words.rb

@@ -5,9 +5,11 @@ module KeywordRuby
     class StopWords
       STOP_WORDS_DIR = File.join(__dir__, "..", "stop_words")
 
-      def initialize(language: :en)
+      def initialize(language: :en, strict: false)
         @language = language
+        @strict = strict
         @words = load_stop_words
+        add_custom_stop_words
       end
 
       def stop_word?(word)
@@ -18,14 +20,26 @@ module KeywordRuby
         words.reject { |w| stop_word?(w) }
       end
 
+      def add(words)
+        words.each { |w| @words.add(w.downcase) }
+      end
+
       private
 
       def load_stop_words
         path = File.join(STOP_WORDS_DIR, "#{@language}.txt")
+        if !File.exist?(path) && @strict
+          raise KeywordRuby::Error, "Stop word file not found for language: #{@language}"
+        end
         return Set.new unless File.exist?(path)
 
         Set.new(File.readlines(path, chomp: true).map(&:downcase).reject(&:empty?))
       end
+
+      def add_custom_stop_words
+        custom = KeywordRuby.configuration.custom_stop_words
+        custom.each { |w| @words.add(w.downcase) } if custom
+      end
     end
   end
 end

data/lib/keyword_ruby/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module KeywordRuby
-  VERSION = "0.1.1"
+  VERSION = "0.2.0"
 end

data/lib/keyword_ruby.rb

@@ -5,15 +5,17 @@ require_relative "keyword_ruby/configuration"
 require_relative "keyword_ruby/keyword"
 require_relative "keyword_ruby/text_processing/stop_words"
 require_relative "keyword_ruby/text_processing/tokenizer"
-require_relative "keyword_ruby/text_processing/sentence_splitter"
 require_relative "keyword_ruby/extractors/base"
 require_relative "keyword_ruby/extractors/rake"
 require_relative "keyword_ruby/extractors/yake"
 require_relative "keyword_ruby/extractors/tfidf"
+require_relative "keyword_ruby/extractors/textrank"
 
 module KeywordRuby
   class Error < StandardError; end
 
+  ALGORITHMS = %i[rake yake tfidf textrank].freeze
+
   class << self
     def configuration
       @configuration ||= Configuration.new
@@ -29,19 +31,33 @@ module KeywordRuby
 
     def extract(text, algorithm: nil, language: nil, top_n: nil, normalize: true)
       algo = algorithm || configuration.default_algorithm
+      build_extractor(algo, language: language, top_n: top_n, normalize: normalize).extract(text)
+    end
 
-      extractor = case algo
-                  when :rake then Extractors::Rake.new(language: language, top_n: top_n, normalize: normalize)
-                  when :yake then Extractors::Yake.new(language: language, top_n: top_n, normalize: normalize)
-                  when :tfidf then Extractors::Tfidf.new(language: language, top_n: top_n, normalize: normalize)
-                  else raise ArgumentError, "Unknown algorithm: #{algo}. Supported: :rake, :yake, :tfidf"
-                  end
+    def extract_batch(documents, algorithm: nil, language: nil, top_n: nil, normalize: true)
+      algo = algorithm || configuration.default_algorithm
 
-      extractor.extract(text)
+      if algo == :tfidf
+        # TF-IDF benefits from shared corpus state
+        extractor = build_extractor(algo, language: language, top_n: top_n, normalize: normalize)
+        extractor.fit(documents)
+        documents.map { |doc| extractor.extract(doc) }
+      else
+        extractor = build_extractor(algo, language: language, top_n: top_n, normalize: normalize)
+        documents.map { |doc| extractor.extract(doc) }
+      end
     end
 
-    def extract_batch(documents, **opts)
-      documents.map { |doc| extract(doc, **opts) }
+    private
+
+    def build_extractor(algo, **opts)
+      case algo
+      when :rake then Extractors::Rake.new(**opts)
+      when :yake then Extractors::Yake.new(**opts)
+      when :tfidf then Extractors::Tfidf.new(**opts)
+      when :textrank then Extractors::TextRank.new(**opts)
+      else raise ArgumentError, "Unknown algorithm: #{algo}. Supported: #{ALGORITHMS.join(', ')}"
+      end
     end
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: keyword-ruby
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.2.0
 platform: ruby
 authors:
 - Johannes Dwi Cahyo
@@ -54,11 +54,20 @@ files:
 - lib/keyword_ruby/configuration.rb
 - lib/keyword_ruby/extractors/base.rb
 - lib/keyword_ruby/extractors/rake.rb
+- lib/keyword_ruby/extractors/textrank.rb
 - lib/keyword_ruby/extractors/tfidf.rb
 - lib/keyword_ruby/extractors/yake.rb
 - lib/keyword_ruby/keyword.rb
+- lib/keyword_ruby/stop_words/ar.txt
+- lib/keyword_ruby/stop_words/de.txt
 - lib/keyword_ruby/stop_words/en.txt
+- lib/keyword_ruby/stop_words/es.txt
+- lib/keyword_ruby/stop_words/fr.txt
 - lib/keyword_ruby/stop_words/id.txt
+- lib/keyword_ruby/stop_words/ja.txt
+- lib/keyword_ruby/stop_words/ms.txt
+- lib/keyword_ruby/stop_words/nl.txt
+- lib/keyword_ruby/stop_words/pt.txt
 - lib/keyword_ruby/text_processing/sentence_splitter.rb
 - lib/keyword_ruby/text_processing/stop_words.rb
 - lib/keyword_ruby/text_processing/tokenizer.rb