keyphrase 0.1.3 → 0.2.0

data/lib/keyphrase/stoplist/yid.rb

@@ -0,0 +1,204 @@
+    class Keyphrase
+      module Stoplist
+        class Yid
+          def self.stopwords
+            @@stopwords ||= [
+ "אבל",
+    "או",
+    "אולי",
+    "אותה",
+    "אותו",
+    "אותי",
+    "אותך",
+    "אותם",
+    "אותן",
+    "אותנו",
+    "אז",
+    "אחר",
+    "אחרות",
+    "אחרי",
+    "אחריכן",
+    "אחרים",
+    "אחרת",
+    "אי",
+    "איזה",
+    "איך",
+    "אין",
+    "איפה",
+    "איתה",
+    "איתו",
+    "איתי",
+    "איתך",
+    "איתכם",
+    "איתכן",
+    "איתם",
+    "איתן",
+    "איתנו",
+    "אך",
+    "אל",
+    "אלה",
+    "אלו",
+    "אם",
+    "אנחנו",
+    "אני",
+    "אס",
+    "אף",
+    "אצל",
+    "אשר",
+    "את",
+    "אתה",
+    "אתכם",
+    "אתכן",
+    "אתם",
+    "אתן",
+    "באיזומידה",
+    "באמצע",
+    "באמצעות",
+    "בגלל",
+    "בין",
+    "בלי",
+    "במידה",
+    "במקוםשבו",
+    "ברם",
+    "בשביל",
+    "בשעהש",
+    "בתוך",
+    "גם",
+    "דרך",
+    "הוא",
+    "היא",
+    "היה",
+    "היכן",
+    "היתה",
+    "היתי",
+    "הם",
+    "הן",
+    "הנה",
+    "הסיבהשבגללה",
+    "הרי",
+    "ואילו",
+    "ואת",
+    "זאת",
+    "זה",
+    "זות",
+    "יהיה",
+    "יוכל",
+    "יוכלו",
+    "יותרמדי",
+    "יכול",
+    "יכולה",
+    "יכולות",
+    "יכולים",
+    "יכל",
+    "יכלה",
+    "יכלו",
+    "יש",
+    "כאן",
+    "כאשר",
+    "כולם",
+    "כולן",
+    "כזה",
+    "כי",
+    "כיצד",
+    "כך",
+    "ככה",
+    "כל",
+    "כלל",
+    "כמו",
+    "כן",
+    "כפי",
+    "כש",
+    "לא",
+    "לאו",
+    "לאיזותכלית",
+    "לאן",
+    "לבין",
+    "לה",
+    "להיות",
+    "להם",
+    "להן",
+    "לו",
+    "לי",
+    "לכם",
+    "לכן",
+    "למה",
+    "למטה",
+    "למעלה",
+    "למקוםשבו",
+    "למרות",
+    "לנו",
+    "לעבר",
+    "לעיכן",
+    "לפיכך",
+    "לפני",
+    "מאד",
+    "מאחורי",
+    "מאיזוסיבה",
+    "מאין",
+    "מאיפה",
+    "מבלי",
+    "מבעד",
+    "מדוע",
+    "מה",
+    "מהיכן",
+    "מול",
+    "מחוץ",
+    "מי",
+    "מכאן",
+    "מכיוון",
+    "מלבד",
+    "מן",
+    "מנין",
+    "מסוגל",
+    "מעט",
+    "מעטים",
+    "מעל",
+    "מצד",
+    "מקוםבו",
+    "מתחת",
+    "מתי",
+    "נגד",
+    "נגר",
+    "נו",
+    "עד",
+    "עז",
+    "על",
+    "עלי",
+    "עליה",
+    "עליהם",
+    "עליהן",
+    "עליו",
+    "עליך",
+    "עליכם",
+    "עלינו",
+    "עם",
+    "עצמה",
+    "עצמהם",
+    "עצמהן",
+    "עצמו",
+    "עצמי",
+    "עצמם",
+    "עצמן",
+    "עצמנו",
+    "פה",
+    "רק",
+    "שוב",
+    "של",
+    "שלה",
+    "שלהם",
+    "שלהן",
+    "שלו",
+    "שלי",
+    "שלך",
+    "שלכה",
+    "שלכם",
+    "שלכן",
+    "שלנו",
+    "שם",
+    "תהיה",
+    "תחת",
+            ]
+          end
+        end
+      end
+    end

data/lib/keyphrase/stoplist/zul.rb

@@ -0,0 +1,39 @@
+    class Keyphrase
+      module Stoplist
+        class Zul
+          def self.stopwords
+            @@stopwords ||= [
+  "futhi",
+    "kahle",
+    "kakhulu",
+    "kanye",
+    "khona",
+    "kodwa",
+    "kungani",
+    "kusho",
+    "la",
+    "lakhe",
+    "lapho",
+    "mina",
+    "ngesikhathi",
+    "nje",
+    "phansi",
+    "phezulu",
+    "u",
+    "ukuba",
+    "ukuthi",
+    "ukuze",
+    "uma",
+    "wahamba",
+    "wakhe",
+    "wami",
+    "wase",
+    "wathi",
+    "yakhe",
+    "zakhe",
+    "zonke",
+            ]
+          end
+        end
+      end
+    end

data/lib/keyphrase/stoplist.rb

@@ -1,21 +1,24 @@
 module Keyphrase::Stoplist
+  class << self
+    # Class variable to store filenames
+    @@file_names = []
+
+    # Method to retrieve the array of filenames
+    def languages
+      @@file_names
+    end
+  end
+
   # Dynamically require all files in the stoplist directory
   Dir[File.join(__dir__, 'stoplist', '*.rb')].each do |file|
     require_relative file
+    @@file_names << File.basename(file, '.rb').to_sym
   end
 
-  def self.stopwords lang, type=:smart
+  def self.stopwords_for_lang lang
     cl = const_get(lang.to_s.capitalize)
 
-    if type == :strict
-      cl.strict
-    else
-      cl.smart
-    end
-  end
-
-  def self.stoplist_classes
-    constants.map { |const|  }
+    cl.stopwords
   end
 
 end

data/lib/keyphrase/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 class Keyphrase
-  VERSION = "0.1.3"
+  VERSION = "0.2.0"
 end

data/lib/keyphrase.rb

@@ -6,8 +6,8 @@ class Keyphrase
 
   autoload :Stoplist, "keyphrase/stoplist"
 
-  CLEAN_REGEX = /([^a-zA-Z0-9\'\- \.]|(?<!\w)\.)/ # don't remove ' because it might be part of a stop word
-  BLACKLIST_REGEX = /(?:^|\s)[^a-zA-Z]+\b|\'|\-/ # remove words with no letters, ie 123.23.12. And last chance to remove ' and -
+  CLEAN_REGEX = /([^\p{L}a-zA-Z0-9\'\- \.]|(?<!\w)\.)/ # don't remove ' because it might be part of a stop word
+  BLACKLIST_REGEX = /(?:^|\s)[^a-zA-Z\p{L}]+\b|\'|\-/ # remove words with no letters, ie 123.23.12. And last chance to remove ' and -
   CLEAN_SPACES_REGEX = /\s+/
   SENTENCES_REGEX = /[+!?,;:&\[\]\{\}\<\>\=\/\n\t\\"\\(\\)\u2019\u2013\|]|-(?!\w)|'(?=s)|(?<!\s)\.(?![a-zA-Z0-9])|(?<!\w)\#(?=\w)/u
 
@@ -16,8 +16,12 @@ class Keyphrase
     @@keyphrase.analyse text, options
   end
 
+  def initialize
+    @cached_regex = {}
+  end
+
   def analyse text, options={}
-    stoplist = options[:stoplist] || :smart
+    stopwords = options[:stopwords]
     lang = options[:lang] || :eng
     clean_regex = options[:clean] || CLEAN_REGEX
     position_bonus = options[:position_bonus] || true
@@ -26,7 +30,7 @@ class Keyphrase
     sentences_regex = options[:sentences_regex] || SENTENCES_REGEX
     clean_spaces_regex = options[:clean_spaces_regex] || CLEAN_SPACES_REGEX
 
-    pattern    = buildStopwordRegExPattern stoplist, lang
+    pattern    = buildStopwordRegExPattern lang, stopwords
     sentences  = text.split sentences_regex
     phrases    = generateCandidateKeywords sentences, pattern, clean_regex, blacklist, clean_spaces_regex
     wordscores = calculateWordScores phrases
@@ -49,16 +53,20 @@ class Keyphrase
 
   # create stopword pattern
   # 1
-  def buildStopwordRegExPattern stopwords, lang
-
-    if stopwords.is_a? Symbol
-      # use caching
-      return Keyphrase::Stoplist.stopwords lang, stopwords
+  def buildStopwordRegExPattern lang, stopwords = nil
+    stopwords ||= Keyphrase::Stoplist.stopwords_for_lang lang
+
+    # Check if the regex for the given language and stopwords is already cached
+    if @cached_regex[lang].nil? || @cached_regex[lang][:stopwords] != stopwords
+      # If not cached or stopwords have changed, recompile the regex and store in the cache
+      @cached_regex[lang] = {
+        stopwords: stopwords,
+        regex: Regexp.new("(?:^|\\s)(?:#{stopwords.join('|')})(?:$|\\s)", Regexp::IGNORECASE | Regexp::MULTILINE)
+      }
     end
 
-    stop_regex = /(?:^|\s)(?:#{stopwords.join('|')})(?:$|\s)/io
-
-    return stop_regex
+    # Return the cached regex
+    @cached_regex[lang][:regex]
   end
 
   # generate candidate keywords

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: keyphrase
 version: !ruby/object:Gem::Version
-  version: 0.1.3
+  version: 0.2.0
 platform: ruby
 authors:
 - Ben D'Angelo
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2023-12-08 00:00:00.000000000 Z
+date: 2023-12-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rspec
@@ -38,7 +38,75 @@ files:
 - Rakefile
 - lib/keyphrase.rb
 - lib/keyphrase/stoplist.rb
+- lib/keyphrase/stoplist/afr.rb
+- lib/keyphrase/stoplist/aka.rb
+- lib/keyphrase/stoplist/amh.rb
+- lib/keyphrase/stoplist/ara.rb
+- lib/keyphrase/stoplist/aze.rb
+- lib/keyphrase/stoplist/bel.rb
+- lib/keyphrase/stoplist/ben.rb
+- lib/keyphrase/stoplist/bul.rb
+- lib/keyphrase/stoplist/cat.rb
+- lib/keyphrase/stoplist/ces.rb
+- lib/keyphrase/stoplist/cmn.rb
+- lib/keyphrase/stoplist/dan.rb
+- lib/keyphrase/stoplist/deu.rb
+- lib/keyphrase/stoplist/ell.rb
 - lib/keyphrase/stoplist/eng.rb
+- lib/keyphrase/stoplist/epo.rb
+- lib/keyphrase/stoplist/est.rb
+- lib/keyphrase/stoplist/fin.rb
+- lib/keyphrase/stoplist/fra.rb
+- lib/keyphrase/stoplist/guj.rb
+- lib/keyphrase/stoplist/heb.rb
+- lib/keyphrase/stoplist/hin.rb
+- lib/keyphrase/stoplist/hrv.rb
+- lib/keyphrase/stoplist/hun.rb
+- lib/keyphrase/stoplist/hye.rb
+- lib/keyphrase/stoplist/ind.rb
+- lib/keyphrase/stoplist/ita.rb
+- lib/keyphrase/stoplist/jav.rb
+- lib/keyphrase/stoplist/jpn.rb
+- lib/keyphrase/stoplist/kan.rb
+- lib/keyphrase/stoplist/kat.rb
+- lib/keyphrase/stoplist/khm.rb
+- lib/keyphrase/stoplist/kor.rb
+- lib/keyphrase/stoplist/lat.rb
+- lib/keyphrase/stoplist/lav.rb
+- lib/keyphrase/stoplist/lit.rb
+- lib/keyphrase/stoplist/mal.rb
+- lib/keyphrase/stoplist/mar.rb
+- lib/keyphrase/stoplist/mkd.rb
+- lib/keyphrase/stoplist/mya.rb
+- lib/keyphrase/stoplist/nep.rb
+- lib/keyphrase/stoplist/nld.rb
+- lib/keyphrase/stoplist/nob.rb
+- lib/keyphrase/stoplist/ori.rb
+- lib/keyphrase/stoplist/pan.rb
+- lib/keyphrase/stoplist/pes.rb
+- lib/keyphrase/stoplist/pol.rb
+- lib/keyphrase/stoplist/por.rb
+- lib/keyphrase/stoplist/ron.rb
+- lib/keyphrase/stoplist/rus.rb
+- lib/keyphrase/stoplist/sin.rb
+- lib/keyphrase/stoplist/slk.rb
+- lib/keyphrase/stoplist/slv.rb
+- lib/keyphrase/stoplist/sna.rb
+- lib/keyphrase/stoplist/spa.rb
+- lib/keyphrase/stoplist/srp.rb
+- lib/keyphrase/stoplist/swe.rb
+- lib/keyphrase/stoplist/tam.rb
+- lib/keyphrase/stoplist/tel.rb
+- lib/keyphrase/stoplist/tgl.rb
+- lib/keyphrase/stoplist/tha.rb
+- lib/keyphrase/stoplist/tuk.rb
+- lib/keyphrase/stoplist/tur.rb
+- lib/keyphrase/stoplist/ukr.rb
+- lib/keyphrase/stoplist/urd.rb
+- lib/keyphrase/stoplist/uzb.rb
+- lib/keyphrase/stoplist/vie.rb
+- lib/keyphrase/stoplist/yid.rb
+- lib/keyphrase/stoplist/zul.rb
 - lib/keyphrase/version.rb
 - sig/keyphrase.rbs
 homepage: https://github.com/bendangelo/keyphrase
@@ -62,7 +130,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.4.22
+rubygems_version: 3.5.3
 signing_key: 
 specification_version: 4
 summary: Extracts keywords from texts using a stoplist and some magic.