keyphrase 0.1.0 → 0.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b93bac15e3ec17af2a33f9f4c92ffc127e6963acf520d7ef253599652662c8ac
-  data.tar.gz: 55e9a1e431414ff360707bc5ba5319978bce1293a4eb9c83941baa3b9051f0e9
+  metadata.gz: 1d9478f7e5c47a826e90eae00884034ff4078262ce6b46a8f6c0a2070824b3c7
+  data.tar.gz: f14706384868b463ce8b968dbb4b6dd7e37bd6214d1d02b68ceb5d629930dec9
 SHA512:
-  metadata.gz: db4181abd7629d99df0be7855dc20d4bf9cc64d1299d56973f474ab48abe940ec121063572843fc566631b32a0a69bda205152b817e8bc92f0ff8b6ce3101d91
-  data.tar.gz: f2be99d0eca63fa493d2347c3121d2e817829ed3bc19dc62f15aaccc376dc1dca25c378d4001b259fe63a117cf4a74c4c701fc084a361f1114a0281d95643ce8
+  metadata.gz: 776507b887f30458a163f15999fcd0c57543b32c7acb8d604372180cc87f740a7c48d240311a9d5dbb9b0bdb0f46c5d02985332d622e10c85b2250bfd8f3af00
+  data.tar.gz: d408f218b406c45fe47c9315633aeb0fbe364cd5a3d597f2b77433bb3dc74289bfe8b90e5c5a187117d677b236f47763478fea4857c5619ce99cfb7f03946f65

data/lib/keyphrase/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 class Keyphrase
-  VERSION = "0.1.0"
+  VERSION = "0.1.1"
 end

data/lib/keyphrase.rb

@@ -6,9 +6,10 @@ class Keyphrase
 
   autoload :Stoplist, "keyphrase/stoplist"
 
-  CLEAN_REGEX = /([^a-zA-Z0-9'\- \.]|(?<!\w)'|(?<!\w)\.)/
-  BLACKLIST_REGEX = /(?:^|\s)[^a-zA-Z\-\']+\b/
-  SENTENCES_REGEX = /[+!?,;:&\[\]\{\}\<\>\=\/\n\t\\"\\(\\)\u2019\u2013\|]|-(?!\w)|(?<!\w)'(?!\w)|(?<!\s)\.(?![a-zA-Z0-9])|(?<!\w)\#(?=\w)/u
+  CLEAN_REGEX = /([^a-zA-Z0-9\'\- \.]|(?<!\w)\.)/ # don't remove ' because it might be part of a stop word
+  BLACKLIST_REGEX = /(?:^|\s)[^a-zA-Z\-]+\b|\'/ # remove words with no letters, ie 123.23.12. And last chance to remove '
+  CLEAN_SPACES_REGEX = /\s+/
+  SENTENCES_REGEX = /[+!?,;:&\[\]\{\}\<\>\=\/\n\t\\"\\(\\)\u2019\u2013\|]|-(?!\w)|'(?=s)|(?<!\s)\.(?![a-zA-Z0-9])|(?<!\w)\#(?=\w)/u
 
   def self.analyse text, options={}
     @@keyphrase ||= Keyphrase.new
@@ -23,10 +24,11 @@ class Keyphrase
     sort = options[:sort] || true
     blacklist = options[:blacklist] || BLACKLIST_REGEX
     sentences_regex = options[:sentences_regex] || SENTENCES_REGEX
+    clean_spaces_regex = options[:clean_spaces_regex] || CLEAN_SPACES_REGEX
 
     pattern    = buildStopwordRegExPattern stoplist, lang
     sentences  = text.split sentences_regex
-    phrases    = generateCandidateKeywords sentences, pattern, clean_regex, blacklist
+    phrases    = generateCandidateKeywords sentences, pattern, clean_regex, blacklist, clean_spaces_regex
     wordscores = calculateWordScores phrases
     candidates = generateCandidateKeywordScores phrases, wordscores, position_bonus
 
@@ -61,14 +63,14 @@ class Keyphrase
 
   # generate candidate keywords
   # 2
-  def generateCandidateKeywords sentences, stopwords_regex, clean_regex, blacklist
+  def generateCandidateKeywords sentences, stopwords_regex, clean_regex, blacklist, clean_spaces_regex
     phrases = Array.new
 
     filtered_sentences = sentences.map { |sentence| sentence.gsub(clean_regex, " ").gsub(stopwords_regex, "|") }
 
     filtered_sentences.each do |parts|
       parts.split("|").each do |part|
-        part = part.gsub(blacklist, " ").strip
+        part = part.gsub(blacklist, " ").gsub(clean_spaces_regex, " ").strip
 
         if !part.empty?
           phrases.push part

metadata

@@ -1,15 +1,29 @@
 --- !ruby/object:Gem::Specification
 name: keyphrase
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.1
 platform: ruby
 authors:
 - Ben D'Angelo
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2023-12-03 00:00:00.000000000 Z
-dependencies: []
+date: 2023-12-06 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.2'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.2'
 description: Implementation of the Rapid Automatic Keyword Extraction (RAKE) algorithm
   in Ruby. Forked from the original rake_text gem.
 email: