flashtext 0.1.0 → 0.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: b9bc0b315226f3f55d68f38f6997ae8be4ad3377
-  data.tar.gz: 9293854acfd5611be5a54c2257927bd53c8ac19f
+  metadata.gz: d8ca3508b7355433ef41fc40eead394cdc2149d2
+  data.tar.gz: 3715210544ee9170cf9c559ab10193b016e63615
 SHA512:
-  metadata.gz: ae6751f2f37175f1271ad6f3aea385213522df7db12b977ae2e03f246d36e06cca88deb3e82ec952528a3110632f97ea41673656d4a4fa565a0e39fc6b40ece1
-  data.tar.gz: 53badf293130a98770d67fc5b20d86bb0830607602559cf77a3943f40f2f1b9b907ca6f340cdfde74c6ce8fcad992a9b7f229e0b69eafc339684bae0cb355e41
+  metadata.gz: 2af1340ae2088f1327f689a0fc9c6036758f85571c3fbe6045b89593d2e5a45df8c5d2bcf705c51aa8fa69aad24504f71edd604454fd7cccfa485a31a605669f
+  data.tar.gz: 0daa3bc37c6820204950c02f6766d2e11ab3850ebfaca8abd75582aa0453a0a2f6df59d2bdf19bd83e4108597f48caec8cb7c224cfc4a5295a106a57c21ac735

data/Gemfile.lock

@@ -0,0 +1,37 @@
+PATH
+  remote: .
+  specs:
+    flashtext (0.1.0)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    diff-lcs (1.3)
+    json (2.1.0)
+    rake (10.5.0)
+    rspec (3.7.0)
+      rspec-core (~> 3.7.0)
+      rspec-expectations (~> 3.7.0)
+      rspec-mocks (~> 3.7.0)
+    rspec-core (3.7.0)
+      rspec-support (~> 3.7.0)
+    rspec-expectations (3.7.0)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.7.0)
+    rspec-mocks (3.7.0)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.7.0)
+    rspec-support (3.7.0)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  bundler (~> 1.16)
+  flashtext!
+  json (~> 2.0)
+  rake (~> 10.0)
+  rspec (~> 3.0)
+
+BUNDLED WITH
+   1.16.0

data/README.md

@@ -1,2 +1,35 @@
-# flashtext
-Ruby Gem for FlashText algorithm
+# Flashtext Ruby Gem
+
+### Ruby (zero dependencies) gem for amazing Python package [flashtext](https://github.com/vi3k6i5/flashtext)
+
+This module can be used to replace keywords in sentences or extract keywords from sentences. It is based on the [FlashText algorithm](https://arxiv.org/abs/1711.00046)
+
+More about Flashtext algorithm can be found here.
+
+The original paper published on [FlashText algorithm](https://arxiv.org/abs/1711.00046>)
+
+The article published on [Medium freeCodeCamp](https://medium.freecodecamp.org/regex-was-taking-5-days-flashtext-does-it-in-15-minutes-55f04411025f)
+
+
+Installation
+------------
+    $ gem install flashtext
+
+
+API doc
+-------
+
+Documentation can be found at [FlashText Read the Docs](http://www.rubydoc.info/gems/flashtext/)
+
+## Usage
+
+Contribute
+----------
+
+- Issue Tracker: https://github.com/imran3180/flashtext/issues
+- Source Code: https://github.com/imran3180/flashtext/issues
+
+
+## License
+
+This code is under MIT license.

data/flashtext.gemspec

@@ -33,4 +33,5 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "bundler", "~> 1.16"
   spec.add_development_dependency "rake", "~> 10.0"
   spec.add_development_dependency "rspec", "~> 3.0"
+  spec.add_development_dependency "json", "~> 2.0"
 end

data/lib/flashtext.rb

@@ -1,5 +1,4 @@
 require "flashtext/version"
-
+require "flashtext/keyword_processor"
 module Flashtext
-  # Your code goes here...
 end

data/lib/flashtext/keyword_processor.rb

@@ -0,0 +1,134 @@
+module Flashtext
+  class KeywordProcessor
+    require 'set'
+
+    attr_accessor :_keyword, :_white_space_chars, :keyword_trie_hash, :case_sensitive, :word_boundaries
+
+    def initialize case_sensitive = false
+      self._keyword = '_keyword_'
+      self._white_space_chars = Set.new(['.', '\t', '\n', '\a', ' ', ','])
+      self.keyword_trie_hash = {}
+      self.case_sensitive = case_sensitive
+      self.word_boundaries = Set.new("0".upto("9").to_a + "A".upto("Z").to_a + "a".upto("z").to_a + ["_"])
+    end
+
+    def add_keyword keyword, clean_name = nil
+      if not clean_name and keyword
+        clean_name = keyword
+      end
+
+      if keyword and clean_name
+        keyword = keyword.downcase if not case_sensitive
+        current_hash = keyword_trie_hash
+        keyword.each_char do |char|
+          current_hash =  if current_hash.has_key?(char)
+                            current_hash[char]
+                          else
+                            current_hash[char] = {}
+                            current_hash[char]
+                          end
+        end
+        current_hash[_keyword] = clean_name
+      end
+    end
+
+    def add_keywords_from_hash keyword_hash
+      raise ArgumentError, "#{keyword_hash} is not hash. argument expected: Hash" unless keyword_hash.instance_of?(Hash)
+      keyword_hash.each do |clean_name, keywords|
+        raise ArgumentError, "#{keyword_hash['clean_name']} is not array. expected: Array" unless keywords.instance_of?(Array)
+        keywords.each do |keyword|
+          add_keyword(keyword.to_s, clean_name.to_s)
+        end
+      end
+    end
+
+    def extract_keywords sentence
+      keywords_extracted = []
+      keywords_extracted if not sentence #if sentence is empty or none just return empty list
+      sentence = sentence.downcase if not case_sensitive
+      current_hash = keyword_trie_hash
+
+      sequence_end_pos = 0
+      idx = 0
+      sentence_len = sentence.length
+
+      while idx < sentence_len
+        char = sentence[idx]
+        # when we reach a character that might denote word end
+        if not word_boundaries.member?(char)
+          # If end is present OR ?? (confused)
+          if current_hash.has_key?(_keyword) or current_hash.has_key?(char)
+            # Update longest sequence found
+            sequence_found = nil
+            longest_sequence_found = nil
+            is_longer_seq_found = false
+
+            if current_hash.has_key?(_keyword)
+              sequence_found = current_hash[_keyword]
+              longest_sequence_found = current_hash[_keyword]
+              sequence_end_pos = idx
+            end
+
+            # re look for longest_sequence from this position
+            if current_hash.has_key?(char)
+              current_hash_continued = current_hash[char]
+
+              idy = idx + 1
+              while idy < sentence_len
+                inner_char = sentence[idy]
+                if not word_boundaries.member?(inner_char) and current_hash_continued.has_key?(_keyword)
+                  # update longest sequence found. This will keep updating longest_sequence if exists.
+                  longest_sequence_found = current_hash_continued[_keyword]
+                  sequence_end_pos = idy
+                  is_longer_seq_found = true
+                end
+                if current_hash_continued.has_key?(inner_char)
+                  current_hash_continued = current_hash_continued[inner_char]
+                else
+                  break
+                end
+                idy = idy + 1
+              end
+              # checked for end of sentenance
+              if idy == sentence_len and current_hash_continued.has_key?(_keyword)
+                # Update longest sequence found
+                longest_sequence_found = current_hash_continued[_keyword]
+                sequence_end_pos = idy
+                is_longer_seq_found = true
+              end
+              idx = sequence_end_pos if is_longer_seq_found
+            end
+            current_hash = keyword_trie_hash # reset
+            if longest_sequence_found
+              keywords_extracted << longest_sequence_found
+            end
+          else
+            # reset current_hash
+            current_hash = keyword_trie_hash
+          end
+        elsif current_hash.has_key?(char)
+          # we can continue from this char
+          current_hash = current_hash[char]
+        else
+          # we reset current_hash
+          current_hash = keyword_trie_hash
+          # skip to end of keyword
+          while idx < sentence_len
+            char = sentence[idx]
+            break if not word_boundaries.member?(char)
+            idx = idx + 1
+          end
+        end
+        # if we are end of sentence and have a sequence discovered
+        if idx + 1 >= sentence_len
+          if current_hash.has_key?(_keyword)
+            sequence_found = current_hash[_keyword]
+            keywords_extracted << sequence_found
+          end
+        end
+        idx = idx + 1 # loop increment.
+      end
+      keywords_extracted
+    end
+  end
+end

data/lib/flashtext/version.rb

@@ -1,3 +1,3 @@
 module Flashtext
-  VERSION = "0.1.0"
+  VERSION = "0.1.1"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: flashtext
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.1
 platform: ruby
 authors:
 - Imran
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2017-11-16 00:00:00.000000000 Z
+date: 2017-11-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -52,6 +52,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: json
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
 description: Ruby implementation of FlashText Algorithm
 email:
 - imranjannatiitkgp@gmail.com
@@ -64,6 +78,7 @@ files:
 - ".travis.yml"
 - CODE_OF_CONDUCT.md
 - Gemfile
+- Gemfile.lock
 - LICENSE.txt
 - README.md
 - Rakefile
@@ -71,6 +86,7 @@ files:
 - bin/setup
 - flashtext.gemspec
 - lib/flashtext.rb
+- lib/flashtext/keyword_processor.rb
 - lib/flashtext/version.rb
 homepage: https://github.com/imran3180/flashtext
 licenses: