soulheart 0.1.4 → 0.2.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 0a9835278571fb659e0eae5ad1d39918bcd41049
-  data.tar.gz: e3dbbeecea70b9cf11327f012f3d47e02f64f04b
+  metadata.gz: 92cb6b53b32cdadf8fe407f940ee941f0c9cc0e4
+  data.tar.gz: 1af49cbdf332eade831a421fa3b47079e130516a
 SHA512:
-  metadata.gz: 01215919be332be2cc67d9885c5bb21380110fa04caf4a424e91f7649aacf696d08c88f24c5a95b0bf88b8329915c93bf1c37976ee3f95b0e96c1cfb7fd2449b
-  data.tar.gz: e5b336efcc86b4250cecdb4d61c8427c831dca3eea2ba25f8bf2e362f2cd7ea00612951da3c1160f9f0eb1f9a0e62833b37aab69d20a7507bfafb3ccfbb105bc
+  metadata.gz: 90287822a0fdb40a18ff3abbbdac18548e7f029886a11dfacad86e5ce6b57f10238d7504e6e5de68199a7a09117071c6f1e1788028dce56dda5ffff3edad40e6
+  data.tar.gz: 24940196b5f5d04374d9e068ef0fdb621dd62559fc01750529651b8b6512a48f9e7ac5cfa7ec03c728c14f997b3418198dd96d8115a8af45339da5e81f127157

data/bin/soulheart

@@ -11,6 +11,13 @@ require 'soulheart'
 require 'optparse'
 require 'tempfile'
 
+@batch_size = 1000
+@no_all = false
+@no_combinatorial = false
+@normalize_regex = false
+@normalize_no_sym = false
+@remove_results = false
+
 parser = OptionParser.new do |opts|
   opts.banner = 'Usage: soulheart [options] COMMAND'
 
@@ -21,54 +28,68 @@ parser = OptionParser.new do |opts|
     Soulheart.redis = host
   end
 
-  opts.on('-s', '--stop-words [FILE]', 'Path to file containing a list of stop words to overwrite defaults - "the", "at", "vs"') do |fn|
-    File.open(fn) do |file|
-      Soulheart.stop_words = file.readlines.map(&:strip).reject(&:empty?)
-    end
-  end
-
   opts.on('-h', '--help', 'Show this message') do
     puts opts
     exit
   end
 
-  opts.on('-A', '--no-all', 'on load: Do not add items into the "all" category (on load)') do |size|
-    NO_ALL = true
+  opts.separator ''
+  opts.separator '  load options:'
+
+  opts.on('-A', '--no-all', 'Do not add items into the "all" category') do |size|
+    @no_all = true
   end
 
-  opts.on('-C', '--no-combinatorial', 'Do not create combined categories, do not add items to combined categories (on load)') do |size|
-    NO_COMBINATORIAL = true
+  opts.on('-C', '--no-combinatorial', 'Do not create combined categories, do not add items to combined categories') do |size|
+    @no_combinatorial = true
   end
 
-  opts.on('-R', '--remove-results', 'Remove results data - breaks the cache, fully clears all loaded data. (on clear)') do |size|
-    REMOVE_RESULTS = true
+  opts.separator ''
+  opts.separator '  normalize options:'
+
+  opts.on('-s', '--with-symbols', 'Do not remove symbols when normalizing terms') do |size|
+    @normalize_no_sym = true
+  end
+
+  opts.on('-x', '--regex', 'Use the first line from the FILE as the regular expression for normalizing terms') do |size|
+    @normalize_regex = true
+  end
+
+  opts.separator ''
+  opts.separator '  clear options:'
+
+  opts.on('-R', '--remove-results', 'Remove results data - breaks the cache, fully clears all loaded data') do |size|
+    @remove_results = true
   end
 
   opts.separator ''
   opts.separator 'Commands:'
-  opts.separator '  load   FILE  Loads data from a FILE - can be a local file or a url. Accepted formats are .json, .tsv and .csv'
-  opts.separator "  reset        Removes all existing data (optionally pass a file to run load after reset)"
+  opts.separator '  load        FILE   Loads data from a FILE - can be a local file or a url. Accepted formats are .json, .tsv and .csv'
+  opts.separator "  stop-words  FILE   Load file containing a list of stop words to overwrite defaults - 'the', 'at' and 'vs'"
+  opts.separator "  normalize          Set the way that terms are normalized. Requires a file when including the --regex option"
+  opts.separator "  clear              Removes existing items and categories from the database"
   opts.separator ''
   opts.separator 'Additional info: https://sethherr.github.io/soulheart/commands/'
   opts.separator ''
 end
 
-
-def load(file)
+def open_file(file)
   require 'uri'
   if file =~ URI.regexp
     require 'open-uri'
-    f = open(file)
+    open(file)
   elsif File.exist?(file)
-    f = File.open(file)
+    File.open(file)
   else
-    puts "Couldn't open file: #{file}"
-    return true
+    raise StandardError, "Couldn't open file: #{file}"
   end
+end
 
+def load(file)
+  f = open_file(file)
   start_time = Time.now.to_i
   count = 0
-  loader = Soulheart::Loader.new({no_all: NO_ALL, no_combinatorial: NO_COMBINATORIAL})
+  loader = Soulheart::Loader.new({no_all: @no_all, no_combinatorial: @no_combinatorial})
   lines = []
   begin
     if file.match(/(c|t)sv\z/i)
@@ -81,10 +102,10 @@ def load(file)
       end
     elsif file.match(/json\z/i)
       puts 'Reading JSON'
-      puts "Loading items in batches of #{BATCH_SIZE} ..."
+      puts "Loading items in batches of #{@batch_size} ..."
       until f.eof?
         lines = []
-        BATCH_SIZE.times do
+        @batch_size.times do
           break if f.eof?
           lines << MultiJson.decode(f.gets)
           count += 1
@@ -100,20 +121,37 @@ def load(file)
   puts "Total time to load:                                  #{Time.now.to_i - start_time} second(s)"
 end
 
-def clear(remove_results)
-  Soulheart::Loader.new.clear(remove_results)
+def stop_words(file)
+  f = open_file(file)
+  Soulheart.stop_words = f.readlines.map(&:strip).reject(&:empty?)
 end
 
-parser.parse!
+def normalize(file=nil)
+  if @normalize_regex
+    f = open_file(file) 
+    puts f.readlines.map(&:strip).reject(&:empty?).first
+    Soulheart.normalizer = f.readlines.map(&:strip).reject(&:empty?).first
+  elsif @normalize_no_sym
+    Soulheart.normalizer = ''
+  else
+    Soulheart.normalizer = Soulheart.default_normalizer
+  end
+end
 
+def clear
+  Soulheart::Loader.new.clear(@remove_results)
+end
+
+parser.parse!
 case ARGV[0]
 when 'load'
-  BATCH_SIZE ||= 1000
-  NO_ALL ||= false
-  NO_COMBINED_CATEGORIES ||= false
   load ARGV[1]
-when 'reset'
-  clear(REMOVE_RESULTS)
+when 'stop-words'
+  stop_words ARGV[1]
+when 'normalize'
+  ARGV[1] ? normalize(ARGV[1]) : normalize
+when 'clear'
+  clear
   load ARGV[1] if ARGV[1]
 else
   puts parser.help

data/lib/soulheart/base.rb

@@ -12,10 +12,6 @@ module Soulheart
       10 * 60 # Setting to 10 minutes, but making it possible to edit down the line
     end
 
-    def base_id
-      ENV['RACK_ENV'] != 'test' ? 'soulheart:' : 'soulheart_test:'
-    end
-
     def sorted_category_array
       redis.smembers(categories_id).map { |c| normalize(c) }.uniq.sort
     end
@@ -39,7 +35,7 @@ module Soulheart
     end
 
     def category_combos_id
-      "#{base_id}category_combos:"
+      "#{Soulheart.base_id}category_combos:"
     end
 
     def category_combos
@@ -47,7 +43,7 @@ module Soulheart
     end
 
     def categories_id
-      "#{base_id}categories:"
+      "#{Soulheart.base_id}categories:"
     end
 
     def hidden_categories_id
@@ -63,11 +59,16 @@ module Soulheart
     end
 
     def results_hashes_id
-      "#{base_id}database:"
+      "#{Soulheart.base_id}database:"
     end
 
+    def normalize_type_id
+      "#{Soulheart.base_id}normalize:"
+    end
+
+
     def cache_id(type = 'all')
-      "#{base_id}cache:#{type}:"
+      "#{Soulheart.base_id}cache:#{type}:"
     end
   end
 end

data/lib/soulheart/config.rb

@@ -3,7 +3,7 @@ require 'redis'
 
 module Soulheart
   module Config
-    DEFAULT_STOP_WORDS = %w(vs at the)
+    
 
     # Accepts:
     #   1. A Redis URL String 'redis://host:port/db'
@@ -37,12 +37,54 @@ module Soulheart
       )
     end
 
+    def base_id
+      ENV['RACK_ENV'] != 'test' ? 'soulheart:' : 'soulheart_test:'
+    end
+
+    def stop_words_id
+      "#{base_id}stop_list:"
+    end
+
+    def default_stop_words
+      %w(vs at the)
+    end
+
+    def redis_stop_words
+      return false unless redis.exists stop_words_id
+      redis.lrange(stop_words_id, 0, -1) 
+    end
+
     def stop_words
-      @stop_words ||= DEFAULT_STOP_WORDS
+      @stop_words ||= redis_stop_words || default_stop_words
     end
 
     def stop_words=(arr)
+      redis.expire stop_words_id, 0
       @stop_words = Array(arr).flatten
+      redis.lpush stop_words_id, @stop_words
+    end
+
+    def normalizer_id
+      "#{base_id}normalizer:"
+    end
+
+    def default_normalizer
+      '[^\p{Word}\ ]'
+    end
+
+    def redis_normalizer
+      return false unless redis.exists normalizer_id
+      redis.get normalizer_id
+    end
+
+    def normalizer
+      @normalizer ||= redis_normalizer || default_normalizer
+    end
+
+    def normalizer=(str)
+      redis.expire normalizer_id, 0
+      @normalizer = str
+      redis.set normalizer_id, @normalizer
     end
   end
 end

data/lib/soulheart/helpers.rb

@@ -3,7 +3,7 @@ module Soulheart
   module Helpers
     def normalize(str)
       # Letter, Mark, Number, Connector_Punctuation (Chinese, Japanese, etc.)
-      str.downcase.gsub(/[^\p{Word}\ ]/i, '').strip
+      str.downcase.gsub(/#{Soulheart.normalizer}/i, '').strip
     end
 
     def prefixes_for_phrase(phrase)

data/lib/soulheart/loader.rb

@@ -39,9 +39,9 @@ module Soulheart
       redis.sadd categories_id, categories
     end
 
-    def delete_data(id="#{base_id}:")
+    def delete_data(id="#{Soulheart.base_id}:")
       # delete the sorted sets for this type
-      phrases = redis.smembers(base_id)
+      phrases = redis.smembers(Soulheart.base_id)
       redis.pipelined do
         phrases.each do |p|
           redis.del("#{id}#{p}")
@@ -72,10 +72,10 @@ module Soulheart
     end
 
     def load(items)
-      # Replace with item return so we know we have category_id
+      Soulheart.stop_words # Load stop words so we don't pipeline redis_stop_words accidentally
       i = 0
       items.each do |item|
-        item.replace(add_item(item))
+        item.replace(add_item(item)) # Replace with item return so we know we have category_id
         i += 1
       end
       set_category_combos_array.each do |category_combo|
@@ -84,10 +84,10 @@ module Soulheart
             next
           elsif category_combo == 'all'
             next if @no_all
-          elsif !category_combo.match(item['category']) 
-            next
           elsif @no_combinatorial
             next
+          elsif !category_combo.match(item['category']) 
+            next
           end
           add_item(item, category_id(category_combo), true) # send it base
           i += 1
@@ -114,7 +114,7 @@ module Soulheart
       item
     end
 
-    def add_item(item, category_base_id=nil, cleaned=false)
+    def add_item(item, category_base_id=nil, cleaned=false)      
       item = clean(item) unless cleaned
       category_base_id ||= category_id(item['category'])
       priority = (-item['priority'])
@@ -125,7 +125,7 @@ module Soulheart
         phrase = ([item['term']] + (item['aliases'] || [])).join(' ')
         # Store all the prefixes
         prefixes_for_phrase(phrase).each do |p|
-          redis.sadd(base_id, p) unless cleaned # remember prefix in a master set
+          redis.sadd(Soulheart.base_id, p) unless cleaned # remember prefix in a master set
           # store the normalized term in the index for each of the categories
           redis.zadd("#{category_base_id}#{p}", priority, item['term'])
         end

data/lib/soulheart/matcher.rb

@@ -61,10 +61,10 @@ module Soulheart
       redis.expire(@cachekey, cache_duration) # cache_duration is set in base.rb
     end
 
-    def matching_hashes(ids)
-      return [] unless ids.size > 0
-      results = redis.hmget(results_hashes_id, *ids)
-      results = results.reject(&:nil?) # handle cached results for ids which have since been deleted
+    def matching_hashes(terms)
+      return [] unless terms.size > 0
+      results = redis.hmget(results_hashes_id, *terms)
+      results = results.reject(&:nil?) # handle cached results for terms which have since been deleted
       results.map { |r| MultiJson.decode(r) }
     end
 
@@ -74,8 +74,8 @@ module Soulheart
       limit = @opts['per_page'].to_i + offset - 1
 
       limit = 0 if limit < 0
-      ids = redis.zrange(@cachekey, offset, limit) # Using 'ids', even though keys are now terms
-      matching_hashes(ids)      
+      terms = redis.zrange(@cachekey, offset, limit)
+      matching_hashes(terms)      
     end
   end
 end

data/lib/soulheart/server.rb

@@ -28,6 +28,8 @@ module Soulheart
         soulheart_version: Soulheart::VERSION,
         current_time: Time.now.utc.strftime('%H:%M:%S UTC'),
         redis_used_memory: info['used_memory_human'],
+        stop_words: Soulheart.stop_words,
+        normalizer: Soulheart.normalizer,
       })
     end
 

data/lib/soulheart/version.rb

@@ -1,3 +1,3 @@
 module Soulheart
-  VERSION = '0.1.4'
+  VERSION = '0.2.1'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: soulheart
 version: !ruby/object:Gem::Version
-  version: 0.1.4
+  version: 0.2.1
 platform: ruby
 authors:
 - Seth Herr
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-07-23 00:00:00.000000000 Z
+date: 2015-07-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: hiredis
@@ -108,20 +108,6 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
-  name: rubocop
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
 description: Simple, fast autocomplete server for Ruby and Rails
 email:
 - seth.william.herr@gmail.com
@@ -164,7 +150,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.8
+rubygems_version: 2.2.2
 signing_key: 
 specification_version: 4
 summary: Simple, fast autocomplete server for Ruby and Rails