soulheart 0.0.1

data/bin/soulheart-web

@@ -0,0 +1,28 @@
+#!/usr/bin/env ruby
+
+$LOAD_PATH.unshift File.expand_path(File.dirname(__FILE__) + '/../lib')
+begin
+  require 'vegas'
+rescue LoadError
+  require 'rubygems'
+  require 'vegas'
+end  
+require 'soulheart/server'
+
+
+Vegas::Runner.new(Soulheart::Server, 'soulheart-web', {
+  :before_run => lambda {|v|
+    # path = (ENV['RESQUECONFIG'] || v.args.first)
+    # load path.to_s.strip if path
+  }
+}) do |runner, opts, app|
+  opts.on("-r", "--redis [HOST:PORT]", "Redis connection string") do |host|
+    runner.logger.info "Using Redis connection string '#{host}'"
+    Soulheart.redis = host
+  end
+  opts.on("-s", "--stop-words [FILE]", "Path to file containing a list of stop words") do |fn|
+    File.open(fn) do |file|
+      Soulheart.stop_words = file.readlines.map{ |l| l.strip }.reject{ |w| w.empty? }
+    end
+  end
+end

data/lib/soulheart.rb

@@ -0,0 +1,12 @@
+require 'multi_json'
+
+require 'soulheart/version'
+require 'soulheart/helpers'
+require 'soulheart/base'
+require 'soulheart/matcher'
+require 'soulheart/loader'
+require 'soulheart/config'
+
+module Soulheart
+  extend Config
+end

data/lib/soulheart/base.rb

@@ -0,0 +1,54 @@
+module Soulheart
+  
+  class Base
+    
+    include Helpers
+    
+    attr_accessor :type
+
+    def redis
+      Soulheart.redis
+    end
+
+    def cache_length
+      10 * 60 # Setting to 10 minutes, but making it possible to edit down the line
+    end
+    
+    def base_id
+      ENV['RACK_ENV'] != 'test' ? "soulheart:" : "soulheart_test:"
+    end
+
+    def set_category_combos_array
+      redis.expire category_combos_id, 0
+      ar = redis.smembers(categories_id).map{ |c| normalize(c) }.uniq.sort
+      ar = 1.upto(ar.size).flat_map {|n| ar.combination(n).map{|el| el.join('')}}
+      ar.last.replace('all')
+      redis.sadd category_combos_id, ar
+      ar
+    end
+
+    def category_combos_id
+      "#{base_id}category_combos:"
+    end
+
+    def categories_id
+      "#{base_id}categories:"
+    end
+
+    def category_id(name='all')
+      "#{categories_id}#{name}:"
+    end
+
+    def no_query_id(category=category_id)
+      "all:#{category}"
+    end
+
+    def results_hashes_id
+      "#{base_id}database:"
+    end
+
+    def cache_id(type='all')
+      "#{base_id}:cache:#{type}:"
+    end
+  end
+end

data/lib/soulheart/config.rb

@@ -0,0 +1,45 @@
+require 'uri'
+require 'redis'
+
+module Soulheart
+  module Config
+    DEFAULT_STOP_WORDS = ["vs", "at", "the"]
+
+    # Accepts:
+    #   1. A Redis URL String 'redis://host:port/db'
+    #   2. An existing instance of Redis, Redis::Namespace, etc.
+    def redis=(server)
+      if server.is_a?(String)
+        @redis = nil
+        @redis_url = server
+      else
+        @redis = server
+      end
+
+      redis
+    end
+
+    # Returns the current Redis connection. If none has been created, will
+    # create a new one.
+    def redis
+      @redis ||= (
+        url = URI(@redis_url || ENV["REDIS_URL"] || "redis://127.0.0.1:6379/0")
+        ::Redis.new({
+          # driver: :hiredis,
+          host: url.host,
+          port: url.port,
+          db: url.path[1..-1],
+          password: url.password
+        })
+      )
+    end
+
+    def stop_words
+      @stop_words ||= DEFAULT_STOP_WORDS
+    end
+
+    def stop_words=(arr)
+      @stop_words = Array(arr).flatten
+    end
+  end
+end

data/lib/soulheart/helpers.rb

@@ -0,0 +1,23 @@
+# coding: utf-8
+
+module Soulheart
+  module Helpers
+    def blank?
+      respond_to?(:empty?) ? !!empty? : !self
+    end
+
+    def normalize(str)
+      # Letter, Mark, Number, Connector_Punctuation (Chinese, Japanese, etc.)
+      str.downcase.gsub(/[^\p{Word}\ ]/i, '').strip
+    end
+    
+    def prefixes_for_phrase(phrase)
+      words = normalize(phrase).split(' ').reject do |w|
+        Soulheart.stop_words.include?(w)
+      end
+      words.map do |w|
+        (0..(w.length-1)).map{ |l| w[0..l] }
+      end.flatten.uniq
+    end
+  end
+end

data/lib/soulheart/loader.rb

@@ -0,0 +1,106 @@
+module Soulheart
+
+  class Loader < Base
+
+    def default_items_hash(text, category)
+      category ||= 'default'
+      {
+        'category' => normalize(category),
+        'priority' => 100,
+        'term' => normalize(text),
+        'aliases' => [],
+        'data' => {
+          'text' => text,
+          'category' => category
+        },
+      }
+    end
+
+    def delete_categories
+      redis.expire categories_id, 0
+    end
+
+    def reset_categories(categories)
+      delete_categories
+      redis.sadd categories_id, categories
+    end
+
+    def delete_data
+      # delete the sorted sets for this type
+      phrases = redis.smembers(base_id)
+      redis.pipelined do
+        phrases.each do |p|
+          redis.del("#{base_id}:#{p}")
+        end
+        redis.del(base_id)
+      end
+
+      # Redis can continue serving cached requests for this type while the reload is
+      # occuring. Some requests may be cached incorrectly as empty set (for requests
+      # which come in after the above delete, but before the loading completes). But
+      # everything will work itself out as soon as the cache expires again.
+
+      # delete the data stored for this type
+      redis.del(results_hashes_id)
+    end
+
+    def load(items)
+      delete_data
+      # Replace with item return so we know we have category_id
+      items.each { |item| item.replace(add_item(item)) }
+      set_category_combos_array.each do |category_combo|
+        items.each do |item|
+          next unless category_combo.match(item['category']) || category_combo == 'all'
+          add_item(item, category_id(category_combo), cleaned: true) # send it base
+        end
+      end
+    end
+
+    def clean(item)
+      raise ArgumentError, "Items must have text" unless item["text"]
+      default_items_hash(item.delete('text'), item.delete('category')).
+        tap{ |i| i['data'].merge!(item.delete('data')) if item['data'] }.
+        merge item
+    end
+
+    def add_item(item, category_base_id=nil, cleaned: false)
+      unless cleaned
+        item = clean(item)
+        category_base_id ||= category_id(item['category'])
+        unless redis.smembers(categories_id).include?(item['category'])
+          redis.sadd categories_id, item['category']
+        end
+      end
+      redis.pipelined do
+        redis.zadd(no_query_id(category_base_id), item["priority"], item["term"]) # Add to master set for queryless searches
+        # store the raw data in a separate key to reduce memory usage, if it's cleaned it's done
+        redis.hset(results_hashes_id, item['term'], MultiJson.encode(item['data'])) unless cleaned
+        phrase = ([item["term"]] + (item["aliases"] || [])).join(' ')
+        # Store all the prefixes
+        prefixes_for_phrase(phrase).each do |p|
+          redis.sadd(base_id, p) unless cleaned # remember prefix in a master set
+          # store the normalized term in the index for each of the categories
+          redis.zadd("#{category_base_id}#{p}", item["priority"], item["term"])
+        end
+      end
+      item
+    end
+
+    # remove only cares about an item's id, but for consistency takes an object
+    def remove(item)
+      prev_item = Soulheart.redis.hget(base_id, item["term"])
+      if prev_item
+        prev_item = MultiJson.decode(prev_item)
+        # undo the operations done in add
+        Soulheart.redis.pipelined do
+          Soulheart.redis.hdel(base_id, prev_item["term"])
+          phrase = ([prev_item["term"]] + (prev_item["aliases"] || [])).join(' ')
+          prefixes_for_phrase(phrase).each do |p|
+            Soulheart.redis.srem(base_id, p)
+            Soulheart.redis.zrem("#{base_id}:#{p}", prev_item["term"])
+          end
+        end
+      end
+    end
+  end
+end

data/lib/soulheart/matcher.rb

@@ -0,0 +1,69 @@
+module Soulheart
+
+  class Matcher < Base
+    def initialize(params={})
+      @opts = self.class.default_params_hash.merge params
+      clean_opts
+    end
+
+    def self.default_params_hash
+      {
+        'page' => 0,
+        'per_page' => 5,
+        'categories' => [],
+        'query' => '',
+        'cache' => true
+      }
+    end
+
+    def clean_opts
+      unless @opts['categories'] == '' || @opts['categories'] == []
+        @opts['categories'] = @opts['categories'].split(/,|\+/) unless @opts['categories'].kind_of?(Array)
+        @opts['categories'] = @opts['categories'].map{ |s| normalize(s) }.uniq.sort
+        @opts['categories'] = [] if @opts['categories'].length == redis.scard(categories_id)
+      end
+      @opts['query'] = normalize(@opts['query']).split(' ')
+      # .reject{ |i| i && i.length > 0 } .split(' ').reject{  Soulmate.stop_words.include?(w) }
+      @opts
+    end
+
+    def categories_string
+      @opts['categories'].empty? ? 'all' : @opts['categories'].join('')
+    end
+
+    def category_id_from_opts
+      category_id(categories_string)
+    end
+
+    def cache_id_from_opts
+      "#{cache_id(categories_string)}#{@opts['query'].join(':')}:"
+    end
+
+    def interkeys_from_opts(cid)
+      # If there isn't a query, we use a special key in redis
+      @opts['query'].empty? ? [no_query_id(cid)] : @opts['query'].map { |w| "#{cid}#{w}" }
+    end
+
+    def matches
+      cachekey = cache_id_from_opts
+      cid = category_id_from_opts
+
+      if !@opts['cache'] || !redis.exists(cachekey) || redis.exists(cachekey) == 0
+        interkeys = interkeys_from_opts(cid)
+        redis.zinterstore(cachekey, interkeys)
+        redis.expire(cachekey, cache_length) # cache_length is set in base.rb
+      end
+      page = @opts['page'].to_i
+      per_page = @opts['per_page'].to_i
+      ids = redis.zrevrange(cachekey, page*per_page, per_page-1) # Using 'ids', even though keys are now terms - because clarity?
+      if ids.size > 0
+        results = redis.hmget(results_hashes_id, *ids)
+        results = results.reject{ |r| r.nil? } # handle cached results for ids which have since been deleted
+        results.map { |r| MultiJson.decode(r) }
+      else
+        []
+      end      
+    end
+
+  end
+end

data/lib/soulheart/server.rb

@@ -0,0 +1,33 @@
+require 'sinatra/base'
+require 'soulheart'
+require 'rack/contrib'
+
+module Soulheart
+
+  class Server < Sinatra::Base
+    include Helpers
+    
+    before do
+      content_type 'application/json', :charset => 'utf-8'
+      headers['Access-Control-Allow-Origin'] = '*'
+      headers['Access-Control-Allow-Methods'] = 'POST, PUT, GET, OPTIONS'
+      headers['Access-Control-Request-Method'] = '*'
+      headers['Access-Control-Allow-Headers'] = 'Origin, X-Requested-With, Content-Type, Accept, Authorization'
+    end
+        
+    get '/' do
+      matches = Matcher.new(params).matches
+      MultiJson.encode({ results:  matches })
+    end
+
+    get '/status' do 
+      MultiJson.encode({ soulheart: Soulheart::VERSION, :status   => "ok" })
+    end
+    
+    not_found do
+      content_type 'application/json', :charset => 'utf-8'
+      MultiJson.encode({ :error => "not found" })
+    end
+    
+  end
+end

data/lib/soulheart/version.rb

@@ -0,0 +1,3 @@
+module Soulheart
+  VERSION = "0.0.1"
+end

data/soulheart.gemspec

@@ -0,0 +1,24 @@
+# -*- encoding: utf-8 -*-
+require File.expand_path('../lib/soulheart/version', __FILE__)
+
+Gem::Specification.new do |gem|
+  gem.version       = Soulheart::VERSION
+  gem.authors       = ["Seth Herr"]
+  gem.email         = ["seth.william.herr@gmail.com"]
+  gem.description   = gem.summary = "Simple, fast autocomplete server for Ruby and Rails"
+  gem.homepage      = "https://github.com/sethherr/soulheart"
+  gem.license       = "MIT"
+  gem.executables   = ["soulheart", "soulheart-web"]
+  gem.files         = `git ls-files | grep -Ev '^(test)'`.split("\n")
+  gem.name          = "soulheart"
+  gem.require_paths = ["lib"]
+  gem.add_dependency                  'hiredis', '~> 0.4.5'
+  gem.add_dependency                  'redis', '>= 3.0.6'
+  gem.add_dependency                  'vegas', '>= 0.1.0'
+  gem.add_dependency                  'json'
+  gem.add_dependency                  'sinatra'
+  gem.add_development_dependency      'rack-contrib'
+  gem.add_development_dependency      'rake'
+  gem.add_development_dependency      'rspec'
+  gem.add_development_dependency      'rubocop'
+end

data/spec/fixtures/multiple_categories.json

@@ -0,0 +1,15 @@
+{"text":"Steel","category":"Frame Materials" }
+{"text":"Brompton Bicycle","priority":51,"category":"Frame Manufacturer","data":{"id":8,"url":"http://www.brompton.com"}}
+{"text":"Jamis","priority":75,"category":"Frame Manufacturer","data":{"id":2222,"url":"http://jamisbikes.com"}}
+{"text":"Surly","priority":102,"category":"Frame Manufacturer"}
+{"text":"Jagwire","priority":40,"category":"Manufacturer"}
+{"text":"Jannd","priority":41,"category":"Manufacturer"}
+{"text":"Sram","priority":50,"category":"Manufacturer","data":{"id":8,"url":"http://sram.com"}}
+{"text":"Brooks England LTD.","priority":50,"category":"Manufacturer","data":{"id":200,"url":"http://www.brooksengland.com/"}}
+{"text":"Dodger Stadium","priority":84,"data":{"id":1,"url":"\/dodger-stadium-tickets\/","subtitle":"Los Angeles, CA"},"aliases":["Chavez Ravine"]}
+{"text":"Angel Stadium","priority":90,"data":{"id":28,"url":"\/angel-stadium-tickets\/","subtitle":"Anaheim, CA"},"aliases":["Edison International Field of Anaheim"]}
+{"text":"Chase Field ","priority":80,"data":{"id":30,"url":"\/chase-field-tickets\/","subtitle":"Phoenix, AZ"},"aliases":["Bank One Ballpark", "Bank One Stadium"]}
+{"text":"Sun Life Stadium","priority":75,"data":{"id":29,"url":"\/sun-life-stadium-tickets\/","subtitle":"Miami, FL"},"aliases":["Dolphins Stadium","Land Shark Stadium"]}
+{"text":"Turner Field","priority":50,"data":{"id":2,"url":"\/turner-field-tickets\/","subtitle":"Atlanta, GA"}}
+{"text":"Citi Field","priority":92,"data":{"id":3,"url":"\/citi-field-tickets\/","subtitle":"Atlanta, GA"},"aliases":["Shea Stadium"]}
+{"text":"中国佛山 李小龙","priority":94,"data":{"id":8,"url":"\/Bruce Lee\/","subtitle":"Chinese Foshan"},"aliases":["Li XiaoLong"]}

data/spec/soulheart/loader_spec.rb

@@ -0,0 +1,96 @@
+require 'spec_helper'
+
+describe Soulheart::Loader do
+
+  describe :clean_data do 
+    it "sets the default category, priority and normalizes term" do 
+      item = { 'text' => '  FooBar' }
+      result = Soulheart::Loader.new.clean(item)
+      expect(result['priority']).to eq(100)
+      expect(result['term']).to eq('foobar')
+      expect(result['category']).to eq('default')
+      expect(result['data']['text']).to eq('  FooBar')
+    end
+    
+    it "doesn't overwrite the submitted params" do 
+      item = {
+        'text' => 'Cool ',
+        'priority' => 50,
+        'category' => 'Gooble',
+        'data' => {
+          'id' => 199
+        }
+      }
+      result = Soulheart::Loader.new.clean(item)
+      expect(result['term']).to eq('cool')
+      expect(result['priority']).to eq(50)
+      expect(result['data']['text']).to eq('Cool ')
+      expect(result['data']['id']).to eq(199)
+      expect(result['category']).to eq('gooble')
+      expect(result['data']['category']).to eq('Gooble')
+    end
+
+    it "raises argument error if text is passed" do 
+      expect{
+        Soulheart::Loader.new.clean({'name' => 'stuff'})
+      }.to raise_error(/must have/i)
+    end
+  end
+
+  describe :add_item do 
+    it "adds an item, adds prefix scopes, adds category" do 
+      item = {
+        'text' => 'Brompton Bicycle',
+        'priority' => 50,
+        'category' => 'Gooble',
+        'data' => {
+          'id' => 199
+        }
+      }
+      loader = Soulheart::Loader.new
+      redis = loader.redis
+      redis.expire loader.results_hashes_id, 0
+      loader.add_item(item)
+      redis = loader.redis 
+      target = "{\"text\":\"Brompton Bicycle\",\"category\":\"Gooble\",\"id\":199}"
+      result = redis.hget(loader.results_hashes_id, 'brompton bicycle')
+      expect(result).to eq(target)
+      prefixed = redis.zrevrange "#{loader.category_id('gooble')}:brom", 0, -1
+      expect(prefixed[0]).to eq('brompton bicycle')
+      expect(redis.smembers(loader.categories_id).include?('gooble')).to be_true
+    end
+  end
+
+  describe :store_terms do 
+    it "stores terms by priority and adds categories for each possible category combination" do
+      items = []
+      file = File.read('spec/fixtures/multiple_categories.json')
+      file.each_line { |l| items << MultiJson.decode(l) }
+      loader = Soulheart::Loader.new
+      redis = loader.redis
+      loader.delete_categories
+      loader.load(items)
+      
+      cat_prefixed = redis.zrevrange "#{loader.category_id('frame manufacturermanufacturer')}:brom", 0, -1
+      expect(cat_prefixed.count).to eq(1)
+      expect(redis.smembers(loader.categories_id).count).to be > 3
+
+      prefixed = redis.zrevrange "#{loader.category_id('all')}:bro", 0, -1
+      expect(prefixed.count).to eq(2)
+      expect(prefixed[0]).to eq('brompton bicycle')
+    end
+
+    it "stores terms by priority and doesn't add run categories if none are present" do
+      items = [
+        {'text' => 'cool thing', 'category' => 'AWESOME'},
+        {'text' => 'Sweet', 'category' => ' awesome'}
+      ]
+      loader = Soulheart::Loader.new
+      redis = loader.redis
+      loader.delete_categories
+      loader.load(items)
+      expect(redis.smembers(loader.category_combos_id).count).to eq(1)
+    end
+  end
+
+end