soulheart 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 701b63e892265647a1615c790e81ab3e4132bb0a
+  data.tar.gz: baa1a417ca1bedf0efff4cd795a5f0f6679c60e8
+SHA512:
+  metadata.gz: 37fdeb72ef3cb8179b4aafc1eb9860e8d96a469a7fe866b51160004fb8298da877496b22d46df5537f0b4f10dd35fb927720c1d3420b7a489179c0a3537b5c46
+  data.tar.gz: 8580e37cbd146a1b03d15dbb4ddd913ed1e30c4d4c08c79f7fa4b7b6b46cc63013c0bd47a4902216235491fc9c3b61e5a41fd6b08cb47ee9f8dff0d5ee744a86

data/.gitignore

@@ -0,0 +1,21 @@
+# rdoc generated
+rdoc
+
+# yard generated
+doc
+.yardoc
+
+# bundler
+.bundle
+
+gem 
+
+test/db/*.rdb
+
+dump.rdb
+tmp/
+
+Gemfile.lock
+
+.rvmrc
+.ruby-*

data/.rspec

@@ -0,0 +1,2 @@
+--color
+--format documentation

data/.travis.yml

@@ -0,0 +1,12 @@
+language: ruby
+rvm:
+- 2.1.0
+bundler_args: "--without=guard"
+notifications:
+  disabled: true
+script:
+- bundle exec rake spec
+# - bundle exec rubocop
+addons:
+  code_climate:
+    repo_token: e72d8fa152922cef05c311cd49d3db9016b82486b712399a8e7c7da2af5e071e

data/Gemfile

@@ -0,0 +1,22 @@
+source 'http://rubygems.org'
+
+group :development do
+  gem 'rspec',      '~> 2.14.1'
+  gem 'bundler'
+  gem 'guard'
+  gem 'guard-rspec', '~> 4.2.8'
+  gem 'rack-contrib'
+  gem 'rubocop'
+end
+
+group :test do 
+  gem 'rack-test'
+  gem "codeclimate-test-reporter", require: nil
+end
+
+gem 'rake'
+gem 'redis',      '>= 3.0.1'
+gem 'hiredis',    '~> 0.4.5'
+gem 'vegas',      '>= 0.1.0'
+gem 'sinatra'
+gem 'multi_json', '>= 1.11.0'

data/Guardfile

@@ -0,0 +1,9 @@
+rspec_opts = {
+  failed_mode: :focus
+}
+
+guard :rspec, cmd: "bundle exec rspec" do
+  watch(%r{^spec/.+_spec\.rb$})
+  watch(%r{^lib/(.+)\.rb$})     { |m| "spec/#{m[1]}_spec.rb" }
+  watch(%r{^lib/soulheart/(.+)\.rb$})     { |m| "spec/soulheart/#{m[1]}_spec.rb" }
+end

data/LICENSE.md

@@ -0,0 +1,20 @@
+Copyright (c) 2011-2015 Eric Waller, Seth Herr, and Soulheart contributors
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.markdown

@@ -0,0 +1,203 @@
+Soulheart [![Build Status](https://travis-ci.org/sethherr/soulheart.svg)](https://travis-ci.org/sethherr/soulheart) [![Code Climate](https://codeclimate.com/github/sethherr/soulheart/badges/gpa.svg)](https://codeclimate.com/github/sethherr/soulheart) [![Test Coverage](https://codeclimate.com/github/sethherr/soulheart/badges/coverage.svg)](https://codeclimate.com/github/sethherr/soulheart/coverage)
+========
+
+This is an updated fork of [Seatgeek/Soulmate](https://github.com/seatgeek/soulmate) to address a few issues - namely [CORS support](../../issues/2), [minimum entry length](../../issues/3) and [playing better with Selectize & Select2](../../issues/4) - also the future.
+
+Since [Seatgeek no longer uses Soulmate](https://news.ycombinator.com/item?id=9317891), and this is a pretty large rewrite that doesn't emphasize backward compatibility, it's a new project and gem.
+
+========
+
+**Every item has to have a unique name**
+
+Loading data: 
+
+Pushing new options via rake task. Use a gist url! Do it with .csvs!
+
+Structuring data
+
+The only required attribute is term, which is the search term to go by. So this would be a valid thing:
+
+```javascript
+[
+  { "term": "Something sweet" },
+  { "term": "The color blue" }
+]
+```
+
+Here are all the possible (non-required) values:
+
+
+| Key         | Type     | Default    |
+| ----------- | -----    | ---------- |
+| `priority`  | integer  | 100        |
+| `types`     | string   | 'default'  |
+| `data`      | hash     | {}         |
+
+e.g.
+
+```javascript
+[
+  {
+    "term": "Something sweet",
+    "priority": 99,
+    "types": "niceness", // A set type for searching by type
+    "data": { // Any key value pair you want, this is the return value
+      "id": 99,
+      "url": "http://example.com",
+      "whatever": "The color purple"
+    }
+  },
+  { "term": "The color blue" }
+]
+```
+
+*If you set `term` in `data`, it will respond with that rather than the term it searches by. I haven't figured out a use case for this yet, but I'm sure one exists.*
+
+======
+
+I'm testing with:
+
+  - ruby >= 2.1
+  - redis >= 3
+
+To test, run `bundle exec guard`. It will live reload the files you change. 
+
+
+======
+
+:x::o::x::o::x::o: *Soulmate's README follows ([issue for making new documentation](../../issues/1))*
+
+Soulmate is a tool to help solve the common problem of developing a fast autocomplete feature. It uses Redis's sorted sets to build an index of partially completed words and the corresponding top matching items, and provides a simple sinatra app to query them. Soulmate finishes your sentences.
+
+Soulmate was designed to be simple and fast, and offers the following:
+
+ * Provide suggestions for multiple types of items in a single query (at SeatGeek we're autocompleting for performers, events, and venues)
+ * Results are ordered by a user-specified score
+ * Arbitrary metadata for each item (at SeatGeek we're storing both a url and a subtitle)
+
+An item is a simple JSON object that looks like:
+
+    {
+      "id": 3,
+      "term": "Citi Field",
+      "score": 81,
+      "data": {
+        "url": "/citi-field-tickets/",
+        "subtitle": "Flushing, NY"
+      }
+    }
+
+Where `id` is a unique identifier (within the specific type), `term` is the phrase you wish to provide completions for, `score` is a user-specified ranking metric (redis will order things lexicographically for items with the same score), and `data` is an optional container for metadata you'd like to return when this item is matched (at SeatGeek we're including a url for the item as well as a subtitle for when we present it in an autocomplete dropdown).
+
+Getting Started
+---------------
+
+As always, kick things off with a `gem install`:
+
+    gem install soulmate
+
+### Loading Items
+
+You can load data into Soulmate by piping items in the JSON lines format into `soulmate load TYPE`.
+
+Here's a sample `venues.json` (one JSON item per line):
+
+    {"id":1,"term":"Dodger Stadium","score":85,"data":{"url":"\/dodger-stadium-tickets\/","subtitle":"Los Angeles, CA"}}
+    {"id":28,"term":"Angel Stadium","score":85,"data":{"url":"\/angel-stadium-tickets\/","subtitle":"Anaheim, CA"}}
+    {"id":30,"term":"Chase Field ","score":85,"data":{"url":"\/chase-field-tickets\/","subtitle":"Phoenix, AZ"}}
+    {"id":29,"term":"Sun Life Stadium","score":84,"data":{"url":"\/sun-life-stadium-tickets\/","subtitle":"Miami, FL"}}
+    {"id":2,"term":"Turner Field","score":83,"data":{"url":"\/turner-field-tickets\/","subtitle":"Atlanta, GA"}}
+
+And here's the load command (Soulmate assumes redis is running locally on the default port, or you can specify a redis connection string with the `--redis` argument):
+
+    $ soulmate load venue --redis=redis://localhost:6379/0 < venues.json
+
+You can also provide an array of strings under the `aliases` key that will also be added to the index for this item.
+
+### Querying for Data
+
+Once it's loaded, we can query this data by starting `soulmate-web`:
+
+    $ soulmate-web --foreground --no-launch --redis=redis://localhost:6379/0
+
+And viewing the service in your browser: http://localhost:5678/search?types[]=venue&term=stad. You should see something like:
+
+    {
+      "term": "stad",
+      "results": {
+        "venue": [
+          {
+            "id": 28,
+            "term": "Angel Stadium",
+            "score": 85,
+            "data": {
+              "url": "/angel-stadium-tickets/",
+              "subtitle": "Anaheim, CA"
+            }
+          },
+          {
+            "id": 1,
+            "term": "Dodger Stadium",
+            "score": 85,
+            "data": {
+              "url": "/dodger-stadium-tickets/",
+              "subtitle": "Los Angeles, CA"
+            }
+          },
+          {
+            "id": 29,
+            "term": "Sun Life Stadium",
+            "score": 84,
+            "data": {
+              "url": "/sun-life-stadium-tickets/",
+              "subtitle": "Miami, FL"
+            }
+          }
+        ]
+      }
+    }
+
+The `/search` method supports multiple `types` as well as an optional `limit`. For example: `http://localhost:5678/search?types[]=event&types[]=venue&types[]=performer&limit=3&term=yank`. You can also add the `callback` parameter to enable JSONP output.
+
+### Mounting soulmate into a rails app
+
+If you are integrating Soulmate into a rails app, an alternative to launching a separate 'soulmate-web' server is to mount the sinatra app inside of rails.
+
+Add this to routes.rb:
+
+    mount Soulmate::Server, :at => "/sm"
+
+Add this to gemfile:
+
+    gem 'rack-contrib'
+    gem 'soulmate', :require => 'soulmate/server'
+
+Then you can query soulmate at the /sm url, for example: http://localhost:3000/sm/search?types[]=venues&limit=6&term=kitten
+
+You can also config your redis instance:
+
+    # config/initializers/soulmate.rb
+    
+    Soulmate.redis = 'redis://127.0.0.1:6379/0'
+    # or you can asign an existing instance of Redis, Redis::Namespace, etc.
+    # Soulmate.redis = $redis
+
+### Rendering an autocompleter
+
+Soulmate doesn't include any client-side code necessary to render an autocompleter, but Mitch Crowe put together a pretty cool looking jquery plugin designed for exactly that: <a href="https://github.com/mcrowe/soulmate.js">soulmate.js</a>.
+
+Contributing to soulmate
+------------------------
+ 
+* Check out the latest master to make sure the feature hasn't been implemented or the bug hasn't been fixed yet
+* Check out the issue tracker to make sure someone already hasn't requested it and/or contributed it
+* Fork the project
+* Start a feature/bugfix branch
+* Commit and push until you are happy with your contribution
+* Please try not to mess with the Rakefile, version, or history. If you want to have your own version, or is otherwise necessary, that is fine, but please isolate to its own commit so I can cherry-pick around it.
+
+Copyright
+---------
+
+Copyright (c) 2011 Eric Waller. See LICENSE.txt for further details.
+

data/Rakefile

@@ -0,0 +1,56 @@
+#!/usr/bin/env rake
+
+require "bundler/gem_helper"
+
+begin
+  Bundler.setup(:default, :development)
+rescue Bundler::BundlerError => e
+  $stderr.puts e.message
+  $stderr.puts "Run `bundle install` to install missing gems"
+  exit e.status_code
+end
+require 'rake'
+
+REDIS_DIR = File.expand_path(File.join("..", "test"), __FILE__)
+REDIS_CNF = File.join(REDIS_DIR, "test.conf")
+REDIS_PID = File.join(REDIS_DIR, "db", "redis.pid")
+REDIS_LOCATION = ENV['REDIS_LOCATION']
+
+desc "Run rcov and manage server start/stop"
+task :rcoverage => [:start, :rcov, :stop]
+
+desc "Start the Redis server"
+task :start do
+  redis_running = \
+    begin
+      File.exists?(REDIS_PID) && Process.kill(0, File.read(REDIS_PID).to_i)
+    rescue Errno::ESRCH
+      FileUtils.rm REDIS_PID
+      false
+    end
+
+  if REDIS_LOCATION
+    system "#{REDIS_LOCATION}/redis-server #{REDIS_CNF}" unless redis_running
+  else
+    system "redis-server #{REDIS_CNF}" unless redis_running
+  end
+end
+
+desc "Stop the Redis server"
+task :stop do
+  if File.exists?(REDIS_PID)
+    Process.kill "INT", File.read(REDIS_PID).to_i
+    FileUtils.rm REDIS_PID
+  end
+end
+
+
+require "rspec/core/rake_task"
+
+desc "Run all specs"
+RSpec::Core::RakeTask.new(:spec) do |t|
+  t.rspec_opts = %w(--color)
+  t.verbose = false
+end
+
+task :default => :spec

data/bin/soulheart

@@ -0,0 +1,181 @@
+#!/usr/bin/env ruby
+
+$LOAD_PATH.unshift File.expand_path(File.dirname(__FILE__) + '/../lib')
+begin
+  require 'redis'
+rescue LoadError
+  require 'rubygems'
+  require 'redis'
+end
+require 'soulheart'
+require 'optparse'
+require 'tempfile'
+
+parser = OptionParser.new do |opts|
+  opts.banner = "Usage: soulheart [options] COMMAND"
+
+  opts.separator ""
+  opts.separator "Options:"
+
+  opts.on("-r", "--redis [HOST:PORT]", "Redis connection string") do |host|
+    Soulheart.redis = host
+  end
+
+  opts.on("-s", "--stop-words [FILE]", "Path to file containing a list of stop words") do |fn|
+    File.open(fn) do |file|
+      Soulheart.stop_words = file.readlines.map{ |l| l.strip }.reject{ |w| w.empty? }
+    end
+  end
+
+  opts.on("-h", "--help", "Show this message") do
+    puts opts
+    exit
+  end
+
+  opts.on("-b", "--batch-size", "Number of lines to read at a time") do |size|
+    BATCH_SIZE = size
+  end
+
+  opts.separator ""
+  opts.separator "Commands:"
+  opts.separator "  load   TYPE FILE  Replaces collection specified by TYPE with items read from FILE in the JSON lines format."
+  opts.separator "  add    TYPE       Adds items to collection specified by TYPE read from stdin in the JSON lines format."
+  opts.separator "  remove TYPE       Removes items from collection specified by TYPE read from stdin in the JSON lines format. Items only require an 'id', all other fields are ignored."
+  opts.separator "  query  TYPE QUERY Queries for items from collection specified by TYPE."
+end
+
+def generate(type, file)
+  include Soulheart::Helpers
+
+  begin
+    temp = Tempfile.new("soulheart")
+
+    if File.exists?(file)
+      start_time = Time.now.to_i
+      base = "soulheart-index:#{type}"
+      database = "soulheart-data:#{type}"
+      # hset = "*4\r\n$4\r\nHSET\r\n$#{database.length}\r\n#{database}\r\n$"
+      # del = "*2\r\n$3\r\nDEL\r\n$"
+      begin
+        f = File.open(file)
+        # cleanup
+        phrases = Soulheart.redis.smembers(base)
+        phrases.each do |phrase|
+          temp << gen_redis_proto("DEL", phrase)
+          # temp << del + phrase.length.to_s + "\r\n" + phrase + "\r\n"
+        end
+        temp << gen_redis_proto("DEL", base)
+        # temp << del + base.length.to_s + "\r\n" + base + "\r\n"
+        while !f.eof?
+          line = f.gets.chomp
+          line =~ /"id":(\d+)/
+          id = $1
+          line =~ /"score":(\d+)/
+          score = $1
+          json = MultiJson.decode(line)
+          temp << gen_redis_proto("HSET", database, id, line)
+          # temp << hset + $1.length.to_s + "\r\n" + $1 + "\r\n$" + line.length.to_s + "\r\n" + line + "\r\n"
+          phrase = json.key?("aliases") ? json["term"] + " " + json["aliases"] : json["term"]
+          prefixes_for_phrase(phrase).each do |p|
+            temp << gen_redis_proto("SADD", base, p)
+            temp << gen_redis_proto("ZADD", base + ":" + p, score, id)
+          end
+        end
+      ensure
+        f.close
+      end
+      puts "Converted in #{Time.now.to_i - start_time} second(s)"
+      puts "Importing into redis ..."
+      `time redis-cli --pipe < #{temp.path}`
+    else
+      puts "Couldn't open file: #{file}"
+    end
+  ensure
+    temp.close
+  end
+end
+
+def load(file)
+  if File.exists?(file)
+    start_time = Time.now.to_i
+    loader = Soulheart::Loader.new
+
+    puts "Loading items in batches of #{BATCH_SIZE} ..."
+    count = 0
+    begin
+      f = File.open(file)
+      while !f.eof?
+        lines = []
+        BATCH_SIZE.times do
+          break if f.eof?
+          lines << MultiJson.decode(f.gets)
+          count += 1
+        end
+        loader.load(lines)
+        puts "Loaded #{count} items ..." unless f.eof?
+      end
+    ensure
+      f.close
+    end
+    puts "Loaded a total of #{count} items in #{Time.now.to_i - start_time} second(s)"
+  else
+    puts "Couldn't open file: #{file}"
+  end
+end
+
+def add(type)
+  puts "Adding items of type #{type}..."
+  loader = Soulheart::Loader.new(type)
+  items = $stdin.read.split("\n").map { |l| MultiJson.decode(l) }
+  items.each do |item|
+    loader.add(item)
+  end
+  puts "Loaded a total of #{items.size} items"
+end
+
+def remove(type)
+  puts "Removing items of type #{type}..."
+  loader = Soulheart::Loader.new(type)
+  items = $stdin.read.split("\n").map { |l| MultiJson.decode(l) }
+  items.each do |item|
+    loader.remove(item)
+  end
+  puts "Removed a total of #{items.size} items"
+end
+
+def query(type, query)
+  puts "> Querying '#{type}' for '#{query}'"
+  matcher = Soulheart::Matcher.new(type)
+  results = matcher.matches_for_term(query, :limit => 0)
+  results.each do |item|
+    puts MultiJson.encode(item)
+  end
+  puts "> Found #{results.size} matches"
+end
+
+def gen_redis_proto(*cmd)
+  proto = "*"+cmd.length.to_s+"\r\n"
+  cmd.each{|arg|
+    proto << "$"+arg.bytesize.to_s+"\r\n"
+    proto << arg+"\r\n"
+  }
+  proto
+end
+
+parser.parse!
+BATCH_SIZE ||= 1000
+
+case ARGV[0]
+when 'generate'
+  generate ARGV[1], ARGV[2]
+when 'load'
+  load ARGV[1]
+when 'add'
+  add ARGV[1]
+when 'remove'
+  remove ARGV[1]
+when 'query'
+  query ARGV[1], ARGV[2]
+else
+  puts parser.help
+end