autosuggest 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 305c68241ebeabdd7eca78f2bb1b4d3e7e1e430d
+  data.tar.gz: 797e7579646943ec7aa324f2f1c748d550914af0
+SHA512:
+  metadata.gz: d137ecf95fcdb4ab5271ad6be3f1cc4776411721a094093e84798a85938b91f92429cbb1090fdcb3bfc9e6e717b39a1197809d16dd79b81a56a53df88bb00bad
+  data.tar.gz: 52a264b4ebb28ef1786a3b9adfa6dff750f063d1eb92307113465a353becd64fd90e0a2b93f63cd81258a84dd998e8cf18e5204c0655c231c3bcf5ea92e1f342

data/.gitignore

@@ -0,0 +1,9 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/

data/Gemfile

@@ -0,0 +1,4 @@
+source "https://rubygems.org"
+
+# Specify your gem's dependencies in autosuggest.gemspec
+gemspec

data/README.md

@@ -0,0 +1,126 @@
+# Autosuggest
+
+Generate autocomplete suggestions based on what your users search
+
+:tangerine: Battle-tested at [Instacart](https://www.instacart.com/opensource)
+
+## How It Works
+
+#### Start with the most popular queries
+
+```ruby
+top_queries = Search.group("LOWER(query)")
+                    .having("COUNT(DISTINCT user_id) >= 5")
+                    .count("DISTINCT user_id")
+# {"bananas" => 353, "apples" => 213, ...
+
+autosuggest = Autosuggest.new(top_queries)
+```
+
+#### Filter duplicates
+
+[Stemming](https://en.wikipedia.org/wiki/Stemming) is used to detect duplicates like `apple` and `apples`.
+
+The most popular query is preferred by default.  To override this, use:
+
+```ruby
+autosuggest.prefer ["apples"]
+```
+
+To fix false positives, use:
+
+```ruby
+autosuggest.not_duplicates [["straws", "straus"]]
+```
+
+#### Filter misspellings
+
+We tried open-source libraries like [Aspell](http://aspell.net) and [Hunspell](http://hunspell.sourceforge.net/) but quickly realized we needed to build a corpus specific to our application.
+
+There are two ways to build the corpus, which can be used together.
+
+1. Add words
+
+  ```ruby
+  autosuggest.parse_words Product.pluck(:name)
+  ```
+
+  Use the `min` option to only add words that appear multiple times.
+
+2. Add concepts
+
+  ```ruby
+  autosuggest.add_concept "brand", Brand.pluck(:name)
+  ```
+
+#### Blacklist words
+
+[Profanity](https://github.com/tjackiw/obscenity/blob/master/config/blacklist.yml) is blacklisted by default.
+
+Add custom words with:
+
+```ruby
+autosuggest.blacklist_words ["boom"]
+```
+
+#### Profit
+
+Get suggestions with:
+
+```ruby
+autosuggest.suggestions
+```
+
+Filter queries without results and you’re set.
+
+We also prefer to have someone manually approve them by hand.
+
+## Full Example
+
+```ruby
+top_queries = Search.group("LOWER(query)")
+                    .having("COUNT(DISTINCT user_id) >= 5")
+                    .count("DISTINCT user_id")
+
+autosuggest = Autosuggest.new(top_queries)
+
+# create corpus with product names and brand names
+autosuggest.parse_words Product.pluck(:name)
+brand_names = Brand.pluck(:name)
+autosuggest.add_concept "brand", brand_names
+
+# prefer brand names
+autosuggest.prefer brand_names
+
+# prevent false positives for duplicates
+autosuggest.not_duplicates [["straws", "straus"]]
+
+# blacklist words
+autosuggest.blacklist_words ["boom"]
+
+# print suggestions
+puts autosuggest.pretty_suggestions
+# or
+p autosuggest.suggestions
+```
+
+## Installation
+
+Add this line to your application’s Gemfile:
+
+```ruby
+gem 'autosuggest'
+```
+
+## TODO
+
+- try Jaro-Winkler for duplicates
+
+## Contributing
+
+Everyone is encouraged to help improve this project. Here are a few ways you can help:
+
+- [Report bugs](https://github.com/ankane/autosuggest/issues)
+- Fix bugs and [submit pull requests](https://github.com/ankane/autosuggest/pulls)
+- Write, clarify, or fix documentation
+- Suggest or add new features

data/Rakefile

@@ -0,0 +1,8 @@
+require "bundler/gem_tasks"
+require "rake/testtask"
+
+task default: :test
+Rake::TestTask.new do |t|
+  t.libs << "test"
+  t.pattern = "test/**/*_test.rb"
+end

data/autosuggest.gemspec

@@ -0,0 +1,26 @@
+# coding: utf-8
+lib = File.expand_path("../lib", __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require "autosuggest/version"
+
+Gem::Specification.new do |spec|
+  spec.name          = "autosuggest"
+  spec.version       = Autosuggest::VERSION
+  spec.authors       = ["Andrew Kane"]
+  spec.email         = ["andrew@chartkick.com"]
+
+  spec.summary       = "Generate autocomplete suggestions based on what your users search"
+  spec.homepage      = "https://github.com/ankane/autosuggest"
+
+  spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency "ruby-stemmer"
+  spec.add_dependency "obscenity"
+
+  spec.add_development_dependency "bundler", "~> 1.7"
+  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "minitest"
+end

data/lib/autosuggest.rb

@@ -0,0 +1,202 @@
+require "autosuggest/version"
+require "lingua/stemmer"
+require "yaml" # for obscenity
+require "obscenity"
+
+class Autosuggest
+  def initialize(top_queries)
+    @top_queries = top_queries
+    @concepts = {}
+    @words = Set.new
+    @non_duplicates = Set.new
+    @blacklisted_words = Set.new
+    @preferred_queries = {}
+    @profane_words = Set.new(Obscenity::Base.blacklist)
+  end
+
+  def add_concept(name, values)
+    @concepts[name] = Set.new(values.compact.uniq.map(&:downcase))
+  end
+
+  def parse_words(phrases, options = {})
+    min = options[:min] || 1
+
+    word_counts = Hash.new(0)
+    phrases.each do |phrase|
+      words = tokenize(phrase)
+      words.each do |word|
+        word_counts[word] += 1
+      end
+    end
+
+    word_counts.select { |_, c| c >= min }.each do |word, _|
+      @words << word
+    end
+
+    word_counts
+  end
+
+  def not_duplicates(pairs)
+    pairs.each do |pair|
+      @non_duplicates << pair.map(&:downcase).sort
+    end
+  end
+
+  def blacklist_words(words)
+    words.each do |word|
+      @blacklisted_words << word.downcase
+    end
+  end
+
+  def prefer(queries)
+    queries.each do |query|
+      @preferred_queries[normalize_query(query)] ||= query
+    end
+  end
+
+  def suggestions
+    stemmed_queries = {}
+    added_queries = Set.new
+    @top_queries.sort_by { |_query, count| -count }.map do |query, count|
+      query = query.to_s
+
+      # TODO do not ignore silently
+      next if query.length < 2
+
+      stemmed_query = normalize_query(query)
+
+      # get preferred term
+      preferred_query = @preferred_queries[stemmed_query]
+      if preferred_query && preferred_query != query
+        original_query, query = query, preferred_query
+      end
+
+      # exclude duplicates
+      duplicate = stemmed_queries[stemmed_query]
+      stemmed_queries[stemmed_query] ||= query
+
+      # also detect possibly misspelled duplicates
+      # TODO use top query as duplicate
+      if !duplicate && query.length > 4
+        edits(query).each do |edited_query|
+          if added_queries.include?(edited_query)
+            duplicate = edited_query
+            break
+          end
+        end
+      end
+      if duplicate && @non_duplicates.include?([duplicate, query].sort)
+        duplicate = nil
+      end
+      added_queries << query unless duplicate
+
+      # find concepts
+      concepts = []
+      @concepts.each do |name, values|
+        concepts << name if values.include?(query)
+      end
+
+      # exclude misspellings that are not brands
+      misspelling = @words.any? && misspellings?(query)
+
+      profane = blacklisted?(query, @profane_words)
+
+      blacklisted = blacklisted?(query, @blacklisted_words)
+
+      notes = []
+      notes << "duplicate of #{duplicate}" if duplicate
+      notes.concat(concepts)
+      notes << "misspelling" if misspelling
+      notes << "profane" if profane
+      notes << "blacklisted" if blacklisted
+      notes << "originally #{original_query}" if original_query
+
+      {
+        query: query,
+        original_query: original_query,
+        score: count,
+        duplicate: duplicate,
+        concepts: concepts,
+        misspelling: misspelling,
+        profane: profane,
+        blacklisted: blacklisted,
+        notes: notes
+      }
+    end
+  end
+
+  def pretty_suggestions
+    str = "%-30s   %5s   %s\n" % %w(Query Score Notes)
+    suggestions.each do |suggestion|
+      str << "%-30s   %5d   %s\n" % [suggestion[:query], suggestion[:score], suggestion[:notes].join(", ")]
+    end
+    str
+  end
+
+  protected
+
+  def misspellings?(query)
+    recurse(tokenize(query)).each do |terms|
+      if terms.all? { |t| @concepts.any? { |_, values| values.include?(t) } || @words.include?(t) }
+        return false
+      end
+    end
+    true
+  end
+
+  def blacklisted?(query, blacklisted_words)
+    recurse(tokenize(query)).each do |terms|
+      return true if terms.any? { |t| blacklisted_words.include?(t) }
+    end
+    false
+  end
+
+  def recurse(words)
+    if words.size == 1
+      [words]
+    else
+      result = [[words.join(" ")]]
+      i = 0
+      while i < words.size - 1
+        recurse(words[0..i]).each do |v1|
+          recurse(words[i + 1..-1]).each do |v2|
+            result << v1 + v2
+          end
+        end
+        i += 1
+      end
+      result.uniq
+    end
+  end
+
+  def tokenize(str)
+    str.to_s.downcase.split(" ")
+  end
+
+  # from http://blog.lojic.com/2008/09/04/how-to-write-a-spelling-corrector-in-ruby/
+  LETTERS = ("a".."z").to_a.join + "'"
+  def edits(word)
+    n = word.length
+    deletion = (0...n).collect { |i| word[0...i] + word[i + 1..-1] }
+    transposition = (0...n - 1).collect { |i| word[0...i] + word[i + 1, 1] + word[i, 1] + word[i + 2..-1] }
+    alteration = []
+    n.times { |i| LETTERS.each_byte { |l| alteration << word[0...i] + l.chr + word[i + 1..-1] } }
+    insertion = []
+    (n + 1).times { |i| LETTERS.each_byte { |l| insertion << word[0...i] + l.chr + word[i..-1] } }
+    deletion + transposition + alteration + insertion
+  end
+
+  def normalize_query(query)
+    tokenize(query.to_s.gsub("&", "and")).map { |q| Lingua.stemmer(q) }.sort.join
+  end
+
+  # TODO remove ActiveSupport dependency
+
+  def singularize(str)
+    str.singularize
+  end
+
+  def pluralize(str)
+    str.pluralize
+  end
+end

data/lib/autosuggest/version.rb

@@ -0,0 +1,3 @@
+class Autosuggest
+  VERSION = "0.1.0"
+end

metadata

@@ -0,0 +1,121 @@
+--- !ruby/object:Gem::Specification
+name: autosuggest
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Andrew Kane
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2015-06-20 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: ruby-stemmer
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: obscenity
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.7'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.7'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: 
+email:
+- andrew@chartkick.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- Gemfile
+- README.md
+- Rakefile
+- autosuggest.gemspec
+- lib/autosuggest.rb
+- lib/autosuggest/version.rb
+homepage: https://github.com/ankane/autosuggest
+licenses: []
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.5
+signing_key: 
+specification_version: 4
+summary: Generate autocomplete suggestions based on what your users search
+test_files: []
+has_rdoc: