autosuggest 0.1.1 → 0.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: bda23aeaee3abc15c45b52ddc2d64943de44a164f11d3a33de234171126b209c
-  data.tar.gz: de6f64ac9ceb628e55e275080a45ac8cf0f5605a5e80767350c264adc92266a6
+  metadata.gz: '083927cb6e763a26bad61351189dd1a1c1e66d31da0a4844635ec5440f4620d0'
+  data.tar.gz: 94245fc7eaeb98f54669b561311d9dee4ac43788f88cef3768a33891c8b530ad
 SHA512:
-  metadata.gz: 9a7a317190aba2237a7b98b6b2cce8ed656eeb44f7c57edd6e19644700fd2c8268c5086201d44d0186f1cabd513e365d471ff2abc4e14d0963989cac55b676f1
-  data.tar.gz: d5dfe7b13390ef216d1f3ab614c4b6c83177d68ff4fef2545e0f34d92936f27cbd077fa44175393a487b6a398c216a51677add321521c97c4db9753e35c6bfbd
+  metadata.gz: f20b852dfcb4cf4249b4c4b2a72e7f58c2c2b642a11b14b33e632ee934736c721d70bcd87f7b6e20659ba5e82f05ce679b69d0e5e72f3b4c64d8187078d2b8c3
+  data.tar.gz: 6e99a02d77dc1ea2cf06e0903d6adebbbe73579496de092efff471369226dfb917df9bd31cee7697bc2d0fcd9392cbe6393186a6c4a747b49a6b03cee348108a

data/CHANGELOG.md

@@ -1,3 +1,8 @@
+## 0.1.2 (2021-11-22)
+
+- Added `filter` option to `suggestions` method
+- Improved performance
+
 ## 0.1.1 (2021-03-15)
 
 - Deprecated `blacklist_words` in favor of `block_words`

data/README.md

@@ -14,16 +14,30 @@ Add this line to your application’s Gemfile:
 gem 'autosuggest'
 ```
 
-## How It Works
+## Getting Started
 
-#### Start with the most popular queries
+#### Prepare your data
+
+Start with a hash of queries and their popularity, like the number of users who have searched it.
+
+```ruby
+top_queries = {
+  "bananas" => 353,
+  "apples"  => 213,
+  "oranges" => 140
+}
+```
+
+With [Searchjoy](https://github.com/ankane/searchjoy), you can do:
 
 ```ruby
-top_queries = Search.group("LOWER(query)")
-                    .having("COUNT(DISTINCT user_id) >= 5")
-                    .count("DISTINCT user_id")
-# {"bananas" => 353, "apples" => 213, ...
+top_queries = Searchjoy::Search.group(:normalized_query)
+  .having("COUNT(DISTINCT user_id) >= 5").distinct.count(:user_id)
+```
 
+Then pass them to Autosuggest.
+
+```ruby
 autosuggest = Autosuggest.new(top_queries)
 ```
 
@@ -76,7 +90,7 @@ autosuggest.block_words ["boom"]
 Get suggestions with:
 
 ```ruby
-autosuggest.suggestions
+autosuggest.suggestions(filter: true)
 ```
 
 Filter queries without results and you’re set. We also prefer to have someone manually approve them by hand.
@@ -84,9 +98,8 @@ Filter queries without results and you’re set. We also prefer to have someone
 ## Full Example
 
 ```ruby
-top_queries = Search.group("LOWER(query)")
-                    .having("COUNT(DISTINCT user_id) >= 5")
-                    .count("DISTINCT user_id")
+top_queries = Searchjoy::Search.group(:normalized_query)
+  .having("COUNT(DISTINCT user_id) >= 5").distinct.count(:user_id)
 product_names = Product.pluck(:name)
 brand_names = Brand.pluck(:name)
 
@@ -99,7 +112,7 @@ autosuggest.block_words ["boom"]
 
 puts autosuggest.pretty_suggestions
 # or
-suggestions = autosuggest.suggestions
+suggestions = autosuggest.suggestions(filter: true)
 ```
 
 ## History

data/lib/autosuggest/version.rb

@@ -1,3 +1,3 @@
 class Autosuggest
-  VERSION = "0.1.1"
+  VERSION = "0.1.2"
 end

data/lib/autosuggest.rb

@@ -15,14 +15,18 @@ class Autosuggest
     @concepts = {}
     @words = Set.new
     @non_duplicates = Set.new
-    @blocked_words = Set.new
-    @blacklisted_words = Set.new
+    @blocked_words = {}
+    @blacklisted_words = {}
     @preferred_queries = {}
-    @profane_words = Set.new(Obscenity::Base.blacklist)
+    @profane_words = {}
+    @concept_tree = {}
+    add_nodes(@profane_words, Obscenity::Base.blacklist)
   end
 
   def add_concept(name, values)
-    @concepts[name] = Set.new(values.compact.uniq.map(&:downcase))
+    values = values.compact.uniq
+    add_nodes(@concept_tree, values)
+    @concepts[name] = Set.new(values.map(&:downcase))
   end
 
   def parse_words(phrases, options = {})
@@ -50,16 +54,14 @@ class Autosuggest
   end
 
   def block_words(words)
-    words.each do |word|
-      @blocked_words << word.downcase
-    end
+    add_nodes(@blocked_words, words)
+    words
   end
 
   def blacklist_words(words)
     warn "[autosuggest] blacklist_words is deprecated. Use block_words instead."
-    words.each do |word|
-      @blacklisted_words << word.downcase
-    end
+    add_nodes(@blacklisted_words, words)
+    words
   end
 
   def prefer(queries)
@@ -68,10 +70,11 @@ class Autosuggest
     end
   end
 
-  def suggestions
+  # TODO add queries method for filter: false and make suggestions use filter: true in 0.2.0
+  def suggestions(filter: false)
     stemmed_queries = {}
     added_queries = Set.new
-    @top_queries.sort_by { |_query, count| -count }.map do |query, count|
+    results = @top_queries.sort_by { |_query, count| -count }.map do |query, count|
       query = query.to_s
 
       # TODO do not ignore silently
@@ -110,12 +113,14 @@ class Autosuggest
         concepts << name if values.include?(query)
       end
 
+      tokens = tokenize(query)
+
       # exclude misspellings that are not brands
-      misspelling = @words.any? && misspellings?(query)
+      misspelling = @words.any? && misspellings?(tokens)
 
-      profane = blocked?(query, @profane_words)
-      blocked = blocked?(query, @blocked_words)
-      blacklisted = blocked?(query, @blacklisted_words)
+      profane = blocked?(tokens, @profane_words)
+      blocked = blocked?(tokens, @blocked_words)
+      blacklisted = blocked?(tokens, @blacklisted_words)
 
       notes = []
       notes << "duplicate of #{duplicate}" if duplicate
@@ -140,6 +145,10 @@ class Autosuggest
       result[:notes] = notes
       result
     end
+    if filter
+      results.reject! { |s| s[:duplicate] || s[:misspelling] || s[:profane] || s[:blocked] }
+    end
+    results
   end
 
   def pretty_suggestions
@@ -152,38 +161,41 @@ class Autosuggest
 
   protected
 
-  def misspellings?(query)
-    recurse(tokenize(query)).each do |terms|
-      if terms.all? { |t| @concepts.any? { |_, values| values.include?(t) } || @words.include?(t) }
-        return false
+  def misspellings?(tokens)
+    pos = [0]
+    while i = pos.shift
+      return false if i == tokens.size
+
+      if @words.include?(tokens[i])
+        pos << i + 1
+      end
+
+      node = @concept_tree[tokens[i]]
+      j = i
+      while node
+        j += 1
+        pos << j if node[:eos]
+        break if j == tokens.size
+        node = node[tokens[j]]
       end
-    end
-    true
-  end
 
-  def blocked?(query, blocked_words)
-    recurse(tokenize(query)).each do |terms|
-      return true if terms.any? { |t| blocked_words.include?(t) }
+      pos.uniq!
     end
-    false
+    true
   end
 
-  def recurse(words)
-    if words.size == 1
-      [words]
-    else
-      result = [[words.join(" ")]]
-      i = 0
-      while i < words.size - 1
-        recurse(words[0..i]).each do |v1|
-          recurse(words[i + 1..-1]).each do |v2|
-            result << v1 + v2
-          end
-        end
-        i += 1
+  def blocked?(tokens, blocked_words)
+    tokens.each_with_index do |token, i|
+      node = blocked_words[token]
+      j = i
+      while node
+        return true if node[:eos]
+        j += 1
+        break if j == tokens.size
+        node = node[tokens[j]]
       end
-      result.uniq
     end
+    false
   end
 
   def tokenize(str)
@@ -206,4 +218,15 @@ class Autosuggest
   def normalize_query(query)
     tokenize(query.to_s.gsub("&", "and")).map { |q| Lingua.stemmer(q) }.sort.join
   end
+
+  def add_nodes(var, words)
+    words.each do |word|
+      node = var
+      tokenize(word).each do |token|
+        node = (node[token] ||= {})
+      end
+      node[:eos] = true
+    end
+    var
+  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: autosuggest
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.1.2
 platform: ruby
 authors:
 - Andrew Kane
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-03-15 00:00:00.000000000 Z
+date: 2021-11-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ruby-stemmer
@@ -68,7 +68,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.2.3
+rubygems_version: 3.2.22
 signing_key:
 specification_version: 4
 summary: Generate autocomplete suggestions based on what your users search