autosuggest 0.1.2 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: '083927cb6e763a26bad61351189dd1a1c1e66d31da0a4844635ec5440f4620d0'
-  data.tar.gz: 94245fc7eaeb98f54669b561311d9dee4ac43788f88cef3768a33891c8b530ad
+  metadata.gz: fe418b5cfaa006d454a8e061ae08b63821de147677175054be5e28ad254399c6
+  data.tar.gz: 10c28b44de11f53fccd66a3e6f547a7f97282e23b529452a1db827b3d2dd0624
 SHA512:
-  metadata.gz: f20b852dfcb4cf4249b4c4b2a72e7f58c2c2b642a11b14b33e632ee934736c721d70bcd87f7b6e20659ba5e82f05ce679b69d0e5e72f3b4c64d8187078d2b8c3
-  data.tar.gz: 6e99a02d77dc1ea2cf06e0903d6adebbbe73579496de092efff471369226dfb917df9bd31cee7697bc2d0fcd9392cbe6393186a6c4a747b49a6b03cee348108a
+  metadata.gz: 236b65f1939693fd076445ebddff62535d1d1197e44561ade442e2c381f2d3c4bd06572daacc29718dbfb087abc1cf3c001d0152f21c9198d059d8c1d933985c
+  data.tar.gz: 1ca979179a176b6a7eca3ee8b9593e6479680ea4706e8ad1c004d8c88efe0a1f1a82b779dee050cb1aa3c6af749323a87fc9f93c9c0c4d8eeb530a9e6a9f11cd

data/CHANGELOG.md

@@ -1,3 +1,15 @@
+## 0.2.0 (2023-01-29)
+
+- Added `language` option
+- Changed `suggestions` method to filter by default
+- Changed `filter: true` to only return query and score
+- Removed `blacklist_words` method
+- Dropped support for Ruby < 2.7
+
+## 0.1.3 (2021-11-23)
+
+- Added model generator
+
 ## 0.1.2 (2021-11-22)
 
 - Added `filter` option to `suggestions` method

data/LICENSE.txt

@@ -1,4 +1,4 @@
-Copyright (c) 2015-2021 Andrew Kane
+Copyright (c) 2015-2023 Andrew Kane
 
 MIT License
 

data/README.md

@@ -4,6 +4,8 @@ Generate autocomplete suggestions based on what your users search
 
 :tangerine: Battle-tested at [Instacart](https://www.instacart.com/opensource)
 
+Autosuggest 0.2 was recently released! See [how to upgrade](#upgrading)
+
 [![Build Status](https://github.com/ankane/autosuggest/workflows/build/badge.svg?branch=master)](https://github.com/ankane/autosuggest/actions)
 
 ## Installation
@@ -11,7 +13,7 @@ Generate autocomplete suggestions based on what your users search
 Add this line to your application’s Gemfile:
 
 ```ruby
-gem 'autosuggest'
+gem "autosuggest"
 ```
 
 ## Getting Started
@@ -38,14 +40,20 @@ top_queries = Searchjoy::Search.group(:normalized_query)
 Then pass them to Autosuggest.
 
 ```ruby
-autosuggest = Autosuggest.new(top_queries)
+autosuggest = Autosuggest::Generator.new(top_queries)
 ```
 
 #### Filter duplicates
 
 [Stemming](https://en.wikipedia.org/wiki/Stemming) is used to detect duplicates like `apple` and `apples`.
 
-The most popular query is preferred by default.  To override this, use:
+Specify the stemming language (defaults to `english`) with:
+
+```ruby
+autosuggest = Autosuggest::Generator.new(top_queries, language: "spanish")
+```
+
+The most popular query is preferred by default. To override this, use:
 
 ```ruby
 autosuggest.prefer ["apples"]
@@ -85,17 +93,90 @@ There are two ways to build the corpus, which can be used together.
 autosuggest.block_words ["boom"]
 ```
 
-#### Profit
+#### Generate suggestions
+
+Generate suggestions with:
+
+```ruby
+suggestions = autosuggest.suggestions
+```
+
+#### Save suggestions
+
+Save suggestions in your database or another data store.
+
+With Rails, you can generate a simple model with:
+
+```sh
+rails generate autosuggest:suggestions
+rails db:migrate
+```
+
+And update suggestions with:
+
+```ruby
+now = Time.now
+records = suggestions.map { |s| s.slice(:query, :score).merge(updated_at: now) }
+Autosuggest::Suggestion.transaction do
+  Autosuggest::Suggestion.upsert_all(records, unique_by: :query)
+  Autosuggest::Suggestion.where("updated_at < ?", now).delete_all
+end
+```
+
+Leave out `unique_by` for MySQL, and use [activerecord-import](https://github.com/zdennis/activerecord-import) for upserts with Rails < 6.
+
+#### Show suggestions
+
+Use a JavaScript autocomplete library like [typeahead.js](https://github.com/twitter/typeahead.js) to show suggestions in the UI.
+
+If you only have a few thousand suggestions, it’s much faster to load them all at once instead of as a user types (eliminates network requests).
+
+With Rails, you can load all suggestions with:
+
+```ruby
+Autosuggest::Suggestion.order(score: :desc).pluck(:query)
+```
+
+And suggestions matching user input with:
+
+```ruby
+input = params[:query]
+Autosuggest::Suggestion
+  .order(score: :desc)
+  .where("query LIKE ?", "%#{Autosuggest::Suggestion.sanitize_sql_like(input.downcase)}%")
+  .pluck(:query)
+```
+
+You can also cache suggestions for performance.
+
+```ruby
+Rails.cache.fetch("suggestions", expires_in: 5.minutes) do
+  Autosuggest::Suggestion.order(score: :desc).pluck(:query)
+end
+```
+
+#### Additional considerations
 
-Get suggestions with:
+You may want to have someone manually approve suggestions:
 
 ```ruby
-autosuggest.suggestions(filter: true)
+Autosuggest::Suggestion.where(status: "approved")
 ```
 
-Filter queries without results and you’re set. We also prefer to have someone manually approve them by hand.
+Or filter suggestions without results:
 
-## Full Example
+```ruby
+Autosuggest::Suggestion.find_each do |suggestion|
+  suggestion.results_count = Product.search(suggestion.query, load: false).count
+  suggestion.save! if suggestion.changed?
+end
+
+Autosuggest::Suggestion.where("results_count > 0")
+```
+
+You can add additional fields to your model/data store to accomplish this.
+
+## Example
 
 ```ruby
 top_queries = Searchjoy::Search.group(:normalized_query)
@@ -103,16 +184,31 @@ top_queries = Searchjoy::Search.group(:normalized_query)
 product_names = Product.pluck(:name)
 brand_names = Brand.pluck(:name)
 
-autosuggest = Autosuggest.new(top_queries)
+autosuggest = Autosuggest::Generator.new(top_queries)
 autosuggest.parse_words product_names
 autosuggest.add_concept "brand", brand_names
 autosuggest.prefer brand_names
 autosuggest.not_duplicates [["straws", "straus"]]
 autosuggest.block_words ["boom"]
 
-puts autosuggest.pretty_suggestions
-# or
-suggestions = autosuggest.suggestions(filter: true)
+suggestions = autosuggest.suggestions
+
+now = Time.now
+records = suggestions.map { |s| s.slice(:query, :score).merge(updated_at: now) }
+Autosuggest::Suggestion.transaction do
+  Autosuggest::Suggestion.upsert_all(records, unique_by: :query)
+  Autosuggest::Suggestion.where("updated_at < ?", now).delete_all
+end
+```
+
+## Upgrading
+
+### 0.2.0
+
+Suggestions are now filtered by default, and only the query and score are returned. To get all queries and fields, use:
+
+```ruby
+autosuggest.suggestions(filter: false)
 ```
 
 ## History

data/lib/autosuggest/generator.rb

@@ -0,0 +1,226 @@
+module Autosuggest
+  class Generator
+    def initialize(top_queries, language: "english")
+      @top_queries = top_queries
+      @concepts = {}
+      @words = Set.new
+      @non_duplicates = Set.new
+      @blocked_words = {}
+      @preferred_queries = {}
+      @profane_words = {}
+      @concept_tree = {}
+      begin
+        @stemmer = Lingua::Stemmer.new(language: language)
+      rescue Lingua::StemmerError
+        raise ArgumentError, "Language not available"
+      end
+      # TODO take language into account for profanity
+      add_nodes(@profane_words, Obscenity::Base.blacklist)
+    end
+
+    def add_concept(name, values)
+      values = values.compact.uniq
+      add_nodes(@concept_tree, values)
+      @concepts[name] = Set.new(values.map(&:downcase))
+    end
+
+    def parse_words(phrases, options = {})
+      min = options[:min] || 1
+
+      word_counts = Hash.new(0)
+      phrases.each do |phrase|
+        words = tokenize(phrase)
+        words.each do |word|
+          word_counts[word] += 1
+        end
+      end
+
+      word_counts.select { |_, c| c >= min }.each do |word, _|
+        @words << word
+      end
+
+      word_counts
+    end
+
+    def not_duplicates(pairs)
+      pairs.each do |pair|
+        @non_duplicates << pair.map(&:downcase).sort
+      end
+    end
+
+    def block_words(words)
+      add_nodes(@blocked_words, words)
+      words
+    end
+
+    def prefer(queries)
+      queries.each do |query|
+        @preferred_queries[normalize_query(query)] ||= query
+      end
+    end
+
+    def suggestions(filter: true)
+      stemmed_queries = {}
+      added_queries = Set.new
+      results = @top_queries.sort_by { |_query, count| -count }.map do |query, count|
+        query = query.to_s
+
+        # TODO do not ignore silently
+        next if query.length < 2
+
+        stemmed_query = normalize_query(query)
+
+        # get preferred term
+        preferred_query = @preferred_queries[stemmed_query]
+        if preferred_query && preferred_query != query
+          original_query, query = query, preferred_query
+        end
+
+        # exclude duplicates
+        duplicate = stemmed_queries[stemmed_query]
+        stemmed_queries[stemmed_query] ||= query
+
+        # also detect possibly misspelled duplicates
+        # TODO use top query as duplicate
+        if !duplicate && query.length > 4
+          edits(query).each do |edited_query|
+            if added_queries.include?(edited_query)
+              duplicate = edited_query
+              break
+            end
+          end
+        end
+        if duplicate && @non_duplicates.include?([duplicate, query].sort)
+          duplicate = nil
+        end
+        added_queries << query unless duplicate
+
+        # find concepts
+        concepts = []
+        @concepts.each do |name, values|
+          concepts << name if values.include?(query)
+        end
+
+        tokens = tokenize(query)
+
+        # exclude misspellings that are not brands
+        misspelling = @words.any? && misspellings?(tokens)
+
+        profane = blocked?(tokens, @profane_words)
+        blocked = blocked?(tokens, @blocked_words)
+
+        notes = []
+        notes << "duplicate of #{duplicate}" if duplicate
+        notes.concat(concepts)
+        notes << "misspelling" if misspelling
+        notes << "profane" if profane
+        notes << "blocked" if blocked
+        notes << "originally #{original_query}" if original_query
+
+        {
+          query: query,
+          original_query: original_query,
+          score: count,
+          duplicate: duplicate,
+          concepts: concepts,
+          misspelling: misspelling,
+          profane: profane,
+          blocked: blocked,
+          notes: notes
+        }
+      end
+
+      results.compact!
+
+      if filter
+        results.filter_map do |s|
+          unless s[:duplicate] || s[:misspelling] || s[:profane] || s[:blocked]
+            s.slice(:query, :score)
+          end
+        end
+      else
+        results
+      end
+    end
+
+    def table
+      str = "%-30s   %5s   %s\n" % %w(Query Score Notes)
+      suggestions(filter: false).each do |suggestion|
+        str << "%-30s   %5d   %s\n" % [suggestion[:query], suggestion[:score], suggestion[:notes].join(", ")]
+      end
+      str
+    end
+    alias_method :pretty_suggestions, :table
+
+    protected
+
+    def misspellings?(tokens)
+      pos = [0]
+      while i = pos.shift
+        return false if i == tokens.size
+
+        if @words.include?(tokens[i])
+          pos << i + 1
+        end
+
+        node = @concept_tree[tokens[i]]
+        j = i
+        while node
+          j += 1
+          pos << j if node[:eos]
+          break if j == tokens.size
+          node = node[tokens[j]]
+        end
+
+        pos.uniq!
+      end
+      true
+    end
+
+    def blocked?(tokens, blocked_words)
+      tokens.each_with_index do |token, i|
+        node = blocked_words[token]
+        j = i
+        while node
+          return true if node[:eos]
+          j += 1
+          break if j == tokens.size
+          node = node[tokens[j]]
+        end
+      end
+      false
+    end
+
+    def tokenize(str)
+      str.to_s.downcase.split(" ")
+    end
+
+    # from https://blog.lojic.com/2008/09/04/how-to-write-a-spelling-corrector-in-ruby/
+    LETTERS = ("a".."z").to_a.join + "'"
+    def edits(word)
+      n = word.length
+      deletion = (0...n).collect { |i| word[0...i] + word[i + 1..-1] }
+      transposition = (0...n - 1).collect { |i| word[0...i] + word[i + 1, 1] + word[i, 1] + word[i + 2..-1] }
+      alteration = []
+      n.times { |i| LETTERS.each_byte { |l| alteration << word[0...i] + l.chr + word[i + 1..-1] } }
+      insertion = []
+      (n + 1).times { |i| LETTERS.each_byte { |l| insertion << word[0...i] + l.chr + word[i..-1] } }
+      deletion + transposition + alteration + insertion
+    end
+
+    def normalize_query(query)
+      tokenize(query.to_s.gsub("&", "and")).map { |q| @stemmer.stem(q) }.sort.join
+    end
+
+    def add_nodes(var, words)
+      words.each do |word|
+        node = var
+        tokenize(word).each do |token|
+          node = (node[token] ||= {})
+        end
+        node[:eos] = true
+      end
+      var
+    end
+  end
+end

data/lib/autosuggest/version.rb

@@ -1,3 +1,3 @@
-class Autosuggest
-  VERSION = "0.1.2"
+module Autosuggest
+  VERSION = "0.2.0"
 end

data/lib/autosuggest.rb

@@ -7,226 +7,11 @@ require "lingua/stemmer"
 require "obscenity"
 
 # modules
-require "autosuggest/version"
+require_relative "autosuggest/generator"
+require_relative "autosuggest/version"
 
-class Autosuggest
-  def initialize(top_queries)
-    @top_queries = top_queries
-    @concepts = {}
-    @words = Set.new
-    @non_duplicates = Set.new
-    @blocked_words = {}
-    @blacklisted_words = {}
-    @preferred_queries = {}
-    @profane_words = {}
-    @concept_tree = {}
-    add_nodes(@profane_words, Obscenity::Base.blacklist)
-  end
-
-  def add_concept(name, values)
-    values = values.compact.uniq
-    add_nodes(@concept_tree, values)
-    @concepts[name] = Set.new(values.map(&:downcase))
-  end
-
-  def parse_words(phrases, options = {})
-    min = options[:min] || 1
-
-    word_counts = Hash.new(0)
-    phrases.each do |phrase|
-      words = tokenize(phrase)
-      words.each do |word|
-        word_counts[word] += 1
-      end
-    end
-
-    word_counts.select { |_, c| c >= min }.each do |word, _|
-      @words << word
-    end
-
-    word_counts
-  end
-
-  def not_duplicates(pairs)
-    pairs.each do |pair|
-      @non_duplicates << pair.map(&:downcase).sort
-    end
-  end
-
-  def block_words(words)
-    add_nodes(@blocked_words, words)
-    words
-  end
-
-  def blacklist_words(words)
-    warn "[autosuggest] blacklist_words is deprecated. Use block_words instead."
-    add_nodes(@blacklisted_words, words)
-    words
-  end
-
-  def prefer(queries)
-    queries.each do |query|
-      @preferred_queries[normalize_query(query)] ||= query
-    end
-  end
-
-  # TODO add queries method for filter: false and make suggestions use filter: true in 0.2.0
-  def suggestions(filter: false)
-    stemmed_queries = {}
-    added_queries = Set.new
-    results = @top_queries.sort_by { |_query, count| -count }.map do |query, count|
-      query = query.to_s
-
-      # TODO do not ignore silently
-      next if query.length < 2
-
-      stemmed_query = normalize_query(query)
-
-      # get preferred term
-      preferred_query = @preferred_queries[stemmed_query]
-      if preferred_query && preferred_query != query
-        original_query, query = query, preferred_query
-      end
-
-      # exclude duplicates
-      duplicate = stemmed_queries[stemmed_query]
-      stemmed_queries[stemmed_query] ||= query
-
-      # also detect possibly misspelled duplicates
-      # TODO use top query as duplicate
-      if !duplicate && query.length > 4
-        edits(query).each do |edited_query|
-          if added_queries.include?(edited_query)
-            duplicate = edited_query
-            break
-          end
-        end
-      end
-      if duplicate && @non_duplicates.include?([duplicate, query].sort)
-        duplicate = nil
-      end
-      added_queries << query unless duplicate
-
-      # find concepts
-      concepts = []
-      @concepts.each do |name, values|
-        concepts << name if values.include?(query)
-      end
-
-      tokens = tokenize(query)
-
-      # exclude misspellings that are not brands
-      misspelling = @words.any? && misspellings?(tokens)
-
-      profane = blocked?(tokens, @profane_words)
-      blocked = blocked?(tokens, @blocked_words)
-      blacklisted = blocked?(tokens, @blacklisted_words)
-
-      notes = []
-      notes << "duplicate of #{duplicate}" if duplicate
-      notes.concat(concepts)
-      notes << "misspelling" if misspelling
-      notes << "profane" if profane
-      notes << "blocked" if blocked
-      notes << "blacklisted" if blacklisted
-      notes << "originally #{original_query}" if original_query
-
-      result = {
-        query: query,
-        original_query: original_query,
-        score: count,
-        duplicate: duplicate,
-        concepts: concepts,
-        misspelling: misspelling,
-        profane: profane,
-        blocked: blocked
-      }
-      result[:blacklisted] = blacklisted if @blacklisted_words.any?
-      result[:notes] = notes
-      result
-    end
-    if filter
-      results.reject! { |s| s[:duplicate] || s[:misspelling] || s[:profane] || s[:blocked] }
-    end
-    results
-  end
-
-  def pretty_suggestions
-    str = "%-30s   %5s   %s\n" % %w(Query Score Notes)
-    suggestions.each do |suggestion|
-      str << "%-30s   %5d   %s\n" % [suggestion[:query], suggestion[:score], suggestion[:notes].join(", ")]
-    end
-    str
-  end
-
-  protected
-
-  def misspellings?(tokens)
-    pos = [0]
-    while i = pos.shift
-      return false if i == tokens.size
-
-      if @words.include?(tokens[i])
-        pos << i + 1
-      end
-
-      node = @concept_tree[tokens[i]]
-      j = i
-      while node
-        j += 1
-        pos << j if node[:eos]
-        break if j == tokens.size
-        node = node[tokens[j]]
-      end
-
-      pos.uniq!
-    end
-    true
-  end
-
-  def blocked?(tokens, blocked_words)
-    tokens.each_with_index do |token, i|
-      node = blocked_words[token]
-      j = i
-      while node
-        return true if node[:eos]
-        j += 1
-        break if j == tokens.size
-        node = node[tokens[j]]
-      end
-    end
-    false
-  end
-
-  def tokenize(str)
-    str.to_s.downcase.split(" ")
-  end
-
-  # from https://blog.lojic.com/2008/09/04/how-to-write-a-spelling-corrector-in-ruby/
-  LETTERS = ("a".."z").to_a.join + "'"
-  def edits(word)
-    n = word.length
-    deletion = (0...n).collect { |i| word[0...i] + word[i + 1..-1] }
-    transposition = (0...n - 1).collect { |i| word[0...i] + word[i + 1, 1] + word[i, 1] + word[i + 2..-1] }
-    alteration = []
-    n.times { |i| LETTERS.each_byte { |l| alteration << word[0...i] + l.chr + word[i + 1..-1] } }
-    insertion = []
-    (n + 1).times { |i| LETTERS.each_byte { |l| insertion << word[0...i] + l.chr + word[i..-1] } }
-    deletion + transposition + alteration + insertion
-  end
-
-  def normalize_query(query)
-    tokenize(query.to_s.gsub("&", "and")).map { |q| Lingua.stemmer(q) }.sort.join
-  end
-
-  def add_nodes(var, words)
-    words.each do |word|
-      node = var
-      tokenize(word).each do |token|
-        node = (node[token] ||= {})
-      end
-      node[:eos] = true
-    end
-    var
+module Autosuggest
+  def self.new(*args, **options)
+    Generator.new(*args, **options)
   end
 end

data/lib/generators/autosuggest/suggestions_generator.rb

@@ -0,0 +1,33 @@
+require "rails/generators/active_record"
+
+module Autosuggest
+  module Generators
+    class SuggestionsGenerator < Rails::Generators::Base
+      include ActiveRecord::Generators::Migration
+      source_root File.join(__dir__, "templates")
+
+      def copy_templates
+        template "model.rb", "app/models/autosuggest/suggestion.rb"
+        migration_template "migration.rb", "db/migrate/create_autosuggest_suggestions.rb", migration_version: migration_version
+      end
+
+      def migration_version
+        "[#{ActiveRecord::VERSION::MAJOR}.#{ActiveRecord::VERSION::MINOR}]"
+      end
+
+      def mysql?
+        adapter =~ /mysql/i
+      end
+
+      # use connection_config instead of connection.adapter
+      # so database connection isn't needed
+      def adapter
+        if ActiveRecord::VERSION::STRING.to_f >= 6.1
+          ActiveRecord::Base.connection_db_config.adapter.to_s
+        else
+          ActiveRecord::Base.connection_config[:adapter].to_s
+        end
+      end
+    end
+  end
+end

data/lib/generators/autosuggest/templates/migration.rb.tt

@@ -0,0 +1,11 @@
+class <%= migration_class_name %> < ActiveRecord::Migration<%= migration_version %>
+  def change
+    create_table :autosuggest_suggestions do |t|
+      t.string :query
+      t.float :score
+      t.datetime :updated_at<%= mysql? ? ", precision: 6" : "" %>
+    end
+
+    add_index :autosuggest_suggestions, :query, unique: true
+  end
+end

data/lib/generators/autosuggest/templates/model.rb.tt

@@ -0,0 +1,3 @@
+class Autosuggest::Suggestion < ApplicationRecord
+  self.table_name = "autosuggest_suggestions"
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: autosuggest
 version: !ruby/object:Gem::Version
-  version: 0.1.2
+  version: 0.2.0
 platform: ruby
 authors:
 - Andrew Kane
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-11-23 00:00:00.000000000 Z
+date: 2023-01-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ruby-stemmer
@@ -48,7 +48,11 @@ files:
 - LICENSE.txt
 - README.md
 - lib/autosuggest.rb
+- lib/autosuggest/generator.rb
 - lib/autosuggest/version.rb
+- lib/generators/autosuggest/suggestions_generator.rb
+- lib/generators/autosuggest/templates/migration.rb.tt
+- lib/generators/autosuggest/templates/model.rb.tt
 homepage: https://github.com/ankane/autosuggest
 licenses:
 - MIT
@@ -61,14 +65,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '2.4'
+      version: '2.7'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.2.22
+rubygems_version: 3.4.1
 signing_key:
 specification_version: 4
 summary: Generate autocomplete suggestions based on what your users search