autosuggest 0.1.3 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 3f04e20f21653bcc9a8941c6da1eddae7726cf775abea699b78022b64fec9e48
-  data.tar.gz: 7a5c362428558d17808310245d7cad1bdf693800419665d650ab9026e5aa8628
+  metadata.gz: 0cd2e072b09ebaecd6858e27f87b198e0909c8d70c0f36250c6eefc022600e59
+  data.tar.gz: '02434900d6a69b6c18ad2d5353c7359681f442fbe19a28f1aacfdac3c60dfe42'
 SHA512:
-  metadata.gz: 62c38fb482638185c18dde9b0b4db1c9aa4deaa6e8f0fb4341220f61c28b218c51f79376b1080fb954982f0aa2b9a979b20b4212bfb704f73fe2e92be8b4373e
-  data.tar.gz: 84383dd6de2c654aefabe546f3c01335f906d5cfb517fd1890b26376ed9f482651b6e7e8d1b5082d120d228603b7b0002227b4293d2668c5de51a464aa831fa5
+  metadata.gz: 22ed0ecf00dae3f077cbd2be2cd024e61f6ceb6cdefca57af9a90d1769499e51c1df301ee37bfbd2344ef69afd6b9da3582aa21714b1b3d5061749eb0d9b190d
+  data.tar.gz: d8b757d31b2c2b9429afc29758f6d70631e9402593dbc127728c47f5553edb2ec04b9007dc79257820a500f0105f93d72254decd1cde3b2123990752f255600e

data/CHANGELOG.md

@@ -1,3 +1,16 @@
+## 0.3.0 (2024-05-22)
+
+- Switched to Mittens for stemming
+- Dropped support for Ruby < 3.1
+
+## 0.2.0 (2023-01-29)
+
+- Added `language` option
+- Changed `suggestions` method to filter by default
+- Changed `filter: true` to only return query and score
+- Removed `blacklist_words` method
+- Dropped support for Ruby < 2.7
+
 ## 0.1.3 (2021-11-23)
 
 - Added model generator

data/LICENSE.txt

@@ -1,4 +1,4 @@
-Copyright (c) 2015-2021 Andrew Kane
+Copyright (c) 2015-2024 Andrew Kane
 
 MIT License
 

data/README.md

@@ -4,14 +4,14 @@ Generate autocomplete suggestions based on what your users search
 
 :tangerine: Battle-tested at [Instacart](https://www.instacart.com/opensource)
 
-[![Build Status](https://github.com/ankane/autosuggest/workflows/build/badge.svg?branch=master)](https://github.com/ankane/autosuggest/actions)
+[![Build Status](https://github.com/ankane/autosuggest/actions/workflows/build.yml/badge.svg)](https://github.com/ankane/autosuggest/actions)
 
 ## Installation
 
 Add this line to your application’s Gemfile:
 
 ```ruby
-gem 'autosuggest'
+gem "autosuggest"
 ```
 
 ## Getting Started
@@ -38,14 +38,20 @@ top_queries = Searchjoy::Search.group(:normalized_query)
 Then pass them to Autosuggest.
 
 ```ruby
-autosuggest = Autosuggest.new(top_queries)
+autosuggest = Autosuggest::Generator.new(top_queries)
 ```
 
 #### Filter duplicates
 
 [Stemming](https://en.wikipedia.org/wiki/Stemming) is used to detect duplicates like `apple` and `apples`.
 
-The most popular query is preferred by default.  To override this, use:
+Specify the stemming language (defaults to `english`) with:
+
+```ruby
+autosuggest = Autosuggest::Generator.new(top_queries, language: "spanish")
+```
+
+The most popular query is preferred by default. To override this, use:
 
 ```ruby
 autosuggest.prefer ["apples"]
@@ -90,7 +96,7 @@ autosuggest.block_words ["boom"]
 Generate suggestions with:
 
 ```ruby
-suggestions = autosuggest.suggestions(filter: true)
+suggestions = autosuggest.suggestions
 ```
 
 #### Save suggestions
@@ -152,18 +158,18 @@ end
 You may want to have someone manually approve suggestions:
 
 ```ruby
-Autosuggest::Suggestion.where(approved: true)
+Autosuggest::Suggestion.where(status: "approved")
 ```
 
 Or filter suggestions without results:
 
 ```ruby
 Autosuggest::Suggestion.find_each do |suggestion|
-  suggestion.has_results = Product.search(suggestion.query, load: false, limit: 1).any?
+  suggestion.results_count = Product.search(suggestion.query, load: false).count
   suggestion.save! if suggestion.changed?
 end
 
-Autosuggest::Suggestion.where(has_results: true)
+Autosuggest::Suggestion.where("results_count > 0")
 ```
 
 You can add additional fields to your model/data store to accomplish this.
@@ -176,14 +182,14 @@ top_queries = Searchjoy::Search.group(:normalized_query)
 product_names = Product.pluck(:name)
 brand_names = Brand.pluck(:name)
 
-autosuggest = Autosuggest.new(top_queries)
+autosuggest = Autosuggest::Generator.new(top_queries)
 autosuggest.parse_words product_names
 autosuggest.add_concept "brand", brand_names
 autosuggest.prefer brand_names
 autosuggest.not_duplicates [["straws", "straus"]]
 autosuggest.block_words ["boom"]
 
-suggestions = autosuggest.suggestions(filter: true)
+suggestions = autosuggest.suggestions
 
 now = Time.now
 records = suggestions.map { |s| s.slice(:query, :score).merge(updated_at: now) }
@@ -193,6 +199,16 @@ Autosuggest::Suggestion.transaction do
 end
 ```
 
+## Upgrading
+
+### 0.2.0
+
+Suggestions are now filtered by default, and only the query and score are returned. Get all queries and fields with:
+
+```ruby
+autosuggest.suggestions(filter: false)
+```
+
 ## History
 
 View the [changelog](https://github.com/ankane/autosuggest/blob/master/CHANGELOG.md)

data/lib/autosuggest/generator.rb

@@ -0,0 +1,226 @@
+module Autosuggest
+  class Generator
+    def initialize(top_queries, language: "english")
+      @top_queries = top_queries
+      @concepts = {}
+      @words = Set.new
+      @non_duplicates = Set.new
+      @blocked_words = {}
+      @preferred_queries = {}
+      @profane_words = {}
+      @concept_tree = {}
+      begin
+        @stemmer = Mittens::Stemmer.new(language: language)
+      rescue ArgumentError
+        raise ArgumentError, "Language not available"
+      end
+      # TODO take language into account for profanity
+      add_nodes(@profane_words, Obscenity::Base.blacklist)
+    end
+
+    def add_concept(name, values)
+      values = values.compact.uniq
+      add_nodes(@concept_tree, values)
+      @concepts[name] = Set.new(values.map(&:downcase))
+    end
+
+    def parse_words(phrases, options = {})
+      min = options[:min] || 1
+
+      word_counts = Hash.new(0)
+      phrases.each do |phrase|
+        words = tokenize(phrase)
+        words.each do |word|
+          word_counts[word] += 1
+        end
+      end
+
+      word_counts.select { |_, c| c >= min }.each do |word, _|
+        @words << word
+      end
+
+      word_counts
+    end
+
+    def not_duplicates(pairs)
+      pairs.each do |pair|
+        @non_duplicates << pair.map(&:downcase).sort
+      end
+    end
+
+    def block_words(words)
+      add_nodes(@blocked_words, words)
+      words
+    end
+
+    def prefer(queries)
+      queries.each do |query|
+        @preferred_queries[normalize_query(query)] ||= query
+      end
+    end
+
+    def suggestions(filter: true)
+      stemmed_queries = {}
+      added_queries = Set.new
+      results = @top_queries.sort_by { |_query, count| -count }.map do |query, count|
+        query = query.to_s
+
+        # TODO do not ignore silently
+        next if query.length < 2
+
+        stemmed_query = normalize_query(query)
+
+        # get preferred term
+        preferred_query = @preferred_queries[stemmed_query]
+        if preferred_query && preferred_query != query
+          original_query, query = query, preferred_query
+        end
+
+        # exclude duplicates
+        duplicate = stemmed_queries[stemmed_query]
+        stemmed_queries[stemmed_query] ||= query
+
+        # also detect possibly misspelled duplicates
+        # TODO use top query as duplicate
+        if !duplicate && query.length > 4
+          edits(query).each do |edited_query|
+            if added_queries.include?(edited_query)
+              duplicate = edited_query
+              break
+            end
+          end
+        end
+        if duplicate && @non_duplicates.include?([duplicate, query].sort)
+          duplicate = nil
+        end
+        added_queries << query unless duplicate
+
+        # find concepts
+        concepts = []
+        @concepts.each do |name, values|
+          concepts << name if values.include?(query)
+        end
+
+        tokens = tokenize(query)
+
+        # exclude misspellings that are not brands
+        misspelling = @words.any? && misspellings?(tokens)
+
+        profane = blocked?(tokens, @profane_words)
+        blocked = blocked?(tokens, @blocked_words)
+
+        notes = []
+        notes << "duplicate of #{duplicate}" if duplicate
+        notes.concat(concepts)
+        notes << "misspelling" if misspelling
+        notes << "profane" if profane
+        notes << "blocked" if blocked
+        notes << "originally #{original_query}" if original_query
+
+        {
+          query: query,
+          original_query: original_query,
+          score: count,
+          duplicate: duplicate,
+          concepts: concepts,
+          misspelling: misspelling,
+          profane: profane,
+          blocked: blocked,
+          notes: notes
+        }
+      end
+
+      results.compact!
+
+      if filter
+        results.filter_map do |s|
+          unless s[:duplicate] || s[:misspelling] || s[:profane] || s[:blocked]
+            s.slice(:query, :score)
+          end
+        end
+      else
+        results
+      end
+    end
+
+    def table
+      str = "%-30s   %5s   %s\n" % %w(Query Score Notes)
+      suggestions(filter: false).each do |suggestion|
+        str << "%-30s   %5d   %s\n" % [suggestion[:query], suggestion[:score], suggestion[:notes].join(", ")]
+      end
+      str
+    end
+    alias_method :pretty_suggestions, :table
+
+    protected
+
+    def misspellings?(tokens)
+      pos = [0]
+      while i = pos.shift
+        return false if i == tokens.size
+
+        if @words.include?(tokens[i])
+          pos << i + 1
+        end
+
+        node = @concept_tree[tokens[i]]
+        j = i
+        while node
+          j += 1
+          pos << j if node[:eos]
+          break if j == tokens.size
+          node = node[tokens[j]]
+        end
+
+        pos.uniq!
+      end
+      true
+    end
+
+    def blocked?(tokens, blocked_words)
+      tokens.each_with_index do |token, i|
+        node = blocked_words[token]
+        j = i
+        while node
+          return true if node[:eos]
+          j += 1
+          break if j == tokens.size
+          node = node[tokens[j]]
+        end
+      end
+      false
+    end
+
+    def tokenize(str)
+      str.to_s.downcase.split(" ")
+    end
+
+    # from https://blog.lojic.com/2008/09/04/how-to-write-a-spelling-corrector-in-ruby/
+    LETTERS = ("a".."z").to_a.join + "'"
+    def edits(word)
+      n = word.length
+      deletion = (0...n).collect { |i| word[0...i] + word[i + 1..-1] }
+      transposition = (0...n - 1).collect { |i| word[0...i] + word[i + 1, 1] + word[i, 1] + word[i + 2..-1] }
+      alteration = []
+      n.times { |i| LETTERS.each_byte { |l| alteration << word[0...i] + l.chr + word[i + 1..-1] } }
+      insertion = []
+      (n + 1).times { |i| LETTERS.each_byte { |l| insertion << word[0...i] + l.chr + word[i..-1] } }
+      deletion + transposition + alteration + insertion
+    end
+
+    def normalize_query(query)
+      tokenize(query.to_s.gsub("&", "and")).map { |q| @stemmer.stem(q) }.sort.join
+    end
+
+    def add_nodes(var, words)
+      words.each do |word|
+        node = var
+        tokenize(word).each do |token|
+          node = (node[token] ||= {})
+        end
+        node[:eos] = true
+      end
+      var
+    end
+  end
+end

data/lib/autosuggest/version.rb

@@ -1,3 +1,3 @@
-class Autosuggest
-  VERSION = "0.1.3"
+module Autosuggest
+  VERSION = "0.3.0"
 end

data/lib/autosuggest.rb

@@ -3,230 +3,15 @@ require "set"
 require "yaml" # for obscenity
 
 # dependencies
-require "lingua/stemmer"
+require "mittens"
 require "obscenity"
 
 # modules
-require "autosuggest/version"
+require_relative "autosuggest/generator"
+require_relative "autosuggest/version"
 
-class Autosuggest
-  def initialize(top_queries)
-    @top_queries = top_queries
-    @concepts = {}
-    @words = Set.new
-    @non_duplicates = Set.new
-    @blocked_words = {}
-    @blacklisted_words = {}
-    @preferred_queries = {}
-    @profane_words = {}
-    @concept_tree = {}
-    add_nodes(@profane_words, Obscenity::Base.blacklist)
-  end
-
-  def add_concept(name, values)
-    values = values.compact.uniq
-    add_nodes(@concept_tree, values)
-    @concepts[name] = Set.new(values.map(&:downcase))
-  end
-
-  def parse_words(phrases, options = {})
-    min = options[:min] || 1
-
-    word_counts = Hash.new(0)
-    phrases.each do |phrase|
-      words = tokenize(phrase)
-      words.each do |word|
-        word_counts[word] += 1
-      end
-    end
-
-    word_counts.select { |_, c| c >= min }.each do |word, _|
-      @words << word
-    end
-
-    word_counts
-  end
-
-  def not_duplicates(pairs)
-    pairs.each do |pair|
-      @non_duplicates << pair.map(&:downcase).sort
-    end
-  end
-
-  def block_words(words)
-    add_nodes(@blocked_words, words)
-    words
-  end
-
-  def blacklist_words(words)
-    warn "[autosuggest] blacklist_words is deprecated. Use block_words instead."
-    add_nodes(@blacklisted_words, words)
-    words
-  end
-
-  def prefer(queries)
-    queries.each do |query|
-      @preferred_queries[normalize_query(query)] ||= query
-    end
-  end
-
-  # TODO add queries method for filter: false and make suggestions use filter: true in 0.2.0
-  def suggestions(filter: false)
-    stemmed_queries = {}
-    added_queries = Set.new
-    results = @top_queries.sort_by { |_query, count| -count }.map do |query, count|
-      query = query.to_s
-
-      # TODO do not ignore silently
-      next if query.length < 2
-
-      stemmed_query = normalize_query(query)
-
-      # get preferred term
-      preferred_query = @preferred_queries[stemmed_query]
-      if preferred_query && preferred_query != query
-        original_query, query = query, preferred_query
-      end
-
-      # exclude duplicates
-      duplicate = stemmed_queries[stemmed_query]
-      stemmed_queries[stemmed_query] ||= query
-
-      # also detect possibly misspelled duplicates
-      # TODO use top query as duplicate
-      if !duplicate && query.length > 4
-        edits(query).each do |edited_query|
-          if added_queries.include?(edited_query)
-            duplicate = edited_query
-            break
-          end
-        end
-      end
-      if duplicate && @non_duplicates.include?([duplicate, query].sort)
-        duplicate = nil
-      end
-      added_queries << query unless duplicate
-
-      # find concepts
-      concepts = []
-      @concepts.each do |name, values|
-        concepts << name if values.include?(query)
-      end
-
-      tokens = tokenize(query)
-
-      # exclude misspellings that are not brands
-      misspelling = @words.any? && misspellings?(tokens)
-
-      profane = blocked?(tokens, @profane_words)
-      blocked = blocked?(tokens, @blocked_words)
-      blacklisted = blocked?(tokens, @blacklisted_words)
-
-      notes = []
-      notes << "duplicate of #{duplicate}" if duplicate
-      notes.concat(concepts)
-      notes << "misspelling" if misspelling
-      notes << "profane" if profane
-      notes << "blocked" if blocked
-      notes << "blacklisted" if blacklisted
-      notes << "originally #{original_query}" if original_query
-
-      result = {
-        query: query,
-        original_query: original_query,
-        score: count,
-        duplicate: duplicate,
-        concepts: concepts,
-        misspelling: misspelling,
-        profane: profane,
-        blocked: blocked
-      }
-      result[:blacklisted] = blacklisted if @blacklisted_words.any?
-      result[:notes] = notes
-      result
-    end
-    if filter
-      results.reject! { |s| s[:duplicate] || s[:misspelling] || s[:profane] || s[:blocked] }
-    end
-    results
-  end
-
-  def pretty_suggestions
-    str = "%-30s   %5s   %s\n" % %w(Query Score Notes)
-    suggestions.each do |suggestion|
-      str << "%-30s   %5d   %s\n" % [suggestion[:query], suggestion[:score], suggestion[:notes].join(", ")]
-    end
-    str
-  end
-
-  protected
-
-  def misspellings?(tokens)
-    pos = [0]
-    while i = pos.shift
-      return false if i == tokens.size
-
-      if @words.include?(tokens[i])
-        pos << i + 1
-      end
-
-      node = @concept_tree[tokens[i]]
-      j = i
-      while node
-        j += 1
-        pos << j if node[:eos]
-        break if j == tokens.size
-        node = node[tokens[j]]
-      end
-
-      pos.uniq!
-    end
-    true
-  end
-
-  def blocked?(tokens, blocked_words)
-    tokens.each_with_index do |token, i|
-      node = blocked_words[token]
-      j = i
-      while node
-        return true if node[:eos]
-        j += 1
-        break if j == tokens.size
-        node = node[tokens[j]]
-      end
-    end
-    false
-  end
-
-  def tokenize(str)
-    str.to_s.downcase.split(" ")
-  end
-
-  # from https://blog.lojic.com/2008/09/04/how-to-write-a-spelling-corrector-in-ruby/
-  LETTERS = ("a".."z").to_a.join + "'"
-  def edits(word)
-    n = word.length
-    deletion = (0...n).collect { |i| word[0...i] + word[i + 1..-1] }
-    transposition = (0...n - 1).collect { |i| word[0...i] + word[i + 1, 1] + word[i, 1] + word[i + 2..-1] }
-    alteration = []
-    n.times { |i| LETTERS.each_byte { |l| alteration << word[0...i] + l.chr + word[i + 1..-1] } }
-    insertion = []
-    (n + 1).times { |i| LETTERS.each_byte { |l| insertion << word[0...i] + l.chr + word[i..-1] } }
-    deletion + transposition + alteration + insertion
-  end
-
-  def normalize_query(query)
-    tokenize(query.to_s.gsub("&", "and")).map { |q| Lingua.stemmer(q) }.sort.join
-  end
-
-  def add_nodes(var, words)
-    words.each do |word|
-      node = var
-      tokenize(word).each do |token|
-        node = (node[token] ||= {})
-      end
-      node[:eos] = true
-    end
-    var
+module Autosuggest
+  def self.new(*args, **options)
+    Generator.new(*args, **options)
   end
 end

data/lib/generators/autosuggest/suggestions_generator.rb

@@ -1,6 +1,6 @@
 require "rails/generators/active_record"
 
-class Autosuggest
+module Autosuggest
   module Generators
     class SuggestionsGenerator < Rails::Generators::Base
       include ActiveRecord::Generators::Migration

metadata

@@ -1,17 +1,17 @@
 --- !ruby/object:Gem::Specification
 name: autosuggest
 version: !ruby/object:Gem::Version
-  version: 0.1.3
+  version: 0.3.0
 platform: ruby
 authors:
 - Andrew Kane
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-11-24 00:00:00.000000000 Z
+date: 2024-05-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: ruby-stemmer
+  name: mittens
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -48,6 +48,7 @@ files:
 - LICENSE.txt
 - README.md
 - lib/autosuggest.rb
+- lib/autosuggest/generator.rb
 - lib/autosuggest/version.rb
 - lib/generators/autosuggest/suggestions_generator.rb
 - lib/generators/autosuggest/templates/migration.rb.tt
@@ -64,14 +65,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '2.4'
+      version: '3.1'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.2.22
+rubygems_version: 3.5.9
 signing_key:
 specification_version: 4
 summary: Generate autocomplete suggestions based on what your users search