polysearch 0.1.0 → 0.2.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 21bf6d369ed9516908e5ac52bb3b5688335e05952e920741cccd40d73b2279d8
-  data.tar.gz: 7257fa7063ad0b250c7d3b602a7acc5bc3b42984b6fefa47202629eedbb03018
+  metadata.gz: babc73df6dc70b27a5f6ae5c0adb98af6adfaa5080ba9072800002d3102af646
+  data.tar.gz: 6dc3742766a2f8d737c6f045741fc1981a0a538ca269bfae11ab28ca4d8897b2
 SHA512:
-  metadata.gz: 8995e6d99585aadc2add2e47b9d18829313dd29853468c706e0297a4361b633acbe593db5fcd34e0a2890cc1832f4758d95f74d3a10ad990fde0dc41e91f348b
-  data.tar.gz: c6e75132f957e688b23ef5499fc8af50189edeb470ed664ad3dab4ca5096a3a343e7e33902d98fb0db2113b7993610b8126ee20653bac026787c7c3397c9d380
+  metadata.gz: fdee3a4765bd4d4ca85671355a84c2f71336a6ef0d7d31bfc1287975e384f0b42148a383c418a13eddade4d2681c80a67c996ba9eed83b73aaad1dc72e764b84
+  data.tar.gz: efc6a17835a5400d5220e434ff6ca332d2ce8a0aa09d0ed2a55fd772da8ea97546389f2fe1be3e4243091b02012d243e26ecd62143f75b11633c774071715568

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    polysearch (0.1.0)
+    polysearch (0.2.4)
       rails (>= 6.0)
 
 GEM
@@ -68,14 +68,14 @@ GEM
       zeitwerk (~> 2.3)
     ast (2.4.2)
     builder (3.2.4)
-    concurrent-ruby (1.1.8)
+    concurrent-ruby (1.1.9)
     crass (1.0.6)
     erubi (1.10.0)
     globalid (0.4.2)
       activesupport (>= 4.2.0)
     i18n (1.8.10)
       concurrent-ruby (~> 1.0)
-    loofah (2.9.1)
+    loofah (2.10.0)
       crass (~> 1.0.2)
       nokogiri (>= 1.5.9)
     magic_frozen_string_literal (1.2.0)
@@ -84,13 +84,13 @@ GEM
     marcel (1.0.1)
     method_source (1.0.0)
     mini_mime (1.0.3)
-    mini_portile2 (2.5.1)
+    mini_portile2 (2.5.3)
     minitest (5.14.4)
     nio4r (2.5.7)
-    nokogiri (1.11.4)
+    nokogiri (1.11.7)
       mini_portile2 (~> 2.5.0)
       racc (~> 1.4)
-    nokogiri (1.11.4-arm64-darwin)
+    nokogiri (1.11.7-arm64-darwin)
       racc (~> 1.4)
     parallel (1.20.1)
     parser (3.0.1.1)
@@ -138,7 +138,7 @@ GEM
       rubocop-ast (>= 1.5.0, < 2.0)
       ruby-progressbar (~> 1.7)
       unicode-display_width (>= 1.4.0, < 3.0)
-    rubocop-ast (1.5.0)
+    rubocop-ast (1.7.0)
       parser (>= 3.0.1.1)
     rubocop-performance (1.11.2)
       rubocop (>= 1.7.0, < 2.0)
@@ -160,7 +160,7 @@ GEM
     tzinfo (2.0.4)
       concurrent-ruby (~> 1.0)
     unicode-display_width (2.0.0)
-    websocket-driver (0.7.3)
+    websocket-driver (0.7.5)
       websocket-extensions (>= 0.1.0)
     websocket-extensions (0.1.5)
     zeitwerk (2.4.2)

data/README.md

@@ -1,6 +1,10 @@
+[![Lines of Code](http://img.shields.io/badge/lines_of_code-237-brightgreen.svg?style=flat)](http://blog.codinghorror.com/the-best-code-is-no-code-at-all/)
+
 # Polysearch
 
-#### Simplified polymorphic full text + similarity search based on postgres
+Simplified polymorphic full text + similarity search based on postgres.
+
+> NOTE: This project is narrower in scope and more opinionated than [pg_search](https://github.com/Casecommons/pg_search).
 
 ## Requirements
 
@@ -43,23 +47,43 @@
       after_save_commit :update_polysearch
 
       def to_tsvectors
-        []
-          .then { |list| list << make_tsvector(first_name, weight: "A") }
-          .then { |list| list << make_tsvector(last_name, weight: "A") }
-          .then { |list| list << make_tsvector(email, weight: "B") }
+        [
+          make_tsvector(first_name, weight: "A"),
+          make_tsvector(last_name, weight: "A"),
+          make_tsvector(nickname, weight: "B")
+        ]
       end
     end
     ```
 
-1. Start searching
+    If you have existing records that need to create/update a polysearch record, you can save them like this.
 
+    ```ruby
+    User.find_each(&:update_polysearch)
     ```
-    User.create first_name: "Nate", last_name: "Hopkins", email: "nhopkins@mailinator.com"
 
-    User.polysearch("nate")
-    User.polysearch("ntae") # misspellings also return results
-    User.polysearch("nate").where(created_at: 1.day.ago..Current.time) # active record chaining
-    User.polysearch("nate").order(created_at: :desc) # chain additional ordering after the polysearch scope
+1. Start searching
+
+    ```ruby
+    User.create first_name: "Shawn", last_name: "Spencer", nickname: "Maverick"
+
+    # find natural language matches (faster)
+    User.full_text_search("shawn")
+
+    # find similarity matches, best for misspelled search terms (slower)
+    User.similarity_search("shwn")
+
+    # perform both a full text search and similarity search
+    User.combined_search("shwn")
+
+    # perform a full text search and fall back to similarity search (faster than combined_search)
+    User.polysearch("shwn")
+
+    # calculate counts (explicitly pass :id to omit search rankings)
+    User.full_text_search("shawn").count(:id)
+    User.similarity_search("shwn").count(:id)
+    User.combined_search("shwn").count(:id)
+    User.polysearch("shwn").count(:id)
     ```
 
 ## License

data/app/models/{record.rb → polysearch/record.rb}

@@ -33,39 +33,57 @@ module Polysearch
 
     # scopes ....................................................................
 
-    scope :select_fts_rank, ->(value, *selects, rank_alias: nil) {
-      value = value.to_s.gsub(/\W/, " ").squeeze(" ").downcase.strip
-      value = Arel::Nodes::SqlLiteral.new(sanitize_sql_array(["?", value.to_s]))
-      plainto_tsquery = Arel::Nodes::NamedFunction.new("plainto_tsquery", [Arel::Nodes::SqlLiteral.new("'simple'"), value])
+    scope :select_full_text_search_rank, ->(value, *selects) {
+      plainto_tsquery = Arel::Nodes::NamedFunction.new("plainto_tsquery", [Arel::Nodes::SqlLiteral.new("'simple'"), arel_search_value(value)])
       ts_rank = Arel::Nodes::NamedFunction.new("ts_rank", [arel_table[:value], plainto_tsquery])
-
-      rank_alias ||= "fts_rank"
       selects << Arel.star if selects.blank?
-      selects << ts_rank.as(rank_alias)
-      select(*selects).order("#{rank_alias} desc")
+      selects << ts_rank.as("search_rank")
+      select(*selects).reorder("search_rank desc")
     }
 
-    scope :fts, ->(value) {
-      value = value.to_s.gsub(/\W/, " ").squeeze(" ").downcase.strip
-      value = Arel::Nodes::SqlLiteral.new(sanitize_sql_array(["?", value.to_s]))
-      plainto_tsquery = Arel::Nodes::NamedFunction.new("plainto_tsquery", [Arel::Nodes::SqlLiteral.new("'simple'"), value])
-      where(Arel::Nodes::InfixOperation.new("@@", arel_table[:value], plainto_tsquery))
+    scope :full_text_search, ->(value) {
+      if value.blank?
+        all
+      else
+        plainto_tsquery = Arel::Nodes::NamedFunction.new("plainto_tsquery", [Arel::Nodes::SqlLiteral.new("'simple'"), arel_search_value(value)])
+        where(Arel::Nodes::InfixOperation.new("@@", arel_table[:value], plainto_tsquery))
+      end
     }
 
-    scope :select_similarity_rank, ->(value, *selects, rank_alias: nil) {
-      value = value.to_s.gsub(/\W/, " ").squeeze(" ").downcase.strip
-      value = Arel::Nodes::SqlLiteral.new(sanitize_sql_array(["?", value.to_s]))
-
-      rank_alias ||= "similarity_rank"
+    scope :select_similarity_rank, ->(value, *selects) {
+      similarity = Arel::Nodes::NamedFunction.new("similarity", [arel_table[:words], arel_search_value(value)])
       selects << Arel.star if selects.blank?
-      selects << Arel::Nodes::NamedFunction.new("similarity", [arel_table[:words], value]).as(rank_alias)
-      select(*selects).order("#{rank_alias} desc")
+      selects << similarity.as("search_rank")
+      select(*selects).order("search_rank desc")
+    }
+
+    scope :similarity_search, ->(value) {
+      if value.blank?
+        all
+      else
+        where Arel::Nodes::NamedFunction.new("similarity", [arel_table[:words], arel_search_value(value)]).gt(0)
+      end
+    }
+
+    scope :combined_search, ->(value) {
+      subquery = <<~SQL
+        (
+          #{select_full_text_search_rank(value).full_text_search(value).except(:order).to_sql}
+          UNION ALL
+          #{select_similarity_rank(value).similarity_search(value).except(:order).to_sql}
+        ) AS #{table_name}
+      SQL
+      from(subquery).order("search_rank desc")
     }
 
-    scope :similar, ->(value, range: 0.01) {
-      value = value.to_s.gsub(/\W/, " ").squeeze(" ").downcase.strip
-      value = Arel::Nodes::SqlLiteral.new(sanitize_sql_array(["?", value.to_s]))
-      where Arel::Nodes::NamedFunction.new("similarity", [arel_table[:words], value]).gteq(range)
+    scope :polysearch, ->(value) {
+      if value.blank?
+        all
+      else
+        full_text_search(value).exists? ?
+          select_full_text_search_rank(value).full_text_search(value) :
+          select_similarity_rank(value).similarity_search(value)
+      end
     }
 
     # additional config (i.e. accepts_nested_attribute_for etc...) ..............
@@ -74,6 +92,10 @@ module Polysearch
 
     # class methods .............................................................
     class << self
+      def arel_search_value(value)
+        value = value.to_s.gsub(/\W/, " ").squeeze(" ").downcase.strip
+        Arel::Nodes::SqlLiteral.new(sanitize_sql_array(["?", value]))
+      end
     end
 
     # public instance methods ...................................................

data/app/models/{concerns → polysearch}/searchable.rb

@@ -36,39 +36,54 @@ module Polysearch
       has_one :polysearch, as: :searchable, class_name: "Polysearch::Record", inverse_of: "searchable"
       after_destroy :destroy_polysearch
 
+      scope :full_text_search, ->(value) {
+        value.blank? ? all : joins(:polysearch).merge(Polysearch::Record.full_text_search(value).select_full_text_search_rank(value))
+      }
+
+      scope :similarity_search, ->(value) {
+        value.blank? ? all : joins(:polysearch).merge(Polysearch::Record.similarity_search(value).select_similarity_rank(value))
+      }
+
+      scope :combined_search, ->(value) {
+        subquery = <<~SQL
+          (
+            SELECT #{table_name}.*, searchable_polysearches.search_rank from (#{Polysearch::Record.combined_search(value).except(:order).to_sql}) searchable_polysearches
+            LEFT JOIN LATERAL (select * from #{table_name} WHERE id = searchable_polysearches.searchable_id) #{table_name} ON TRUE
+          ) #{table_name}
+        SQL
+        from(subquery).order("search_rank desc")
+      }
+
       scope :polysearch, ->(value) {
-        if value.blank?
-          all
-        else
-          fts_rank_alias = "#{table_name.singularize}_fts_rank"
-          similarity_rank_alias = "#{table_name.singularize}_similarity_rank"
-
-          fts = Polysearch::Record
-            .select_fts_rank(value, :searchable_id, rank_alias: fts_rank_alias)
-            .select_similarity_rank(value, :searchable_id, rank_alias: similarity_rank_alias)
-            .where(searchable_type: name)
-            .fts(value).or(Polysearch::Record.similar(value))
-
-          query = <<~SQL
-            SELECT searchables.*, fts.#{fts_rank_alias}, fts.#{similarity_rank_alias} from (#{fts.to_sql}) fts
-            LEFT JOIN LATERAL (select * from #{table_name} WHERE id = fts.searchable_id) searchables ON TRUE
-          SQL
-
-          select(Arel.star).from(Arel::Nodes::SqlLiteral.new("(#{query})").as(table_name))
-            .reorder(fts_rank_alias => :desc, similarity_rank_alias => :desc)
-        end
+        value.blank? ? all : joins(:polysearch).merge(Polysearch::Record.polysearch(value))
       }
     end
 
     def update_polysearch
+      return unless persisted?
       tsvectors = to_tsvectors.compact.uniq
       return if tsvectors.blank?
+
       tsvectors.pop while tsvectors.size > 500
       tsvectors.concat similarity_words_tsvectors
       tsvector = tsvectors.join(" || ")
-      fts = Polysearch::Record.where(searchable: self).first_or_create
-      fts.update_value tsvector
-      fts.update_columns words: similarity_words.join(" ")
+
+      attributes = {
+        searchable_type: self.class.name,
+        searchable_id: id,
+        words: similarity_words.join(" "),
+        created_at: Time.current,
+        updated_at: Time.current
+      }
+
+      result = Polysearch::Record.upsert(
+        attributes,
+        unique_by: [:searchable_type, :searchable_id],
+        returning: :id
+      )
+
+      record = Polysearch::Record.find_by(id: result.first["id"])
+      record.update_value tsvector
     end
 
     # Polysearch::Searchable#to_tsvectors is abstract... a noop by default
@@ -95,7 +110,7 @@ module Polysearch
         Arel::Nodes::SqlLiteral.new(sanitize_sql_value("SELECT #{tsvector}"))
       ])
       length = Arel::Nodes::NamedFunction.new("length", [Arel::Nodes::SqlLiteral.new(quote_column_name(:word))])
-      query = self.class.select(:word).from(ts_stat.to_sql).where(length.gteq(3)).to_sql
+      query = self.class.unscoped.select(:word).from(ts_stat.to_sql).where(length.gteq(3)).to_sql
       result = self.class.connection.execute(query)
       result.values.flatten
     end

data/bin/loc

@@ -0,0 +1,3 @@
+#!/usr/bin/env bash
+
+cloc --exclude-dir=test --include-ext=rb .

data/lib/polysearch.rb

@@ -1,8 +1,8 @@
 # frozen_string_literal: true
 
 require_relative "polysearch/version"
-require_relative "../app/models/record"
-require_relative "../app/models/concerns/searchable"
+require_relative "../app/models/polysearch/record"
+require_relative "../app/models/polysearch/searchable"
 
 module Polysearch
   class Engine < Rails::Engine

data/lib/polysearch/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Polysearch
-  VERSION = "0.1.0"
+  VERSION = "0.2.4"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: polysearch
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.4
 platform: ruby
 authors:
 - Nathan Hopkins
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-05-18 00:00:00.000000000 Z
+date: 2021-06-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rails
@@ -92,9 +92,10 @@ files:
 - LICENSE.txt
 - README.md
 - Rakefile
-- app/models/concerns/searchable.rb
-- app/models/record.rb
+- app/models/polysearch/record.rb
+- app/models/polysearch/searchable.rb
 - bin/console
+- bin/loc
 - bin/setup
 - bin/standardize
 - lib/generators/polysearch/migration_generator.rb