hayfork 1.0.0

data/lib/generators/hayfork/templates/migrations/create.rb

@@ -0,0 +1,51 @@
+class Create<%= table_name.camelize %> < ActiveRecord::Migration<%= migration_version %>
+  def change
+    enable_extension :unaccent
+
+    # Full-Text Search in Postgres can be configured extensively.
+    #
+    # Custom Parsers are able to match entities in documents and handle them
+    # differently. Custom Dictionaries can be used to tune search for different
+    # languages and to ignore certain words.
+    #
+    # Learn more at https://www.postgresql.org/docs/10/textsearch.html
+    #
+    # This block creates a TEXT SEARCH CONFIGURATION specifically for use
+    # by Hayfork that uses an English dictionary and normalizes characters
+    # by stripping accent marks and converting special characters (like smart
+    # quotes) to their easily-typed ASCII counterparts (like straight quotes).
+    #
+    execute <<~SQL
+      CREATE TEXT SEARCH CONFIGURATION public.hayfork ( COPY = pg_catalog.english );
+      ALTER TEXT SEARCH CONFIGURATION public.hayfork ALTER MAPPING FOR asciiword, asciihword, hword_asciipart, hword, hword_part, word WITH unaccent, english_stem;
+    SQL
+
+    create_table :<%= table_name %>, id: false do |t|
+      t.string :<%= Hayfork::SEARCH_RESULT_TYPE %>, null: false
+      t.integer :<%= Hayfork::SEARCH_RESULT_ID %>, null: false
+
+      t.string :<%= Hayfork::SOURCE_TYPE %>, null: false
+      t.integer :<%= Hayfork::SOURCE_ID %>, null: false
+      t.string :<%= Hayfork::FIELD %>, null: false
+
+      t.tsvector :<%= Hayfork::SEARCH_VECTOR %>
+      t.text :<%= Hayfork::TEXT %>, null: false
+
+      # Add additional columns to <%= table_name %> here.
+      #
+      # For example, to allow users to search only documents of their
+      # own creation:
+      #
+      #    t.belongs_to :user, null: false
+      #
+
+      # If you add columns that will always be used in searches (like `user_id`),
+      # consider including them in this index. For example:
+      #
+      #    enable_extension :btree_gist
+      #    t.index [:tenant_id, :<%= Hayfork::SEARCH_VECTOR %>], using: "gist"
+      #
+      t.index :<%= Hayfork::SEARCH_VECTOR %>, using: "gist"
+    end
+  end
+end

data/lib/generators/hayfork/templates/migrations/rebuild.rb

@@ -0,0 +1,8 @@
+require "triggers"
+
+class Rebuild<%= table_name.camelize %> < ActiveRecord::Migration<%= migration_version %>
+  def up
+    execute <%= class_name %>.triggers.replace
+    execute <%= class_name %>.triggers.rebuild.lines
+  end
+end

data/lib/generators/hayfork/templates/model.rb

@@ -0,0 +1,14 @@
+class <%= class_name %> < ActiveRecord::Base
+<% unless table_name == class_name.tableize -%>
+  self.table_name = "<%= table_name %>"
+<% end -%>
+
+  belongs_to :search_result, polymorphic: true
+
+  def self.search(querystring)
+    ::Query.parse(querystring).against(all)
+      .select(:search_result_type, :search_result_id).distinct
+      .preload(:search_result).map(&:search_result)
+  end
+
+end

data/lib/generators/hayfork/templates/query.rb

@@ -0,0 +1,10 @@
+require_relative "query/parser"
+require_relative "query/object"
+
+module Query
+
+  def self.parse(querystring)
+    Query::Parser.new(Query::Object, querystring).parse!
+  end
+
+end

data/lib/generators/hayfork/templates/query/exact_phrase.rb

@@ -0,0 +1,19 @@
+module Query
+  class ExactPhrase < Struct.new(:words)
+
+    def apply(set)
+      set.where(Arel::Nodes::InfixOperation.new("@@",
+        set.arel_table[Hayfork::SEARCH_VECTOR],
+        to_tsquery(Hayfork.default_dictionary, words.join(" <-> "))))
+    end
+
+  private
+
+    def to_tsquery(dictionary, querystring)
+      Arel::Nodes::NamedFunction.new("to_tsquery", [
+        Arel::Nodes.build_quoted(dictionary),
+        Arel::Nodes.build_quoted(querystring) ])
+    end
+
+  end
+end

data/lib/generators/hayfork/templates/query/object.rb

@@ -0,0 +1,4 @@
+module Query
+  class Object < Hayfork::QueryObject
+  end
+end

data/lib/generators/hayfork/templates/query/parser.rb

@@ -0,0 +1,33 @@
+require_relative "exact_phrase"
+
+module Query
+  class Parser < Hayfork::QueryParser
+
+    def parse_phrase(querystring, phrases)
+      tokenize_words(querystring).each do |word|
+        phrases << Query::ExactPhrase.new([ word ])
+      end
+    end
+
+    def parse_exact_phrase(querystring, phrases)
+      phrases << Query::ExactPhrase.new(tokenize_words(querystring))
+    end
+
+    def tokenize_words(querystring)
+      # Postgres does not handle hyphens well.
+      #
+      # Notice how, in the following example, the way it breaks up
+      # the hyphenated word throws off the index (Jesus is the fifth word
+      # not the third or fourth). This prevents you from constructing
+      # an exact-phrase query for a hyphenated word:
+      #
+      #   > select to_tsvector('hayfork', 'thou long-expected jesus');
+      #   { 'expect':4 'jesus':5 'long':3 'long-expect':2 'thou':1 }
+      #
+      #
+      # We'll coerce Postgres into treating hyphenated words as two words.
+      querystring.to_s.scan(/\w+/)
+    end
+
+  end
+end

data/lib/generators/hayfork/templates/triggers.rb

@@ -0,0 +1,21 @@
+Hayfork.maintain(<%= class_name %>) do
+
+  # Define triggers here.
+  # Learn more at https://github.com/boblail/hayfork#libhaystack_triggersrb
+  #
+  # Example:
+  #
+  #    Hayfork.maintain(Haystack) do
+  #
+  #      foreach(Employee) do
+  #        insert(:full_name)
+  #      end
+  #
+  #      foreach(Project) do
+  #        insert(:title)
+  #      end
+  #
+  #    end
+  #
+
+end

data/lib/hayfork.rb

@@ -0,0 +1,35 @@
+require "hayfork/errors"
+require "hayfork/join"
+require "hayfork/query_object"
+require "hayfork/query_parser"
+require "hayfork/trigger_builder"
+require "hayfork/triggers"
+require "hayfork/unaccent"
+require "hayfork/version"
+
+module Hayfork
+  extend Hayfork::Unaccent, Hayfork::Join
+
+  TEXT = "text".freeze
+  SEARCH_VECTOR = "search_vector".freeze
+  SEARCH_RESULT_TYPE = "search_result_type".freeze
+  SEARCH_RESULT_ID = "search_result_id".freeze
+  SOURCE_TYPE = "source_type".freeze
+  SOURCE_ID = "source_id".freeze
+  FIELD = "field".freeze
+
+  @default_weight = "C".freeze
+  @default_dictionary = "hayfork".freeze
+
+  class << self
+    attr_accessor :default_weight, :default_dictionary
+
+    def maintain(haystack, &block)
+      triggers = Triggers.new(haystack)
+      TriggerBuilder.new(triggers).instance_eval(&block)
+      haystack.singleton_class.send(:attr_reader, :triggers)
+      haystack.instance_variable_set :@triggers, triggers
+    end
+  end
+
+end

data/lib/hayfork/binding.rb

@@ -0,0 +1,67 @@
+module Hayfork
+  class Binding < Struct.new(:statement, :column, :raw_value)
+
+    def to_sql
+      "#{quoted_value} \"#{column.name}\""
+    end
+    alias to_s to_sql
+
+    def column_name
+      column.name
+    end
+    alias key column_name
+
+    def quoted_value
+      case raw_value
+      when Arel::Attributes::Attribute
+        value_column = raw_value.relation.send(:type_caster).send(:types).columns_hash[raw_value.name.to_s]
+        fail Hayfork::ColumnNotFoundError, "'#{raw_value.name}' is not a column on '#{raw_value.relation.name}'" unless value_column
+
+        value = "#{raw_value.relation.name}.#{raw_value.name}"
+
+        unless column.sql_type == value_column.sql_type
+          type = SPECIAL_CASTS.fetch(column.sql_type, column.sql_type)
+          value = "#{value}::#{type}"
+        end
+
+        if statement.unnest? && [Hayfork::SEARCH_VECTOR, Hayfork::TEXT].member?(column.name)
+          value = "unnest(string_to_array(#{value}, E'\\n'))"
+        end
+
+        if column.type == :tsvector
+
+          # Postgres does not handle hyphens well.
+          #
+          # Notice how, in the following example, the way it breaks up
+          # those words throws off the index (Jesus is the fifth word
+          # not the third or fourth). This prevents you from constructing
+          # an exact-phrase query for a hyphenated word:
+          #
+          #   > select to_tsvector('hayfork', 'thou long-expected jesus');
+          #   { 'expect':4 'jesus':5 'long':3 'long-expect':2 'thou':1 }
+          #
+          #
+          # We'll coerce Postgres into treating hyphenated words as two words.
+
+          value = "setweight(to_tsvector('#{statement.dictionary}', replace(#{value}, '-', ' ')), '#{statement.weight}')"
+        end
+
+        value
+
+      when Arel::Nodes::Node
+        raw_value.to_sql
+
+      else
+        type = SPECIAL_CASTS.fetch(column.sql_type, column.sql_type)
+        "#{statement.haystack.connection.quote(raw_value)}::#{type}"
+
+      end
+    end
+
+    SPECIAL_CASTS = {
+      "character varying" => "varchar",
+      "tsvector" => "varchar"
+    }.freeze
+
+  end
+end

data/lib/hayfork/delete_sql.rb

@@ -0,0 +1,24 @@
+module Hayfork
+  class DeleteSql
+    attr_reader :haystack, :relation, :bindings
+
+    def initialize(haystack, relation, bindings)
+      @haystack = haystack
+      @relation = relation
+      @bindings = bindings.reject { |binding| binding.key == Hayfork::SEARCH_VECTOR || binding.key == Hayfork::TEXT }
+    end
+
+    def to_sql
+      select_statement = relation.select(bindings.map(&:to_s))
+      select_statement = select_statement.from("(SELECT OLD.*) \"#{relation.table_name}\"")
+
+      constraints = bindings.map { |binding| "#{haystack.table_name}.#{binding.key}=x.#{binding.key}" }.join(" AND ")
+
+      <<~SQL
+        DELETE FROM #{haystack.table_name} USING (#{select_statement.to_sql}) "x" WHERE #{constraints};
+      SQL
+    end
+    alias to_s to_sql
+
+  end
+end

data/lib/hayfork/errors.rb

@@ -0,0 +1,4 @@
+module Hayfork
+  class AssociationNotFoundError < ArgumentError; end
+  class ColumnNotFoundError < ArgumentError; end
+end

data/lib/hayfork/insert_sql.rb

@@ -0,0 +1,22 @@
+module Hayfork
+  class InsertSql
+    attr_reader :haystack, :relation, :bindings
+
+    def initialize(haystack, relation, bindings)
+      @haystack = haystack
+      @relation = relation
+      @bindings = bindings
+    end
+
+    def to_sql(from: true)
+      select_statement = relation.select(bindings.map(&:to_s))
+      select_statement = select_statement.from("(SELECT NEW.*) \"#{relation.table_name}\"") if from
+
+      <<~SQL
+        INSERT INTO #{haystack.table_name} (#{bindings.map(&:key).join(", ")}) SELECT * FROM (#{select_statement.to_sql}) "x" WHERE "x"."#{Hayfork::TEXT}" != '';
+      SQL
+    end
+    alias to_s to_sql
+
+  end
+end

data/lib/hayfork/join.rb

@@ -0,0 +1,32 @@
+module Hayfork
+  module Join
+
+    def join(relation, join_value)
+      case join_value
+      when String
+        relation.joins(join_value)
+
+      when Symbol
+        reflection = reflection_for(relation.model, join_value)
+        case reflection.macro
+        when :has_many, :has_and_belongs_to_many
+          relation.left_outer_joins(join_value).where(reflection.klass.arel_table[:id].not_eq(nil))
+        when :belongs_to, :has_one
+          relation.joins(join_value)
+        else
+          fail NotImplementedError, "Joins haven't been implemented for #{reflection.macro.inspect} associations"
+        end
+
+      else
+        fail NotImplementedError, "Statement#joins does not yet accept #{join_value.class} params like #{join_value.inspect}. Will this scenario work with `has_many through:` or `has_one through:`?"
+      end
+    end
+
+    def reflection_for(model, association)
+      reflection = model.reflect_on_association(association)
+      fail AssociationNotFoundError, "Association ':#{association}' not found on '#{model}'" unless reflection
+      reflection
+    end
+
+  end
+end

data/lib/hayfork/query_object.rb

@@ -0,0 +1,38 @@
+module Hayfork
+  class QueryObject
+    attr_reader :phrases
+
+    def initialize(phrases)
+      @phrases = phrases
+    end
+
+    def against(set)
+      return set.none if phrases.none?
+      return phrases.first.apply(set) if phrases.one?
+
+      # The haystack may contain more than one hit per result.
+      #
+      # A composite query may match more than one hit for a result.
+      # For example, it may match both a hymn's author and its tune.
+      #
+      # If we search more than one phrase, we want to find all of
+      # the hits that match *_any_* of the search phrases but only
+      # return hits for results that match *_all_* of the phrases.
+      filter_hits_by_any_phrase(filter_results_that_match_all_phrases(set))
+    end
+
+  private
+
+    def filter_hits_by_any_phrase(set)
+      phrases[1..-1].inject(phrases.first.apply(set)) { |memo, phrase| memo.or(phrase.apply(set)) }
+    end
+
+    def filter_results_that_match_all_phrases(set)
+      scope = set.select(Hayfork::SEARCH_RESULT_ID)
+      set.where(
+        set.arel_table[Hayfork::SEARCH_RESULT_ID].in(
+          phrases[1..-1].inject(phrases.first.apply(scope).arel) { |memo, phrase| Arel::Nodes::Intersect.new(memo, phrase.apply(scope).arel) }))
+    end
+
+  end
+end

data/lib/hayfork/query_parser.rb

@@ -0,0 +1,35 @@
+module Hayfork
+  class QueryParser
+    attr_reader :klass, :querystring
+
+    def initialize(klass, querystring)
+      @klass = klass
+      @querystring = querystring
+    end
+
+    def parse!
+      phrases = []
+      scanner = StringScanner.new(Hayfork.unaccent(querystring))
+
+      until scanner.eos?
+        parse_phrase(scanner.scan(/[^"]+/), phrases)
+        break if scanner.eos?
+
+        scanner.getch # "
+        parse_exact_phrase(scanner.scan(/[^"]+/), phrases)
+        scanner.getch # "
+      end
+
+      klass.new(phrases)
+    end
+
+    def parse_phrase(querystring, phrases)
+      raise NotImplementedError
+    end
+
+    def parse_exact_phrase(querystring, phrases)
+      raise NotImplementedError
+    end
+
+  end
+end