runestone 1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 3756ae6e1dcf043dc6d4ca6e1312a0409ceac0fd15a90e6a7859c7d5cfec0ef9
+  data.tar.gz: a2e9bfd44ae564d10ddd6ee9f25179b267b12979dd7bfe2311825ce2bc348d97
+SHA512:
+  metadata.gz: 4c6b1cd82ea806fd419ab2b9986b27d2fabb0d0d898decf69068bb9b824c642d9ef333e8c2f6874ca5151d5250ef5f4d81796cce7813b2bfdb66dbb47ed5a01d
+  data.tar.gz: ec0d633c0eaab16dccea6a340f53f6160e6932a530d6d1398ce3462b9366abbc018548e5789cccf2974c8f45495e78949a865a3b270f09ab7d61b9738a5e7db6

data/.gitignore

@@ -0,0 +1,4 @@
+coverage
+.byebug_history
+*.gem
+Gemfile.lock

data/.tm_properties

@@ -0,0 +1 @@
+exclude = '{$exclude,log,bin,tmp,.tm_properties,public/system,coverage}'

data/Gemfile

@@ -0,0 +1,4 @@
+source "https://rubygems.org"
+
+# Specify your gem's dependencies in sunstone.gemspec
+gemspec

data/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2020 Jon Bracy
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/README.md

@@ -0,0 +1,10 @@
+# Runestone
+
+Runestone provides full text search PostgreSQL's full text search capabilities.
+It was inspired by [Postgres full-text search is Good Enough!][1] and
+[Super Fuzzy Searching on PostgreSQL][2]
+
+
+
+[1]: http://rachbelaid.com/postgres-full-text-search-is-good-enough/
+[2]: http://www.www-old.bartlettpublishing.com/site/bartpub/blog/3/entry/350

data/Rakefile

@@ -0,0 +1,14 @@
+require 'bundler/setup'
+require "bundler/gem_tasks"
+Bundler.require(:development)
+
+require 'fileutils'
+require "rake/testtask"
+
+# Test Task
+Rake::TestTask.new do |t|
+    t.libs << 'lib' << 'test'
+    t.test_files = FileList['test/**/*_test.rb']
+    # t.warning = true
+    # t.verbose = true
+end

data/db/migrate/20181101150207_create_ts_tables.rb

@@ -0,0 +1,31 @@
+class CreateRunestoneTables < ActiveRecord::Migration[6.0]
+
+  def change
+    enable_extension 'pgcrypto'
+    enable_extension 'pg_trgm'
+    enable_extension 'fuzzystrmatch'
+
+    create_table :runestones, id: :uuid do |t|
+      t.belongs_to  :record, type: :uuid, polymorphic: true, null: false
+      t.string      :name
+      t.string      :dictionary
+      t.jsonb       :data,    null: false
+      t.tsvector    :vector,  null: false
+    end
+
+    add_index :runestones, [:record_type, :record_id, :name, :dictionary], unique: true, name: 'index_runestones_for_uniqueness'
+    add_index :runestones, :vector, using: :gin
+
+    execute <<-SQL
+      CREATE TABLE runestone_corpus ( word varchar, CONSTRAINT word UNIQUE(word) );
+
+      CREATE INDEX runestone_corpus_trgm_idx ON runestone_corpus USING GIN (word gin_trgm_ops);
+
+      CREATE TEXT SEARCH CONFIGURATION simple_unaccent (COPY = simple);
+      ALTER TEXT SEARCH CONFIGURATION simple_unaccent
+        ALTER MAPPING FOR hword, hword_part, word
+        WITH unaccent, simple;
+    SQL
+  end
+
+end

data/lib/runestone.rb

@@ -0,0 +1,103 @@
+require 'arel/extensions'
+
+module Runestone
+  autoload :Model, "#{File.dirname(__FILE__)}/runestone/model"
+  autoload :Settings, "#{File.dirname(__FILE__)}/runestone/settings"
+  autoload :WebSearch, "#{File.dirname(__FILE__)}/runestone/web_search"
+  autoload :IndexingJob, "#{File.dirname(__FILE__)}/runestone/indexing_job"
+  
+  mattr_accessor :dictionary, default: :simple_unaccent
+  mattr_accessor :runner, default: :inline
+  mattr_accessor :job_queue, default: :runestone_indexing
+  mattr_accessor :typo_tolerances, default: { 1 => 4..7, 2 => 8.. }
+  
+  mattr_reader :synonyms do
+    { }
+  end
+  
+  def self.normalize(string)
+    string = string.downcase
+    string = string.unicode_normalize!
+    string
+  rescue Encoding::CompatibilityError
+    string
+  end
+  
+  def self.add_synonyms(dictionary)
+    dictionary.each do |k, v|
+      add_synonym(k, *v)
+    end
+  end
+  
+  def self.add_synonym(word, *replacements)
+    word = normalize(word)
+    replacements.map! { |r| normalize(r) }
+
+    word = word.split(/\s+/)
+    last = word.pop
+
+    syn = synonyms
+    word.each do |w|
+      syn = if syn.has_key?(w) && h = syn[w].find { |i| i.is_a?(Hash) }
+        h
+      else
+        h = {}
+        syn[w] ||= []
+        syn[w] << h
+        h
+      end
+    end
+  
+    syn[last] ||= []
+    syn[last] += replacements
+    syn[last].uniq!
+  end
+  
+  def search(query, dictionary: nil, prefix: :last)
+    exact_search = Runestone::WebSearch.parse(query, prefix: prefix)
+    typo_search = exact_search.typos
+    syn_search = typo_search.synonymize
+    
+    tsqueries = [exact_search, typo_search, syn_search].map(&:to_s).uniq.map do |q|
+      ts_query(q, dictionary: dictionary)
+    end
+    
+    q = if select_values.empty?
+      select(
+        klass.arel_table[Arel.star],
+        *tsqueries.each_with_index.map { |q, i| Arel::Nodes::As.new(ts_rank_cd(:vector, q, dictionary: dictionary), Arel::Nodes::SqlLiteral.new("rank#{i}")) }
+      )
+    else
+      select(
+        *tsqueries.each_with_index.map { |q, i| Arel::Nodes::As.new(ts_rank_cd(:vector, q, dictionary: dictionary), Arel::Nodes::SqlLiteral.new("rank#{i}")) }
+      )
+    end
+
+    q = if klass == Runestone::Model
+      q.where(ts_match(:vector, tsqueries.last, dictionary: dictionary))
+    else
+      q.joins(:runestones).where(ts_match(TS::Model.arel_table['vector'], tsqueries.last, dictionary: dictionary))
+    end
+  
+    q = q.where(dictionary: dictionary) if dictionary
+      
+    q.order(
+      *tsqueries.each_with_index.map { |q, i| Arel::Nodes::Descending.new(Arel::Nodes::SqlLiteral.new("rank#{i}")) }
+    )
+  end
+
+  
+end
+
+require 'runestone/corpus'
+require 'runestone/active_record/base_methods'
+require 'runestone/active_record/relation_methods'
+
+require 'active_record'
+require 'active_record/relation'
+require 'active_record/querying'
+ActiveRecord::Base.include Runestone::ActiveRecord::BaseMethods
+ActiveRecord::Relation.include Runestone::ActiveRecord::RelationMethods
+ActiveRecord::Querying.delegate :search, to: :all
+
+require 'runestone/engine' if defined?(Rails)

data/lib/runestone/active_record/base_methods.rb

@@ -0,0 +1,135 @@
+require 'active_support/concern'
+
+module Runestone::ActiveRecord
+  module BaseMethods
+
+    extend ActiveSupport::Concern
+  
+    included do
+      class_attribute :runestone_settings, instance_accessor: true
+    end
+  
+    class_methods do
+
+      def runestone(name: :default, dictionary: nil, runner: nil, &block)
+        runner ||= Runestone.runner
+        dictionary  ||= Runestone.dictionary
+        
+        if self.runestone_settings.nil?
+          self.runestone_settings = {}
+
+          class_eval do
+            has_many :runestones, class_name: 'Runestone::Model', as: :record, dependent: :destroy
+            
+            case runner
+            when :active_job
+              after_commit :create_runestones, on: :create
+              after_commit :update_runestones, on: :update
+            else
+              after_create :create_runestones!
+              after_update :update_runestones!
+            end
+          end
+        end
+        
+        self.runestone_settings[name] ||= []
+        self.runestone_settings[name] << Runestone::Settings.new(base_class.name, name: name, dictionary: dictionary, &block)
+      end
+
+      def reindex!
+        conn = Runestone::Model.connection
+        model_table = conn.quote_table_name(table_name)
+        
+        conn.execute(<<-SQL)
+          DELETE FROM runestones
+          USING runestones AS t2
+          LEFT OUTER JOIN #{model_table} ON
+            t2.record_type = #{conn.quote(base_class.name)}
+            AND t2.record_id = #{model_table}.id
+          WHERE runestones.record_type = #{conn.quote(base_class.name)}
+            AND runestones.record_id = t2.record_id
+            AND #{model_table}.id IS NULL;
+        SQL
+
+        find_each(&:update_runestones!)
+      end
+
+      def highlights(name: :default, dictionary: nil)
+        dictionary ||= Runestone.dictionary
+
+        rsettings = self.runestone_settings[name].find { |s| s.dictionary.to_s == dictionary.to_s}
+        @highlights ||= highlight_indexes(rsettings.indexes.values.flatten.map{ |i| i.to_s.split('.') })
+      end
+
+      def highlight_indexes(indexes)
+        str = {}
+        indexes.sort.group_by { |i| i[0] }.each do |key, value|
+          value.each(&:shift)
+          value.reject!(&:empty?)
+          str[key] = value.empty? ? true : highlight_indexes(value)
+        end
+        str
+      end
+
+    end
+
+    def create_runestones
+      Runestone::IndexingJob.perform_later(self, :create_runestones!)
+    end
+    
+    def create_runestones!
+      conn = Runestone::Model.connection
+      self.runestone_settings.each do |index_name, settings|
+        settings.each do |setting|
+          rdata = setting.extract_attributes(self)
+
+          ts_column_names = %w(record_type record_id name dictionary data vector).map { |name| conn.quote_column_name(name) }
+          ts_values = [
+            conn.quote(conn.send(:type_map).lookup('varchar').serialize(self.class.base_class.name)),
+            conn.quote(conn.send(:type_map).lookup('uuid').serialize(id)),
+            index_name == :default ? 'NULL' : conn.quote(conn.send(:type_map).lookup('varchar').serialize(index_name.to_s)),
+            conn.quote(conn.send(:type_map).lookup('varchar').serialize(setting.dictionary)),
+            conn.quote(conn.send(:type_map).lookup('jsonb').serialize(rdata)),
+            setting.vectorize(rdata).join(' || ')
+          ]
+          conn.execute(<<-SQL)
+            INSERT INTO #{Runestone::Model.quoted_table_name} (#{ts_column_names.join(",")})
+            VALUES (#{ts_values.join(',')})
+          SQL
+
+          Runestone::Corpus.add(*setting.corpus(rdata))
+        end
+      end
+    end
+    
+    def update_runestones
+      Runestone::IndexingJob.preform_later(self, :update_runestones!)
+    end
+    
+    def update_runestones!
+      conn = Runestone::Model.connection
+      self.runestone_settings.each do |index_name, settings|
+        settings.each do |setting|
+          rdata = setting.extract_attributes(self)
+
+          if conn.execute(<<-SQL).cmd_tuples == 0
+              UPDATE #{Runestone::Model.quoted_table_name}
+              SET
+                data = #{conn.quote(conn.send(:type_map).lookup('jsonb').serialize(rdata))},
+                vector = #{setting.vectorize(rdata).join(' || ')}
+              WHERE record_type = #{conn.quote(conn.send(:type_map).lookup('varchar').serialize(self.class.base_class.name))}
+              AND record_id = #{conn.quote(conn.send(:type_map).lookup('integer').serialize(id))}
+              AND name #{index_name == :default ? 'IS NULL' : "= " + conn.quote(conn.send(:type_map).lookup('integer').serialize(index_name))}
+              AND dictionary = #{conn.quote(conn.send(:type_map).lookup('integer').serialize(setting.dictionary))}
+              SQL
+            create_runestones!
+          else
+            Runestone::Corpus.add(*setting.corpus(rdata))
+          end
+
+        end
+      end
+    end
+
+  end
+end

data/lib/runestone/active_record/relation_methods.rb

@@ -0,0 +1,83 @@
+module Runestone::ActiveRecord
+  module RelationMethods
+  
+    def ts_query(query, dictionary: nil)
+      dictionary ||= Runestone.dictionary
+    
+      if query.is_a?(Arel::Nodes::TSQuery)
+        query
+      else
+        Arel::Nodes::TSQuery.new(query, language: dictionary)
+      end
+    end
+
+    def ts_vector(column_name, dictionary: nil)
+      # if column_name.is_a?(String) || column_name.is_a?(Symbol)
+      #   column = columns_hash[column_name.to_s]
+      #   if column.type == :tsvector
+      #     arel_table[column.name]
+      #   else
+      #     Arel::Nodes::TSVector.new(arel_table[column.name], language)
+      #   end
+      # else
+      #   column_name
+      # end
+      Runestone::Model.arel_table[:vector]
+    end
+
+    def ts_match(vector, query, dictionary: nil)
+      Arel::Nodes::TSMatch.new(
+        ts_vector(vector, dictionary: dictionary),
+        ts_query(query, dictionary: dictionary)
+      )
+    end
+
+    def ts_rank(vector, query, dictionary: nil)
+      Arel::Nodes::TSRank.new(
+        ts_vector(vector, dictionary: dictionary),
+        ts_query(query, dictionary: dictionary)
+      )
+    end
+
+    def ts_rank_cd(vector, query, dictionary: nil)
+      Arel::Nodes::TSRankCD.new(
+        ts_vector(vector, dictionary: dictionary),
+        ts_query(query, dictionary: dictionary)
+      )
+    end
+
+    def search(query, dictionary: nil, prefix: nil)
+      exact_search = Runestone::WebSearch.parse(query, prefix: prefix)
+      typo_search = exact_search.typos
+      syn_search = typo_search.synonymize
+      
+      tsqueries = [exact_search, typo_search, syn_search].map(&:to_s).uniq.map do |q|
+        ts_query(q, dictionary: dictionary)
+      end
+      
+      q = if select_values.empty?
+        select(
+          klass.arel_table[Arel.star],
+          *tsqueries.each_with_index.map { |q, i| Arel::Nodes::As.new(ts_rank_cd(:vector, q, dictionary: dictionary), Arel::Nodes::SqlLiteral.new("rank#{i}")) }
+        )
+      else
+        select(
+          *tsqueries.each_with_index.map { |q, i| Arel::Nodes::As.new(ts_rank_cd(:vector, q, dictionary: dictionary), Arel::Nodes::SqlLiteral.new("rank#{i}")) }
+        )
+      end
+
+      q = if klass == Runestone::Model
+        q.where(ts_match(:vector, tsqueries.last, dictionary: dictionary))
+      else
+        q.joins(:runestones).where(ts_match(Runestone::Model.arel_table['vector'], tsqueries.last, dictionary: dictionary))
+      end
+
+      q = q.where(dictionary: dictionary) if dictionary
+        
+      q.order(
+        *tsqueries.each_with_index.map { |q, i| Arel::Nodes::Descending.new(Arel::Nodes::SqlLiteral.new("rank#{i}")) }
+      )
+    end
+    
+  end
+end

data/lib/runestone/corpus.rb

@@ -0,0 +1,45 @@
+module Runestone::Corpus
+
+  def self.add(*words)
+    return if words.size == 0
+
+    conn = Runestone::Model.connection
+    conn.execute(<<-SQL)
+      INSERT INTO runestone_corpus ( word )
+      VALUES (#{words.map { |w| conn.quote(w.downcase) }.join('),(')})
+      ON CONFLICT DO NOTHING
+    SQL
+  end
+
+  def self.similar_words(*words)
+    lut = {}
+    words = words.inject([]) do |ws, w|
+      tt = typo_tolerance(w)
+      ws << "#{Runestone::Model.connection.quote(w)}, #{Runestone::Model.connection.quote(w.downcase)}, #{tt}" if tt > 0
+      ws
+    end
+    return lut if words.size == 0
+    
+    result = Runestone::Model.connection.execute(<<-SQL)
+      WITH  tokens (token, token_downcased, typo_tolerance) AS (VALUES (#{words.join('), (')}))
+      SELECT token, word, levenshtein(runestone_corpus.word, tokens.token_downcased)
+      FROM tokens
+      JOIN runestone_corpus ON runestone_corpus.word % tokens.token_downcased
+      WHERE
+        runestone_corpus.word != tokens.token_downcased
+        AND levenshtein(runestone_corpus.word, tokens.token_downcased) <= tokens.typo_tolerance
+    SQL
+    result.each_row do |t, w, l|
+      w.gsub!(/\(|\)|:|\||!|\&|\*/, '')
+      next if w == t
+      lut[t] ||= []
+      lut[t] << w
+    end
+    lut
+  end
+
+  def self.typo_tolerance(word)
+    Runestone.typo_tolerances.find { |k,v| v.member?(word.length) }&.first || 0
+  end
+  
+end