runestone 1.0

data/lib/runestone/engine.rb

@@ -0,0 +1,21 @@
+class Runestone::Engine < Rails::Engine
+  config.runestone = ActiveSupport::OrderedOptions.new
+
+  initializer :append_migrations do |app|
+    unless app.root.to_s.match root.to_s
+      config.paths["db/migrate"].expanded.each do |expanded_path|
+        app.config.paths["db/migrate"] << expanded_path
+      end
+    end
+  end
+  
+  initializer "runestone.set_configs" do |app|
+    options = app.config.runestone
+
+    Runestone.runner = options.runner if options.runner
+    Runestone.dictionary = options.dictionary if options.dictionary
+    Runestone.job_queue = options.job_queue if options.job_queue
+    Runestone.typo_tolerances = options.typo_tolerances if options.typo_tolerances
+  end
+  
+end

data/lib/runestone/indexing_job.rb

@@ -0,0 +1,8 @@
+class Runestone::IndexingJob < ActiveJob::Base
+  queue_as { Runestone.job_queue }
+  
+  def perform(record, indexing_method)
+    record.public_send(indexing_method)
+  end
+  
+end

data/lib/runestone/model.rb

@@ -0,0 +1,92 @@
+class Runestone::Model < ActiveRecord::Base
+
+  self.table_name = :runestones
+
+  attr_accessor :highlights
+  
+  belongs_to :record, polymorphic: true
+  
+  def self.highlight(records, query, prefix: nil)
+    return [] if records.empty?
+    
+    binds = []
+    records.each do |record|
+      binds += get_binds(record.data, record.record_type.constantize.highlights(dictionary: records.first.dictionary))
+    end
+
+    hlites = binds.uniq
+
+    newbinds = []
+    binds.each_with_index do |b|
+      newbinds << hlites.index(b)
+    end
+    binds = newbinds
+
+    hlites = get_highlights(hlites, query, prefix: prefix, dictionary: records.first.dictionary)
+
+    binds.map! { |x| hlites[x] }
+
+    records.each do |record|
+      record.highlights = highlight_data(
+        record.data,
+        binds,
+        record.record_type.constantize.highlights
+      )
+    end
+  end
+  
+  def self.highlight_data(data, hlights, indexes)
+    str = {}
+    indexes.each do |key, value|
+      next unless data[key]
+
+      if data[key].is_a?(Hash)
+        str[key] = highlight_data(data[key], hlights, indexes[key])
+      elsif data[key].is_a?(Array)
+        str[key] = data[key].map { |i|
+          if i.is_a?(Hash)
+            highlight_data(i, hlights, indexes[key])
+          else
+            hlights.shift
+          end
+        }
+      else
+        str[key] = hlights.shift
+      end
+    end
+    str
+  end
+
+  def self.get_highlights(words, query, prefix: nil, dictionary: nil)
+    dictionary ||= Runestone.dictionary
+    
+    query = Arel::Nodes::TSQuery.new(Runestone::WebSearch.parse(query, prefix: prefix).typos.synonymize.to_s, language: dictionary).to_sql
+    connection.exec_query(<<-SQL).cast_values
+      SELECT ts_headline(#{connection.quote(dictionary)}, words, #{query}, 'ShortWord=2')
+      FROM unnest(ARRAY[ #{words.map{ |t| connection.quote(t) }.join(', ')} ]::varchar[]) AS words
+    SQL
+  end
+  
+  def self.get_binds(hash, highlight)
+    rt = []
+    highlight.each do |k, v|
+      next unless hash[k]
+
+      if hash[k].is_a?(Hash)
+        rt += get_binds(hash[k], highlight[k])
+      elsif hash[k].is_a?(Array)
+        hash[k].each do |i|
+          if i.is_a?(Hash)
+            rt += get_binds(i, highlight[k])
+          else
+            rt += i.is_a?(Array) ? i : [i]
+          end
+        end
+      else
+        rt << hash[k].to_s
+      end
+    end
+    rt
+  end
+
+end

data/lib/runestone/settings.rb

@@ -0,0 +1,106 @@
+class Runestone::Settings
+
+  attr_reader :indexes, :dictionary
+  
+  def initialize(model, name: , dictionary: , &block)
+    @name = name
+    @dictionary = dictionary
+    @indexes = {}
+    instance_exec(&block)
+  end
+
+  def index(*args, weight: 1)
+    @indexes[weight] = args.map(&:to_s)
+  end
+
+  def attribute(*names, &block)
+    raise ArgumentError.new('Cannot pass multiple attribute names if block given') if block_given? and names.length > 1
+
+    @attributes ||= {}
+    names.each do |name|
+      @attributes[name.to_sym] = block ? block : nil
+    end
+  end
+  alias :attributes :attribute
+
+  def extract_attributes(record)
+    attributes = {}
+
+    @attributes.each do |name, value|
+      attributes[name] = if value.is_a?(Proc)
+        record.instance_exec(&value)
+      else
+        rv = record.send(name)
+      end
+    end
+
+    remove_nulls(attributes)
+  end
+
+  def vectorize(data)
+    conn = Runestone::Model.connection
+    tsvector = []
+
+    @indexes.each do |weight, paths|
+      tsweight = {4 => 'D', 3 => 'C', 2 => 'B', 1 => 'A'}[weight]
+      paths.each do |path|
+        path = path.to_s.split('.')
+        
+        dig(data, path).each do |value|
+          next if !value
+          language = value.to_s.size <= 5 ? 'simple' : @dictionary
+          tsvector << "setweight(to_tsvector(#{conn.quote(language)}, #{conn.quote(value.to_s.downcase)}), #{conn.quote(tsweight)})"
+        end
+      end
+    end
+    tsvector.empty? ? ["to_tsvector('')"] : tsvector
+  end
+  
+  def corpus(data)
+    words = []
+    
+    @indexes.each do |weight, paths|
+      paths.each do |path|
+        dig(data, path.to_s.split('.')).each do |value|
+          next if !value
+          value.to_s.split(/\s+/).each do |word|
+            words << word.downcase.gsub(/\A\W/, '').gsub(/\W\Z/, '')
+          end
+        end
+      end
+    end
+    
+    words
+  end
+
+  def remove_nulls(value)
+    if value.is_a?(Hash)
+      nh = {}
+      value.each do |k, v|
+        nh[k] = if v.is_a?(Hash) || v.is_a?(Array)
+          remove_nulls(v)
+        elsif !v.nil?
+          v.is_a?(String) ? v.unicode_normalize(:nfc) : v
+        end
+        nh.delete(k) if nh[k].nil? || (nh[k].is_a?(Hash) && nh[k].empty?)
+      end
+      nh
+    elsif value.is_a?(Array)
+      value.select{|i| !i.nil? && !i.empty? }.map { |i| remove_nulls(i) }
+    else
+      value
+    end
+  end
+
+  def dig(data, keys)
+    if data.is_a?(Hash)
+      key = keys.shift
+      dig(data[key.to_sym] || data[key.to_s], keys)
+    elsif data.is_a?(Array)
+      data.map{ |d| dig(d, keys.dup) }.flatten.compact
+    else
+      [data]
+    end
+  end
+
+end

data/lib/runestone/version.rb

@@ -0,0 +1,3 @@
+module Runestone
+  VERSION = '1.0'
+end

data/lib/runestone/web_search.rb

@@ -0,0 +1,203 @@
+class Runestone::WebSearch
+  autoload :Or, "#{File.dirname(__FILE__)}/web_search/or"
+  autoload :And, "#{File.dirname(__FILE__)}/web_search/and"
+  autoload :Token, "#{File.dirname(__FILE__)}/web_search/token"
+  autoload :Phrase, "#{File.dirname(__FILE__)}/web_search/phrase"
+  
+  class Match
+    attr_accessor :index, :substitution
+    def initialize(index, substitution)
+      @index = index
+      @substitution = substitution
+    end
+  end
+
+  class PartialMatch
+    attr_accessor :start_index, :end_index, :substitution
+    def initialize(start_index, end_index, substitution)
+      @start_index = start_index
+      @end_index = end_index
+      @substitution = substitution
+    end
+  end
+
+  attr_accessor :values
+
+  # prefix options: :all, :last, :none (default: :last)
+  def self.parse(query, prefix: :last)
+    prefix ||= :last
+    begin
+      query.unicode_normalize!
+    rescue Encoding::CompatibilityError
+    end
+    query.downcase!
+
+    q = []
+    stack = []
+    knot = false
+    tokens = query.gsub(/\"\s+\"/, '""').split(' ')
+    tokens.each_with_index do |token, i|
+      token.gsub!(/\(|\)|:|\||!|\&|\*/, '')
+      if token.start_with?('-')
+        knot = true
+        token.delete_prefix!('-')
+      else
+        knot = false
+      end
+  
+      next if token.empty? || token == '""' || %w(' ").include?(token)
+    
+      if token.start_with?('"') && token.end_with?('"')
+        token.delete_prefix!('"')
+        token.delete_suffix!('"')
+      
+        q << Phrase.new([token], negative: knot)
+      elsif token.start_with?('"')
+        token.delete_prefix!('"')
+        stack.push(:phrase)
+        q << Phrase.new([Token.new(token)], negative: knot)
+      elsif token.end_with?('"')
+        token.delete_suffix!('"')
+        q.last.values << Token.new(token)
+        stack.pop
+      else
+        token = Token.new(token, negative: knot)
+        if !knot && prefix == :last && tokens.size - 1 == i
+          token.prefix = true
+        elsif !knot && prefix == :all
+          token.prefix = true
+        end
+      
+        if stack.last == :phrase
+          q.last.values << token
+        else
+          q << token
+        end
+      end
+    end
+    
+    new(q)
+  end
+  
+  def initialize(values)
+    @values = values
+  end
+  
+  def typos
+    tokens = @values.select{|t| t.is_a?(Token) && !t.negative }
+    sw = Runestone::Corpus.similar_words(*tokens.map(&:value))
+    q = @values.map do |t|
+      if t.is_a?(Token) && sw.has_key?(t.value)
+        Token.new(t.value, prefix: t.prefix, negative: t.negative, alts: sw[t.value])
+      else
+        t
+      end
+    end
+    
+    Runestone::WebSearch.new(q)
+  end
+  
+  def synonymize
+    parts = []
+    @values.each do |token|
+      if token.is_a?(Phrase) || token.negative
+        parts << token
+      else
+        parts << [] if parts.empty? || parts.last.is_a?(Phrase) || (!parts.last.is_a?(Array) && parts.last.negative)
+        parts.last << token
+      end
+    end
+
+    parts.map! do |part|
+      if !part.is_a?(Phrase) && (part.is_a?(Array) || !part.negative)
+        synonymize_part(part)
+      else
+        part
+      end
+    end
+
+    Runestone::WebSearch.new(parts)
+  end
+
+  def synonymize_part(part)
+    pending_matches = []
+    matches = []
+    
+    part.each_with_index do |token, i|
+      
+      pending_matches.select! do |match|
+        if match.end_index + 1 == i && match.substitution[token.value]
+          match.substitution[token.value].map do |nm|
+            if nm.is_a?(Hash)
+              match.end_index = i
+              match.alts = nm
+              true
+            else
+              matches << Match.new(match.start_index..i, Phrase.new(Array(nm), distance: 1))
+              false
+            end
+          end
+        else
+          false
+        end
+      end
+
+      if match = Runestone.synonyms[token.value]
+        match.each do |m|
+          if m.is_a?(Hash)
+            pending_matches << PartialMatch.new(i, i, m)
+          else
+            matches << Match.new(i, Phrase.new(m.split(/\s+/), distance: 1))
+          end
+        end
+      end
+      
+    end
+
+    matches.select! do |match|
+      if match.index.is_a?(Integer)
+        case part[match.index]
+        when Or
+          part[match.index].values << match.substitution
+        else
+          part[match.index] = Or.new([part[match.index], match.substitution])
+        end
+
+        false
+      else
+        true
+      end
+    end
+
+    groups = matches.inject([]) do |memo, match|
+      if memo.empty?
+        memo << [match]
+      elsif i = memo.index { |k| k.none? { |j| j.index.overlaps?(match.index) } }
+        memo[i] << match
+      else
+        memo << [match]
+      end
+      memo
+    end
+
+    if groups.empty?
+      And.new(part)
+    else
+      orrs = Or.new([])
+      groups.each do |g|
+        p = []
+        p << And.new(part[0..g.first.index.begin-1]) if g.first.index.begin > 0
+        g.each do |m|
+          p << Or.new([And.new(part[m.index]), m.substitution])
+        end
+        p << And.new(part[g.last.index.end+1..-1]) if g.last.index.end < part.size
+        orrs.values << And.new(p)
+      end
+      orrs
+    end
+  end
+
+  def to_s(use_synonyms: true, allow_typos: true)
+    self.values.join(' & ')
+  end
+end

data/lib/runestone/web_search/and.rb

@@ -0,0 +1,17 @@
+class Runestone::WebSearch::And
+  attr_accessor :values, :negative
+  def initialize(values, negative: false)
+    @values = values
+    @negative = negative
+  end
+  
+  def to_s
+    v = if values.size == 1
+      values.first.to_s
+    else
+      values.map(&:to_s).join(' & ')
+    end
+
+    negative ? "!#{v}" : v
+  end
+end