runestone 1.0

data/lib/runestone/web_search/or.rb

@@ -0,0 +1,11 @@
+class Runestone::WebSearch::Or
+  attr_accessor :values
+  def initialize(values, negative: false)
+    @values = values
+    @negative = negative
+  end
+  
+  def to_s
+    "(#{values.map(&:to_s).join(' | ')})"
+  end
+end

data/lib/runestone/web_search/phrase.rb

@@ -0,0 +1,19 @@
+class Runestone::WebSearch::Phrase
+  attr_accessor :values, :prefix, :negative, :distance
+  def initialize(values, prefix: false, negative: false, distance: nil)
+    @values = values
+    @prefix = prefix
+    @negative = negative
+    @distance = distance
+  end
+  
+  def to_s
+    v = if values.size == 1
+      values.first.to_s
+    else
+      seperator = distance ? " <#{distance}> " : ' <-> '
+      "(#{values.map(&:to_s).join(seperator)})"
+    end
+    negative ? "!#{v}" : v
+  end
+end

data/lib/runestone/web_search/token.rb

@@ -0,0 +1,27 @@
+class Runestone::WebSearch::Token
+  attr_accessor :value, :prefix, :negative, :alts
+  def initialize(value, prefix: false, negative: false, alts: nil)
+    @value = value
+    @prefix = prefix
+    @negative = negative
+    @alts = alts || []
+  end
+
+  def to_s
+    if negative
+      "!#{value}"
+    elsif prefix
+      if alts.empty?
+        "#{value}:*"
+      else
+        "(#{value}:* | #{alts.join(' | ')})"
+      end
+    else
+      if alts.empty?
+        value
+      else
+        "(#{value} | #{alts.join(' | ')})"
+      end
+    end
+  end
+end

data/runestone.gemspec

@@ -0,0 +1,32 @@
+require File.expand_path("../lib/runestone/version", __FILE__)
+
+Gem::Specification.new do |s|
+  s.name        = "runestone"
+  s.version     = Runestone::VERSION
+  s.authors     = ["Jon Bracy"]
+  s.email       = ["jonbracy@gmail.com"]
+  s.homepage    = "https://github.com/malomalo/runestone"
+  s.summary     = %q{Full Text Search for Active Record / Rails}
+  s.description = %q{PostgreSQL Full Text Search for Active Record and Rails}
+  s.license     = 'MIT'
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+
+  # Developoment 
+  s.add_development_dependency 'rake'
+  s.add_development_dependency 'bundler'
+  s.add_development_dependency 'minitest'
+  s.add_development_dependency 'minitest-reporters'
+  s.add_development_dependency 'pg'
+  s.add_development_dependency 'byebug'
+  s.add_development_dependency 'faker'
+  s.add_development_dependency 'simplecov'
+  s.add_development_dependency 'activejob', '>= 6.0'
+
+  # Runtime
+  s.add_runtime_dependency 'arel-extensions', '>= 6.0'
+  s.add_runtime_dependency 'activerecord', '>= 6.0'
+end

data/test/corpus_test.rb

@@ -0,0 +1,42 @@
+require 'test_helper'
+
+class CorpusTest < ActiveSupport::TestCase
+  
+  test 'similar_words' do
+    Address.create(name: 'Address name broccolini')
+    
+    assert_equal(
+      {},
+      Runestone::Corpus.similar_words('nam')
+    )
+    
+    assert_equal(
+      {'addresz' => ['address']},
+      Runestone::Corpus.similar_words('addresz')
+    )
+    
+    assert_equal(
+      {'brockolinl' => ['broccolini']},
+      Runestone::Corpus.similar_words('brockolinl')
+    )
+
+    assert_equal(
+      {
+        'addresz' => ['address'],
+        'brockolinl' => ['broccolini']
+      },
+      Runestone::Corpus.similar_words('nam', 'addresz', 'brockolinl')
+    )
+  end
+  
+  test 'adding words to corpus downcases them' do
+    Runestone::Corpus.add('Allée')
+    assert_equal(
+      {
+        "Allee" => ["allée"]
+      },
+      Runestone::Corpus.similar_words('Allee')
+    )
+  end
+  
+end

data/test/database.rb

@@ -0,0 +1,119 @@
+GlobalID.app = 'TestApp'
+
+task = ActiveRecord::Tasks::PostgreSQLDatabaseTasks.new({
+  'adapter' => 'postgresql',
+  'database' => "arel-extensions-test"
+})
+task.drop
+task.create
+
+ActiveRecord::Base.establish_connection({
+  adapter:  "postgresql",
+  database: "arel-extensions-test",
+  encoding: "utf8"
+})
+
+ActiveRecord::Migration.suppress_messages do
+  ActiveRecord::Schema.define do
+    enable_extension 'pgcrypto'
+    enable_extension 'pg_trgm'
+    enable_extension 'unaccent'
+    enable_extension 'fuzzystrmatch'
+
+    create_table :addresses, id: :uuid, force: :cascade do |t|
+      t.string  "name"
+      t.uuid    "property_id"
+    end
+    
+    create_table :properties, id: :uuid, force: :cascade do |t|
+      t.string   "name",                 limit: 255
+    end
+
+    create_table :regions, id: :uuid, force: :cascade do |t|
+      t.string   "name",                 limit: 255
+    end
+
+    create_table :buildings, id: :uuid, force: :cascade do |t|
+      t.string   "name_en",              limit: 255
+      t.string   "name_ru",              limit: 255
+    end
+
+    create_table :runestones, id: :uuid, force: :cascade do |t|
+      t.string    :record_type,     null: false
+      t.uuid      :record_id,        null: false
+      t.string    :name
+      t.string    :dictionary
+      t.jsonb     :data,      null: false
+      t.tsvector  :vector,    null: false
+    end
+
+    add_index :runestones, [:record_type, :record_id, :name, :dictionary], unique: true, name: 'index_runestones_for_uniqueness'
+    add_index :runestones, :vector, using: :gin
+    
+    execute <<-SQL
+    CREATE TABLE runestone_corpus ( word varchar, CONSTRAINT word UNIQUE(word) );
+
+    CREATE INDEX runestone_corpus_trgm_idx ON runestone_corpus USING GIN (word gin_trgm_ops);
+
+
+      CREATE TEXT SEARCH CONFIGURATION simple_unaccent (COPY = simple);
+      ALTER TEXT SEARCH CONFIGURATION simple_unaccent
+        ALTER MAPPING FOR hword, hword_part, word
+        WITH unaccent, simple;
+    SQL
+  end
+end
+
+class Address < ActiveRecord::Base
+  
+  belongs_to :property
+
+  runestone do
+    index 'name'
+
+    attribute(:name)
+  end
+
+end
+
+class Region < ActiveRecord::Base
+  
+  include GlobalID::Identification
+  
+  runestone runner: :active_job do
+    index 'name'
+
+    attribute(:name)
+  end
+  
+end
+
+class Property < ActiveRecord::Base
+  
+  has_many :addresses
+
+  runestone do
+    index :name
+    index 'addresses.name', weight: 3
+
+    attribute(:name)
+    attribute(:addresses) { addresses.map{ |a| a&.attributes&.slice('id', 'name') } }
+  end
+
+end
+
+class Building < ActiveRecord::Base
+  
+  runestone dictionary: 'english' do
+    index :name
+
+    attribute(:name) { name_en }
+  end
+  
+  runestone dictionary: 'russian' do
+    index :name
+
+    attribute(:name) { name_ru }
+  end
+
+end

data/test/delayed_index_test.rb

@@ -0,0 +1,34 @@
+require 'test_helper'
+
+class DelayedIndexingTest < ActiveSupport::TestCase
+  
+  test 'simple_unaccent index' do
+    region = assert_no_difference 'Runestone::Model.count' do
+      assert_no_sql(/setweight\(to_tsvector\('simple_unaccent', 'address name'\), 'A'\)/) do
+        Region.create(name: 'Region name')
+      end
+    end
+    
+    job = assert_enqueued_with(
+      job: Runestone::IndexingJob,
+      args: [region, :create_runestones!],
+      queue: 'runestone_indexing'
+    )
+    
+    job.perform_now
+
+    assert_equal([[
+      'Region', region.id,
+      {"name" => "Region name"},
+      "'name':2A 'region':1A"
+    ]], region.runestones.map { |runestone|
+      [
+        runestone.record_type,
+        runestone.record_id,
+        runestone.data,
+        runestone.vector
+      ]
+    })
+  end
+  
+end

data/test/helper_test.rb

@@ -0,0 +1,40 @@
+require 'test_helper'
+
+class WebSearchTest < ActiveSupport::TestCase
+
+  test '::parse' do
+    assert_equal "the & fat & rats:*", Runestone::WebSearch.parse('The fat rats').to_s
+    assert_equal "(supernovae <-> stars) & !crab", Runestone::WebSearch.parse('"supernovae stars" -crab').to_s
+    # assert_equal "(sad <-> cat) | (fat <-> rat)", TS.websearch_to_tsquery('"sad cat" || "fat rat"')
+    assert_equal "signal & !(segmentation <-> fault)", Runestone::WebSearch.parse('signal -"segmentation fault"').to_s
+  end
+  
+  test '::parse(query, prefix: :all)' do
+    assert_equal "the:* & fat:* & rats:*", Runestone::WebSearch.parse('The fat rats', prefix: :all).to_s
+    assert_equal "(supernovae <-> stars) & !crab", Runestone::WebSearch.parse('"supernovae stars" -crab', prefix: :all).to_s
+    # assert_equal "(sad <-> cat) | (fat <-> rat)", TS.websearch_to_tsquery('"sad cat" || "fat rat"')
+    assert_equal "signal:* & !(segmentation <-> fault)", Runestone::WebSearch.parse('signal -"segmentation fault"', prefix: :all).to_s
+  end
+  
+  test '::parse(query, prefix: :last)' do
+    assert_equal "the & fat & rats:*", Runestone::WebSearch.parse('The fat rats', prefix: :last).to_s
+    assert_equal "(supernovae <-> stars) & !crab", Runestone::WebSearch.parse('"supernovae stars" -crab', prefix: :last).to_s
+    # assert_equal "(sad <-> cat) | (fat <-> rat)", TS.websearch_to_tsquery('"sad cat" || "fat rat"')
+    assert_equal "signal & !(segmentation <-> fault)", Runestone::WebSearch.parse('signal -"segmentation fault"', prefix: :last).to_s
+  end
+  
+  test '::parse(query, prefix: :none)' do
+    assert_equal "the & fat & rats", Runestone::WebSearch.parse('The fat rats', prefix: :none).to_s
+    assert_equal "(supernovae <-> stars) & !crab", Runestone::WebSearch.parse('"supernovae stars" -crab', prefix: :none).to_s
+    # assert_equal "(sad <-> cat) | (fat <-> rat)", TS.websearch_to_tsquery('"sad cat" || "fat rat"')
+    assert_equal "signal & !(segmentation <-> fault)", Runestone::WebSearch.parse('signal -"segmentation fault"', prefix: :none).to_s
+  end
+  
+  test '::parse(weird query, prefix: :none)' do
+    assert_equal "(supernovae <-> stars)", Runestone::WebSearch.parse('"supernovae stars', prefix: :none).to_s
+    assert_equal "signal", Runestone::WebSearch.parse('signal -', prefix: :none).to_s
+    assert_equal "signal", Runestone::WebSearch.parse('signal -"', prefix: :none).to_s
+    assert_equal "signal", Runestone::WebSearch.parse('signal -""', prefix: :none).to_s
+  end
+
+end

data/test/highlight_test.rb

@@ -0,0 +1,26 @@
+require 'test_helper'
+
+class HighlightTest < ActiveSupport::TestCase
+
+  test '::highlights(query)' do
+    Property.create(name: 'Empire state building', addresses: [Address.create(name: 'address uno')])
+    Property.create(name: 'Big state building', addresses: [Address.create(name: 'address of state duo')])
+    
+    tsmodels = Runestone::Model.search('state')
+    Runestone::Model.highlight(tsmodels, 'state')
+    assert_equal([
+      {
+        "name"=>"Big <b>state</b> building",
+        "addresses"=> [{"name"=>"address of <b>state</b> duo"}]
+      },
+      {
+        "name"=>"Empire <b>state</b> building",
+        "addresses"=> [{"name"=>"address uno"}]
+      },
+      {
+        "name"=>"address of <b>state</b> duo"
+      }
+    ], tsmodels.map(&:highlights))
+  end
+  
+end

data/test/indexing_test.rb

@@ -0,0 +1,151 @@
+require 'test_helper'
+
+class IndexingTest < ActiveSupport::TestCase
+  
+  test 'simple_unaccent index' do
+    address = assert_difference 'Runestone::Model.count', 1 do
+      assert_sql(/setweight\(to_tsvector\('simple_unaccent', 'address name'\), 'A'\)/) do
+        Address.create(name: 'Address name')
+      end
+    end
+
+    assert_equal([[
+      'Address', address.id,
+      {"name" => "Address name"},
+      "'address':1A 'name':2A"
+    ]], address.runestones.map { |runestone|
+      [
+        runestone.record_type,
+        runestone.record_id,
+        runestone.data,
+        runestone.vector
+      ]
+    })
+  end
+
+  test 'empty index' do
+    address = assert_difference 'Runestone::Model.count', 1 do
+      Address.create(name: nil)
+    end
+
+    assert_equal([[
+      'Address', address.id,
+      {},
+      ""
+    ]], address.runestones.map { |runestone|
+      [
+        runestone.record_type,
+        runestone.record_id,
+        runestone.data,
+        runestone.vector
+      ]
+    })
+  end
+
+  test 'complex index' do
+    property = assert_difference 'Runestone::Model.count', 3 do
+      Property.create(name: 'Property name', addresses: [
+        Address.create(name: 'Address 1'),
+        Address.create(name: 'Address 2')
+      ])
+    end
+
+    assert_equal([[
+      'Property', property.id,
+      {
+        "name"=>"Property name",
+        "addresses"=>[
+          { "id" => property.addresses.first.id, "name" => "Address 1" },
+          { "id" => property.addresses.last.id,  "name" => "Address 2" }
+        ]
+      },
+      "'1':4C '2':6C 'address':3C,5C 'name':2A 'property':1A"
+    ]], property.runestones.map { |runestone|
+      [
+        runestone.record_type,
+        runestone.record_id,
+        runestone.data,
+        runestone.vector
+      ]
+    })
+  end
+  
+  test 'index Unicode strings get normalized' do
+    address = Address.create(name: "on\u0065\u0301")
+
+    assert_equal(["'on\u00e9':1A"], address.reload.runestones.map(&:vector))
+  end
+
+  test 'index gets created on Model.create' do
+    address = assert_difference 'Runestone::Model.count', 1 do
+      Address.create(name: 'Address name')
+    end
+    
+    assert_corpus('address', 'name')
+  end
+
+  test 'index gets updated on Model.create' do
+    address = Address.create(name: 'Address name')
+    assert_no_difference 'Runestone::Model.count' do
+      address.update!(name: 'Address name two')
+    end
+
+    assert_equal([[
+      'Address', address.id,
+      {"name" => "Address name two"},
+      "'address':1A 'name':2A 'two':3A"
+    ]], address.runestones.map { |runestone|
+      [
+        runestone.record_type,
+        runestone.record_id,
+        runestone.data,
+        runestone.vector
+      ]
+    })
+    
+    assert_corpus('address', 'name', 'two')
+  end
+
+  test 'index gets deleted on Model.destroy' do
+    address = Address.create(name: 'Address name')
+    assert_difference 'Runestone::Model.count', -1 do
+      address.destroy!
+    end
+  end
+
+  test 'reindex! deleted removed records' do
+    a1 = Address.create(name: 'one')
+    a2 = Address.create(name: 'two')
+
+    assert_no_difference 'Runestone::Model.count' do
+      a2.delete
+    end
+
+    assert_difference 'Runestone::Model.count', -1 do
+      Address.reindex!
+    end
+  end
+
+  test 'reindex! updates runestone on outdated indexes' do
+    address = Address.create(name: 'one')
+    address.update_columns(name: 'two')
+
+    assert_equal(["'one':1A"], address.runestones.map(&:vector))
+    assert_no_difference 'Runestone::Model.count' do
+      Address.reindex!
+    end
+    assert_equal(["'two':1A"], address.runestones.map { |runestone| runestone.reload.vector })
+  end
+
+  test 'reindex! creates index if not there' do
+    address = Address.create(name: 'one')
+    address.runestones.each(&:delete)
+
+    assert_equal 0, address.reload.runestones.size
+    assert_difference 'Runestone::Model.count', 1 do
+      Address.reindex!
+    end
+    assert_equal(["'one':1A"], address.reload.runestones.map(&:vector))
+  end
+  
+end