searchcraft 0.4.0

data/lib/searchcraft/builder.rb

@@ -0,0 +1,218 @@
+class SearchCraft::Builder
+  extend SearchCraft::Annotate
+  include SearchCraft::DependsOn
+  include SearchCraft::DumpSchema
+
+  # Subclass must implement view_scope or view_select_sql
+  def view_scope
+    raise NotImplementedError, "Subclass must implement view_scope or view_select_sql"
+  end
+
+  # By default, assumes subclass implements view_scope to return
+  # an ActiveRecord::Relation.
+  # Alternately, override view_select_sql to return a SQL string.
+  def view_select_sql
+    @_view_select_sql ||= view_scope.to_sql
+  end
+
+  # Override if a Builder SQL has dependencies, such as extensions or text search config
+  # that are required first.
+  def dependencies_ready?
+    true
+  end
+
+  class << self
+    # Iterate through subclasses, and invoke recreate_view_if_changed!
+    def rebuild_any_if_changed!
+      SearchCraft::ViewHashStore.setup_table_if_needed!
+
+      sorted_builders = sort_builders_by_dependency
+
+      # If tests, and after rails db:schema:load, the ViewHashStore table is empty.
+      # So just drop any views created from the schema.rb and we'll recreate them.
+
+      unless SearchCraft::ViewHashStore.any?
+        sorted_builders.each { |builder| builder.new.drop_view! }
+      end
+
+      builders_changed = []
+      sorted_builders.each do |builder|
+        changed = builder.new.recreate_view_if_changed!(builders_changed: builders_changed)
+        builders_changed << builder if changed
+      end
+
+      annotate_models!
+    end
+
+    def rebuild_all!
+    end
+
+    def recreate_indexes!
+      sorted_builders = sort_builders_by_dependency
+      sorted_builders.each { |builder| builder.new.recreate_indexes! }
+    end
+
+    def builders_to_rebuild
+      if SearchCraft.config.explicit_builder_class_names
+        SearchCraft.config.explicit_builder_class_names.map(&:constantize)
+      elsif Object.const_defined?(:Rails) && Rails.application
+        find_subclasses_via_rails_eager_load_paths.map(&:constantize)
+      else
+        subclasses
+      end
+    end
+
+    # Looks for subclasses of SearchCraft::Builder in Rails eager load paths
+    # and then any subclasses of those.
+    # Returns an array of class names
+    def find_subclasses_via_rails_eager_load_paths(known_subclass_names: [])
+      subclass_names = []
+
+      potential_superclass_names = known_subclass_names + ["SearchCraft::Builder"]
+      potential_superclass_regex = Regexp.new(potential_superclass_names.join("|"))
+
+      Rails.configuration.eager_load_paths.each do |load_path|
+        Dir.glob("#{load_path}/**/*.rb").each do |file|
+          File.readlines(file).each do |line|
+            if (match = line.match(/class\s+([\w:]+)\s*<\s*#{potential_superclass_regex}/))
+              class_name = match[1]
+              warn "Found #{class_name} in #{file}" unless known_subclass_names.include?(class_name)
+              subclass_names << class_name
+            end
+          end
+        end
+      end
+
+      newly_found_subclass_names = subclass_names - known_subclass_names
+      if newly_found_subclass_names.any?
+        return find_subclasses_via_rails_eager_load_paths(known_subclass_names: subclass_names)
+      end
+
+      subclass_names
+    end
+  end
+
+  # Produces the SQL that will create the materialized view
+  def view_sql
+    # remove trailing ; from view_sql
+    inner_sql = view_select_sql.gsub(/;\s*$/, "")
+    "CREATE MATERIALIZED VIEW #{view_name} AS (#{inner_sql}) WITH DATA;"
+  end
+
+  # After materialized view created, do you need indexes on its columns?
+  def view_indexes
+    {}
+  end
+
+  # To indicate if view has changed, we store a hash of the SQL used to create it
+  # TODO: include the indexes SQL too
+  def view_sql_hash
+    Digest::SHA256.hexdigest(view_sql)
+  end
+
+  # If missing or changed, drop and create view
+  # Returns false if no change required
+  def recreate_view_if_changed!(builders_changed: [])
+    if SearchCraft.debug?
+      warn "#{self.class.name}#recreate_view_if_changed!"
+      warn "  builders_changed: #{builders_changed.map(&:name).join(", ")}" if builders_changed.any?
+    end
+    return unless dependencies_ready?
+
+    dependencies_changed = (@@dependencies[self.class.name] || []) & builders_changed.map(&:name)
+    return false unless dependencies_changed.any? ||
+      SearchCraft::ViewHashStore.changed?(builder: self)
+
+    if SearchCraft.debug?
+      if !SearchCraft::ViewHashStore.exists?(builder: self)
+        warn "Creating #{view_name} because it doesn't yet exist"
+      elsif dependencies_changed.any?
+        warn "Recreating #{view_name} because dependencies changed: #{dependencies_changed.join(" ")}"
+      else
+        warn "Recreating #{view_name} because SQL changed"
+      end
+    end
+
+    drop_view!
+    create_view!
+    update_hash_store!
+    dump_schema!
+
+    true
+  end
+
+  def create_view!
+    create_sequence!
+    sql_execute(view_sql)
+    create_indexes!
+  end
+
+  # Finds and drops all indexes and sequences on view, and then drops view
+  def drop_view!
+    sql_execute("DROP MATERIALIZED VIEW IF EXISTS #{view_name} CASCADE;")
+
+    sql_execute("DROP SEQUENCE IF EXISTS #{view_id_sequence_name};")
+
+    SearchCraft::ViewHashStore.reset!(builder: self)
+  end
+
+  def recreate_indexes!
+    drop_indexes!
+    create_indexes!
+  end
+
+  # Pluralized table name of class
+  def view_name
+    base_sql_name
+  end
+
+  protected
+
+  # CREATE SEQUENCE #{view_id_sequence_name} CYCLE;
+  # DROP SEQUENCE #{view_id_sequence_name};
+  def view_id_sequence_name
+    "#{base_sql_name}_seq"
+  end
+
+  # ProductSearchBuilder name becomes product_searches
+  def base_sql_name
+    self.class.name.gsub(/Builder$/, "").tableize.tr("/", "_")
+  end
+
+  def base_idx_name
+    "idx_#{base_sql_name}"
+  end
+
+  def create_sequence!
+    sql_execute("CREATE SEQUENCE #{view_id_sequence_name} CYCLE;")
+  end
+
+  def drop_indexes!
+    simple_view_name = view_name.gsub(/^.+\./, "")
+    indexes = sql_execute("SELECT indexname FROM pg_indexes WHERE tablename = '#{simple_view_name}';")
+    indexes.each do |index|
+      warn "DROP INDEX IF EXISTS #{index["indexname"]};" if SearchCraft.debug?
+      sql_execute("DROP INDEX IF EXISTS #{index["indexname"]};")
+    end
+  end
+
+  def create_indexes!
+    view_indexes.each do |index_name, index_options|
+      columns = index_options[:columns]
+      name = "#{base_idx_name}_#{index_name}"
+      options = index_options.except(:columns).merge({name: name})
+
+      warn "ActiveRecord::Base.connection.add_index(#{view_name.inspect}, #{columns.inspect}, #{options.inspect})" if SearchCraft.debug?
+      ActiveRecord::Base.connection.add_index(view_name, columns, **options)
+    end
+  end
+
+  def update_hash_store!
+    SearchCraft::ViewHashStore.update_for(builder: self)
+  end
+
+  def sql_execute(sql)
+    warn sql if SearchCraft.debug?
+    ActiveRecord::Base.connection.execute(sql)
+  end
+end

data/lib/searchcraft/configuration.rb

@@ -0,0 +1,12 @@
+module SearchCraft
+  class Configuration
+    attr_accessor :disable_autorebuild
+    attr_accessor :debug
+    attr_accessor :explicit_builder_class_names
+    attr_accessor :explicit_model_class_names
+
+    def autorebuild?
+      !disable_autorebuild
+    end
+  end
+end

data/lib/searchcraft/depends_on.rb

@@ -0,0 +1,42 @@
+module SearchCraft::DependsOn
+  extend ActiveSupport::Concern
+
+  class_methods do
+    @@dependencies = {}
+
+    def depends_on(*builder_names)
+      @@dependencies[name] = builder_names
+    end
+
+    # TODO: implement .add_index instead of #view_indexes below
+    def add_index(index_name, columns, unique: false, name: nil)
+      @indexes ||= {}
+      # TODO: also get indexes from @@dependencies[name]
+      @indexes[index_name] = {columns: columns, unique: unique, name: name}
+    end
+
+    def sort_builders_by_dependency
+      sorted = []
+      visited = {}
+
+      builders_to_rebuild.each do |builder|
+        visit(builder, visited, sorted)
+      end
+
+      sorted
+    end
+
+    def visit(builder, visited, sorted)
+      return if visited[builder.name]
+
+      dependency_names = @@dependencies[builder.name] || []
+      dependency_names.each do |dependency_name|
+        dependency = Object.const_get(dependency_name)
+        visit(dependency, visited, sorted)
+      end
+
+      visited[builder.name] = true
+      sorted << builder
+    end
+  end
+end

data/lib/searchcraft/dump_schema.rb

@@ -0,0 +1,19 @@
+module SearchCraft::DumpSchema
+  extend ActiveSupport::Concern
+
+  # If in Rails, dump schema.rb after rebuilding views
+  def dump_schema!
+    return unless Rails.env.development?
+    require "active_record/tasks/database_tasks"
+
+    env = Rails.env
+    db_configs = ActiveRecord::Base.configurations.configs_for(env_name: env)
+    db_configs.each do |db_config|
+      ActiveRecord::Tasks::DatabaseTasks.dump_schema(db_config, ActiveRecord.schema_format)
+    end
+  rescue ActiveRecord::NoDatabaseError
+  rescue => e
+    warn "Error dumping schema: #{e.message}"
+    pp e.backtrace
+  end
+end

data/lib/searchcraft/model.rb

@@ -0,0 +1,50 @@
+require "scenic"
+
+module SearchCraft::Model
+  extend ActiveSupport::Concern
+
+  included do
+    def read_only?
+      true
+    end
+    self.table_name = name.tableize.tr("/", "_")
+  end
+
+  class_methods do
+    def refresh!
+      Scenic.database.refresh_materialized_view(table_name, concurrently: @refresh_concurrently, cascade: false)
+    end
+
+    def refresh_concurrently=(value)
+      @refresh_concurrently = value
+    end
+  end
+
+  # Maintain a list of classes that include this module
+  @included_classes = []
+
+  class << self
+    # Class method to add a class to the list of included classes
+    def included(base)
+      @included_classes << base
+      super
+    end
+
+    def included_classes
+      if SearchCraft.config.explicit_model_class_names
+        return SearchCraft.config.explicit_model_class_names.map(&:constantize)
+      end
+      @included_classes
+    end
+
+    # Runs .refresh! on all classes that include SearchCraft::Model
+    # TODO: eager load all classes that include SearchCraft::Model;
+    # perhaps via Builder eager loading?
+    def refresh_all!
+      included_classes.each do |klass|
+        warn "Refreshing materialized view #{klass.table_name}..." unless Rails.env.test?
+        klass.refresh!
+      end
+    end
+  end
+end

data/lib/searchcraft/railtie.rb

@@ -0,0 +1,22 @@
+require "rails/railtie"
+
+module SearchCraft
+  class Railtie < Rails::Railtie
+    initializer "searchcraft.reloader_hook" do
+      ActiveSupport::Reloader.to_prepare do
+        next unless SearchCraft.database_ready?
+        next unless SearchCraft.config.autorebuild?
+        next unless SearchCraft.dependencies_ready?
+
+        warn "[#{Rails.env}] running: SearchCraft::Builder.rebuild_any_if_changed!" if SearchCraft.debug?
+
+        SearchCraft::Builder.rebuild_any_if_changed!
+      rescue => e
+        if SearchCraft.debug?
+          puts "Preparing SearchCraft: #{e.message}"
+          puts e.backtrace
+        end
+      end
+    end
+  end
+end

data/lib/searchcraft/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module SearchCraft
+  VERSION = "0.4.0"
+end

data/lib/searchcraft/view_hash_store.rb

@@ -0,0 +1,61 @@
+# Stores the sha265 hash of the SQL used to create each view.
+# This allows the Builder to determine if a view needs
+# to be recreated. We don't want to use ActiveRecord Migrations,
+# instead want the view to be automatically recreated when
+# any Builder's #view_select_sql SQL changes.
+#
+# The view SQL hashes are stored in a table named
+# search_craft_view_hash_stores, which is automatically
+# created by SearchCraft.
+class SearchCraft::ViewHashStore < ActiveRecord::Base
+  self.table_name = "search_craft_view_hash_stores"
+
+  class << self
+    def update_for(builder:)
+      setup_table_if_needed!
+      view_sql_hash = builder.view_sql_hash
+      view_hash_store = find_or_initialize_by(view_name: builder.view_name)
+      view_hash_store.update!(view_sql_hash: view_sql_hash)
+    end
+
+    def changed?(builder:)
+      setup_table_if_needed!
+
+      view_sql_hash = builder.view_sql_hash
+      view_hash_store = exists?(builder: builder)
+      !view_hash_store || view_hash_store.view_sql_hash != view_sql_hash
+    end
+
+    def exists?(builder:)
+      setup_table_if_needed!
+      connection.schema_cache.data_source_exists?(builder.view_name) &&
+        find_by(view_name: builder.view_name)
+    end
+
+    def reset!(builder:)
+      view_hash_store = find_by(view_name: builder.view_name)
+      view_hash_store&.destroy!
+    end
+
+    def setup_table_if_needed!
+      return if table_exists?
+
+      # TODO: store its own sha265 hash in a table,
+      # so we can detect if it changes during development
+      # and recreate table if table is empty -- just in case
+
+      # Migrate table
+      create_table_sql = <<~SQL
+        CREATE TABLE search_craft_view_hash_stores (
+          id serial primary key,
+          view_name varchar(255) not null,
+          view_sql_hash varchar(255) not null,
+          created_at timestamp not null default now(),
+          updated_at timestamp not null default now()
+        );
+      SQL
+      ActiveRecord::Base.connection.execute(create_table_sql)
+      reset_column_information
+    end
+  end
+end

data/lib/searchcraft.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+module SearchCraft
+  class Error < StandardError; end
+
+  extend self
+
+  def configure
+    yield(config)
+  end
+
+  def config
+    @config ||= Configuration.new
+  end
+
+  def database_ready?
+    ActiveRecord::Base.connection.table_exists?("schema_migrations")
+  rescue
+    false
+  end
+
+  def dependencies_ready?
+    config.explicit_builder_class_names.all? do |builder_class_name|
+      builder_class_name.constantize.new.dependencies_ready?
+    end
+  end
+
+  def debug?
+    config.debug
+  end
+
+  def load_tasks
+    return if @tasks_loaded
+
+    Dir[File.join(File.dirname(__FILE__), "tasks", "**/*.rake")].each do |rake|
+      load rake
+    end
+
+    @tasks_loaded = true
+  end
+end
+
+require "active_record"
+
+require_relative "searchcraft/version"
+require_relative "searchcraft/configuration"
+require_relative "searchcraft/annotate"
+require_relative "searchcraft/depends_on"
+require_relative "searchcraft/dump_schema"
+require_relative "searchcraft/builder"
+require_relative "searchcraft/model"
+require_relative "searchcraft/view_hash_store"
+require_relative "searchcraft/railtie" if defined?(Rails)

data/lib/tasks/builders.rake

@@ -0,0 +1,25 @@
+namespace :searchcraft do
+  desc "Recreates search builders' materialized views if necessary"
+  task rebuild: :environment do
+    puts "Rebuilding search builders' materialized views if necessary"
+    puts Benchmark.measure {
+      SearchCraft::Builder.rebuild_any_if_changed!
+    }
+  end
+
+  desc "Recreates all materialized views' indices"
+  task recreate_indexes: :environment do
+    puts "Recreating search builders' indices"
+    puts Benchmark.measure {
+      SearchCraft::Builder.recreate_indexes!
+    }
+  end
+
+  desc "Force recreate all materialize views, including indexes and sequences"
+  task force_recreate: :environment do
+    puts "Force recreateing search builders' materialized views"
+    puts Benchmark.measure {
+      SearchCraft::Builder.rebuild_all!
+    }
+  end
+end

data/lib/tasks/refresh.rake

@@ -0,0 +1,12 @@
+namespace :searchcraft do
+  desc "Refresh searchcraft materialized views"
+  task refresh: :environment do
+    SearchCraft::Builder.rebuild_any_if_changed!
+    require "benchmark"
+    SearchCraft.config.explicit_model_class_names.each do |model_class_name|
+      klass = model_class_name.constantize
+      puts "Refreshing materialized views for #{klass.name}"
+      puts Benchmark.measure { klass.refresh! }
+    end
+  end
+end

data/sig/searchcraft.rbs

@@ -0,0 +1,4 @@
+module SearchCraft
+  VERSION: String
+  # See the writing guide of rbs: https://github.com/ruby/rbs#guides
+end

metadata

@@ -0,0 +1,152 @@
+--- !ruby/object:Gem::Specification
+name: searchcraft
+version: !ruby/object:Gem::Version
+  version: 0.4.0
+platform: ruby
+authors:
+- Dr Nic Williams
+autorequire:
+bindir: exe
+cert_chain: []
+date: 2023-10-23 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: activerecord
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: scenic
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.7'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.7'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '13.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '13.0'
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.0'
+- !ruby/object:Gem::Dependency
+  name: pg
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Use your SQL database to power instant search for your Rails app with
+  materalized views
+email:
+- drnicwilliams@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".DS_Store"
+- ".overcommit.yml"
+- ".standard.yml"
+- CHANGELOG.md
+- CODE_OF_CONDUCT.md
+- LICENSE.txt
+- README.md
+- Rakefile
+- lib/search_craft.rb
+- lib/searchcraft.rb
+- lib/searchcraft/annotate.rb
+- lib/searchcraft/builder.rb
+- lib/searchcraft/configuration.rb
+- lib/searchcraft/depends_on.rb
+- lib/searchcraft/dump_schema.rb
+- lib/searchcraft/model.rb
+- lib/searchcraft/railtie.rb
+- lib/searchcraft/version.rb
+- lib/searchcraft/view_hash_store.rb
+- lib/tasks/builders.rake
+- lib/tasks/refresh.rake
+- sig/searchcraft.rbs
+homepage: https://github.com/drnic/searchcraft
+licenses:
+- MIT
+metadata:
+  allowed_push_host: https://rubygems.org
+  homepage_uri: https://github.com/drnic/searchcraft
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '3.0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.4.20
+signing_key:
+specification_version: 4
+summary: Instant search for Rails and ActiveRecord
+test_files: []