activerecord-text-search-configuration 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: afbdb8cf9b7b89a5f0825b33393d58a5fb0478e6a031646a4dbbf4afbb89ba34
+  data.tar.gz: 90574a92d626ba0861cc522dfe8dd6d9bb4d5e9f8397fe80e8efceaeb08926d2
+SHA512:
+  metadata.gz: d13520cbe86371cf2df820aca696cd482ea69a9205e7a665fea3a4597fd8ae9503339b102cc9c41a428d79c699693498bedeaa21632a2f1f3afee2ca672e3bce
+  data.tar.gz: e61a3191d201d6fdbebadb7c36248d1419daf75a474e62c956dc29ae8b8996b37be92fe3954f694ea41a72e610e3ddf745e3328955e6e04e4e5a2a248274b473

data/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright © Fingertips <manfred@fngtps.com>
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/README.md

@@ -0,0 +1,123 @@
+# Active Record – Text Search Configuration
+
+Allows basic management of text search configurations in Postgres.
+
+```ruby
+# Assumes you already have the unaccent extension created for the database.
+class CreateSimpleUnaccentSearchConfiguration < ActiveRecord::Migration[8.1]
+  def change
+    create_text_search_configuration(
+      :simple_unaccent,
+      # Start from the simple search configuration.
+      copy: :simple,
+      # The tsearch parser assigns a type to each token. We configure how
+      # values for each type are mapped to an internal value before
+      # creating a search vector.
+      mappings: {
+      	%w[hword hword_part word] => %w[unaccent simple],
+        %w[asciiword] => %w[simple]
+      }
+    )
+  end
+end
+```
+
+After this we can use the search configuration every time we create a vector:
+
+```ruby
+create_table :documents do |t|
+  t.text :title
+  t.virtual(
+  	:title_tsvector,
+    type: :tsvector,
+    stored: true,
+    as: "to_tsvector('simple_unaccent', coalesce(title, ''))"
+  )
+end
+```
+
+We store a vector for query performance reasons and the original text for presentation. Having a predefined search configuration means we can re-use the same settings when using a function like `ts_headline` on the text:
+
+    ts_headline('simple_unaccent', "documents"."title", …)
+
+## Setup
+
+Add `activerecord-text-search-configuration` to your dependencies.
+
+## Defining a search configuration and mapping
+
+You probably want to start by reading the [PostgreSQL documentation](https://www.postgresql.org/docs/current/textsearch-controls.html) about controlling text search.
+
+A configuration starts from either a parser, without any mappings:
+
+    create_text_search_configuration(parser: :default)
+    
+Or it starts as a copy of an existing configuration:
+
+    create_text_search_configuration(copy: :simple)
+
+You can get a full list of token types from the server by running `psql`:
+
+    \dFp+ default
+
+In schema changes you can use `change_text_search_configuration_mapping(name, token_types, dictionaries)` and `drop_text_search_configuration(name)` to remove a configuration. Pass `force: :cascade` to also drop generated columns and indexes that depend on it. 
+
+Note that `create_text_search_configuration` is reversible; `drop_text_search_configuration` is reversible when you pass `parser:` or `copy:` with `mappings:`.
+
+If you're still lost, you probably want something like this:
+
+```ruby
+create_text_search_configuration(
+  'simple_unaccent',
+  parser: 'default',
+  mappings: {
+    %w[
+      asciihword
+      asciiword
+      email
+      file
+      float
+      host
+      hword_asciipart
+      hword_numpart
+      int
+      numhword
+      numword
+      protocol
+      sfloat
+      uint
+      url
+      url_path
+      version
+    ] => ['simple'],
+    %w[word hword_part hword] => %w[unaccent simple]
+  }
+)
+```
+
+This leaves out `blank`, `entity`, and `tag`. We probably don't care about whitespace and entities (eg. `&gt;`) and tags (eg. `<xml>`) need to pre-processing to handle in a useful way if you want to make them searchable so the parser should never find them.
+
+You will need to do a lot of work to massage documents and queries so you get the type of query features you want, but it's outside of the scope of this README to discuss all these details.
+
+## Development
+
+Run tests in this repository with `rake`. Migrations are exercised by running them in the dummy application.
+
+```
+bundle install
+cd test/dummy
+rm -f db/schema.rb && RAILS_ENV=test bundle exec rake db:drop db:create db:migrate
+cd ../../
+bundle exec rake
+```
+
+If you also want to verify running tests on Rails Edge, you can set `BUNDLE_GEMFILE` and run everything again.
+
+```
+export BUNDLE_GEMFILE=$(pwd)/Gemfile.edge
+bundle install
+cd test/dummy
+rm -f db/schema.rb && RAILS_ENV=test bundle exec rake db:drop db:create db:migrate
+cd ../../
+bundle exec rake
+```

data/lib/active_record/text_search_configuration/command_recorder.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+module ActiveRecord
+  module TextSearchConfiguration
+    # Records text search configuration statements in migrations so they can be reverted.
+    module CommandRecorder
+      def create_text_search_configuration(*args, &)
+        record(:create_text_search_configuration, args, &)
+      end
+
+      def drop_text_search_configuration(*args, &)
+        record(:drop_text_search_configuration, args, &)
+      end
+
+      private
+
+      def invert_create_text_search_configuration(args)
+        [:drop_text_search_configuration, [args.first]]
+      end
+
+      # Dropping can only be reverted when the migration passes everything needed to create the
+      # configuration again.
+      def invert_drop_text_search_configuration(args)
+        options = args.extract_options!
+        if options[:parser].nil? && options[:copy].nil?
+          raise(
+            ActiveRecord::IrreversibleMigration,
+            'To make drop_text_search_configuration reversible, pass parser: or copy: and ' \
+            'mappings:.'
+          )
+        end
+
+        [:create_text_search_configuration, [args.first, options.except(:if_exists, :force)]]
+      end
+    end
+  end
+end

data/lib/active_record/text_search_configuration/railtie.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module ActiveRecord
+  module TextSearchConfiguration
+    class Railtie < Rails::Railtie # :nodoc:
+      initializer 'active_record.text_search_configuration' do
+        ActiveSupport.on_load(:active_record_postgresqladapter) do
+          ActiveRecord::ConnectionAdapters::PostgreSQL::SchemaDumper.include(
+            ActiveRecord::TextSearchConfiguration::SchemaDumper
+          )
+          ActiveSupport.on_load(:active_record) do
+            ActiveRecord::ConnectionAdapters::PostgreSQL::SchemaStatements.include(
+              ActiveRecord::TextSearchConfiguration::SchemaStatements
+            )
+            ActiveRecord::Migration::CommandRecorder.include(
+              ActiveRecord::TextSearchConfiguration::CommandRecorder
+            )
+          end
+        end
+      end
+    end
+  end
+end

data/lib/active_record/text_search_configuration/schema_dumper.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+module ActiveRecord
+  module TextSearchConfiguration
+    # Overrides the PostgresAdapter schema dumper to include text search configurations.
+    module SchemaDumper
+      extend ActiveSupport::Concern
+
+      def text_search_configurations(stream)
+        configurations = @connection.text_search_configurations
+        return unless configurations.any?
+
+        configurations.each do |configuration|
+          text_search_configuration(stream, configuration)
+        end
+        stream.puts
+      end
+
+      def tables(stream)
+        text_search_configurations(stream)
+        super
+      end
+
+      private
+
+      def text_search_configuration(stream, configuration)
+        stream.puts("  create_text_search_configuration(\n")
+        stream.puts("    #{configuration[:name].inspect},")
+        stream.puts("    parser: #{configuration[:parser].inspect},")
+        stream.puts('    force: :cascade,')
+        stream.puts('    mappings: {')
+        stream.puts(
+          configuration[:mappings].map do |token_types, dictionaries|
+            "      #{token_types.inspect} => #{dictionaries.inspect}"
+          end.join(",\n")
+        )
+        stream.puts('    }')
+        stream.puts('  )')
+      end
+    end
+  end
+end

data/lib/active_record/text_search_configuration/schema_statements.rb

@@ -0,0 +1,104 @@
+# frozen_string_literal: true
+
+module ActiveRecord
+  module TextSearchConfiguration
+    # Defines methods to create, drop, and inspect text search configurations on a schema in
+    # Postgres.
+    module SchemaStatements
+      # Creates a text search configuration.
+      def create_text_search_configuration(
+        name, parser: nil, copy: nil, mappings: {}, force: false
+      )
+        drop_text_search_configuration(name, if_exists: true, force: force) if force
+        execute(
+          "CREATE TEXT SEARCH CONFIGURATION #{quote_column_name(name)} " \
+          "(#{text_search_configuration_source(parser, copy)})"
+        )
+        mappings.each do |token_types, dictionaries|
+          change_text_search_configuration_mapping(name, token_types, dictionaries)
+        end
+      end
+
+      # Replaces the dictionaries for one or more token types. An empty list of dictionaries
+      # removes the mapping.
+      def change_text_search_configuration_mapping(name, token_types, dictionaries)
+        token_types = quote_column_names(token_types)
+        execute(
+          "ALTER TEXT SEARCH CONFIGURATION #{quote_column_name(name)} " \
+          "DROP MAPPING IF EXISTS FOR #{token_types}"
+        )
+        return if dictionaries.blank?
+
+        execute(
+          "ALTER TEXT SEARCH CONFIGURATION #{quote_column_name(name)} " \
+          "ADD MAPPING FOR #{token_types} " \
+          "WITH #{quote_column_names(dictionaries)}"
+        )
+      end
+
+      # Drops a text search configuration.
+      def drop_text_search_configuration(name, if_exists: false, force: false, **)
+        execute(
+          "DROP TEXT SEARCH CONFIGURATION #{'IF EXISTS ' if if_exists}" \
+          "#{quote_column_name(name)}#{' CASCADE' if force == :cascade}"
+        )
+      end
+
+      # Returns the text search configurations in the public schema.
+      def text_search_configurations
+        rows = select_all(text_search_configurations_sql, 'SCHEMA')
+        rows.group_by { |row| row['name'] }.map do |name, configuration_rows|
+          {
+            name: name,
+            parser: configuration_rows.first['parser'],
+            mappings: text_search_configuration_mappings(configuration_rows)
+          }
+        end
+      end
+
+      private
+
+      def text_search_configuration_source(parser, copy)
+        if parser && copy
+          raise ArgumentError, 'Pass either parser: or copy:, a configuration needs one source.'
+        end
+
+        return "COPY = #{quote_column_name(copy)}" if copy
+
+        "PARSER = #{quote_column_name(parser || 'default')}"
+      end
+
+      def quote_column_names(names)
+        Array(names).map { |name| quote_column_name(name) }.join(', ')
+      end
+
+      def text_search_configuration_mappings(rows)
+        rows
+          .select { |row| row['token_type'] }
+          .group_by { |row| row['dictionaries'] }
+          .to_h do |dictionaries, grouped|
+            [grouped.pluck('token_type'), dictionaries.split(',')]
+          end
+      end
+
+      def text_search_configurations_sql
+        <<~SQL
+          SELECT
+            config.cfgname AS name,
+            parser.prsname AS parser,
+            token.alias AS token_type,
+            string_agg(dictionary.dictname, ',' ORDER BY map.mapseqno) AS dictionaries
+          FROM pg_ts_config config
+          JOIN pg_namespace namespace
+          ON config.cfgnamespace = namespace.oid AND namespace.nspname = 'public'
+          JOIN pg_ts_parser parser ON config.cfgparser = parser.oid
+          LEFT JOIN pg_ts_config_map map ON map.mapcfg = config.oid
+          LEFT JOIN ts_token_type(config.cfgparser) token ON token.tokid = map.maptokentype
+          LEFT JOIN pg_ts_dict dictionary ON dictionary.oid = map.mapdict
+          GROUP BY config.cfgname, parser.prsname, token.tokid, token.alias
+          ORDER BY config.cfgname, token.tokid
+        SQL
+      end
+    end
+  end
+end

data/lib/active_record/text_search_configuration/version.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+module ActiveRecord
+  module TextSearchConfiguration
+    VERSION = '0.0.1'
+  end
+end

data/lib/active_record/text_search_configuration.rb

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+module ActiveRecord
+  # Implements tools to manage and use text search configurations in Postgres.
+  module TextSearchConfiguration
+  end
+end
+
+require 'active_record/text_search_configuration/command_recorder'
+require 'active_record/text_search_configuration/schema_dumper'
+require 'active_record/text_search_configuration/schema_statements'
+require 'active_record/text_search_configuration/version'

data/lib/activerecord-text-search-configuration.rb

@@ -0,0 +1,4 @@
+# frozen_string_literal: true
+
+require 'active_record/text_search_configuration'
+require 'active_record/text_search_configuration/railtie'

metadata

@@ -0,0 +1,66 @@
+--- !ruby/object:Gem::Specification
+name: activerecord-text-search-configuration
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Manfred Stienstra
+bindir: bin
+cert_chain: []
+date: 1980-01-02 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: activerecord
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: |2
+    Adds methods like create_text_search_configuration to manage search configurations in your
+    Ruby on Rails project.
+email:
+- manfred@fngtps.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- LICENSE
+- README.md
+- lib/active_record/text_search_configuration.rb
+- lib/active_record/text_search_configuration/command_recorder.rb
+- lib/active_record/text_search_configuration/railtie.rb
+- lib/active_record/text_search_configuration/schema_dumper.rb
+- lib/active_record/text_search_configuration/schema_statements.rb
+- lib/active_record/text_search_configuration/version.rb
+- lib/activerecord-text-search-configuration.rb
+homepage: https://github.com/Fingertips/activerecord-text-search-configuration
+licenses:
+- MIT
+metadata:
+  rubygems_mfa_required: 'true'
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">"
+    - !ruby/object:Gem::Version
+      version: '3.3'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 4.0.10
+specification_version: 4
+summary: Text search configuration support for Active Record.
+test_files: []