pg_fulltext 0.1.0 → 0.2.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 49c8f3b3250945a7e3e600a100951abffe80a0537e14fcf392d96778d9648777
-  data.tar.gz: efef8f9be9540e961015472dc3f498203c62a37856ef4f7c66f00fad30e7e5a2
+  metadata.gz: 2f4f614bb0fa282ac0bb258122139efd64f81ac8f04eb3dd4fd2c0e80c9f9d5a
+  data.tar.gz: c4292d5984bf2e3774b7f73948173ea2b4f022d2d29989b3a57222b3b516b02a
 SHA512:
-  metadata.gz: 2c324ce42e53dd725bec4ee7e11a8763927fe8080059a504ba31e5546b8d528ee76c040ae4a93c6b272a744d413f695583d413d991d1db6afb846f2e8750162e
-  data.tar.gz: 90e7cd23c65e9ee883c4c65c8fb5c21d08078eedf134f9d42cf989faf8928e2edc05a40ab008c07b367191326873773704de5bc458588aa5c0bcff7edce2ae11
+  metadata.gz: 1890385e76798f005d56c8005aa32b23badbd9ad770822b61962f37016cbe9997ebe9dc090d0948654c29a0178b939c6c472ed80b4c4fed671e0015781444ed5
+  data.tar.gz: af5b0b8d46c2a5ee73ee5de50cdc5c6b9f8e36bcb616830207db36cd32072a6a2fe7b173baf35da5fb9f938b80da14cda44cc9d70f966e6e3704a8b230ab0a2e

data/README.md

@@ -0,0 +1,69 @@
+# pg_fulltext
+
+A pretty reasonable PostgreSQL fulltext implementation with minimal configuration
+
+## Installation
+
+```shell
+gem install pg_fulltext
+```
+
+or add the following to your `Gemfile`:
+
+```shell
+gem 'pg_fulltext'
+```
+
+and run `bundle install`
+
+## Rails Configuration
+
+This implementation assumes you have a `tsv` column on your model, and that you're generating a string appropriate for 
+the language you're using (we don't specify a default, so it will probably default to `'english'` depending on your 
+Postgres implementation):
+
+```ruby
+class MyModel
+  include PgFulltext::ActiveRecord
+
+  add_search_scope
+end
+```
+
+You can then use the `search` method (configurable via the first parameter of the `add_search_scope` method):
+
+```ruby
+MyModel.search('foo bar "include this phrase" !butnotthis !"and and also not this"')
+```
+
+The defaults for this include support for negation, phrases, phrase negation, and prefix searches, but those can be 
+configured per the following options:
+
+| Option            | Default  | Description |
+| ----------------  | :------- | :----------- |
+| `tsvector_column` | `tsv`    | If you have a different column containing your tsvector, specify it here. |
+| `search_type`     | `simple` | Your PostgreSQL probably defaults to `'english'`, but the default for this option is `simple` to ensure the most predictable behavior. IMPORTANT: Set this to match the tsvector you've generated. |
+| `order`           | `true`   | Whether or not the `order` method should be applied against the generated `rank` for the fulltext query. If you just care about returning matches and not their respective rank, set this to `false`. |
+| `prefix`          | `true`   | Default search will match partial words as well as whole words. Set this to `false` if only whole words should be matched. |
+| `reorder`         | `false`  | If you already have `order` set on this relation, it will take precedence over the fulltext `rank`. `reorder` will call clear, effectively clearing the existing order and applying `rank`. |
+| `any_word`        | `false`  | Default search uses the `&` operator, ensuring that all terms are matched in the query.  If you want to match _any_ term in the query, set this to `true`. |
+| `ignore_accents`  | `false`  | By default, search queries with accents will be sent through as-is. Setting this to `true` will `unaccent()` the query, which helps match `tsv` columns that have also been unaccented.  Alternatively, you can have your `tsv` column be a combination of both, and this option will be unnecesary. Requires the `unaccent` Postgres extension. |
+
+## Standalone Configuration
+
+There's not much, here, but the `PgFulltext::Query.to_tsquery_string` method will generate a nice `tsvector`-compatible 
+string for you to use as you wish.  
+
+Something this should do the trick:
+
+```ruby
+db = PG.connect(dbname: 'mydb')
+search_string = 'foo bar "include this phrase" !butnotthis !"and and also not this"'
+tsv_query = db.escape_string(PgFulltext::Query.to_tsquery_string(search_string))
+sql = <<~SQL
+  SELECT *
+  FROM my_model
+  WHERE tsv @@ to_tsquery('portuguese', '#{tsv_query}')
+SQL
+db.exec(sql)
+```

data/Rakefile

@@ -0,0 +1,7 @@
+require 'bundler'
+Bundler::GemHelper.install_tasks
+
+require 'rspec/core/rake_task'
+RSpec::Core::RakeTask.new(:spec)
+
+task default: %w[spec]

data/lib/pg_fulltext/active_record.rb

@@ -21,13 +21,13 @@ module PgFulltext
         tsvector_column: :tsv,
         search_type: :simple,
         order: true,
+        prefix: true,
         reorder: false,
         any_word: false,
         ignore_accents: false
       )
         serial = SecureRandom.hex(4)
-        table_quoted = connection.quote_table_name(table_name)
-        pk_quoted = "#{table_quoted}.#{connection.quote_column_name(primary_key)}"
+        pk_quoted = "#{quoted_table_name}.#{connection.quote_column_name(primary_key)}"
         fulltext_join_name = "pg_fulltext_#{serial}"
 
         # Build the search relation to join on
@@ -37,6 +37,7 @@ module PgFulltext
           tsvector_column: tsvector_column,
           search_type: search_type,
           any_word: any_word,
+          prefix: prefix,
           ignore_accents: ignore_accents,
         )
 
@@ -58,13 +59,13 @@ module PgFulltext
         tsvector_column: :tsv,
         search_type: nil,
         any_word: false,
+        prefix: true,
         ignore_accents: false
       )
-        tsquery_string_quoted = connection.quote(PgFulltext::Query.to_tsquery_string(query, operator: any_word ? '|' : '&'))
+        tsquery_string_quoted = connection.quote(PgFulltext::Query.to_tsquery_string(query, operator: any_word ? '|' : '&', prefix: prefix))
         tsquery_string_quoted = "unaccent(#{tsquery_string_quoted})" if ignore_accents
-        table_quoted = connection.quote_table_name(table_name)
         column_quoted = connection.quote_column_name(tsvector_column)
-        fqc_quoted = "#{table_quoted}.#{column_quoted}"
+        fqc_quoted = "#{quoted_table_name}.#{column_quoted}"
         tsquery = "to_tsquery(#{"#{connection.quote search_type}, " if search_type.present?}#{tsquery_string_quoted})"
 
         relation

data/lib/pg_fulltext/query.rb

@@ -1,55 +1,47 @@
+require 'rltk/lexer'
+
 module PgFulltext
   module Query
-    def self.to_tsquery_string(query, prefix: true, operator: '&')
-
-      # Parse out all [unicode] non-word and non-quote characters
-      query.gsub!(/[^\s\p{L}"!]/, '')
-      query.gsub!(/"+/, '"')
-      query.gsub!(/\s+/, ' ')
-
-      # Collect terms
-      terms = []
+    class Lexer < RLTK::Lexer
+      rule(/!+/)
+      rule(/"+/)
 
-      # Phrase mode
-      if query.count('"') > 0 && query.count('"') % 2 == 0
-        phrase_terms = []
-        negate_phrase = false
+      rule(/(!?)[0-9\p{L}!]+/) { |v| [:WORD, v] }
+      rule(/"[\p{L}\s!]+"/) { |v| [:PHRASE, v[1..-2]] }
+      rule(/!"[\p{L}\s!]+"/) { |v| [:NOT_PHRASE, v[2..-2]] }
 
-        query_parts = query.split(' ')
-        query_parts.each do |term|
+      rule(/\s+/)
+      rule(/[^\p{L}\s"]+/)
+    end
 
-          # Skip if completely comprised of non-unicode word characters
-          next if term.gsub(/[^\s\p{L}]/, '') == ''
+    def self.to_tsquery_string(query, prefix: true, operator: '&')
+      query = normalize_query(query)
 
-          if term.start_with?('!"') && !term.end_with?('"')
-            phrase_terms << format_term(term[2..-1], prefix: true)
-            negate_phrase = true
-          elsif term.start_with?('"') && !term.end_with?('"')
-            phrase_terms << format_term(term[1..-1], prefix: true)
-          elsif phrase_terms.length > 0
-            if term.end_with?('"')
-              phrase_terms << format_term(term[0..-2], prefix: prefix)
-              terms << "#{'!' if negate_phrase}(#{reject_falsy(phrase_terms, prefix: prefix).join(' <-> ')})"
-              phrase_terms = []
-              negate_phrase = false
-            else
-              phrase_terms << format_term(term, prefix: prefix)
-            end
-          else
-            terms << format_term(term, prefix: prefix)
-          end
+      terms = []
+      Lexer.lex(query).each do |token|
+        if token.type == :WORD
+          terms << format_term(token.value, prefix: prefix)
+        elsif %i[PHRASE NOT_PHRASE].include?(token.type)
+          phrase_terms = Lexer.lex(token.value).map do |phrase_term|
+            phrase_term.value.nil? ? nil : format_term(phrase_term.value, prefix: prefix)
+          end.compact
+          terms << "#{'!' if token.type == :NOT_PHRASE}(#{phrase_terms.join(' <-> ')})"
         end
-      else
-        query.gsub! /["]/, ''
-        terms = reject_falsy(query.split(' ').map { |v| format_term(v, prefix: prefix) }, prefix: prefix)
       end
 
-      # Join terms with operator
       terms.join(" #{operator} ")
     end
 
     private
 
+    def self.normalize_query(query)
+      query
+        .gsub(/[.,]/, ' ')            # Replace all periods and commas with spaces (reasonable delimiters)
+        .gsub(/[^\s\p{L}0-9"!]/, '')  # Remove all non-unicode, whitespace, numbers, quotes ("), and bangs (!)
+        .gsub(/\s+/, ' ')             # Replace repeat whitespace occurrences with single spaces
+        .strip                        # Strip space from beginning and end of line
+    end
+
     def self.format_term(term, prefix: true)
       # Remove any ! that's not at the beginning of the term, as it will break the query
       term.gsub!(/(?<!^)!/, '')
@@ -58,7 +50,7 @@ module PgFulltext
       "#{term}#{':*' if prefix}"
     end
 
-    def self.reject_falsy(terms, prefix: true)
+    def self.reject_falsy(terms)
       false_values = [nil, '', '"', '!', ':*', '":*', '!:*']
       terms.reject { |v| false_values.include?(v) }
     end

metadata

@@ -1,28 +1,102 @@
 --- !ruby/object:Gem::Specification
 name: pg_fulltext
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.3
 platform: ruby
 authors:
 - Adam Robertson
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-10-11 00:00:00.000000000 Z
-dependencies: []
+date: 2021-11-22 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: activerecord
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '5.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '5.0'
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '5.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '5.0'
+- !ruby/object:Gem::Dependency
+  name: rltk
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 3.0.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 3.0.1
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: with_model
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: Allows simple searching with a variety of options
 email: adam@arcreative.net
 executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- README.md
+- Rakefile
 - lib/pg_fulltext.rb
 - lib/pg_fulltext/active_record.rb
 - lib/pg_fulltext/query.rb
 homepage: https://github.com/arcreative/pg_fulltext
 licenses:
 - MIT
-metadata: {}
+metadata:
+  homepage_uri: https://github.com/arcreative/pg_fulltext
+  source_code_uri: https://github.com/arcreative/pg_fulltext
 post_install_message:
 rdoc_options: []
 require_paths:
@@ -38,7 +112,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.4
+rubygems_version: 3.1.6
 signing_key:
 specification_version: 4
 summary: PostgreSQL fulltext search