ar-tsvectors 0.0.1 → 0.0.2

data/README.md

@@ -1,6 +1,6 @@
 # ActiveRecord support for ts_vectors
 
-This small library extends ActiveRecord to support PostgreSQL's `tsvector` type. PostgreSQL's `tsvector` stores an array of keywords that can be efficiently queried. The `tsvector` type is therefore particularly suitable for storing tags.
+This small library extends ActiveRecord to support PostgreSQL's `tsvector` type. PostgreSQL's `tsvector` stores an array of keywords that can be efficiently queried. The `tsvector` type is therefore particularly suitable for storing tags, since it's possible to store all tags in a single column instead of a table.
 
 ## Requirements
 
@@ -13,9 +13,9 @@ Add gem to your `Gemfile`:
 
     gem 'ar-tsvectors', :require => 'activerecord_tsvectors'
 
-To test, add a `tsvector` column to your table in a migration:
+To test, add a `text` column to your table in a migration:
 
-    add_column :posts, :tags, :tsvector
+    add_column :posts, :tags, :text
 
 Then extend your ActiveRecord model with a declaration:
 
@@ -34,10 +34,18 @@ You can now query based on the tags:
     Post.with_all_tags('pizza')
     Post.with_all_tags(['pizza', 'pepperoni'])
 
-Note that for queries to use indexes, you need to create an index on the column. This is slightly more complicated; again, in a migration:
+For queries to perform well, you must add an index on the column. This is slightly more complicated; again, in a migration:
 
     execute("create index index_posts_on_tags on posts using gin(to_tsvector('simple', tags))")
 
+## Options
+
+The `ts_vector` mixin takes the following options:
+
+* `:configuation` - the name of the PostgreSQL text configuration to use. Defaults to `simple`, which is appropriate for keyword-type indexing.
+* `:format` - the format of the column, must be either `:text` (the default) or `:tsvector`.
+* `:normalize` - a proc that overrides the default normalization. It must take a single string and return a normalized copy of the string.
+
 ## Methods
 
 Declaring a `tsvector` column `tags` will dynamically add the following methods:
@@ -77,10 +85,24 @@ The values are normalized both when performing queries, and when assigning new v
     post.tags
     => ['wtf']
 
-## Limitations
+## Using ts_vector directly
+
+It's possible to use the `tsvector` type for the column directly. In other words, in your migration would have something like:
+
+    add_column :posts, :tags, :tsvector
+
+This is supported by supplying a format option to the `ts_vector` declaration in your model:
+
+    class Post < ActiveRecord::Base
+      ts_vector :tags, :format => :tsvector
+    end
+
+When you do this, the way you define a table index is slightly different:
+
+    execute("create index index_posts_on_tags on posts using gin(tags)")
 
-Currently, the library will always use the built-in `simple` configuration, which only performs basic normalization, and does not perform stemming.
+While using a `tsvector` column works, it has some drawbacks:
 
-Due to a limitation in ActiveRecord, stored column values (on `INSERT` and `UPDATE`) are passed to PostgreSQL as strings, and are therefore *not* normalized using the text configuration's rules. This means that if you override the normalization function, you must make sure you always strip and downcase in addition to whatever other normalization you do, otherwise queries will potentially *not* match all rows.
+* The `tsvector` type is lossy. When you assign values to a `tsvector`, the text will be tokenized by PostgreSQL's text parser, which will remove all punctuation and quotes from the text. In this system, a quoted tag such as `"New York"` will be split into two different tags `New` and `York`, and it's no longer possible to search for `New York` as a distinct tag.
 
-A forthcoming version of ActiveRecord will provide the plumbing that will allow us to solve this issue.
+* Due to a limitation in ActiveRecord, stored column values (on `INSERT` and `UPDATE`) are passed to PostgreSQL as strings, and are therefore *not* normalized using the text configuration's rules. This means that if you override the normalization function, you must make sure you always strip and downcase in addition to whatever other normalization you do, otherwise queries will potentially *not* match all rows. A forthcoming version of ActiveRecord will provide the plumbing that will allow us to solve this issue.

data/lib/activerecord_tsvectors.rb

@@ -1,5 +1,6 @@
 require 'active_record'
 
+require 'ts_vectors/attribute'
 require 'ts_vectors/model'
 
 class ActiveRecord::Base

data/lib/ts_vectors/attribute.rb

@@ -0,0 +1,82 @@
+module TsVectors
+
+  class Attribute
+    def initialize(name, options = {})
+      options.assert_valid_keys(:normalize, :configuration, :format)
+      @name, @options = name, options
+      if (config = @options[:configuration])
+        @configuration = config
+      else
+        @configuration = 'simple'
+      end
+      if (format = @options[:format])
+        @format = format.to_sym
+      else
+        @format = :text
+      end
+    end
+
+    def format_operand
+      @operand ||= begin
+        config = @configuration
+        if @format == :text
+          "to_tsvector('#{config}', #{@name})"
+        else
+          @name
+        end
+      end
+    end
+
+    def format_query
+      @query ||= "to_tsquery('#{@configuration}', ?)"
+    end
+
+    def serialize_values(values)
+      if values and values.length > 0
+        values.join(' ')
+      else
+        nil
+      end
+    end
+
+    def parse_values(string)
+      if string
+        values = string.scan(/(?:([^'\s,]+(?::\d+)?)|'([^']+)(?::\d+)?')\s*/u).flatten
+        values.reject! { |v| v.blank? }
+        values
+      else
+        []
+      end
+    end
+
+    def normalize_values(values)
+      values = [values] unless values.is_a?(Enumerable)
+      values = values.map { |v| normalize(v) }
+      values.compact!
+      values
+    end
+
+    def normalize(value)
+      if value
+        if (normalize = @options[:normalize])
+          value = normalize.call(value)
+        else
+          value = value.strip.downcase
+        end
+        if value.blank?
+          nil
+        elsif value =~ /\s/
+          %('#{value}')
+        else
+          value
+        end
+      end
+    end
+
+    attr_reader :name
+    attr_reader :options
+    attr_reader :format
+    attr_reader :configuration
+  end
+
+end

data/lib/ts_vectors/model.rb

@@ -5,50 +5,6 @@ module TsVectors
     included do
     end
 
-    class Attribute
-      def initialize(name, options = {})
-        options.assert_valid_keys(:normalize)
-        @name, @options = name, options
-      end
-
-      def parse_values(string)
-        if string
-          values = string.scan(/(?:([^'\s,]+)|'([^']+)')\s*/u).flatten
-          values.reject! { |v| v.blank? }
-          values
-        else
-          []
-        end
-      end
-
-      def normalize_values(values)
-        values = [values] unless values.is_a?(Enumerable)
-        values = values.map { |v| normalize(v) }
-        values.compact!
-        values
-      end
-
-      def normalize(value)
-        if value
-          if (normalize = @options[:normalize])
-            value = normalize.call(value)
-          else
-            value = value.strip.downcase
-          end
-          if value.blank?
-            nil
-          elsif value =~ /\s/
-            %('#{value}')
-          else
-            value
-          end
-        end
-      end
-
-      attr_reader :name
-      attr_reader :options
-    end
-
     module ClassMethods
 
       def ts_vector(attribute, options = {})
@@ -60,7 +16,7 @@ module TsVectors
         scope "with_all_#{attribute}", lambda { |values|
           values = attr.normalize_values(values)
           if values.any?
-            where("#{attribute} @@ to_tsquery('simple', ?)", values.join(' & '))
+            where("#{attr.format_operand} @@ #{attr.format_query}", values.join(' & '))
           else
             where('false')
           end
@@ -69,7 +25,7 @@ module TsVectors
         scope "with_any_#{attribute}", lambda { |values|
           values = attr.normalize_values(values)
           if values.any?
-            where("#{attribute} @@ to_tsquery('simple', ?)", values.join(' | '))
+            where("#{attr.format_operand} @@ #{attr.format_query}", values.join(' | '))
           else
             where('false')
           end
@@ -78,7 +34,7 @@ module TsVectors
         scope "without_all_#{attribute}", lambda { |values|
           values = attr.normalize_values(values)
           if values.any?
-            where("#{attribute} @@ (!! to_tsquery('simple', ?))", values.join(' & '))
+            where("#{attr.format_operand} @@ (!! #{attr.format_query})", values.join(' & '))
           else
             where('false')
           end
@@ -87,7 +43,7 @@ module TsVectors
         scope "without_any_#{attribute}", lambda { |values|
           values = attr.normalize_values(values)
           if values.any?
-            where("#{attribute} @@ (!! to_tsquery('simple', ?))", values.join(' | '))
+            where("#{attr.format_operand} @@ (!! #{attr.format_query})", values.join(' | '))
           else
             where('false')
           end
@@ -98,7 +54,7 @@ module TsVectors
           values = attr.normalize_values(values)
           if values.any?
             order(sanitize_sql_array([
-              "ts_rank(#{attribute}, to_tsquery('simple', ?)) #{direction}", values.join(' | ')]))
+              "ts_rank(#{attr.format_operand}, #{attr.format_query}) #{direction}", values.join(' | ')]))
           else
             order('false')
           end
@@ -109,12 +65,8 @@ module TsVectors
         end
 
         define_method("#{attribute}=") do |values|
-          values = attr.normalize_values(values)
-          if values.any?
-            write_attribute(attribute, values.join(' '))
-          else
-            write_attribute(attribute, nil)
-          end
+          write_attribute(attribute, attr.serialize_values(
+            attr.normalize_values(values)))
         end
       end
 

data/lib/ts_vectors/version.rb

@@ -1,3 +1,3 @@
 module TsVectors
-  VERSION = '0.0.1'
+  VERSION = '0.0.2'
 end

data/spec/spec_helper.rb

@@ -22,15 +22,15 @@ RSpec.configure do |c|
 
   c.before :each do
     ActiveRecord::Base.connection.execute %(
-      create table if not exists things (
+      create table if not exists models_with_tsvector_format (
         id serial primary key,
         tags tsvector
       )
     )
     ActiveRecord::Base.connection.execute %(
-      create table if not exists thangs (
+      create table if not exists models_with_text_format (
         id serial primary key,
-        tags tsvector
+        tags text
       )
     )
   end

data/spec/ts_vectors_spec.rb

@@ -1,98 +1,127 @@
 require 'spec_helper'
 
-class Thing < ActiveRecord::Base
+class ModelUsingTsVectorFormat < ActiveRecord::Base
+  self.table_name = 'models_with_tsvector_format'
+  FORMAT = 'tsvector'
   ts_vector :tags
 end
 
-class Thang < ActiveRecord::Base
+class ModelUsingTextFormat < ActiveRecord::Base
+  self.table_name = 'models_with_text_format'
+  FORMAT = 'text'
+  ts_vector :tags, :format => :text
+end
+
+class ModelWithNormalization < ActiveRecord::Base
+  self.table_name = 'models_with_tsvector_format'
   ts_vector :tags, :normalize => lambda { |v| v.downcase.gsub(/\d/, '') }
 end
 
 describe TsVectors::Model do
-  let :thing do
-    Thing.new
-  end
-
   context 'attribute accessor' do
+    let :model do
+      ModelUsingTsVectorFormat.new
+    end
+
     it "sets empty array when assigning nil" do
-      thing.tags = nil
-      thing.tags.should == []
+      model.tags = nil
+      model.tags.should == []
     end
 
     it "sets empty array when assigning empty array" do
-      thing.tags = []
-      thing.tags.should == []
+      model.tags = []
+      model.tags.should == []
     end
 
     it "sets empty array when assigning empty string" do
-      thing.tags = ""
-      thing.tags.should == []
+      model.tags = ""
+      model.tags.should == []
     end
 
     it "sets empty array when assigning string containing only spaces" do
-      thing.tags = "  "
-      thing.tags.should == []
+      model.tags = "  "
+      model.tags.should == []
     end
 
     it "sets empty array when assigning array of strings containing only spaces" do
-      thing.tags = ["  "]
-      thing.tags.should == []
-    end
-  end
-
-  context 'with_all_XXX' do
-    it 'matches rows' do
-      thing.tags = %w(foo bar)
-      thing.save!
-      Thing.with_all_tags(%w(foo bar)).first.should == thing
-      Thing.with_all_tags(%w(foo bar baz)).first.should == nil
-      Thing.with_all_tags(%w(FOO BAR)).first.should == thing
-      Thing.with_all_tags(%w(FOO BAR BAZ)).first.should == nil
-      Thing.with_all_tags(%w(baz)).first.should == nil
+      model.tags = ["  "]
+      model.tags.should == []
     end
   end
 
-  context 'with_any_XXX' do
-    it 'matches rows' do
-      thing.tags = %w(foo bar)
-      thing.save!
-      Thing.with_any_tags(%w(foo bar)).first.should == thing
-      Thing.with_any_tags(%w(foo bar baz)).first.should == thing
-      Thing.with_any_tags(%w(FOO BAR)).first.should == thing
-      Thing.with_any_tags(%w(FOO BAR BAZ)).first.should == thing
-      Thing.with_any_tags(%w(baz)).first.should == nil
-    end
-  end
-
-  context 'without_all_XXX' do
-    it 'does not match rows' do
-      thing1 = Thing.create!(:tags => %w(foo bar))
-      thing2 = Thing.create!(:tags => %w(foo baz))
-      thing3 = Thing.create!(:tags => %w(baz))
-
-      Thing.without_all_tags(%w(foo)).sort.should == [thing3]
-      Thing.without_all_tags(%w(foo bar)).sort.should == [thing2, thing3]
-      Thing.without_all_tags(%w(baz)).sort.should == [thing1]
-      Thing.without_all_tags(%w(foo baz)).sort.should == [thing1, thing3]
-    end
-  end
-
-  context 'without_any_XXX' do
-    it 'does not match rows' do
-      thing1 = Thing.create!(:tags => %w(foo bar))
-      thing2 = Thing.create!(:tags => %w(foo baz))
-      thing3 = Thing.create!(:tags => %w(baz))
-
-      Thing.without_any_tags(%w(foo)).sort.should == [thing3]
-      Thing.without_any_tags(%w(foo bar)).sort.should == [thing3]
-      Thing.without_any_tags(%w(baz)).sort.should == [thing1]
-      Thing.without_any_tags(%w(foo baz)).should == []
+  [ModelUsingTsVectorFormat, ModelUsingTextFormat].each do |klass|
+    describe "using #{klass.const_get('FORMAT')} column" do
+      context 'with_all_XXX' do
+        let :model do
+          klass.new
+        end
+
+        it 'matches rows' do
+          model.tags = %w(foo bar)
+          model.save!
+          klass.with_all_tags(%w(foo bar)).first.should == model
+          klass.with_all_tags(%w(foo bar baz)).first.should == nil
+          klass.with_all_tags(%w(FOO BAR)).first.should == model
+          klass.with_all_tags(%w(FOO BAR BAZ)).first.should == nil
+          klass.with_all_tags(%w(baz)).first.should == nil
+        end
+      end
+
+      context 'with_any_XXX' do
+        let :model do
+          klass.new
+        end
+
+        it 'matches rows' do
+          model.tags = %w(foo bar)
+          model.save!
+          klass.with_any_tags(%w(foo bar)).first.should == model
+          klass.with_any_tags(%w(foo bar baz)).first.should == model
+          klass.with_any_tags(%w(FOO BAR)).first.should == model
+          klass.with_any_tags(%w(FOO BAR BAZ)).first.should == model
+          klass.with_any_tags(%w(baz)).first.should == nil
+        end
+      end
+
+      context 'without_all_XXX' do
+        let :model do
+          klass.new
+        end
+
+        it 'does not match rows' do
+          thing1 = klass.create!(:tags => %w(foo bar))
+          thing2 = klass.create!(:tags => %w(foo baz))
+          thing3 = klass.create!(:tags => %w(baz))
+
+          klass.without_all_tags(%w(foo)).sort.should == [thing3]
+          klass.without_all_tags(%w(foo bar)).sort.should == [thing2, thing3]
+          klass.without_all_tags(%w(baz)).sort.should == [thing1]
+          klass.without_all_tags(%w(foo baz)).sort.should == [thing1, thing3]
+        end
+      end
+
+      context 'without_any_XXX' do
+        let :model do
+          klass.new
+        end
+
+        it 'does not match rows' do
+          thing1 = klass.create!(:tags => %w(foo bar))
+          thing2 = klass.create!(:tags => %w(foo baz))
+          thing3 = klass.create!(:tags => %w(baz))
+
+          klass.without_any_tags(%w(foo)).sort.should == [thing3]
+          klass.without_any_tags(%w(foo bar)).sort.should == [thing3]
+          klass.without_any_tags(%w(baz)).sort.should == [thing1]
+          klass.without_any_tags(%w(foo baz)).should == []
+        end
+      end
     end
   end
 
   context 'normalization' do
     let :thang do
-      Thang.new
+      ModelWithNormalization.new
     end
 
     it 'calls normalizing function on assignment' do
@@ -103,10 +132,10 @@ describe TsVectors::Model do
     it 'normalizes query' do
       thang.tags = %w(foo123 BAR123)
       thang.save!
-      Thang.with_all_tags(%w(foo123)).first.should == thang
-      Thang.with_all_tags(%w(BAR123)).first.should == thang
-      Thang.with_all_tags(%w(foo)).first.should == thang
-      Thang.with_all_tags(%w(bar)).first.should == thang
+      ModelWithNormalization.with_all_tags(%w(foo123)).first.should == thang
+      ModelWithNormalization.with_all_tags(%w(BAR123)).first.should == thang
+      ModelWithNormalization.with_all_tags(%w(foo)).first.should == thang
+      ModelWithNormalization.with_all_tags(%w(bar)).first.should == thang
     end
   end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: ar-tsvectors
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.0.2
   prerelease: 
 platform: ruby
 authors:
@@ -90,6 +90,7 @@ files:
 - Rakefile
 - ar-tsvectors.gemspec
 - lib/activerecord_tsvectors.rb
+- lib/ts_vectors/attribute.rb
 - lib/ts_vectors/model.rb
 - lib/ts_vectors/version.rb
 - spec/spec_helper.rb