ar-tsvectors 0.0.1

data/.gitignore

@@ -0,0 +1,5 @@
+*.gem
+.bundle
+Gemfile.lock
+pkg/*
+/spec/database.yml

data/.rspec

@@ -0,0 +1 @@
+--color

data/Gemfile

@@ -0,0 +1,2 @@
+source :rubygems
+gemspec

data/README.md

@@ -0,0 +1,86 @@
+# ActiveRecord support for ts_vectors
+
+This small library extends ActiveRecord to support PostgreSQL's `tsvector` type. PostgreSQL's `tsvector` stores an array of keywords that can be efficiently queried. The `tsvector` type is therefore particularly suitable for storing tags.
+
+## Requirements
+
+* ActiveRecord
+* PostgreSQL with `tsvector` support. (This has been included out of the box since 8.3.)
+
+## Usage
+
+Add gem to your `Gemfile`:
+
+    gem 'ar-tsvectors', :require => 'activerecord_tsvectors'
+
+To test, add a `tsvector` column to your table in a migration:
+
+    add_column :posts, :tags, :tsvector
+
+Then extend your ActiveRecord model with a declaration:
+
+    class Post < ActiveRecord::Base
+      ts_vector :tags
+    end
+
+Now you can assign tags:
+
+    post = Post.new
+    post.tags = ['pizza', 'pepperoni', 'Italian', 'mushrooms']
+    post.save
+
+You can now query based on the tags:
+
+    Post.with_all_tags('pizza')
+    Post.with_all_tags(['pizza', 'pepperoni'])
+
+Note that for queries to use indexes, you need to create an index on the column. This is slightly more complicated; again, in a migration:
+
+    execute("create index index_posts_on_tags on posts using gin(to_tsvector('simple', tags))")
+
+## Methods
+
+Declaring a `tsvector` column `tags` will dynamically add the following methods:
+
+* `with_all_tags(t)` - returns a scope that searches for `t`, which may be either a single value or an array of values. Only rows matching _all_ values will be returned.
+* `without_all_tags(t)` - returns a scope that excludes `t`, which may be either a single value or an array of values. Only rows matching _all_ values will be excluded.
+* `with_any_tags(t)` - returns a scope that searches for `t`, which may be either a single value or an array of values. Rows matching at least one of the provided values will be returned.
+* `without_any_tags(t)` - returns a scope that excludes `t`, which may be either a single value or an array of values. Rows matching at least one of the provided values will be excluded.
+* `order_by_tags_rank(t, direction = 'DESC')` - returns a scope that orders the rows by the rank, ie. a score computed based on the overlap of `t` and the row's value. See below for an example.
+
+## Ranking
+
+It's trivial to rank results according to how many values match a given row:
+
+    @posts = Post.with_any_tags(["pizza", "pepperoni"]).
+      order_by_tags_rank(["pizza", "pepperoni"])
+
+This orders the rows such that rows matching _both_ `pizza` and `pepperoni` will be ordered above rows matching _either_ `pizza` or `pepperoni` but not both.
+
+## Normalization
+
+The default behaviour is to downcase and strip leading and trailing spaces from assigned values. By default, matches are case-sensitive; so something tagged with `PIZZA` will not match a search for `pizza`. To ensure case insensitivity, you can provide a normalization function, like so:
+
+    class Post < ActiveRecord::Base
+      ts_vector :tags, :normalize => lambda { |v| v.downcase }
+    end
+
+You can also use this to strip unwanted characters:
+
+    class Post < ActiveRecord::Base
+      ts_vector :tags, :normalize => lambda { |v| v.downcase.gsub(/[^\w]/, '') }
+    end
+
+The values are normalized both when performing queries, and when assigning new values:
+
+    post.tags = ['  WTF#$%^   &*??!']
+    post.tags
+    => ['wtf']
+
+## Limitations
+
+Currently, the library will always use the built-in `simple` configuration, which only performs basic normalization, and does not perform stemming.
+
+Due to a limitation in ActiveRecord, stored column values (on `INSERT` and `UPDATE`) are passed to PostgreSQL as strings, and are therefore *not* normalized using the text configuration's rules. This means that if you override the normalization function, you must make sure you always strip and downcase in addition to whatever other normalization you do, otherwise queries will potentially *not* match all rows.
+
+A forthcoming version of ActiveRecord will provide the plumbing that will allow us to solve this issue.

data/Rakefile

@@ -0,0 +1 @@
+require "bundler/gem_tasks"

data/ar-tsvectors.gemspec

@@ -0,0 +1,26 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
+
+require 'ts_vectors/version'
+
+Gem::Specification.new do |s|
+  s.name        = "ar-tsvectors"
+  s.version      = TsVectors::VERSION
+  s.authors     = ["Alexander Staubo"]
+  s.email       = ["alex@bengler.no"]
+  s.homepage    = ""
+  s.summary     = %q{Support for PostgreSQL's ts_vector data type in ActiveRecord}
+  s.description = %q{Support for PostgreSQL's ts_vector data type in ActiveRecord. Perfect for indexing tags, arrays etc.}
+  
+  s.rubyforge_project = "ar-tsvectors"
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {spec}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+
+  s.add_dependency "activerecord", ">= 3.0"
+  s.add_development_dependency "rspec"
+  s.add_development_dependency "simplecov"
+  s.add_development_dependency "activerecord-postgresql-adapter"
+end

data/lib/activerecord_tsvectors.rb

@@ -0,0 +1,7 @@
+require 'active_record'
+
+require 'ts_vectors/model'
+
+class ActiveRecord::Base
+  include ::TsVectors::Model
+end

data/lib/ts_vectors/model.rb

@@ -0,0 +1,124 @@
+module TsVectors
+  module Model
+    extend ActiveSupport::Concern
+
+    included do
+    end
+
+    class Attribute
+      def initialize(name, options = {})
+        options.assert_valid_keys(:normalize)
+        @name, @options = name, options
+      end
+
+      def parse_values(string)
+        if string
+          values = string.scan(/(?:([^'\s,]+)|'([^']+)')\s*/u).flatten
+          values.reject! { |v| v.blank? }
+          values
+        else
+          []
+        end
+      end
+
+      def normalize_values(values)
+        values = [values] unless values.is_a?(Enumerable)
+        values = values.map { |v| normalize(v) }
+        values.compact!
+        values
+      end
+
+      def normalize(value)
+        if value
+          if (normalize = @options[:normalize])
+            value = normalize.call(value)
+          else
+            value = value.strip.downcase
+          end
+          if value.blank?
+            nil
+          elsif value =~ /\s/
+            %('#{value}')
+          else
+            value
+          end
+        end
+      end
+
+      attr_reader :name
+      attr_reader :options
+    end
+
+    module ClassMethods
+
+      def ts_vector(attribute, options = {})
+        attr = Attribute.new(attribute, options)
+
+        @ts_vectors ||= {}
+        @ts_vectors[attribute] = attr
+
+        scope "with_all_#{attribute}", lambda { |values|
+          values = attr.normalize_values(values)
+          if values.any?
+            where("#{attribute} @@ to_tsquery('simple', ?)", values.join(' & '))
+          else
+            where('false')
+          end
+        }
+
+        scope "with_any_#{attribute}", lambda { |values|
+          values = attr.normalize_values(values)
+          if values.any?
+            where("#{attribute} @@ to_tsquery('simple', ?)", values.join(' | '))
+          else
+            where('false')
+          end
+        }
+
+        scope "without_all_#{attribute}", lambda { |values|
+          values = attr.normalize_values(values)
+          if values.any?
+            where("#{attribute} @@ (!! to_tsquery('simple', ?))", values.join(' & '))
+          else
+            where('false')
+          end
+        }
+
+        scope "without_any_#{attribute}", lambda { |values|
+          values = attr.normalize_values(values)
+          if values.any?
+            where("#{attribute} @@ (!! to_tsquery('simple', ?))", values.join(' | '))
+          else
+            where('false')
+          end
+        }
+
+        scope "order_by_#{attribute}_rank", lambda { |values, direction = nil|
+          direction = 'DESC' unless %w(asc ascending desc descending).include?(direction.try(:downcase))
+          values = attr.normalize_values(values)
+          if values.any?
+            order(sanitize_sql_array([
+              "ts_rank(#{attribute}, to_tsquery('simple', ?)) #{direction}", values.join(' | ')]))
+          else
+            order('false')
+          end
+        }
+
+        define_method(attribute) do
+          attr.parse_values(read_attribute(attribute))
+        end
+
+        define_method("#{attribute}=") do |values|
+          values = attr.normalize_values(values)
+          if values.any?
+            write_attribute(attribute, values.join(' '))
+          else
+            write_attribute(attribute, nil)
+          end
+        end
+      end
+
+    end
+
+  end
+end

data/lib/ts_vectors/version.rb

@@ -0,0 +1,3 @@
+module TsVectors
+  VERSION = '0.0.1'
+end

data/spec/spec_helper.rb

@@ -0,0 +1,44 @@
+require 'simplecov'  # Must be required first
+
+require 'activerecord_tsvectors'
+
+require 'logger'
+
+ActiveRecord::Base.logger = Logger.new(STDOUT)
+ActiveRecord::Base.logger.level = Logger::INFO
+
+begin
+  database_config = YAML.load(
+    File.open(File.expand_path("../database.yml", __FILE__)))
+rescue Errno::ENOENT
+  abort "You need to create an ActiveRecord database configuration in #{File.expand_path("../database.yml", __FILE__)}."
+else
+  connection = ActiveRecord::Base.establish_connection(database_config).connection
+  connection.execute("set client_min_messages = warning")
+end
+
+RSpec.configure do |c|
+  c.mock_with :rspec
+
+  c.before :each do
+    ActiveRecord::Base.connection.execute %(
+      create table if not exists things (
+        id serial primary key,
+        tags tsvector
+      )
+    )
+    ActiveRecord::Base.connection.execute %(
+      create table if not exists thangs (
+        id serial primary key,
+        tags tsvector
+      )
+    )
+  end
+
+  c.around(:each) do |example|
+    ActiveRecord::Base.connection.transaction do
+      example.run 
+      raise ActiveRecord::Rollback
+    end
+  end
+end

data/spec/ts_vectors_spec.rb

@@ -0,0 +1,113 @@
+require 'spec_helper'
+
+class Thing < ActiveRecord::Base
+  ts_vector :tags
+end
+
+class Thang < ActiveRecord::Base
+  ts_vector :tags, :normalize => lambda { |v| v.downcase.gsub(/\d/, '') }
+end
+
+describe TsVectors::Model do
+  let :thing do
+    Thing.new
+  end
+
+  context 'attribute accessor' do
+    it "sets empty array when assigning nil" do
+      thing.tags = nil
+      thing.tags.should == []
+    end
+
+    it "sets empty array when assigning empty array" do
+      thing.tags = []
+      thing.tags.should == []
+    end
+
+    it "sets empty array when assigning empty string" do
+      thing.tags = ""
+      thing.tags.should == []
+    end
+
+    it "sets empty array when assigning string containing only spaces" do
+      thing.tags = "  "
+      thing.tags.should == []
+    end
+
+    it "sets empty array when assigning array of strings containing only spaces" do
+      thing.tags = ["  "]
+      thing.tags.should == []
+    end
+  end
+
+  context 'with_all_XXX' do
+    it 'matches rows' do
+      thing.tags = %w(foo bar)
+      thing.save!
+      Thing.with_all_tags(%w(foo bar)).first.should == thing
+      Thing.with_all_tags(%w(foo bar baz)).first.should == nil
+      Thing.with_all_tags(%w(FOO BAR)).first.should == thing
+      Thing.with_all_tags(%w(FOO BAR BAZ)).first.should == nil
+      Thing.with_all_tags(%w(baz)).first.should == nil
+    end
+  end
+
+  context 'with_any_XXX' do
+    it 'matches rows' do
+      thing.tags = %w(foo bar)
+      thing.save!
+      Thing.with_any_tags(%w(foo bar)).first.should == thing
+      Thing.with_any_tags(%w(foo bar baz)).first.should == thing
+      Thing.with_any_tags(%w(FOO BAR)).first.should == thing
+      Thing.with_any_tags(%w(FOO BAR BAZ)).first.should == thing
+      Thing.with_any_tags(%w(baz)).first.should == nil
+    end
+  end
+
+  context 'without_all_XXX' do
+    it 'does not match rows' do
+      thing1 = Thing.create!(:tags => %w(foo bar))
+      thing2 = Thing.create!(:tags => %w(foo baz))
+      thing3 = Thing.create!(:tags => %w(baz))
+
+      Thing.without_all_tags(%w(foo)).sort.should == [thing3]
+      Thing.without_all_tags(%w(foo bar)).sort.should == [thing2, thing3]
+      Thing.without_all_tags(%w(baz)).sort.should == [thing1]
+      Thing.without_all_tags(%w(foo baz)).sort.should == [thing1, thing3]
+    end
+  end
+
+  context 'without_any_XXX' do
+    it 'does not match rows' do
+      thing1 = Thing.create!(:tags => %w(foo bar))
+      thing2 = Thing.create!(:tags => %w(foo baz))
+      thing3 = Thing.create!(:tags => %w(baz))
+
+      Thing.without_any_tags(%w(foo)).sort.should == [thing3]
+      Thing.without_any_tags(%w(foo bar)).sort.should == [thing3]
+      Thing.without_any_tags(%w(baz)).sort.should == [thing1]
+      Thing.without_any_tags(%w(foo baz)).should == []
+    end
+  end
+
+  context 'normalization' do
+    let :thang do
+      Thang.new
+    end
+
+    it 'calls normalizing function on assignment' do
+      thang.tags = %w(foo123 BAR123)
+      thang.tags.should == %w(foo bar)
+    end
+
+    it 'normalizes query' do
+      thang.tags = %w(foo123 BAR123)
+      thang.save!
+      Thang.with_all_tags(%w(foo123)).first.should == thang
+      Thang.with_all_tags(%w(BAR123)).first.should == thang
+      Thang.with_all_tags(%w(foo)).first.should == thang
+      Thang.with_all_tags(%w(bar)).first.should == thang
+    end
+  end
+
+end

metadata

@@ -0,0 +1,121 @@
+--- !ruby/object:Gem::Specification
+name: ar-tsvectors
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+  prerelease: 
+platform: ruby
+authors:
+- Alexander Staubo
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2012-06-08 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: activerecord
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: simplecov
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: activerecord-postgresql-adapter
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Support for PostgreSQL's ts_vector data type in ActiveRecord. Perfect
+  for indexing tags, arrays etc.
+email:
+- alex@bengler.no
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- .rspec
+- Gemfile
+- README.md
+- Rakefile
+- ar-tsvectors.gemspec
+- lib/activerecord_tsvectors.rb
+- lib/ts_vectors/model.rb
+- lib/ts_vectors/version.rb
+- spec/spec_helper.rb
+- spec/ts_vectors_spec.rb
+homepage: ''
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: ar-tsvectors
+rubygems_version: 1.8.24
+signing_key: 
+specification_version: 3
+summary: Support for PostgreSQL's ts_vector data type in ActiveRecord
+test_files: []