friendly_postgres 0.4.5 → 0.5.1

data/CHANGELOG.md

@@ -1,7 +1,11 @@
 Changelog
 =========
 
-### (0.4.4)
+### 0.5
+  
+  * (jamesgolick) Add offline index building.
+
+### 0.4.4
 
   * (jamesgolick) Make it possible to query with order, but no conditions.
   * (jamesgolick) Add change tracking. This is mostly to facilitate arbitrary caches.

data/README.md

@@ -204,6 +204,29 @@ You can also use any other Friendly::Scope method like scope chaining.
 
 See the section above or the Friendly::Scope docs for more details.
 
+Offline Indexing
+================
+
+Friendly includes support for building an index in the background, without taking your app offline.
+
+All you have to do is declare the index in your model. If we wanted to add an index on :name, :created_at in a User model, we'd do it like this:
+
+    class User
+      # ...snip...
+
+      indexes :name, :created_at
+    end
+
+Then, make sure to run Friendly.create_tables! to create the table in the database. This won't overwrite any of your existing tables, so don't worry.
+
+    >> Friendly.create_tables!
+
+Now that the the new table has been created, you need to copy the .rake file included with Friendly (lib/tasks/friendly.rake) to somewhere that will get picked up by your main Rakefile (lib/tasks if it's a rails project). Then, run:
+
+    $ KLASS=User FIELDS=name,created_at rake friendly:build_index
+
+If you're running this in production, you'll probably want to fire up GNU screen so that it'll keep running even if you lose your SSH connection. When this task completes, the index is populated and ready to go!
+
 Installation
 ============
 

data/Rakefile

@@ -18,6 +18,7 @@ begin
     gem.add_dependency "json_pure"
     gem.add_dependency "activesupport"
     gem.add_dependency "will_paginate"
+    gem.add_dependency "pg"
     # gem is a Gem::Specification... see http://www.rubygems.org/read/chapter/20 for additional settings
   end
 rescue LoadError

data/VERSION

@@ -1 +1 @@
-0.4.5
+0.5.1

data/friendly.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = %q{friendly}
-  s.version = "0.4.5"
+  s.version = "0.5.1"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["James Golick"]
-  s.date = %q{2010-01-16}
+  s.date = %q{2010-01-25}
   s.description = %q{}
   s.email = %q{jamesgolick@gmail.com}
   s.extra_rdoc_files = [
@@ -47,6 +47,7 @@ Gem::Specification.new do |s|
      "lib/friendly/document/storage.rb",
      "lib/friendly/document_table.rb",
      "lib/friendly/index.rb",
+     "lib/friendly/indexer.rb",
      "lib/friendly/memcached.rb",
      "lib/friendly/newrelic.rb",
      "lib/friendly/query.rb",
@@ -61,6 +62,7 @@ Gem::Specification.new do |s|
      "lib/friendly/time.rb",
      "lib/friendly/translator.rb",
      "lib/friendly/uuid.rb",
+     "lib/tasks/friendly.rake",
      "rails/init.rb",
      "spec/config.yml.example",
      "spec/fakes/data_store_fake.rb",
@@ -81,6 +83,7 @@ Gem::Specification.new do |s|
      "spec/integration/has_many_spec.rb",
      "spec/integration/index_spec.rb",
      "spec/integration/named_scope_spec.rb",
+     "spec/integration/offline_indexing_spec.rb",
      "spec/integration/pagination_spec.rb",
      "spec/integration/scope_chaining_spec.rb",
      "spec/integration/table_creator_spec.rb",
@@ -174,6 +177,7 @@ Gem::Specification.new do |s|
      "spec/integration/has_many_spec.rb",
      "spec/integration/index_spec.rb",
      "spec/integration/named_scope_spec.rb",
+     "spec/integration/offline_indexing_spec.rb",
      "spec/integration/pagination_spec.rb",
      "spec/integration/scope_chaining_spec.rb",
      "spec/integration/table_creator_spec.rb",

data/friendly_postgres.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = %q{friendly_postgres}
-  s.version = "0.4.5"
+  s.version = "0.5.1"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["James Golick"]
-  s.date = %q{2010-01-24}
+  s.date = %q{2010-02-14}
   s.description = %q{}
   s.email = %q{jamesgolick@gmail.com}
   s.extra_rdoc_files = [
@@ -48,6 +48,7 @@ Gem::Specification.new do |s|
      "lib/friendly/document/storage.rb",
      "lib/friendly/document_table.rb",
      "lib/friendly/index.rb",
+     "lib/friendly/indexer.rb",
      "lib/friendly/memcached.rb",
      "lib/friendly/newrelic.rb",
      "lib/friendly/query.rb",
@@ -62,6 +63,7 @@ Gem::Specification.new do |s|
      "lib/friendly/time.rb",
      "lib/friendly/translator.rb",
      "lib/friendly/uuid.rb",
+     "lib/tasks/friendly.rake",
      "rails/init.rb",
      "spec/config.yml.example",
      "spec/fakes/data_store_fake.rb",
@@ -82,6 +84,7 @@ Gem::Specification.new do |s|
      "spec/integration/has_many_spec.rb",
      "spec/integration/index_spec.rb",
      "spec/integration/named_scope_spec.rb",
+     "spec/integration/offline_indexing_spec.rb",
      "spec/integration/pagination_spec.rb",
      "spec/integration/scope_chaining_spec.rb",
      "spec/integration/table_creator_spec.rb",
@@ -175,6 +178,7 @@ Gem::Specification.new do |s|
      "spec/integration/has_many_spec.rb",
      "spec/integration/index_spec.rb",
      "spec/integration/named_scope_spec.rb",
+     "spec/integration/offline_indexing_spec.rb",
      "spec/integration/pagination_spec.rb",
      "spec/integration/scope_chaining_spec.rb",
      "spec/integration/table_creator_spec.rb",
@@ -213,6 +217,7 @@ Gem::Specification.new do |s|
       s.add_runtime_dependency(%q<json_pure>, [">= 0"])
       s.add_runtime_dependency(%q<activesupport>, [">= 0"])
       s.add_runtime_dependency(%q<will_paginate>, [">= 0"])
+      s.add_runtime_dependency(%q<pg>, [">= 0"])
     else
       s.add_dependency(%q<rspec>, [">= 1.2.9"])
       s.add_dependency(%q<cucumber>, [">= 0"])
@@ -222,6 +227,7 @@ Gem::Specification.new do |s|
       s.add_dependency(%q<json_pure>, [">= 0"])
       s.add_dependency(%q<activesupport>, [">= 0"])
       s.add_dependency(%q<will_paginate>, [">= 0"])
+      s.add_dependency(%q<pg>, [">= 0"])
     end
   else
     s.add_dependency(%q<rspec>, [">= 1.2.9"])
@@ -232,6 +238,7 @@ Gem::Specification.new do |s|
     s.add_dependency(%q<json_pure>, [">= 0"])
     s.add_dependency(%q<activesupport>, [">= 0"])
     s.add_dependency(%q<will_paginate>, [">= 0"])
+    s.add_dependency(%q<pg>, [">= 0"])
   end
 end
 

data/lib/friendly.rb

@@ -7,6 +7,7 @@ require 'friendly/data_store'
 require 'friendly/document'
 require 'friendly/document_table'
 require 'friendly/index'
+require 'friendly/indexer'
 require 'friendly/memcached'
 require 'friendly/query'
 require 'friendly/sequel_monkey_patches'

data/lib/friendly/indexer.rb

@@ -0,0 +1,50 @@
+module Friendly
+  class Indexer
+    class << self
+      attr_accessor :objects_per_iteration
+
+      def populate(klass, *fields)
+        instance.populate(klass, klass.storage_proxy.index_for_fields(fields))
+      end
+
+      def instance
+        @instance ||= new
+      end
+    end
+
+    self.objects_per_iteration = 100
+
+    attr_reader :datastore, :translator
+
+    def initialize(datastore  = Friendly.datastore, translator = Translator.new)
+      @datastore  = datastore
+      @translator = translator
+    end
+
+    def populate(klass, index)
+      count  = 0
+      loop do
+        rows = datastore.all(klass, Query.new(:offset! => count, 
+                                              :limit!  => objects_per_iteration, 
+                                              :order!  => :added_id.asc))
+        rows.each do |attrs|
+          begin
+            index.create(translator.to_object(klass, attrs))
+          rescue Sequel::DatabaseError
+            # we can safely swallow this exception because if we've gotten
+            # to this point, we can be pretty sure that it's a duplicate
+            # key error, which just means that the object already exists
+            # in the index
+          end
+        end
+        break if rows.length < objects_per_iteration
+        count += objects_per_iteration
+      end
+    end
+
+    protected
+      def objects_per_iteration
+        self.class.objects_per_iteration
+      end
+  end
+end

data/lib/friendly/storage_proxy.rb

@@ -67,6 +67,14 @@ module Friendly
       index
     end
 
+    def index_for_fields(fields)
+      tables.detect do |t|
+        t.respond_to?(:fields) && t.fields == fields
+      end.tap do |i|
+        raise MissingIndex, "No index found matching #{fields.join(", ")}." if i.nil?
+      end
+    end
+
     protected
       def each_store
         stores.each { |s| yield(s) }

data/lib/friendly/uuid.rb

@@ -29,7 +29,12 @@ module Friendly
           raise TypeError, "Expected #{bytes.inspect} to cast to a #{self.class} (malformed UUID representation)" if elements.size != 5
           @bytes = Array(elements.join).pack('H32')
         else
-          raise TypeError, "Expected #{bytes.inspect} to cast to a #{self.class} (invalid bytecount)"
+          unescaped = PGconn.unescape_bytea(bytes)
+          if unescaped.size == 16
+            @bytes = unescaped
+          else
+            raise TypeError, "Expected #{bytes.inspect} to cast to a #{self.class} (invalid bytecount)"
+          end
         end
 
       when Integer

data/lib/tasks/friendly.rake

@@ -0,0 +1,7 @@
+namespace :friendly do
+  task :build_index do
+    klass  = ENV['KLASS'].constantize
+    fields = ENV['FIELDS'].split(',').map { |f| f.to_sym }
+    Friendly::Indexer.populate(klass, *fields)
+  end
+end

data/spec/integration/offline_indexing_spec.rb

@@ -0,0 +1,53 @@
+require File.expand_path("../../spec_helper", __FILE__)
+
+describe "Building an index offline" do
+  before do
+    $db.drop_table :awesome_things if $db.table_exists?(:awesome_things)
+
+    if $db.table_exists?(:index_awesome_things_on_name)
+      $db.drop_table :index_awesome_things_on_name
+    end
+
+    @klass = Class.new do
+      def self.name; "AwesomeThing"; end
+      def self.table_name; "awesome_things"; end
+
+      include Friendly::Document
+
+      attribute :name, String
+    end
+    @klass.create_tables!
+
+    @jameses = [@klass.create(:name => "James"), @klass.create(:name => "James")]
+
+    @klass.indexes :name
+    @klass.create_tables!
+  end
+
+  describe "" do
+    before do
+      Friendly::Indexer.populate(@klass, :name)
+    end
+
+    it "builds the missing index rows for all the rows in the doc table" do
+      @klass.all(:name => "James").should == @jameses
+    end
+
+    it "ignores records that are already in the index" do
+      lambda {
+        Friendly::Indexer.populate(@klass, :name)
+      }.should_not raise_error
+    end
+  end
+
+  describe "with more than `Indexer.objects_per_iteration` objects" do
+    before do
+      Friendly::Indexer.objects_per_iteration = 1
+      Friendly::Indexer.populate(@klass, :name)
+    end
+
+    it "builds the missing index rows for all the rows in the doc table" do
+      @klass.all(:name => "James").should == @jameses
+    end
+  end
+end

data/spec/unit/storage_proxy_spec.rb

@@ -215,4 +215,30 @@ describe "Friendly::StorageProxy" do
       @storage.count(:x => 1).should == 10
     end
   end
+
+  describe "getting the index that matches a set of fields" do
+    before do
+      @storage_factory = stub(:document_table => @document_table)
+
+      [[:name, :created_at], [:name]].each do |fields|
+        @storage_factory.stubs(:index).
+          with(@klass, *fields).returns(stub(:fields => fields))
+      end
+
+      @storage = Friendly::StorageProxy.new(@klass, @storage_factory)
+      @storage.add(:name, :created_at)
+      @storage.add(:name)
+    end
+
+    it "matches if and only if all the fields are present and in order" do
+      @storage.index_for_fields([:name, :created_at]).fields.should == 
+        [:name, :created_at]
+    end
+
+    it "raises missing index if there's no matching index" do
+      lambda {
+        @storage.index_for_fields([:created_at, :name])
+      }.should raise_error(Friendly::MissingIndex)
+    end
+  end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: friendly_postgres
 version: !ruby/object:Gem::Version 
-  version: 0.4.5
+  version: 0.5.1
 platform: ruby
 authors: 
 - James Golick
@@ -9,7 +9,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-01-24 00:00:00 -05:00
+date: 2010-02-14 00:00:00 -05:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -92,6 +92,16 @@ dependencies:
       - !ruby/object:Gem::Version 
         version: "0"
     version: 
+- !ruby/object:Gem::Dependency 
+  name: pg
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+    version: 
 description: ""
 email: jamesgolick@gmail.com
 executables: []
@@ -133,6 +143,7 @@ files:
 - lib/friendly/document/storage.rb
 - lib/friendly/document_table.rb
 - lib/friendly/index.rb
+- lib/friendly/indexer.rb
 - lib/friendly/memcached.rb
 - lib/friendly/newrelic.rb
 - lib/friendly/query.rb
@@ -147,6 +158,7 @@ files:
 - lib/friendly/time.rb
 - lib/friendly/translator.rb
 - lib/friendly/uuid.rb
+- lib/tasks/friendly.rake
 - rails/init.rb
 - spec/config.yml.example
 - spec/fakes/data_store_fake.rb
@@ -167,6 +179,7 @@ files:
 - spec/integration/has_many_spec.rb
 - spec/integration/index_spec.rb
 - spec/integration/named_scope_spec.rb
+- spec/integration/offline_indexing_spec.rb
 - spec/integration/pagination_spec.rb
 - spec/integration/scope_chaining_spec.rb
 - spec/integration/table_creator_spec.rb
@@ -282,6 +295,7 @@ test_files:
 - spec/integration/has_many_spec.rb
 - spec/integration/index_spec.rb
 - spec/integration/named_scope_spec.rb
+- spec/integration/offline_indexing_spec.rb
 - spec/integration/pagination_spec.rb
 - spec/integration/scope_chaining_spec.rb
 - spec/integration/table_creator_spec.rb