friendly_postgres 0.4.5 → 0.5.1

Sign up to get free protection for your applications and to get access to all the features.
@@ -1,7 +1,11 @@
1
1
  Changelog
2
2
  =========
3
3
 
4
- ### (0.4.4)
4
+ ### 0.5
5
+
6
+ * (jamesgolick) Add offline index building.
7
+
8
+ ### 0.4.4
5
9
 
6
10
  * (jamesgolick) Make it possible to query with order, but no conditions.
7
11
  * (jamesgolick) Add change tracking. This is mostly to facilitate arbitrary caches.
data/README.md CHANGED
@@ -204,6 +204,29 @@ You can also use any other Friendly::Scope method like scope chaining.
204
204
 
205
205
  See the section above or the Friendly::Scope docs for more details.
206
206
 
207
+ Offline Indexing
208
+ ================
209
+
210
+ Friendly includes support for building an index in the background, without taking your app offline.
211
+
212
+ All you have to do is declare the index in your model. If we wanted to add an index on :name, :created_at in a User model, we'd do it like this:
213
+
214
+ class User
215
+ # ...snip...
216
+
217
+ indexes :name, :created_at
218
+ end
219
+
220
+ Then, make sure to run Friendly.create_tables! to create the table in the database. This won't overwrite any of your existing tables, so don't worry.
221
+
222
+ >> Friendly.create_tables!
223
+
224
+ Now that the the new table has been created, you need to copy the .rake file included with Friendly (lib/tasks/friendly.rake) to somewhere that will get picked up by your main Rakefile (lib/tasks if it's a rails project). Then, run:
225
+
226
+ $ KLASS=User FIELDS=name,created_at rake friendly:build_index
227
+
228
+ If you're running this in production, you'll probably want to fire up GNU screen so that it'll keep running even if you lose your SSH connection. When this task completes, the index is populated and ready to go!
229
+
207
230
  Installation
208
231
  ============
209
232
 
data/Rakefile CHANGED
@@ -18,6 +18,7 @@ begin
18
18
  gem.add_dependency "json_pure"
19
19
  gem.add_dependency "activesupport"
20
20
  gem.add_dependency "will_paginate"
21
+ gem.add_dependency "pg"
21
22
  # gem is a Gem::Specification... see http://www.rubygems.org/read/chapter/20 for additional settings
22
23
  end
23
24
  rescue LoadError
data/VERSION CHANGED
@@ -1 +1 @@
1
- 0.4.5
1
+ 0.5.1
@@ -5,11 +5,11 @@
5
5
 
6
6
  Gem::Specification.new do |s|
7
7
  s.name = %q{friendly}
8
- s.version = "0.4.5"
8
+ s.version = "0.5.1"
9
9
 
10
10
  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
11
11
  s.authors = ["James Golick"]
12
- s.date = %q{2010-01-16}
12
+ s.date = %q{2010-01-25}
13
13
  s.description = %q{}
14
14
  s.email = %q{jamesgolick@gmail.com}
15
15
  s.extra_rdoc_files = [
@@ -47,6 +47,7 @@ Gem::Specification.new do |s|
47
47
  "lib/friendly/document/storage.rb",
48
48
  "lib/friendly/document_table.rb",
49
49
  "lib/friendly/index.rb",
50
+ "lib/friendly/indexer.rb",
50
51
  "lib/friendly/memcached.rb",
51
52
  "lib/friendly/newrelic.rb",
52
53
  "lib/friendly/query.rb",
@@ -61,6 +62,7 @@ Gem::Specification.new do |s|
61
62
  "lib/friendly/time.rb",
62
63
  "lib/friendly/translator.rb",
63
64
  "lib/friendly/uuid.rb",
65
+ "lib/tasks/friendly.rake",
64
66
  "rails/init.rb",
65
67
  "spec/config.yml.example",
66
68
  "spec/fakes/data_store_fake.rb",
@@ -81,6 +83,7 @@ Gem::Specification.new do |s|
81
83
  "spec/integration/has_many_spec.rb",
82
84
  "spec/integration/index_spec.rb",
83
85
  "spec/integration/named_scope_spec.rb",
86
+ "spec/integration/offline_indexing_spec.rb",
84
87
  "spec/integration/pagination_spec.rb",
85
88
  "spec/integration/scope_chaining_spec.rb",
86
89
  "spec/integration/table_creator_spec.rb",
@@ -174,6 +177,7 @@ Gem::Specification.new do |s|
174
177
  "spec/integration/has_many_spec.rb",
175
178
  "spec/integration/index_spec.rb",
176
179
  "spec/integration/named_scope_spec.rb",
180
+ "spec/integration/offline_indexing_spec.rb",
177
181
  "spec/integration/pagination_spec.rb",
178
182
  "spec/integration/scope_chaining_spec.rb",
179
183
  "spec/integration/table_creator_spec.rb",
@@ -5,11 +5,11 @@
5
5
 
6
6
  Gem::Specification.new do |s|
7
7
  s.name = %q{friendly_postgres}
8
- s.version = "0.4.5"
8
+ s.version = "0.5.1"
9
9
 
10
10
  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
11
11
  s.authors = ["James Golick"]
12
- s.date = %q{2010-01-24}
12
+ s.date = %q{2010-02-14}
13
13
  s.description = %q{}
14
14
  s.email = %q{jamesgolick@gmail.com}
15
15
  s.extra_rdoc_files = [
@@ -48,6 +48,7 @@ Gem::Specification.new do |s|
48
48
  "lib/friendly/document/storage.rb",
49
49
  "lib/friendly/document_table.rb",
50
50
  "lib/friendly/index.rb",
51
+ "lib/friendly/indexer.rb",
51
52
  "lib/friendly/memcached.rb",
52
53
  "lib/friendly/newrelic.rb",
53
54
  "lib/friendly/query.rb",
@@ -62,6 +63,7 @@ Gem::Specification.new do |s|
62
63
  "lib/friendly/time.rb",
63
64
  "lib/friendly/translator.rb",
64
65
  "lib/friendly/uuid.rb",
66
+ "lib/tasks/friendly.rake",
65
67
  "rails/init.rb",
66
68
  "spec/config.yml.example",
67
69
  "spec/fakes/data_store_fake.rb",
@@ -82,6 +84,7 @@ Gem::Specification.new do |s|
82
84
  "spec/integration/has_many_spec.rb",
83
85
  "spec/integration/index_spec.rb",
84
86
  "spec/integration/named_scope_spec.rb",
87
+ "spec/integration/offline_indexing_spec.rb",
85
88
  "spec/integration/pagination_spec.rb",
86
89
  "spec/integration/scope_chaining_spec.rb",
87
90
  "spec/integration/table_creator_spec.rb",
@@ -175,6 +178,7 @@ Gem::Specification.new do |s|
175
178
  "spec/integration/has_many_spec.rb",
176
179
  "spec/integration/index_spec.rb",
177
180
  "spec/integration/named_scope_spec.rb",
181
+ "spec/integration/offline_indexing_spec.rb",
178
182
  "spec/integration/pagination_spec.rb",
179
183
  "spec/integration/scope_chaining_spec.rb",
180
184
  "spec/integration/table_creator_spec.rb",
@@ -213,6 +217,7 @@ Gem::Specification.new do |s|
213
217
  s.add_runtime_dependency(%q<json_pure>, [">= 0"])
214
218
  s.add_runtime_dependency(%q<activesupport>, [">= 0"])
215
219
  s.add_runtime_dependency(%q<will_paginate>, [">= 0"])
220
+ s.add_runtime_dependency(%q<pg>, [">= 0"])
216
221
  else
217
222
  s.add_dependency(%q<rspec>, [">= 1.2.9"])
218
223
  s.add_dependency(%q<cucumber>, [">= 0"])
@@ -222,6 +227,7 @@ Gem::Specification.new do |s|
222
227
  s.add_dependency(%q<json_pure>, [">= 0"])
223
228
  s.add_dependency(%q<activesupport>, [">= 0"])
224
229
  s.add_dependency(%q<will_paginate>, [">= 0"])
230
+ s.add_dependency(%q<pg>, [">= 0"])
225
231
  end
226
232
  else
227
233
  s.add_dependency(%q<rspec>, [">= 1.2.9"])
@@ -232,6 +238,7 @@ Gem::Specification.new do |s|
232
238
  s.add_dependency(%q<json_pure>, [">= 0"])
233
239
  s.add_dependency(%q<activesupport>, [">= 0"])
234
240
  s.add_dependency(%q<will_paginate>, [">= 0"])
241
+ s.add_dependency(%q<pg>, [">= 0"])
235
242
  end
236
243
  end
237
244
 
@@ -7,6 +7,7 @@ require 'friendly/data_store'
7
7
  require 'friendly/document'
8
8
  require 'friendly/document_table'
9
9
  require 'friendly/index'
10
+ require 'friendly/indexer'
10
11
  require 'friendly/memcached'
11
12
  require 'friendly/query'
12
13
  require 'friendly/sequel_monkey_patches'
@@ -0,0 +1,50 @@
1
+ module Friendly
2
+ class Indexer
3
+ class << self
4
+ attr_accessor :objects_per_iteration
5
+
6
+ def populate(klass, *fields)
7
+ instance.populate(klass, klass.storage_proxy.index_for_fields(fields))
8
+ end
9
+
10
+ def instance
11
+ @instance ||= new
12
+ end
13
+ end
14
+
15
+ self.objects_per_iteration = 100
16
+
17
+ attr_reader :datastore, :translator
18
+
19
+ def initialize(datastore = Friendly.datastore, translator = Translator.new)
20
+ @datastore = datastore
21
+ @translator = translator
22
+ end
23
+
24
+ def populate(klass, index)
25
+ count = 0
26
+ loop do
27
+ rows = datastore.all(klass, Query.new(:offset! => count,
28
+ :limit! => objects_per_iteration,
29
+ :order! => :added_id.asc))
30
+ rows.each do |attrs|
31
+ begin
32
+ index.create(translator.to_object(klass, attrs))
33
+ rescue Sequel::DatabaseError
34
+ # we can safely swallow this exception because if we've gotten
35
+ # to this point, we can be pretty sure that it's a duplicate
36
+ # key error, which just means that the object already exists
37
+ # in the index
38
+ end
39
+ end
40
+ break if rows.length < objects_per_iteration
41
+ count += objects_per_iteration
42
+ end
43
+ end
44
+
45
+ protected
46
+ def objects_per_iteration
47
+ self.class.objects_per_iteration
48
+ end
49
+ end
50
+ end
@@ -67,6 +67,14 @@ module Friendly
67
67
  index
68
68
  end
69
69
 
70
+ def index_for_fields(fields)
71
+ tables.detect do |t|
72
+ t.respond_to?(:fields) && t.fields == fields
73
+ end.tap do |i|
74
+ raise MissingIndex, "No index found matching #{fields.join(", ")}." if i.nil?
75
+ end
76
+ end
77
+
70
78
  protected
71
79
  def each_store
72
80
  stores.each { |s| yield(s) }
@@ -29,7 +29,12 @@ module Friendly
29
29
  raise TypeError, "Expected #{bytes.inspect} to cast to a #{self.class} (malformed UUID representation)" if elements.size != 5
30
30
  @bytes = Array(elements.join).pack('H32')
31
31
  else
32
- raise TypeError, "Expected #{bytes.inspect} to cast to a #{self.class} (invalid bytecount)"
32
+ unescaped = PGconn.unescape_bytea(bytes)
33
+ if unescaped.size == 16
34
+ @bytes = unescaped
35
+ else
36
+ raise TypeError, "Expected #{bytes.inspect} to cast to a #{self.class} (invalid bytecount)"
37
+ end
33
38
  end
34
39
 
35
40
  when Integer
@@ -0,0 +1,7 @@
1
+ namespace :friendly do
2
+ task :build_index do
3
+ klass = ENV['KLASS'].constantize
4
+ fields = ENV['FIELDS'].split(',').map { |f| f.to_sym }
5
+ Friendly::Indexer.populate(klass, *fields)
6
+ end
7
+ end
@@ -0,0 +1,53 @@
1
+ require File.expand_path("../../spec_helper", __FILE__)
2
+
3
+ describe "Building an index offline" do
4
+ before do
5
+ $db.drop_table :awesome_things if $db.table_exists?(:awesome_things)
6
+
7
+ if $db.table_exists?(:index_awesome_things_on_name)
8
+ $db.drop_table :index_awesome_things_on_name
9
+ end
10
+
11
+ @klass = Class.new do
12
+ def self.name; "AwesomeThing"; end
13
+ def self.table_name; "awesome_things"; end
14
+
15
+ include Friendly::Document
16
+
17
+ attribute :name, String
18
+ end
19
+ @klass.create_tables!
20
+
21
+ @jameses = [@klass.create(:name => "James"), @klass.create(:name => "James")]
22
+
23
+ @klass.indexes :name
24
+ @klass.create_tables!
25
+ end
26
+
27
+ describe "" do
28
+ before do
29
+ Friendly::Indexer.populate(@klass, :name)
30
+ end
31
+
32
+ it "builds the missing index rows for all the rows in the doc table" do
33
+ @klass.all(:name => "James").should == @jameses
34
+ end
35
+
36
+ it "ignores records that are already in the index" do
37
+ lambda {
38
+ Friendly::Indexer.populate(@klass, :name)
39
+ }.should_not raise_error
40
+ end
41
+ end
42
+
43
+ describe "with more than `Indexer.objects_per_iteration` objects" do
44
+ before do
45
+ Friendly::Indexer.objects_per_iteration = 1
46
+ Friendly::Indexer.populate(@klass, :name)
47
+ end
48
+
49
+ it "builds the missing index rows for all the rows in the doc table" do
50
+ @klass.all(:name => "James").should == @jameses
51
+ end
52
+ end
53
+ end
@@ -215,4 +215,30 @@ describe "Friendly::StorageProxy" do
215
215
  @storage.count(:x => 1).should == 10
216
216
  end
217
217
  end
218
+
219
+ describe "getting the index that matches a set of fields" do
220
+ before do
221
+ @storage_factory = stub(:document_table => @document_table)
222
+
223
+ [[:name, :created_at], [:name]].each do |fields|
224
+ @storage_factory.stubs(:index).
225
+ with(@klass, *fields).returns(stub(:fields => fields))
226
+ end
227
+
228
+ @storage = Friendly::StorageProxy.new(@klass, @storage_factory)
229
+ @storage.add(:name, :created_at)
230
+ @storage.add(:name)
231
+ end
232
+
233
+ it "matches if and only if all the fields are present and in order" do
234
+ @storage.index_for_fields([:name, :created_at]).fields.should ==
235
+ [:name, :created_at]
236
+ end
237
+
238
+ it "raises missing index if there's no matching index" do
239
+ lambda {
240
+ @storage.index_for_fields([:created_at, :name])
241
+ }.should raise_error(Friendly::MissingIndex)
242
+ end
243
+ end
218
244
  end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: friendly_postgres
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.4.5
4
+ version: 0.5.1
5
5
  platform: ruby
6
6
  authors:
7
7
  - James Golick
@@ -9,7 +9,7 @@ autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
11
 
12
- date: 2010-01-24 00:00:00 -05:00
12
+ date: 2010-02-14 00:00:00 -05:00
13
13
  default_executable:
14
14
  dependencies:
15
15
  - !ruby/object:Gem::Dependency
@@ -92,6 +92,16 @@ dependencies:
92
92
  - !ruby/object:Gem::Version
93
93
  version: "0"
94
94
  version:
95
+ - !ruby/object:Gem::Dependency
96
+ name: pg
97
+ type: :runtime
98
+ version_requirement:
99
+ version_requirements: !ruby/object:Gem::Requirement
100
+ requirements:
101
+ - - ">="
102
+ - !ruby/object:Gem::Version
103
+ version: "0"
104
+ version:
95
105
  description: ""
96
106
  email: jamesgolick@gmail.com
97
107
  executables: []
@@ -133,6 +143,7 @@ files:
133
143
  - lib/friendly/document/storage.rb
134
144
  - lib/friendly/document_table.rb
135
145
  - lib/friendly/index.rb
146
+ - lib/friendly/indexer.rb
136
147
  - lib/friendly/memcached.rb
137
148
  - lib/friendly/newrelic.rb
138
149
  - lib/friendly/query.rb
@@ -147,6 +158,7 @@ files:
147
158
  - lib/friendly/time.rb
148
159
  - lib/friendly/translator.rb
149
160
  - lib/friendly/uuid.rb
161
+ - lib/tasks/friendly.rake
150
162
  - rails/init.rb
151
163
  - spec/config.yml.example
152
164
  - spec/fakes/data_store_fake.rb
@@ -167,6 +179,7 @@ files:
167
179
  - spec/integration/has_many_spec.rb
168
180
  - spec/integration/index_spec.rb
169
181
  - spec/integration/named_scope_spec.rb
182
+ - spec/integration/offline_indexing_spec.rb
170
183
  - spec/integration/pagination_spec.rb
171
184
  - spec/integration/scope_chaining_spec.rb
172
185
  - spec/integration/table_creator_spec.rb
@@ -282,6 +295,7 @@ test_files:
282
295
  - spec/integration/has_many_spec.rb
283
296
  - spec/integration/index_spec.rb
284
297
  - spec/integration/named_scope_spec.rb
298
+ - spec/integration/offline_indexing_spec.rb
285
299
  - spec/integration/pagination_spec.rb
286
300
  - spec/integration/scope_chaining_spec.rb
287
301
  - spec/integration/table_creator_spec.rb