model_iterator 1.0.1 → 1.0.2

data/README.md

@@ -4,17 +4,21 @@ Basic library for iterating through large ActiveRecord datasets.  For instance,
 let's say you add a new feature, and you need to backfill data for existing
 records:
 
-    iter = ModelIterator.new(User, :redis => $redis)
-    iter.each do |user|
-      backfill(user)
-    end
+```ruby
+iter = ModelIterator.new(User, :redis => $redis)
+iter.each do |user|
+  backfill(user)
+end
+```
 
 ModelIterator selects the records in batches (100 by default), and loops
 through the table filtering based on the ID.
 
-    SELECT * FROM users WHERE id > 0 LIMIT 100
-    SELECT * FROM users WHERE id > 100 LIMIT 100
-    SELECT * FROM users WHERE id > 200 LIMIT 100
+```sql
+SELECT * FROM users WHERE id > 0 LIMIT 100
+SELECT * FROM users WHERE id > 100 LIMIT 100
+SELECT * FROM users WHERE id > 200 LIMIT 100
+```
 
 Each record's ID is tracked in Redis immediately after being processed.  If
 jobs crash, you can fix code, and re-run from where you left off.

data/lib/model_iterator.rb

@@ -1,6 +1,6 @@
 # Iterates over large models, storing state in Redis.
 class ModelIterator
-  VERSION = "1.0.1"
+  VERSION = "1.0.2"
 
   class MaxIterations < StandardError
     attr_reader :iterator
@@ -83,6 +83,8 @@ class ModelIterator
   #                        at a time.
   #           :limit     - Fixnum limit of objects to fetch from the db.
   #                        Default: 100
+  #           :conditions - Array of String SQL WHERE clause and optional values
+  #                         (Will override clause/values given in arguments.)
   #
   #   ModelIterator.new(Repository, :start_id => 5000)
   #   ModelIterator.new(Repository, 'public=?', true, :start_id => 1000)
@@ -101,10 +103,15 @@ class ModelIterator
     op = @order == :asc ? '>' : '<'
     @max = @options[:max].to_i
     @joins = @options[:joins]
-    @clause = args.empty? ?
-      "#{@id_clause} #{op} ?" :
-      "#{@id_clause} #{op} ? AND (#{args.shift})"
-    @clause_args = args
+    @clause =  "#{@id_clause} #{op} ?"
+    if @options[:conditions]
+      conditions = Array(@options[:conditions])
+      @clause += " AND (#{conditions.first})"
+      @clause_args = conditions[1..-1]
+    elsif !args.empty?
+      @clause += " AND (#{args.shift})"
+      @clause_args = args
+    end
     @current_id = @options[:start_id]
     @limit = @options[:limit] || 100
     @job = @prefix = @key = nil

data/model_iterator.gemspec

@@ -12,8 +12,8 @@ Gem::Specification.new do |s|
   ## If your rubyforge_project name is different, then edit it and comment out
   ## the sub! line in the Rakefile
   s.name              = 'model_iterator'
-  s.version           = '1.0.1'
-  s.date              = '2012-09-07'
+  s.version           = '1.0.2'
+  s.date              = '2013-03-19'
   s.rubyforge_project = 'model_iterator'
 
   ## Make sure your summary is short. The description may be as long
@@ -46,6 +46,8 @@ Gem::Specification.new do |s|
     Rakefile
     lib/model_iterator.rb
     model_iterator.gemspec
+    script/bootstrap
+    script/test
     test/helper.rb
     test/init_test.rb
     test/iterate_test.rb

data/script/bootstrap

@@ -0,0 +1,6 @@
+#!/bin/sh
+# Usage: script/bootstrap
+# Ensures all gems are in vendor/cache and installed locally.
+
+bundle package --all
+bundle install --binstubs --local --path=vendor/gems

data/script/test

@@ -0,0 +1,5 @@
+#!/bin/sh
+# Usage: script/test
+# Runs the library's test suite.
+
+bundle exec rake test

data/test/init_test.rb

@@ -87,3 +87,78 @@ class InitializationTestWithCustomWhereClause < ModelIterator::TestCase
   end
 end
 
+class InitializationTestWithCustomWhereClauseInOptions < ModelIterator::TestCase
+  def setup
+    @iter = ModelIterator.new Model,
+      :redis => RedisClient.new, :start_id => 5, :limit => 10,
+      :conditions => ['public = ?', true]
+  end
+
+  def test_sets_klass
+    assert_equal Model, @iter.klass
+  end
+
+  def test_sets_current_id
+    assert_equal 5, @iter.current_id
+  end
+
+  def test_sets_conditions
+    assert_equal ['models.id > ? AND (public = ?)', @iter.current_id, true], @iter.conditions
+  end
+
+  def test_sets_limit
+    assert_equal 10, @iter.limit
+  end
+end
+
+class InitializationTestWithCustomWhereClauseInOptionsOverridingArguments < ModelIterator::TestCase
+  def setup
+    @iter = ModelIterator.new Model,
+      'unknown = ?',
+      false,
+      :redis => RedisClient.new, :start_id => 5, :limit => 10,
+      :conditions => ['public = ?', true]
+  end
+
+  def test_sets_klass
+    assert_equal Model, @iter.klass
+  end
+
+  def test_sets_current_id
+    assert_equal 5, @iter.current_id
+  end
+
+  def test_sets_conditions
+    assert_equal ['models.id > ? AND (public = ?)', @iter.current_id, true], @iter.conditions
+  end
+
+  def test_sets_limit
+    assert_equal 10, @iter.limit
+  end
+end
+
+class InitializationTestWithCustomWhereClauseInOptionsAsSql < ModelIterator::TestCase
+  def setup
+    @iter = ModelIterator.new Model,
+      :redis => RedisClient.new, :start_id => 5, :limit => 10,
+      :conditions => 'public = true'
+  end
+
+  def test_sets_klass
+    assert_equal Model, @iter.klass
+  end
+
+  def test_sets_current_id
+    assert_equal 5, @iter.current_id
+  end
+
+  def test_sets_conditions
+    assert_equal ['models.id > ? AND (public = true)', @iter.current_id], @iter.conditions
+  end
+
+  def test_sets_limit
+    assert_equal 10, @iter.limit
+  end
+end
+
+

data/test/iterate_test.rb

@@ -27,6 +27,30 @@ class IterateTest < ModelIterator::TestCase
     assert_equal %w(b c), names
   end
 
+  def test_loops_through_filtered_records_from_options
+    names = []
+    iter = ModelIterator.new Model,
+      :redis => RedisClient.new, :limit => 1,
+      :conditions => ['name != ?', 'a']
+    iter.each do |m|
+      names << m.name
+    end
+
+    assert_equal %w(b c), names
+  end
+
+  def test_loops_through_filtered_records_from_options_as_plain_sql
+    names = []
+    iter = ModelIterator.new Model,
+      :redis => RedisClient.new, :limit => 1,
+      :conditions => "name != 'a'"
+    iter.each do |m|
+      names << m.name
+    end
+
+    assert_equal %w(b c), names
+  end
+
   def test_loops_through_records_in_reverse
     names = []
     iter = ModelIterator.new Model, :redis => RedisClient.new, :limit => 1,

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: model_iterator
 version: !ruby/object:Gem::Version
-  version: 1.0.1
+  version: 1.0.2
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-09-07 00:00:00.000000000 Z
+date: 2013-03-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -55,6 +55,8 @@ files:
 - Rakefile
 - lib/model_iterator.rb
 - model_iterator.gemspec
+- script/bootstrap
+- script/test
 - test/helper.rb
 - test/init_test.rb
 - test/iterate_test.rb
@@ -72,7 +74,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: -4149675607368163527
+      hash: 1002102300965983738
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements: