model_iterator 1.0.0

data/Gemfile

@@ -0,0 +1,10 @@
+source 'https://rubygems.org'
+
+gemspec
+
+group :dev do
+  gem 'rake'
+  gem 'activerecord'
+  gem 'sqlite3'
+end
+

data/LICENSE.md

@@ -0,0 +1,20 @@
+Copyright (c) 2012-* rick olson
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,35 @@
+# ModelIterator
+
+Basic library for iterating through large ActiveRecord datasets.  For instance,
+let's say you add a new feature, and you need to backfill data for existing
+records:
+
+    iter = ModelIterator.new(User, :redis => $redis)
+    iter.each do |user|
+      backfill(user)
+    end
+
+ModelIterator selects the records in batches (100 by default), and loops
+through the table filtering based on the ID.
+
+    SELECT * FROM users WHERE id > 0 LIMIT 100
+    SELECT * FROM users WHERE id > 100 LIMIT 100
+    SELECT * FROM users WHERE id > 200 LIMIT 100
+
+Each record's ID is tracked in Redis immediately after being processed.  If
+jobs crash, you can fix code, and re-run from where you left off.
+
+This code was ported from GitHub, where it's been frequently used for nearly
+two years.
+
+## Note on Patches/Pull Requests
+
+1. Fork the project.
+2. Make your feature addition or bug fix.
+3. Add tests for it. This is important so I don't break it in a future version
+   unintentionally.
+4. Commit, do not mess with rakefile, version, or history. (if you want to have
+   your own version, that is fine but bump version in a commit by itself I can
+   ignore when I pull)
+5. Send me a pull request. Bonus points for topic branches.
+

data/Rakefile

@@ -0,0 +1,134 @@
+#!/usr/bin/env rake
+
+require 'date'
+
+#############################################################################
+#
+# Helper functions
+#
+#############################################################################
+
+def name
+  @name ||= Dir['*.gemspec'].first.split('.').first
+end
+
+def version
+  line = File.read("lib/#{name}.rb")[/^\s*VERSION\s*=\s*.*/]
+  line.match(/.*VERSION\s*=\s*['"](.*)['"]/)[1]
+end
+
+def date
+  Date.today.to_s
+end
+
+def rubyforge_project
+  name
+end
+
+def gemspec_file
+  "#{name}.gemspec"
+end
+
+def gem_file
+  "#{name}-#{version}.gem"
+end
+
+def replace_header(head, header_name)
+  head.sub!(/(\.#{header_name}\s*= ').*'/) { "#{$1}#{send(header_name)}'"}
+end
+
+#############################################################################
+#
+# Standard tasks
+#
+#############################################################################
+
+task :default => :test
+
+require 'rake/testtask'
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.pattern = 'test/**/*_test.rb'
+  test.verbose = true
+end
+
+desc "Open an irb session preloaded with this library"
+task :console do
+  sh "irb -rubygems -r ./lib/#{name}.rb"
+end
+
+#############################################################################
+#
+# Custom tasks (add your own tasks here)
+#
+#############################################################################
+
+
+
+#############################################################################
+#
+# Packaging tasks
+#
+#############################################################################
+
+desc "Create tag v#{version} and build and push #{gem_file} to Rubygems"
+task :release => :build do
+  unless `git branch` =~ /^\* master$/
+    puts "You must be on the master branch to release!"
+    exit!
+  end
+  sh "git commit --allow-empty -a -m 'Release #{version}'"
+  sh "git tag v#{version}"
+  sh "git push origin master"
+  sh "git push origin v#{version}"
+  sh "gem push pkg/#{gem_file}"
+end
+
+desc "Build #{gem_file} into the pkg directory"
+task :build => :gemspec do
+  sh "mkdir -p pkg"
+  sh "gem build #{gemspec_file}"
+  sh "mv #{gem_file} pkg"
+end
+
+desc "Generate #{gemspec_file}"
+task :gemspec => :validate do
+  # read spec file and split out manifest section
+  spec = File.read(gemspec_file)
+  head, manifest, tail = spec.split("  # = MANIFEST =\n")
+
+  # replace name version and date
+  replace_header(head, :name)
+  replace_header(head, :version)
+  replace_header(head, :date)
+  #comment this out if your rubyforge_project has a different name
+  replace_header(head, :rubyforge_project)
+
+  # determine file list from git ls-files
+  files = `git ls-files`.
+    split("\n").
+    sort.
+    reject { |file| file =~ /^\./ }.
+    reject { |file| file =~ /^(rdoc|pkg)/ }.
+    map { |file| "    #{file}" }.
+    join("\n")
+
+  # piece file back together and write
+  manifest = "  s.files = %w[\n#{files}\n  ]\n"
+  spec = [head, manifest, tail].join("  # = MANIFEST =\n")
+  File.open(gemspec_file, 'w') { |io| io.write(spec) }
+  puts "Updated #{gemspec_file}"
+end
+
+desc "Validate #{gemspec_file}"
+task :validate do
+  libfiles = Dir['lib/*'] - ["lib/#{name}.rb", "lib/#{name}"]
+  unless libfiles.empty?
+    puts "Directory `lib` should only contain a `#{name}.rb` file and `#{name}` dir."
+    exit!
+  end
+  unless Dir['VERSION*'].empty?
+    puts "A `VERSION` file at root level violates Gem best practices."
+    exit!
+  end
+end

data/lib/model_iterator.rb

@@ -0,0 +1,213 @@
+# Iterates over large models, storing state in Redis.
+class ModelIterator
+  VERSION = "1.0.0"
+
+  class MaxIterations < StandardError
+    attr_reader :iterator
+    def initialize(iter)
+      @iterator = iter
+      super "Hit the max (#{iter.max}), stopping at id #{iter.current_id}."
+    end
+  end
+
+  # Gets a reference to the ActiveRecord::Base class that is iterated.
+  #
+  # Returns a Class.
+  attr_reader :klass
+
+  # Gets or sets the number of records that are returned in each database
+  # query.
+  #
+  # Returns a Fixnum.
+  attr_accessor :limit
+
+  # Gets a String SQL Where clause fragment.  Use `?` for variable 
+  # substitution.
+  #
+  # Returns a String.
+  attr_reader :clause
+
+  # Gets an Array of values to be sql-escaped and joined with the clause.
+  #
+  # Returns an Array of unescaped sql values.
+  attr_reader :clause_args
+
+  # Gets a String used to prefix the redis keys used by this object.
+  attr_reader :prefix
+
+  # Gets a Fixnum value of the maximum iterations to run, or 0.
+  attr_reader :max
+
+  # Gets the String name of the ID field.
+  attr_reader :id_field
+
+  # Gets the String fully qualified ID field (with the table name).
+  attr_reader :id_clause
+
+  # Gets the :joins value for ActiveRecord::Base.find.
+  attr_reader :joins
+
+  # Gets or sets a Proc that is called with each model instance while
+  # iterating.  This is set automatically by #each.
+  attr_accessor :job
+
+  # Gets or Sets the Redis client object.
+  attr_accessor :redis
+
+  # Initializes a ModelIterator instance.
+  #
+  # klass   - ActiveRecord::Base class to iterate.
+  # clause  - String SQL WHERE clause, with '?' placeholders for values.
+  # *values - Optional array of values to be added to a custom SQL WHERE
+  #           clause.
+  # options - Optional Hash options.
+  #           :redis     - A Redis object for storing the state.
+  #           :order     - Symbol specifying the order to iterate.  :asc or
+  #                        :desc.  Default: :asc
+  #           :id_field  - String name of the ID column.  Default: "id"
+  #           :id_clause - String name of the fully qualified ID column.
+  #                        Prepends the model's table name to the front of
+  #                        the ID field.  Default: "table_name.id"
+  #           :start_id  - Fixnum to start iterating from.  Default: 1
+  #           :prefix    - Custom String prefix for redis keys.
+  #           :select    - Optional String of the columns to retrieve.
+  #           :joins     - Optional Symbol or Hash :joins option for 
+  #                        ActiveRecord::Base.find.
+  #           :max       - Optional Fixnum of the maximum number of iterations.
+  #                        Use max * limit to process a known number of records
+  #                        at a time.
+  #           :limit     - Fixnum limit of objects to fetch from the db.
+  #                        Default: 100
+  #
+  #   ModelIterator.new(Repository, :start_id => 5000)
+  #   ModelIterator.new(Repository, 'public=?', true, :start_id => 1000)
+  #
+  def initialize(klass, *args)
+    @klass = klass
+    @options = if args.last.respond_to?(:fetch)
+      args.pop
+    else
+      {}
+    end
+    @redis = @options[:redis]
+    @id_field = @options[:id_field] || klass.primary_key
+    @id_clause = @options[:id_clause] || "#{klass.table_name}.#{@id_field}"
+    @order = @options[:order] == :desc ? :desc : :asc
+    op = @order == :asc ? '>' : '<'
+    @max = @options[:max].to_i
+    @joins = @options[:joins]
+    @clause = args.empty? ?
+      "#{@id_clause} #{op} ?" :
+      "#{@id_clause} #{op} ? AND (#{args.shift})"
+    @clause_args = args
+    @current_id = @options[:start_id]
+    @limit = @options[:limit] || 100
+    @job = @prefix = @key = nil
+  end
+
+  # Public: Points to the latest record that was yielded, by database ID.
+  #
+  # refresh - Boolean that determines if the instance variable cache should
+  #           be reset first.  Default: false.
+  #
+  # Returns a Fixnum.
+  def current_id(refresh = false)
+    @current_id = nil if refresh
+    @current_id ||= @redis.get(key).to_i
+  end
+
+  # Public: Sets the latest processed Integer ID.
+  attr_writer :current_id
+
+  # Public: Iterates through the whole dataset, yielding individual records as
+  # they are received.  This calls #records multiple times, setting the
+  # #current_id after each run.  If an exception is raised, the ModelIterator
+  # instance can safely be restarted, since all state is stored in Redis.
+  #
+  # &block - Block that gets called with each ActiveRecord::Base instance.
+  #
+  # Returns nothing.
+  def each
+    @job = block = (block_given? ? Proc.new : @job)
+    each_set do |records|
+      records.each do |record|
+        block.call(record)
+        @current_id = record.send(@id_field)
+      end
+    end
+    cleanup
+  end
+
+  # Public: Iterates through the whole dataset.  This calls #records multiple
+  # times, but does not set the #current_id after each record.
+  #
+  # &block - Block that gets called with each ActiveRecord::Base instance.
+  #
+  # Returns nothing.
+  def each_set(&block)
+    loops = 0
+    while records = self.records
+      begin
+        block.call(records)
+        loops += 1
+        if @max > 0 && loops >= @max
+          raise MaxIterations, self
+        end
+      ensure
+        @redis.set(key, @current_id) if @current_id
+      end
+    end
+  end
+
+  # Public: Simple alias for #each with no block.  Useful if the job errors,
+  # and you want to retry it again from where it left off.
+  alias run each
+
+  # Public: Cleans up any redis keys.
+  #
+  # Returns nothing.
+  def cleanup
+    @redis.del(key)
+    @current_id = nil
+  end
+
+  def prefix
+    @prefix = [@options[:prefix], self.class.name, @klass.name].
+      compact.join(":")
+  end
+
+  def key
+    @key ||= "#{prefix}:current"
+  end
+
+  # Public: Gets an ActiveRecord :connections value, ready for
+  # ActiveRecord::Base.all.
+  #
+  # Returns an Array with a String query clause, and unescaped db values.
+  def conditions
+    [@clause, current_id, *@clause_args]
+  end
+
+  # Public: Queries the database for the next page of records.
+  #
+  # Returns an Array of ActiveRecord::Base instances if any results are
+  # returned, or nil.
+  def records
+    arr = @klass.all(find_options)
+    arr.empty? ? nil : arr
+  end
+
+  # Public: Builds the ActiveRecord::Base.find options for a single query.
+  #
+  # Returns a Hash.
+  def find_options
+    opt = {:conditions => conditions, :limit => @limit, :order => "#{@id_clause} #{@order}"}
+    if columns = @options[:select]
+      opt[:select] = columns
+    end
+    opt[:joins] = @joins if @joins
+    opt
+  end
+end
+
+

data/model_iterator.gemspec

@@ -0,0 +1,58 @@
+## This is the rakegem gemspec template. Make sure you read and understand
+## all of the comments. Some sections require modification, and others can
+## be deleted if you don't need them. Once you understand the contents of
+## this file, feel free to delete any comments that begin with two hash marks.
+## You can find comprehensive Gem::Specification documentation, at
+## http://docs.rubygems.org/read/chapter/20
+Gem::Specification.new do |s|
+  s.specification_version = 2 if s.respond_to? :specification_version=
+  s.required_rubygems_version = Gem::Requirement.new(">= 1.3.5") if s.respond_to? :required_rubygems_version=
+
+  ## Leave these as is they will be modified for you by the rake gemspec task.
+  ## If your rubyforge_project name is different, then edit it and comment out
+  ## the sub! line in the Rakefile
+  s.name              = 'model_iterator'
+  s.version           = '1.0.0'
+  s.date              = '2012-09-07'
+  s.rubyforge_project = 'model_iterator'
+
+  ## Make sure your summary is short. The description may be as long
+  ## as you like.
+  s.summary     = "Iterate through large ActiveRecord datasets"
+  s.description = "Iterate through large ActiveRecord datasets"
+
+  ## List the primary authors. If there are a bunch of authors, it's probably
+  ## better to set the email to an email list or something. If you don't have
+  ## a custom homepage, consider using your GitHub URL or the like.
+  s.authors  = ["Rick Olson"]
+  s.email    = 'technoweenie@gmail.com'
+  s.homepage = 'http://github.com/technoweenie/model_iterator'
+
+  ## This gets added to the $LOAD_PATH so that 'lib/NAME.rb' can be required as
+  ## require 'NAME.rb' or'/lib/NAME/file.rb' can be as require 'NAME/file.rb'
+  s.require_paths = %w[lib]
+
+  s.add_development_dependency 'rake'
+  s.add_development_dependency 'test-unit'
+
+  ## Leave this section as-is. It will be automatically generated from the
+  ## contents of your Git repository via the gemspec task. DO NOT REMOVE
+  ## THE MANIFEST COMMENTS, they are used as delimiters by the task.
+  # = MANIFEST =
+  s.files = %w[
+    Gemfile
+    LICENSE.md
+    README.md
+    Rakefile
+    lib/model_iterator.rb
+    model_iterator.gemspec
+    test/helper.rb
+    test/init_test.rb
+    test/iterate_test.rb
+  ]
+  # = MANIFEST =
+
+  ## Test files will be grabbed from the file list. Make sure the path glob
+  ## matches what you actually use.
+  s.test_files = s.files.select { |path| path =~ %r{^test/*/.+\.rb} }
+end

data/test/helper.rb

@@ -0,0 +1,40 @@
+require 'rubygems'
+require 'test/unit'
+require 'active_record'
+require File.expand_path("../../lib/model_iterator", __FILE__)
+
+class ModelIterator::TestCase < Test::Unit::TestCase
+  class Model < ActiveRecord::Base
+    establish_connection :adapter => 'sqlite3', :database => ':memory:'
+    connection.create_table table_name do |c|
+      c.column :name, :string
+    end
+
+    %w(a b c).each do |s|
+      create!(:name => s)
+    end
+  end
+
+  class RedisClient
+    def initialize(hash = nil)
+      @hash = hash || {}
+    end
+
+    def [](key)
+      @hash[key]
+    end
+    alias get []
+
+    def []=(key, value)
+      @hash[key] = value
+    end
+    alias set []=
+
+    def delete(key)
+      @hash.delete(key)
+    end
+
+    alias del delete
+  end
+end
+

data/test/init_test.rb

@@ -0,0 +1,80 @@
+require File.expand_path("../helper", __FILE__)
+
+class InitializationTest < ModelIterator::TestCase
+  def setup
+    @iter = ModelIterator.new(Model, :redis => RedisClient.new)
+  end
+
+  def test_sets_klass
+    assert_equal Model, @iter.klass
+  end
+
+  def test_sets_current_id
+    assert_equal 0, @iter.current_id
+  end
+
+  def test_sets_conditions
+    assert_equal ['models.id > ?', @iter.current_id], @iter.conditions
+  end
+
+  def test_sets_limit
+    assert_equal 100, @iter.limit
+  end
+
+  def test_sets_redis_prefix
+    assert_equal 'ModelIterator:ModelIterator::TestCase::Model', @iter.prefix
+  end
+end
+
+class InitializationTestWithOptions < ModelIterator::TestCase
+  def setup
+    @iter = ModelIterator.new Model, :redis => RedisClient.new,
+      :start_id => 5, :limit => 10, :prefix => 'foo'
+  end
+
+  def test_sets_klass
+    assert_equal Model, @iter.klass
+  end
+
+  def test_sets_current_id
+    assert_equal 5, @iter.current_id
+  end
+
+  def test_sets_conditions
+    assert_equal ['models.id > ?', @iter.current_id], @iter.conditions
+  end
+
+  def test_sets_limit
+    assert_equal 10, @iter.limit
+  end
+
+  def test_sets_redis_prefix
+    assert_equal 'foo:ModelIterator:ModelIterator::TestCase::Model', @iter.prefix
+  end
+end
+
+class InitializationTestWithCustomWhereClause < ModelIterator::TestCase
+  def setup
+    @iter = ModelIterator.new Model,
+      'public = ?',
+      true,
+      :redis => RedisClient.new, :start_id => 5, :limit => 10
+  end
+
+  def test_sets_klass
+    assert_equal Model, @iter.klass
+  end
+
+  def test_sets_current_id
+    assert_equal 5, @iter.current_id
+  end
+
+  def test_sets_conditions
+    assert_equal ['models.id > ? AND (public = ?)', @iter.current_id, true], @iter.conditions
+  end
+
+  def test_sets_limit
+    assert_equal 10, @iter.limit
+  end
+end
+

data/test/iterate_test.rb

@@ -0,0 +1,83 @@
+require File.expand_path("../helper", __FILE__)
+
+class IterateTest < ModelIterator::TestCase
+  def test_finds_current_iteration_of_records
+    iter = ModelIterator.new Model, :redis => RedisClient.new
+    assert_equal %w(a b c), iter.records.map(&:name)
+  end
+
+  def test_loops_through_all_records
+    names = []
+    iter = ModelIterator.new Model, :redis => RedisClient.new, :limit => 1
+    iter.each do |m|
+      names << m.name
+    end
+
+    assert_equal %w(a b c), names
+  end
+
+  def test_loops_through_filtered_records
+    names = []
+    iter = ModelIterator.new Model, 'name != ?', 'a',
+      :redis => RedisClient.new, :limit => 1
+    iter.each do |m|
+      names << m.name
+    end
+
+    assert_equal %w(b c), names
+  end
+
+  def test_loops_through_records_in_reverse
+    names = []
+    iter = ModelIterator.new Model, :redis => RedisClient.new, :limit => 1,
+      :start_id => 100000, :order => :desc
+    iter.each do |m|
+      names << m.name
+    end
+
+    assert_equal %w(c b a), names
+  end
+
+  def test_loops_through_known_number_of_records
+    names = []
+    iter = ModelIterator.new Model, :redis => RedisClient.new,
+      :limit => 1, :start_id => 0, :max => 2
+
+    assert_raises ModelIterator::MaxIterations do
+      iter.each do |m|
+        names << m.name
+      end
+    end
+
+    assert_equal %w(a b), names
+  end
+
+  def test_allows_restart_of_records_after_error
+    redis = RedisClient.new
+    names = []
+    iter = ModelIterator.new Model, :redis => redis, :start_id => 0
+    badjob = lambda do |m|
+      raise(ExpectedError) if m.id != 1
+      names << m.name
+    end
+
+    2.times do
+      assert_raises ExpectedError do
+        iter.each(&badjob)
+      end
+    end
+
+    assert_equal badjob, iter.job
+    assert_equal %w(a), names
+
+    iter = ModelIterator.new Model, :redis => redis, :limit => 1
+    assert_equal 1, iter.current_id
+    iter.job = lambda { |m| names << m.name }
+    iter.run
+
+    assert_equal %w(a b c), names
+  end
+
+  class ExpectedError < StandardError; end
+end
+

metadata

@@ -0,0 +1,91 @@
+--- !ruby/object:Gem::Specification
+name: model_iterator
+version: !ruby/object:Gem::Version
+  version: 1.0.0
+  prerelease: 
+platform: ruby
+authors:
+- Rick Olson
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2012-09-07 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: test-unit
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Iterate through large ActiveRecord datasets
+email: technoweenie@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- Gemfile
+- LICENSE.md
+- README.md
+- Rakefile
+- lib/model_iterator.rb
+- model_iterator.gemspec
+- test/helper.rb
+- test/init_test.rb
+- test/iterate_test.rb
+homepage: http://github.com/technoweenie/model_iterator
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+      segments:
+      - 0
+      hash: 3722461387104018248
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: 1.3.5
+requirements: []
+rubyforge_project: model_iterator
+rubygems_version: 1.8.23
+signing_key: 
+specification_version: 2
+summary: Iterate through large ActiveRecord datasets
+test_files:
+- test/helper.rb
+- test/init_test.rb
+- test/iterate_test.rb