batcher 0.0.1

data/.gitignore

@@ -0,0 +1,4 @@
+*.gem
+.bundle
+Gemfile.lock
+pkg/*

data/Gemfile

@@ -0,0 +1,4 @@
+source "http://rubygems.org"
+
+# Specify your gem's dependencies in batcher.gemspec
+gemspec

data/LICENSE

@@ -0,0 +1,19 @@
+Copyright (c) 2011 Harold Gimenez and Stattleship
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included
+in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,31 @@
+Batcher
+======
+
+Processing ActiveRecord data is impossible with large amounts of data because the memory footprint of tons of ActiveRecord objects is too large and unmanagable. ActiveRecord models provide a `find_each` method for batch processing of data; unfortunately it only allows you to slice the data by primary key.
+
+Batcher is a tiny gem that allows you to batch process data and slice it by whatever you require.
+
+Example
+-------
+
+```ruby
+class User < ActiveRecord::Base
+end
+
+Batcher(User.order('influence desc')).each do |user|
+  user.do_something
+end
+```
+
+The deafult batch size is 1000, but you can pass in a different batch size to tweak accordingly:
+
+```ruby
+Batcher(User.order('influence desc'), batch_size: 2000).each do |user|
+  user.do_something
+end
+```
+
+License
+-------
+
+Batcher is copyright 2011 Harold Gimenez and Stattleship. It is free software, and may be distributed under the terms specified in the LICENSE file.

data/Rakefile

@@ -0,0 +1 @@
+require "bundler/gem_tasks"

data/batcher.gemspec

@@ -0,0 +1,25 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
+require "batcher/version"
+
+Gem::Specification.new do |s|
+  s.name        = "batcher"
+  s.version     = Batcher::VERSION
+  s.authors     = ["Harold Giménez"]
+  s.email       = ["hgimenez@thoughtbot.com"]
+  s.homepage    = ""
+  s.summary     = %q{Data batch processing for ActiveRecord}
+  s.description = %q{Allows you to process many records in batches. Similar to #find_each in ActiveRecord, but allows you to specify a slicing attribute that is not the table's primary key}
+
+  s.rubyforge_project = "batcher"
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+
+  s.add_dependency "activerecord"
+
+  s.add_development_dependency "rspec"
+  s.add_development_dependency "pg"
+end

data/lib/batcher.rb

@@ -0,0 +1,6 @@
+require "batcher/version"
+require "batcher/process"
+
+def Batcher(scope, opts = {})
+  Batcher::Process.new(scope, opts)
+end

data/lib/batcher/process.rb

@@ -0,0 +1,29 @@
+module Batcher
+  class Process
+    attr_accessor :batch_size
+
+    def initialize(scope, opts = {})
+      @scope      =  scope
+      @batch_size = opts[:batch_size] || 1000
+    end
+
+    def each(&block)
+      iterator.each do |object|
+        yield object
+      end
+    end
+
+    def iterator
+      Enumerator.new do |yielder|
+        offset = 0
+        count  = @scope.count
+        begin
+          @scope.limit(batch_size).offset(offset).each do |record|
+            yielder << record
+          end
+          offset += batch_size
+        end until (batch_size + offset - 1) > count
+      end
+    end
+  end
+end

data/lib/batcher/version.rb

@@ -0,0 +1,3 @@
+module Batcher
+  VERSION = "0.0.1"
+end

data/spec/batcher_spec.rb

@@ -0,0 +1,11 @@
+require 'spec_helper'
+
+class User < ActiveRecord::Base
+end
+
+describe Batcher do
+  it 'provides an interface to Process' do
+    batcher = Batcher(User.where(name: 'foo'))
+    batcher.should be_kind_of(Batcher::Process)
+  end
+end

data/spec/process_spec.rb

@@ -0,0 +1,32 @@
+require 'spec_helper'
+require 'batcher'
+
+class User < ActiveRecord::Base
+end
+
+describe Batcher::Process do
+  it 'can be set a batch size' do
+    batcher = Batcher::Process.new(stub, batch_size: 42)
+    batcher.batch_size.should be == 42
+  end
+
+  it 'defaults to a batch size of 1000' do
+    batcher = Batcher::Process.new(stub)
+    batcher.batch_size.should be == 1000
+  end
+
+  it 'yields each object in the provided scope' do
+    jimis = 3.times.map { User.create(name: 'Jimi') }
+    3.times { User.create(name: 'Eric') }
+
+    batcher = Batcher::Process.new(User.where(name: 'Jimi'), batch_size: 2)
+
+    yielded_objects = []
+    batcher.each do |object|
+      yielded_objects << object
+    end
+
+    yielded_objects.should have(3).objects
+    yielded_objects.should =~ jimis
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,33 @@
+require 'rspec'
+require 'active_record'
+
+module DBHelpers
+  def build_and_migrate_database
+    create_db
+    ActiveRecord::Schema.define do
+      create_table :users, force: true do |t|
+        t.string :name
+      end
+    end
+  end
+
+  def create_db
+    ActiveRecord::Base.establish_connection(
+      :adapter  => 'postgresql',
+      :encoding => 'unicode',
+      :database => 'template1'
+    )
+    result = ActiveRecord::Base.connection.select_all %{SELECT * FROM pg_catalog.pg_database WHERE datname = 'batcher_test'}
+    unless result.size > 0
+      ActiveRecord::Base.connection.execute 'CREATE DATABASE batcher_test;'
+    end
+  end
+end
+
+RSpec.configure do |config|
+  config.include DBHelpers
+
+  config.before(:all) do
+    build_and_migrate_database
+  end
+end

metadata

@@ -0,0 +1,95 @@
+--- !ruby/object:Gem::Specification
+name: batcher
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+  prerelease: 
+platform: ruby
+authors:
+- Harold Giménez
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2011-12-16 00:00:00.000000000Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: activerecord
+  requirement: &2157706000 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *2157706000
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: &2157705580 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: *2157705580
+- !ruby/object:Gem::Dependency
+  name: pg
+  requirement: &2157705040 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: *2157705040
+description: ! 'Allows you to process many records in batches. Similar to #find_each
+  in ActiveRecord, but allows you to specify a slicing attribute that is not the table''s
+  primary key'
+email:
+- hgimenez@thoughtbot.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- LICENSE
+- README.md
+- Rakefile
+- batcher.gemspec
+- lib/batcher.rb
+- lib/batcher/process.rb
+- lib/batcher/version.rb
+- spec/batcher_spec.rb
+- spec/process_spec.rb
+- spec/spec_helper.rb
+homepage: ''
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: batcher
+rubygems_version: 1.8.6
+signing_key: 
+specification_version: 3
+summary: Data batch processing for ActiveRecord
+test_files:
+- spec/batcher_spec.rb
+- spec/process_spec.rb
+- spec/spec_helper.rb