mongo_batch 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 9343db89a622725ac3532e5e7b16ff128b5d04b2
+  data.tar.gz: 5a2a6233dbc88941e86f53659020a8d4d1d4d516
+SHA512:
+  metadata.gz: 7fc6a20a065fe187c8980584c50dcaed5597b7ffae396c6c3d091249e7fe5ee6a38f7537005a8b0044c247f25477c996f9cc0f5b4543f70873c75d08c1d66b63
+  data.tar.gz: 17ed373f8e1f0cf2cddf265e3604df9ea42c207da69169346c2eefed1e30781a7140278271a0585e04360c6acb7e1cef7d5173a4228676056ce55118bcc84de1

data/README.md

@@ -0,0 +1,102 @@
+# MongoBatch [![Build Status](https://travis-ci.org/altmetric/mongo_batch.svg?branch=master)](https://travis-ci.org/altmetric/mongo_batch)
+
+A Ruby library to run Mongoid queries on large collections in batches.
+
+**Supported Ruby versions:** 2.1, 2.2
+
+**Supported Mongoid versions:** 4.0, 5.0
+
+## Installation
+
+```
+gem install mongo_batch -v '~> 0.1'
+```
+
+Or, in your `Gemfile`:
+
+```ruby
+gem 'mongo_batch', '~> 0.1'
+```
+
+## Usage
+
+Extend your Mongoid models with `MongoBatch` to be able to call
+`find_in_batches` on your models. The method will yield each batch of
+records to the given block.
+
+```ruby
+require 'mongo_batch'
+
+class Post
+  extend MongoBatch
+end
+
+Post.find_in_batches do |batch|
+  batch.each do |post|
+    post.update(body: 'Hello world!')
+  end
+end
+```
+
+If you do not pass a block to `find_in_batches`, the method will
+return an [Enumerator](http://ruby-doc.org/core-2.2.2/Enumerator.html).
+
+```ruby
+Post.find_in_batches.with_index.each do |batch, index|
+  batch.each do |post|
+    post.update(body: "Hello world! #{index}")
+  end
+end
+```
+
+The default batch size is 1,000 records, but `find_in_batches` accepts
+an option to configure a different batch size, as well as options to
+limit the records to process, sorting criteria and an initial offset.
+
+```ruby
+Post
+  .find_in_batches(batch_size: 500, to: 2_000, offset: 100, order_by: { _id: :desc })
+  .each do |batch|
+    batch.each do |post|
+      post.update(body: "Hola mundo!")
+    end
+end
+
+```
+
+If you have more complex queries or prefer not to extend your models
+with `MongoBatch`,
+you can use `MongoBatch.in_batches` and supply the query you want to batch.
+
+```ruby
+MongoBatch
+  .in_batches(Post.where(:body.exists => true).no_timeout)
+  .each do |batch|
+    batch.each do |post|
+      post.update(body: 'Hello world!')
+    end
+end
+```
+
+`MongoBatch.in_batches` also accepts values to configure the batch size, limit of
+records to process, sorting criteria or an initial offset.
+
+```ruby
+MongoBatch
+  .in_batches(Post.where(:body.exists => true).no_timeout,
+              to: 2_000, offset: 100, order_by: { _id::desc })
+  .each do |batch|
+     batch.each do |post|
+       post.update(body: 'Hi mum!')
+     end
+  end
+```
+
+## License
+
+Copyright © 2015 Altmetric LLP
+
+Distributed under the MIT License.
+
+[URI]: http://ruby-doc.org/stdlib/libdoc/uri/rdoc/URI.html
+

data/lib/mongo_batch.rb

@@ -0,0 +1,43 @@
+module MongoBatch
+  class Batcher
+    attr_reader :query, :batch_size, :to, :offset, :order_by
+
+    def initialize(query, options = {})
+      @query = query
+      @batch_size = options.fetch(:batch_size) { 1_000 }
+      @to = options.fetch(:to) { query.count }
+      @offset = options.fetch(:offset) { 0 }
+      @order_by = options.fetch(:order_by) { { _id: :asc } }
+    end
+
+    def batches
+      Enumerator.new(to) do |yielder|
+        processed_so_far = offset
+
+        offset.step(by: batch_size, to: to - batch_size).each do |offset|
+          yielder << query.order_by(order_by).limit(batch_size).skip(offset)
+          processed_so_far += batch_size
+        end
+
+        if processed_so_far < to
+          last_limit = to - processed_so_far
+          yielder << query.order(order_by).limit(last_limit).skip(processed_so_far)
+        end
+      end
+    end
+  end
+
+  def self.in_batches(query, options = {})
+    Batcher.new(query, options).batches
+  end
+
+  def find_in_batches(options = {}, &block)
+    batcher = Batcher.new(self, options)
+
+    if block
+      batcher.batches.each(&block)
+    else
+      batcher.batches
+    end
+  end
+end

data/spec/factories/posts.rb

@@ -0,0 +1,5 @@
+FactoryGirl.define do
+  factory :post do
+    sequence(:id)
+  end
+end

data/spec/mongo_batch_spec.rb

@@ -0,0 +1,126 @@
+require 'spec_helper'
+require 'mongo_batch'
+
+describe MongoBatch do
+  class Post
+    include Mongoid::Document
+    include Mongoid::Timestamps
+    extend MongoBatch
+
+    field :body, type: String
+  end
+
+  describe '#find_in_batches' do
+    it 'yields to the given block for each batch of records' do
+      posts = FactoryGirl.create_list(:post, 10).sort_by(&:id)
+
+      expect do |block|
+        Post.find_in_batches(batch_size: 2).each(&block)
+      end.to yield_successive_args(posts[(0..1)],
+                                   posts[(2..3)],
+                                   posts[(4..5)],
+                                   posts[(6..7)],
+                                   posts[(8..9)])
+    end
+
+    it 'returns an enumerator if a block is not given' do
+      expect(Post.find_in_batches(batch_size: 2)).to be_an(Enumerator)
+    end
+  end
+
+  describe '.in_batches' do
+    it 'returns an enumerator if a block is not given' do
+      expect(described_class.in_batches(batch_size: 2)).to be_an(Enumerator)
+    end
+
+    it 'starts from the first record if "offset" is not specified' do
+      posts = FactoryGirl.create_list(:post, 4).sort_by(&:id)
+
+      batches = described_class.in_batches(Post).map(&:to_a)
+
+      expect(batches.first).to start_with(posts[0])
+    end
+
+    it 'skips the number of records indicated in "offset"' do
+      posts = FactoryGirl.create_list(:post, 4)
+
+      posts_in_batches = described_class.in_batches(Post, batch_size: 2, offset: 3).map(&:to_a)
+
+      expect(posts_in_batches).to eq([[posts[3]]])
+    end
+
+    it 'calculates the count of records if "to" is not specified' do
+      posts = FactoryGirl.create_list(:post, 4)
+
+      posts_in_batches = described_class.in_batches(Post, batch_size: 2).map(&:to_a)
+
+      expect(posts_in_batches).to eq([posts[(0..1)], posts[(2..3)]])
+    end
+
+    it 'does not calculate the count of records if "to" is specified' do
+      FactoryGirl.create_list(:post, 4)
+
+      expect(Post).not_to receive(:count)
+
+      described_class.in_batches(Post, to: 1).map(&:to_a)
+    end
+
+    it 'orders the elements ascending by _id by default' do
+      posts = FactoryGirl.create_list(:post, 10).sort_by(&:id)
+
+      ids = Post.find_in_batches(batch_size: 2).map(&:to_a).flatten.map(&:id)
+
+      expect(ids).to eq(posts.map(&:id))
+    end
+
+    it 'allows to specify a custom sorting order' do
+      posts = FactoryGirl.create_list(:post, 10).sort_by(&:id)
+
+      ids = described_class
+            .in_batches(Post, batch_size: 2, order_by: { id: 'desc' })
+            .map(&:to_a)
+            .flatten
+            .map(&:id)
+
+      expect(ids).to eq(posts.map(&:id).reverse)
+    end
+
+    it 'preserves any scopes previously applied' do
+      posts = FactoryGirl.create_list(:post, 5, body: 'Hello world!')
+      FactoryGirl.create_list(:post, 1)
+
+      posts_in_batches = described_class
+                         .in_batches(Post.where(:body.exists => true), batch_size: 2)
+                         .map(&:to_a)
+
+      expect(posts_in_batches).to eq([posts[(0..1)], posts[(2..3)], posts[(4..5)]])
+    end
+
+    context 'when "to" is a multiple of the batch size' do
+      it 'splits the query into uniform batches covering all the records in the query' do
+        posts = FactoryGirl.create_list(:post, 10)
+
+        posts_in_batches = described_class.in_batches(Post, batch_size: 2, to: 10).map(&:to_a)
+
+        expect(posts_in_batches).to eq([posts[(0..1)],
+                                        posts[(2..3)],
+                                        posts[(4..5)],
+                                        posts[(6..7)],
+                                        posts[(8..9)]])
+      end
+    end
+
+    context 'when "to" is not a multiple of the batch size' do
+      it 'splits the query into batches covering all the records in the query' do
+        posts = FactoryGirl.create_list(:post, 10)
+
+        posts_in_batches = described_class.in_batches(Post, batch_size: 3, to: 10).map(&:to_a)
+
+        expect(posts_in_batches).to eq([posts[(0..2)],
+                                        posts[(3..5)],
+                                        posts[(6..8)],
+                                        posts[(9..9)]])
+      end
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,18 @@
+require 'mongoid'
+require 'database_cleaner'
+require 'factory_girl'
+require 'factories/posts'
+
+Mongoid.load!(File.expand_path('../../config/mongoid.yml', __FILE__), 'test')
+
+RSpec.configure do |config|
+  config.before(:suite) do
+    DatabaseCleaner[:mongoid].strategy = :truncation
+  end
+
+  config.around(:each) do |example|
+    DatabaseCleaner.cleaning do
+      example.run
+    end
+  end
+end

metadata

@@ -0,0 +1,139 @@
+--- !ruby/object:Gem::Specification
+name: mongo_batch
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Oliver Martell
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2015-11-09 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.10'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.10'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.3'
+- !ruby/object:Gem::Dependency
+  name: mongoid
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.0'
+- !ruby/object:Gem::Dependency
+  name: database_cleaner
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.5'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.5'
+- !ruby/object:Gem::Dependency
+  name: factory_girl
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.5'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.5'
+description: |2
+      A library to iterate over entire Mongo collections or large queries
+      exposing an API to control things like batch size, order and limit.
+email:
+- support@altmetric.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- README.md
+- lib/mongo_batch.rb
+- spec/factories/posts.rb
+- spec/mongo_batch_spec.rb
+- spec/spec_helper.rb
+homepage: https://github.com/altmetric/mongo_batch
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.5
+signing_key: 
+specification_version: 4
+summary: A library to batch Mongo queries
+test_files:
+- spec/factories/posts.rb
+- spec/mongo_batch_spec.rb
+- spec/spec_helper.rb
+has_rdoc: