scripterator 0.1.0

checksums.yaml

@@ -0,0 +1,15 @@
+---
+!binary "U0hBMQ==":
+  metadata.gz: !binary |-
+    N2ViZDdmMmFjMWUxYzIyMTlmMDg4OTYzZmFlZWNkZDkyNWExMzEyYQ==
+  data.tar.gz: !binary |-
+    MDEzMzJkNjdkNjRhZWVhODE2YTIwMjY2NWY2NDdiNGRhM2VmYmM0ZA==
+SHA512:
+  metadata.gz: !binary |-
+    MGQxNmIzOTBlMjg5MjE5ZTJlZGVkNjM3ZTNjNzg0MGI1NjFmODZhMDBjNzgy
+    NjE3OGNlNmY3MGRjNDg5MjA1ODc2ODgwOTNmMzY0MmZlMWMzN2MzZjk5M2M5
+    ZWYzZjVhYWFjZTU2MDQ1YjFiMzg5NWZhMGJhYzkyOWMxNThmZmI=
+  data.tar.gz: !binary |-
+    ZGYxYzQyZWUwMGFjZTJmZWE5MmMyMjRhMDZiMmRjMWU0NTgyYjZkNTU2Y2Qx
+    YmM3YTE0OWZmNDJhYmMyOWE2ZTNmZTkwMDA3YTg4NDU4MTA1MjI0M2YwMzYx
+    ZDk4ZTViYzc2NThmNWYwMjZlYWU3ZTYzZDI0MTQwZjA3ZWRhNWY=

data/.gitignore

@@ -0,0 +1,17 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp

data/.rspec

@@ -0,0 +1,2 @@
+--color
+--format progress

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in scripterator.gemspec
+gemspec

data/LICENSE

@@ -0,0 +1,22 @@
+Copyright (c) 2013 t ddddddd
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,123 @@
+# Scripterator
+"Helping you scripterate over all the things"
+
+A lightweight script harness and DSL for iterating over and running operations on ActiveRecord model records, with Redis hooks for managing subsets, failures, and retries.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'scripterator'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install scripterator
+
+Works with ActiveRecord 3.* (Rails 4 support coming).
+
+## Usage
+
+Create a .rb file with your script code:
+```ruby
+Scripterator.run "Convert users from legacy auth data" do
+
+  before do
+    User.skip_some_callbacks_we_want_to_avoid_during_script_running
+  end
+
+  for_each_user do |user|
+    user.do_legacy_conversion
+  end
+
+  after do
+    # some code to run after everything's finished
+  end
+
+end
+```
+
+Run your script for a given set of IDs:
+
+    $ START=10000 END=19999 bundle exec rails runner my_script.rb >out.txt
+
+This will produce output of the form:
+
+```
+Starting at 2013-09-24 14:53:39 -0700...
+2013-09-24 14:53:40 -0700: Checked 0 rows, 0 migrated.
+2013-09-24 14:53:41 -0700: Checked 10000 rows, 10000 migrated.
+2013-09-24 14:53:41 -0700: Checked 20000 rows, 20000 migrated.
+2013-09-24 14:53:42 -0700: Checked 30000 rows, 30000 migrated.
+done
+Finished at 2013-09-24 14:53:43 -0700...
+
+Total rows migrated: 34903 / 34903
+0 rows previously migrated and skipped
+0 errors
+```
+
+Retrieve set information about checked and failed records:
+
+```
+> Scripterator.failed_ids_for "Convert users from legacy auth data"
+=> [14011, 15634, 17301, 17302]
+
+> Scripterator.already_run_for?("Convert users from legacy auth data", 15000)
+=> true
+```
+
+User-definable blocks:
+
+Required:
+
+- `for_each_(.+)`: code to run for every record. This block should return `true` (or a truthy value) if the operation ran successfully, or `false` (or a falsy value) if the record was skipped/ineligible. Errors and Exceptions will be caught by Scripterator and tabulated/output.
+
+Optional:
+
+- `model`: code with which model should be loaded, e.g., `model { User.includes(:profile, :roles) }`; if this block is not supplied, the model class is inferred from the `for_each_*` block, e.g., `for_each_post_comment` will cause the model `PostComment` to be loaded
+- `before`: code to run before iteration begins
+- `after`: code to run after iteration finishes
+
+Environment variable options:
+
+- `START`: first model ID to scripterate
+- `END`: last model ID to scripterate
+- `REDIS_EXPIRATION`: amount of time (in seconds) before Redis result sets (checked IDs and failed IDs) are expired
+
+Either a starting or an ending ID must be provided.
+
+## Configuration
+
+Within an optional Rails initializer, configure Scripterator further as follows (`config/initializers/scripterator.rb`):
+
+```ruby
+Scripterator.configure do |config|
+  # alternate Redis instance
+  config.redis = MyRedis.new
+
+  # turn off Redis
+  config.redis = nil
+
+  # change default Redis set expiration time
+  config.redis_expiration = 5.days
+
+  # set redis_expiration to 0 to turn off expiration
+  config.redis_expiration = 0
+end
+```
+
+## Running tests
+
+    $ bundle exec rspec
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Added some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/lib/scripterator.rb

@@ -0,0 +1,33 @@
+require "redis" unless defined? Redis
+
+require "scripterator/version"
+require "scripterator/configuration"
+require "scripterator/runner"
+
+module Scripterator
+  class << self
+    def configure
+      yield config
+    end
+
+    def config
+      @config ||= Scripterator::Configuration.new
+    end
+
+    def run(description, &block)
+      options = {}.tap do |o|
+        o[:start_id]         = ENV['START'].try(:to_i)
+        o[:end_id]           = ENV['END'].try(:to_i)
+        o[:redis_expiration] = ENV['REDIS_EXPIRATION'].try(:to_i) || config.redis_expiration
+      end
+
+      Runner.new(description, &block).run(options)
+    end
+
+    %w(already_run_for? checked_ids failed_ids).each do |runner_method|
+      define_method(runner_method) do |description, *args|
+        Runner.new(description).send(runner_method, *args)
+      end
+    end
+  end
+end

data/lib/scripterator/configuration.rb

@@ -0,0 +1,21 @@
+module Scripterator
+  class Configuration
+    attr_accessor :redis_expiration
+    attr_reader   :redis
+
+    # set config.redis = nil to use NilRedis implementation
+    def redis=(r)
+      @redis = r || Scripterator::NilRedis.new
+    end
+  end
+
+  class NilRedis
+    def smembers(*args)
+      []
+    end
+
+    %w(expire sadd sismember).each do |redis_method|
+      define_method(redis_method) { |*args| nil }
+    end
+  end
+end

data/lib/scripterator/runner.rb

@@ -0,0 +1,130 @@
+require 'forwardable'
+require_relative 'script_redis'
+
+module Scripterator
+  class Runner
+    extend Forwardable
+    def_delegators :script_redis,
+      :already_run_for?, :checked_ids, :expire_redis_sets, :failed_ids,
+      :mark_as_failed_for, :mark_as_run_for, :script_key
+
+    def initialize(description, &config_block)
+      @script_description = description
+
+      self.instance_eval(&config_block) if config_block
+
+      @model ||= Proc.new { eval("#{@inferred_model_name}") } # constantize
+    end
+
+    def run(options = {})
+      unless options[:start_id] || options[:end_id]
+        raise 'You must provide either a start ID or end ID'
+      end
+      @start_id         = options[:start_id] || 1
+      @end_id           = options[:end_id]
+      @redis_expiration = options[:redis_expiration]
+      @output_stream    = options[:output_stream] || $stdout
+
+      raise 'No per_record code defined' unless @per_record
+
+      init_vars
+      run_blocks
+      output_stats
+    end
+
+    %w(model before per_record after).each do |callback|
+      define_method callback do |&block|
+        instance_variable_set "@#{callback}", block
+      end
+    end
+
+    def method_missing(method_name, *args, &block)
+      if model_name = /for_each_(.+)/.match(method_name)[1]
+        @per_record          = block
+        @inferred_model_name = model_name.split('_').map(&:capitalize).join
+      else
+        super
+      end
+    end
+
+    private
+
+    def init_vars
+      @success_count = 0
+      @total_checked = 0
+      @already_done  = 0
+      @errors        = []
+    end
+
+    def fetch_record(id)
+      model_finder.find_by_id(id)
+    end
+
+    def model_finder
+      @model_finder ||= self.instance_eval(&@model)
+    end
+
+    def output_progress
+      if (@total_checked + @already_done) % 10000 == 0
+        output "#{Time.now}: Checked #{@total_checked} rows, #{@success_count} migrated."
+      end
+    end
+
+    def output_stats
+      output "Total rows migrated: #{@success_count} / #{@total_checked}"
+      output "#{@already_done} rows previously migrated and skipped"
+      output "#{@errors.count} errors"
+      if @errors.count > 0 && !failed_ids.empty?
+        output "  Retrieve failed IDs with redis: SMEMBERS #{script_key(:failed)}"
+      end
+    end
+
+    def output(*args)
+      @output_stream.puts(*args)
+    end
+
+    def run_blocks
+      self.instance_eval(&@before) if @before
+
+      output "Starting at #{Time.now}..."
+      run_loop
+      output 'done'
+      output "Finished at #{Time.now}...\n\n"
+
+      self.instance_eval(&@after) if @after
+    end
+
+    def run_loop
+      if @end_id
+        (@start_id..@end_id).each { |id| transform_one_record(fetch_record(id)) }
+      else
+        model_finder.find_each(start: @start_id) { |record| transform_one_record(record) }
+      end
+
+      expire_redis_sets
+    end
+
+    def transform_one_record(record)
+      return if record.nil?
+
+      output_progress
+
+      if already_run_for? record.id
+        @already_done += 1
+      else
+        mark_as_run_for record.id
+        @total_checked += 1
+        @success_count += 1 if self.instance_exec record, &@per_record
+      end
+    rescue
+      errmsg = "Record #{record.id}: #{$!}"
+      output "Error: #{errmsg}"
+      @errors << errmsg
+      mark_as_failed_for record.id
+    end
+
+    def script_redis
+      @script_redis ||= ScriptRedis.new(@script_description, redis_expiration: @redis_expiration)
+    end
+  end
+end

data/lib/scripterator/script_redis.rb

@@ -0,0 +1,46 @@
+module Scripterator
+  class ScriptRedis
+    DEFAULT_EXPIRATION = 3 * 30 * 24 * 60 * 60 # 3 months
+
+    def initialize(script_description, options = {})
+      @key_prefix = "one_timer_script:#{script_description.downcase.split.join('_')}"
+      @redis_expiration = options[:redis_expiration] || DEFAULT_EXPIRATION
+    end
+
+    def checked_ids
+      redis.smembers(script_key(:checked)).map &:to_i
+    end
+
+    def failed_ids
+      redis.smembers(script_key(:failed)).map &:to_i
+    end
+
+    def already_run_for?(id)
+      redis.sismember script_key(:checked), id
+    end
+
+    def expire_redis_sets
+      unless @redis_expiration <= 0
+        %w(checked failed).each { |set| redis.expire script_key(set), @redis_expiration }
+      end
+    end
+
+    def mark_as_failed_for(id)
+      redis.sadd script_key(:failed), id
+    end
+
+    def mark_as_run_for(id)
+      redis.sadd script_key(:checked), id
+    end
+
+    def script_key(set_name)
+      "#{@key_prefix}:#{set_name}"
+    end
+
+    private
+
+    def redis
+      @redis ||= Scripterator.config.redis || Redis.new
+    end
+  end
+end

data/lib/scripterator/version.rb

@@ -0,0 +1,3 @@
+module Scripterator
+  VERSION = "0.1.0"
+end

data/scripterator.gemspec

@@ -0,0 +1,24 @@
+# -*- encoding: utf-8 -*-
+require File.expand_path('../lib/scripterator/version', __FILE__)
+
+Gem::Specification.new do |gem|
+  gem.authors       = ["Ted Dumitrescu"]
+  gem.email         = ["ted@lumoslabs.com"]
+  gem.description   = %q{Script iterator for ActiveRecord models}
+  gem.summary       = %q{DSL for running operations on each of a set of models}
+  gem.homepage      = "http://lumosity.com"
+
+  gem.files         = `git ls-files`.split($\)
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(spec)/})
+  gem.name          = "scripterator"
+  gem.require_paths = ["lib"]
+  gem.version       = Scripterator::VERSION
+
+  gem.add_dependency "activerecord", "~> 3.0"
+  gem.add_dependency "redis",        "~> 3.0"
+
+  gem.add_development_dependency "fakeredis", "~> 0.4"
+  gem.add_development_dependency "rspec",     "~> 2.13"
+  gem.add_development_dependency "sqlite3",   "~> 1.3.8"
+end

data/spec/runner_spec.rb

@@ -0,0 +1,138 @@
+require 'spec_helper'
+
+describe Scripterator::Runner do
+  let(:runner)      { Scripterator::Runner.new(description, &awesome_script) }
+  let(:description) { 'Convert widgets to eggplant parmigiana' }
+  let(:options)     { {start_id: start_id, output_stream: StringIO.new} }
+  let(:start_id)    { 1 }
+
+  let(:awesome_script) do
+    Proc.new do
+      before          { Widget.before_stuff }
+      for_each_widget { |widget| Widget.transform_a_widget(widget) }
+    end
+  end
+
+  subject { runner.run(options) }
+
+  shared_examples_for 'raises an error' do
+    specify do
+      expect { subject }.to raise_error
+    end
+  end
+
+  it 'infers the model from the for_each block' do
+    runner.send(:model_finder).should == Widget
+  end
+
+  context 'when a model block is defined' do
+    let(:awesome_script) do
+      Proc.new do
+        model           { Widget.where(name: 'bla') }
+        before          { Widget.before_stuff }
+        for_each_widget { |widget| Widget.transform_a_widget(widget) }
+      end
+    end
+    let!(:widget1) { Widget.create(name: 'foo') }
+    let!(:widget2) { Widget.create(name: 'bla') }
+
+    it 'uses the given model finder code' do
+      Widget.should_receive(:transform_a_widget).once.with(widget2)
+      Widget.should_not_receive(:transform_a_widget).with(widget1)
+      subject
+    end
+  end
+
+  context 'when no start or end ID is passed' do
+    let(:options) { {} }
+
+    it_behaves_like 'raises an error'
+  end
+
+  context 'when no per-record block is defined' do
+    let(:awesome_script) do
+      Proc.new do
+        model  { Widget }
+        before { Widget.before_stuff }
+      end
+    end
+
+    it_behaves_like 'raises an error'
+
+    it 'does not run any other given blocks' do
+      Widget.should_not_receive :before_stuff
+      subject rescue nil
+    end
+  end
+
+  context 'when there are no records for the specified model' do
+    it 'does not run the per-record block' do
+      Widget.should_not_receive :transform_a_widget
+      subject
+    end
+  end
+
+  context 'when there are records for the specified model' do
+    let(:num_widgets) { 3 }
+
+    before { num_widgets.times { Widget.create! } }
+
+    it 'runs the given script blocks' do
+      Widget.should_receive :before_stuff
+      Widget.should_receive(:transform_a_widget).exactly(num_widgets).times
+      subject
+    end
+
+    context 'when not all records are checked' do
+      let(:start_id) { Widget.last.id }
+
+      it 'marks only the checked IDs as checked' do
+        subject
+        Scripterator.already_run_for?(description, Widget.first.id).should be_false
+        Scripterator.checked_ids(description).should_not include Widget.first.id
+        Scripterator.already_run_for?(description, Widget.last.id).should be_true
+        Scripterator.checked_ids(description).should include Widget.last.id
+      end
+    end
+
+    context 'when some records have already been checked' do
+      let(:checked_ids) { [Widget.first.id] }
+
+      before do
+        Scripterator.stub(checked_ids: checked_ids)
+        Scripterator::ScriptRedis.any_instance.stub(already_run_for?: false)
+        Scripterator::ScriptRedis.any_instance.stub(:already_run_for?).with(Widget.first.id).and_return(true)
+      end
+
+      it 'only runs the per-record code for unchecked records' do
+        Widget.should_receive(:transform_a_widget).exactly(num_widgets - 1).times
+        subject
+      end
+    end
+
+    context 'when the code for some records fails' do
+      before do
+        Widget.stub :transform_a_widget do |widget|
+          raise 'Last widget expl0de' if widget.id == Widget.last.id
+          true
+        end
+      end
+
+      it 'marks only the failed IDs as failed' do
+        subject
+        Scripterator.failed_ids(description).should_not include Widget.first.id
+        Scripterator.failed_ids(description).should include Widget.last.id
+      end
+    end
+
+    context 'when Redis client is set to nil' do
+      before { Scripterator.configure { |config| config.redis = nil } }
+      after  { Scripterator.instance_variable_set(:@config, nil) }
+
+      it 'runs without Redis' do
+        expect { subject }.not_to raise_error
+        Scripterator.checked_ids(description).should be_empty
+      end
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,28 @@
+# This file was generated by the `rspec --init` command. Conventionally, all
+# specs live under a `spec` directory, which RSpec adds to the `$LOAD_PATH`.
+# Require this file using `require "spec_helper"` to ensure that it is only
+# loaded once.
+
+require 'fakeredis/rspec'
+require 'scripterator'
+require 'support/widget'
+
+# See http://rubydoc.info/gems/rspec-core/RSpec/Core/Configuration
+RSpec.configure do |config|
+  config.treat_symbols_as_metadata_keys_with_true_values = true
+  config.run_all_when_everything_filtered = true
+  config.filter_run :focus
+
+  # Run specs in random order to surface order dependencies. If you find an
+  # order dependency and want to debug it, you can fix the order by providing
+  # the seed, which is printed after each run.
+  #     --seed 1234
+  config.order = 'random'
+
+  config.around do |example|
+    ActiveRecord::Base.transaction do
+      example.run
+      raise ActiveRecord::Rollback
+    end
+  end
+end

data/spec/support/widget.rb

@@ -0,0 +1,14 @@
+require 'active_record'
+
+ActiveRecord::Base.establish_connection adapter: "sqlite3", database: ":memory:"
+
+ActiveRecord::Migration.create_table :widgets do |t|
+  t.string :name
+  t.timestamps
+end
+
+class Widget < ActiveRecord::Base
+  # some dummy methods for setting message expectations in specs
+  def self.before_stuff; end
+  def self.transform_a_widget(widget); true; end
+end

metadata

@@ -0,0 +1,131 @@
+--- !ruby/object:Gem::Specification
+name: scripterator
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Ted Dumitrescu
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-09-24 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: activerecord
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: redis
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: fakeredis
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '0.4'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '0.4'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '2.13'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '2.13'
+- !ruby/object:Gem::Dependency
+  name: sqlite3
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.3.8
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.3.8
+description: Script iterator for ActiveRecord models
+email:
+- ted@lumoslabs.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- .rspec
+- Gemfile
+- LICENSE
+- README.md
+- lib/scripterator.rb
+- lib/scripterator/configuration.rb
+- lib/scripterator/runner.rb
+- lib/scripterator/script_redis.rb
+- lib/scripterator/version.rb
+- scripterator.gemspec
+- spec/runner_spec.rb
+- spec/spec_helper.rb
+- spec/support/widget.rb
+homepage: http://lumosity.com
+licenses: []
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.1.4
+signing_key: 
+specification_version: 4
+summary: DSL for running operations on each of a set of models
+test_files:
+- spec/runner_spec.rb
+- spec/spec_helper.rb
+- spec/support/widget.rb
+has_rdoc: