hekenga 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 1f8ad94b462c9140e11f83e4f4ad0f0a16a9ab83
+  data.tar.gz: 6be1fcf20d2a9f1b76c826c6624828feba9c5ad5
+SHA512:
+  metadata.gz: 5dfa4bb023c546776514ebf1cd5818688de1937513ba6d2b847b95475052b6f408fd33f3464fcb150d28808c4b0fb7b6282824f81323391514eedafdbe5e4171
+  data.tar.gz: 6d9fb7bed2f662c107773e17f109374a9767f69e6c2c99546fc90e9a74b5c22fbfde767c1fb47065ec38facd31559876b57fe7937c0fcf2b2352d66c7cf2fe27

data/.gitignore

@@ -0,0 +1,9 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/

data/.rspec

@@ -0,0 +1,2 @@
+--format documentation
+--color

data/.travis.yml

@@ -0,0 +1,5 @@
+sudo: false
+language: ruby
+rvm:
+  - 2.3.3
+before_install: gem install bundler -v 1.13.6

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in hekenga.gemspec
+gemspec

data/README.md

@@ -0,0 +1,41 @@
+# Hekenga
+
+An attempt at a migration framework for MongoDB that supports parallel document
+processing via ActiveJob, chained jobs and error recovery.
+
+**Note that this gem is currently in pre-alpha - assume most things have a high
+chance of being broken.**
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'hekenga'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install hekenga
+
+## Usage
+
+CLI instructions:
+
+    $ hekenga help
+
+Migration DSL documentation TBD, for now please look at spec/
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/tzar/hekenga.

data/Rakefile

@@ -0,0 +1,11 @@
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new(:spec)
+
+desc "Open a pry session preloaded with this library"
+task :console do
+  sh "bin/console", verbose: false
+end
+
+task :default => :spec

data/bin/console

@@ -0,0 +1,13 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "hekenga"
+require "pry"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+Pry.start

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/examples/simple_example.rb

@@ -0,0 +1,52 @@
+# Stub
+class MyModel
+  def self.where(*args)
+    self
+  end
+end
+# You can stack multiple tasks within one overall logical migration
+Hekenga.migration do
+  description "Example usage"
+  created "2016-04-03 14:00"
+
+  # Simple tasks have an up and a down and run in one go
+  task "Set foo->bar by default on MyModel" do
+    up do
+      MyModel.all.set(foo: 'bar')
+    end
+    down do
+      MyModel.all.unset(:foo)
+    end
+  end
+
+  # Per document tasks run a block of code per document, with the ability to
+  # filter which documents are loaded by:
+  # - scope
+  # - arbitrary block
+  # Jobs can be run in parallel via ActiveJob.
+  # Callbacks can be disabled for the context of the job either globally via
+  # disable_callbacks or specifically via disable_callback, with multiple models
+  # optionally targetted via the `on` param.
+  # A setup block is also provided (this must be able to be run multiple times!)
+  # per_document migrations should be resumable/retryable..
+  # errors should never result in data loss, and should be logged to a migration
+  # output model
+  per_document "Set MyModel.zap to a random number if unset" do
+    scope MyModel.where(zap: nil)
+    parallel!
+    timeless!
+    disable_callback :reindex, on: MyModel
+
+    setup do
+      @max_rand = 100
+    end
+
+    filter do |document|
+      document.zap.nil?
+    end
+
+    up do |document|
+      document.zap = rand(@max_rand)
+    end
+  end
+end

data/exe/hekenga

@@ -0,0 +1,139 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+
+if File.exists?(File.expand_path("config/environment.rb"))
+  require File.expand_path("config/environment.rb")
+end
+
+require "hekenga"
+require "thor"
+
+class HekengaCLI < Thor
+  desc "status", "Show which migrations have run and their status."
+  def status
+    Hekenga.load_all!
+    Hekenga.registry.sort_by {|x| x.stamp}.each do |migration|
+      status = case Hekenga.status(migration)
+               when :running
+                 "ACTIVE"
+               when :failed
+                 "FAILED"
+               when :complete
+                 "COMPLT"
+               else
+                 "UN-RUN"
+               end
+      puts "[#{status}] #{migration.to_key}"
+    end
+  end
+
+  desc "run_all!", "Run all migrations that have not yet run, in date order."
+  def run_all!
+    bail_if_errors
+    Hekenga.load_all!
+    Hekenga.registry.sort_by {|x| x.stamp}.each do |migration|
+      migration.perform!
+      bail_if_errors
+    end
+  end
+
+  desc "run! PATH_OR_PKEY --test", "Run a migration (optionally in test mode)."
+  option :test, default: false, type: :boolean
+  def run!(path_or_pkey)
+    bail_if_errors
+    migration = load_migration(path_or_pkey)
+    migration.test_mode! if options[:test]
+    migration.perform!
+    if options[:test]
+      if Hekenga::Failure.where(pkey: migration.to_key).any?
+        puts "Logs have been preserved for debugging. To reset migration state run:"
+        puts "  hekenga clear! #{path_or_pkey}"
+      else
+        puts "Migration test run completed successfully."
+        clear!(path_or_pkey)
+      end
+    end
+  end
+
+  desc "cancel", "Cancel all active migrations."
+  def cancel
+    Hekenga::Log.where(done: false).set(cancel: true)
+    puts "Sent :cancel to all active hekenga jobs."
+  end
+
+  desc "cleanup", "Remove any failure logs."
+  def cleanup
+    Hekenga::Failure.all.delete_all
+    puts "Removed all failure logs."
+  end
+
+  desc "clear! PATH_OR_PKEY", "Clear the logs and failure for a migration. Dangerous!"
+  def clear!(path_or_pkey)
+    migration = load_migration(path_or_pkey)
+    puts "Clearing #{migration.to_key}.."
+    Hekenga::Log.where(pkey: migration.to_key).delete_all
+    Hekenga::Failure.where(pkey: migration.to_key).delete_all
+    puts "Done!"
+  end
+
+  desc "rollback", "Rollback a migration."
+  def rollback
+    todo "rollback"
+  end
+
+  desc "recover", "Attempt to resume a failed migration."
+  def recover
+    todo "recover"
+  end
+
+  desc "errors", "Print the errors associated with a failed migration."
+  def errors
+    todo "errors"
+  end
+
+  desc "skip PATH_OR_PKEY", "Skip a migration so that it won't run."
+  def skip(path_or_pkey)
+    migration = load_migration(path_or_pkey)
+    puts "Skipping #{migration.to_key}.."
+    migration.tasks.each.with_index do |task, idx|
+      log = Hekenga::Log.where(pkey: task.to_key, task_idx: idx).first ||
+        Hekenga::Log.new(migration: migration, task_idx: idx)
+
+      log.done = true
+      log.skip = true
+
+      log.save!
+    end
+    puts "Done!"
+  end
+
+  private
+
+  def todo(op)
+    puts "#{op.capitalize} has not yet been implemented."
+    exit(99)
+  end
+  def bail_if_errors
+    if Hekenga.any_fatal?
+      puts "Refusing to run migrations while there is an existing cancelled migration."
+      exit(1)
+    end
+  end
+  def load_migration(path_or_pkey)
+    if File.exists?(File.expand_path(path_or_pkey))
+      require File.expand_path(path_or_pkey)
+      migration = Hekenga.registry.last
+    else
+      Hekenga.load_all!
+      unless migration = Hekenga.find_migration(path_or_pkey)
+        puts "Can't find migration #{path_or_pkey}. Available migrations:"
+        puts Hekenga.registry.map {|x| "- #{x.to_key}"}
+        exit(2)
+      end
+    end
+    migration
+  end
+end
+
+HekengaCLI.start

data/hekenga.gemspec

@@ -0,0 +1,32 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'hekenga/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "hekenga"
+  spec.version       = Hekenga::VERSION
+  spec.authors       = ["Tapio Saarinen"]
+  spec.email         = ["admin@bitlong.org"]
+
+  spec.summary       = %q{Sophisticated migration framework for mongoid, with the ability to parallelise via ActiveJob.}
+  spec.homepage      = "https://github.com/tzar/hekenga"
+
+  spec.files         = `git ls-files -z`.split("\x0").reject do |f|
+    f.match(%r{^(test|spec|features)/})
+  end
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.13"
+  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "rspec", "~> 3.0"
+  spec.add_development_dependency "database_cleaner"
+  spec.add_development_dependency "pry"
+  spec.add_development_dependency "pry-byebug"
+
+  spec.add_runtime_dependency "mongoid", "~> 5"
+  spec.add_runtime_dependency "activejob", "~> 4"
+  spec.add_runtime_dependency "thor"
+end

data/lib/hekenga/base_error.rb

@@ -0,0 +1,4 @@
+module Hekenga
+  class BaseError < StandardError
+  end
+end

data/lib/hekenga/config.rb

@@ -0,0 +1,13 @@
+module Hekenga
+  class Config
+    attr_accessor :dir, :root, :report_sleep
+    def initialize
+      @report_sleep = 10
+      @root         = Dir.pwd
+      @dir          = ["db", "hekenga"]
+    end
+    def abs_dir
+      File.join(@root, *[@dir].flatten)
+    end
+  end
+end

data/lib/hekenga/context.rb

@@ -0,0 +1,14 @@
+module Hekenga
+  class Context
+    def initialize(test_run)
+      @__test_run = test_run
+    end
+
+    def test?
+      !!@__test_run
+    end
+    def actual?
+      !@__test_run
+    end
+  end
+end

data/lib/hekenga/document_task.rb

@@ -0,0 +1,38 @@
+require 'hekenga/irreversible'
+module Hekenga
+  class DocumentTask
+    attr_reader :ups, :downs, :setups, :filters
+    attr_accessor :parallel, :disable_rules, :scope, :timeless
+    attr_accessor :description, :invalid_strategy, :skip_prepare
+    def initialize
+      @ups              = []
+      @downs            = []
+      @disable_rules    = []
+      @setups           = []
+      @filters          = []
+      @invalid_strategy = :prompt
+      @skip_prepare     = false
+    end
+
+    def validate!
+      raise Hekenga::Invalid.new(self, :ups, "missing") unless ups.any?
+    end
+
+    def up!(context, document)
+      @ups.each do |block|
+        context.instance_exec(document, &block)
+      end
+    end
+
+    def down!(context, document)
+      raise Hekenga::Irreversible.new(self) unless reversible?
+      @downs.each do |block|
+        context.instance_eval(document, &block)
+      end
+    end
+
+    def reversible?
+      downs.any?
+    end
+  end
+end

data/lib/hekenga/dsl/document_task.rb

@@ -0,0 +1,53 @@
+require 'hekenga/document_task'
+module Hekenga
+  class DSL
+    class DocumentTask < Hekenga::DSL
+      configures Hekenga::DocumentTask
+
+      INVALID_BEHAVIOR_STRATEGIES = [:prompt, :cancel, :stop, :continue]
+
+      def when_invalid(val)
+        unless INVALID_BEHAVIOR_STRATEGIES.include?(val)
+          raise "Invalid value #{val}. Valid values for invalid_behavior are: #{INVALID_BEHAVIOR_STRATEGIES.join(", ")}."
+        end
+        @object.invalid_strategy = val
+      end
+
+      def scope(scope)
+        @object.scope = scope
+      end
+      def parallel!
+        @object.parallel = true
+      end
+      def timeless!
+        @object.timeless = true
+      end
+      def skip_prepare!
+        @object.skip_prepare = true
+      end
+      def disable_callback(callback, args = {})
+        [args[:on]].flatten.compact.each do |model|
+          @object.disable_rules.push({
+            klass:    model,
+            callback: callback
+          })
+        end
+      end
+      def setup(&block)
+        @object.setups.push block
+      end
+
+      def filter(&block)
+        @object.filters.push block
+      end
+
+      def up(&block)
+        @object.ups.push block
+      end
+
+      def down(&block)
+        @object.downs.push block
+      end
+    end
+  end
+end

data/lib/hekenga/dsl/migration.rb

@@ -0,0 +1,32 @@
+require 'hekenga/migration'
+require 'time'
+module Hekenga
+  class DSL
+    class Migration < Hekenga::DSL
+      configures Hekenga::Migration
+
+      def batch_size(size)
+        unless size.is_a?(Fixnum) && size > 0
+          raise "Invalid batch size #{size.inspect}"
+        end
+        @object.batch_size = size
+      end
+      def created(stamp = nil)
+        @object.stamp = Time.parse(stamp)
+      end
+      def task(description = nil, &block)
+        @object.tasks.push Hekenga::DSL::SimpleTask.new(description, &block).object
+      end
+      def per_document(description = nil, &block)
+        @object.tasks.push Hekenga::DSL::DocumentTask.new(description, &block).object
+      end
+
+      def inspect
+        "<#{self.class} - #{@object.description} (#{@object.stamp.strftime("%Y-%m-%d %H:%M")})>"
+      end
+    end
+  end
+end
+
+require 'hekenga/dsl/simple_task'
+require 'hekenga/dsl/document_task'

data/lib/hekenga/dsl/simple_task.rb

@@ -0,0 +1,14 @@
+require 'hekenga/simple_task'
+module Hekenga
+  class DSL
+    class SimpleTask < Hekenga::DSL
+      configures Hekenga::SimpleTask
+      def up(&block)
+        @object.ups.push(block)
+      end
+      def down(&block)
+        @object.downs.push(block)
+      end
+    end
+  end
+end

data/lib/hekenga/dsl.rb

@@ -0,0 +1,24 @@
+module Hekenga
+  class DSL
+    attr_reader :object
+    def initialize(description = nil, &block)
+      @object = self.class.build_klass&.new
+      description(description) if description
+      instance_exec(&block)
+      @object.validate! if @object.respond_to?(:validate!)
+    end
+    def description(desc = nil)
+      @object.description = desc if @object && desc
+    end
+    def inspect
+      "<#{self.class} - #{self.description}>"
+    end
+    def self.configures(klass)
+      @build_klass = klass
+    end
+    def self.build_klass
+      @build_klass
+    end
+  end
+end
+require 'hekenga/dsl/migration'

data/lib/hekenga/failure/cancelled.rb

@@ -0,0 +1,8 @@
+module Hekenga
+  class Failure
+    class Cancelled < Failure
+      field :document_ids, type: Array
+      field :batch_start
+    end
+  end
+end

data/lib/hekenga/failure/error.rb

@@ -0,0 +1,11 @@
+module Hekenga
+  class Failure
+    class Error < Failure
+      field :message
+      field :backtrace
+      field :document
+      field :batch_start
+      field :simple, default: false
+    end
+  end
+end

data/lib/hekenga/failure/validation.rb

@@ -0,0 +1,9 @@
+module Hekenga
+  class Failure
+    class Validation < Failure
+      field :doc_id, type: BSON::ObjectId
+      field :errs
+      field :document
+    end
+  end
+end

data/lib/hekenga/failure/write.rb

@@ -0,0 +1,11 @@
+module Hekenga
+  class Failure
+    class Write < Failure
+      field :message
+      field :backtrace
+      field :documents
+      field :document_ids, type: Array
+      field :batch_start
+    end
+  end
+end

data/lib/hekenga/failure.rb

@@ -0,0 +1,23 @@
+module Hekenga
+  class Failure
+    include Mongoid::Document
+    belongs_to :log, class_name: "Hekenga::Log"
+
+    # Internal tracking
+    field :pkey
+    field :task_idx
+
+    validates_presence_of [:pkey, :task_idx, :log_id]
+
+    index({pkey:   1})
+    index({log_id: 1})
+
+    def self.lookup(log_id, task_idx)
+      where(log_id: log_id, task_idx: task_idx)
+    end
+  end
+end
+require 'hekenga/failure/error'
+require 'hekenga/failure/write'
+require 'hekenga/failure/validation'
+require 'hekenga/failure/cancelled'

data/lib/hekenga/invalid.rb

@@ -0,0 +1,8 @@
+require 'hekenga/base_error'
+module Hekenga
+  class Invalid < Hekenga::BaseError
+    def initialize(instance, field, reason)
+      super("#{instance.class.to_s} has #{reason} #{field}")
+    end
+  end
+end

data/lib/hekenga/irreversible.rb

@@ -0,0 +1,8 @@
+require 'hekenga/base_error'
+module Hekenga
+  class Irreversible < Hekenga::BaseError
+    def initialize(task)
+      super("#{task.inspect} is not a reversible.")
+    end
+  end
+end

data/lib/hekenga/log.rb

@@ -0,0 +1,56 @@
+require 'hekenga/failure'
+
+module Hekenga
+  class Log
+    include Mongoid::Document
+    # Internal tracking
+    field :pkey
+    field :description
+    field :stamp
+    field :task_idx
+
+    validates_presence_of [:pkey, :description, :stamp, :task_idx]
+
+    # Status flags
+    field :done,   default: false
+    field :error,  default: false
+    field :cancel, default: false
+    field :skip,   default: false
+
+    # Used by document tasks
+    field :total
+    field :processed, default: 0
+    field :skipped,   default: 0
+    field :unvalid,   default: 0
+    field :started,   default: ->{ Time.now }
+    field :finished,  type:    Time
+
+    has_many :failures, class_name: "Hekenga::Failure"
+
+    index({pkey: 1, task_idx: 1}, unique: true)
+
+    def migration=(migration)
+      self.pkey        = migration.to_key
+      self.description = migration.description
+      self.stamp       = migration.stamp
+    end
+
+    def add_failure(attrs, klass)
+      self.failures.create({
+        pkey:     self.pkey,
+        task_idx: self.task_idx
+      }.merge(attrs), klass)
+    end
+
+    def incr_and_return(fields)
+      doc = self.class.where(_id: self.id).find_one_and_update({
+        :$inc => fields
+      }, return_document: :after, projection: fields.keys.map {|x| [x, 1]}.to_h)
+      fields.map do |field, _|
+        value = doc.send(field)
+        send("#{field}=", value)
+        [field, value]
+      end.to_h
+    end
+  end
+end