ntl-orchestra 0.9.0

data/lib/orchestra.rb

@@ -0,0 +1,35 @@
+require "forwardable"
+require "invokr"
+require "observer"
+require "securerandom"
+
+module Orchestra
+  extend self
+
+  def configure &block
+    Configuration.module_eval &block
+  end
+
+  def define &block
+    builder = DSL::Operations::Builder.new
+    DSL::Operations::Context.evaluate builder, &block
+    builder.build_operation
+  end
+
+  def perform operation, inputs = {}
+    Conductor.new.perform operation, inputs
+  end
+
+  def replay_recording operation, store, input = {}
+    store = Util.recursively_symbolize store
+    input = input.merge store[:input]
+    svc_recordings = store[:service_recordings]
+    Recording.replay operation, input, svc_recordings
+  end
+
+  load File.expand_path('../orchestra/errors.rb', __FILE__)
+end
+
+Dir[File.expand_path '../orchestra/**/*.rb', __FILE__].each do |rb_file|
+  load rb_file
+end

data/orchestra.gemspec

@@ -0,0 +1,26 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift lib unless $LOAD_PATH.include?(lib)
+require 'orchestra/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "ntl-orchestra"
+  spec.version       = Orchestra::VERSION
+  spec.authors       = ["ntl"]
+  spec.email         = ["nathanladd+github@gmail.com"]
+  spec.summary       = %q{Orchestrate complex operations with ease.}
+  spec.description   = %q{Orchestra is an orchestration framework for designing complex operations in an object oriented fashion.}
+  spec.homepage      = "https://github.com/ntl/orchestra"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files -z`.split("\x0")
+  spec.executables   = []
+  spec.test_files    = spec.files.grep(%r{^test/})
+  spec.require_paths = %w(lib)
+
+  spec.add_dependency "invokr"
+
+  spec.add_development_dependency "bundler", "~> 1.6"
+  spec.add_development_dependency "pry"
+  spec.add_development_dependency "rake", "~> 10.0"
+end

data/test/examples/fizz_buzz.rb

@@ -0,0 +1,32 @@
+module Examples
+  FizzBuzz = Orchestra.define do
+    node :make_array do
+      depends_on :up_to
+      provides :array
+      perform do
+        up_to.times.to_a
+      end
+    end
+
+    node :apply_fizzbuzz do
+      iterates_over :array
+      provides :fizzbuzz
+      perform do |num|
+        next if num == 0 # filter 0 from the output
+        str = ''
+        str << "Fizz" if num % 3 == 0
+        str << "Buzz" if num % 5 == 0
+        str << num.to_s if str.empty?
+        str
+      end
+    end
+
+    finally :print do
+      depends_on :io
+      iterates_over :fizzbuzz
+      perform do |str|
+        io.puts str
+      end
+    end
+  end
+end

data/test/examples/invitation_service.rb

@@ -0,0 +1,118 @@
+module Examples
+  InvitationService = Orchestra.define do
+    DEFAULT_MESSAGE = "I would really love for you to try out MyApp."
+    ROBOT_FOLLOWER_THRESHHOLD = 500
+
+    node :fetch_followers do
+      depends_on :account_name, :http
+      provides :followers
+      perform do
+        json = http.get "flutter.io", "/users/#{account_name}/followers"
+          JSON.parse json
+      end
+    end
+
+    node :fetch_blacklist do
+      depends_on :db
+      provides :blacklist
+      perform do
+        rows = db.execute "SELECT account_name FROM blacklists"
+        rows.map do |row| row.fetch 0 end
+      end
+    end
+
+    node :remove_blacklisted_followers do
+      depends_on :blacklist
+      modifies :followers
+      perform do
+        followers.reject! do |follower|
+          account_name = follower.fetch 'username'
+          blacklist.include? account_name
+        end
+      end
+    end
+
+    node :filter_robots do
+      depends_on :http
+      modifies :followers, :collection => true
+      perform do |follower|
+        account_name = follower.fetch 'username'
+        json = http.get "flutter.io", "/users/#{account_name}"
+        account = JSON.load json
+        next unless account['following'] > ROBOT_FOLLOWER_THRESHHOLD
+        next unless account['following'] > (account['followers'] / 2)
+        follower
+      end
+    end
+
+    finally :deliver_emails do
+      depends_on :smtp, :message => DEFAULT_MESSAGE
+      iterates_over :followers
+      perform do |follower|
+        email = follower.fetch 'email_address'
+        smtp.deliver message, :to => email
+      end
+    end
+  end
+
+  module InvitationService::TestSetup
+    private
+
+    def build_example_database
+      db = SQLite3::Database.new ':memory:'
+      db.execute <<-SQL
+      CREATE TABLE blacklists (
+        id INTEGER PRIMARY KEY AUTOINCREMENT,
+        account_name VARCHAR(255)
+      )
+      SQL
+      db.execute 'INSERT INTO blacklists(account_name) VALUES("mister_ed")'
+      db.execute 'INSERT INTO blacklists(account_name) VALUES("palpatine4")'
+      db
+    end
+
+    def build_example_smtp
+      SMTPCatcher.new
+    end
+
+    def stub_accounts_requests
+      requests = {
+        'mister_ed'        => { 'following' => 5192,  'followers' => 4820 },
+        'captain_sheridan' => { 'following' => 12840, 'followers' => 523  },
+      }
+
+      requests.each do |account_name, response|
+        stub = stub_request :get, "http://flutter.io/users/#{account_name}"
+        stub.to_return :body => response.to_json
+        stub.times 1
+        stub
+      end
+    end
+
+    def stub_followers_request
+      response = [
+        { 'username' => 'mister_ed', 'email_address' => 'ed@mistered.com' },
+        { 'username' => 'captain_sheridan', 'email_address' => 'captain_sheridan@babylon5.earth.gov' },
+      ]
+
+      followers_stub = stub_request :get, "http://flutter.io/users/realntl/followers"
+      followers_stub.to_return :body => response.to_json
+      followers_stub.times 1
+      followers_stub
+    end
+
+    class SMTPCatcher
+      attr :delivered
+
+      def initialize
+        @delivered = {}
+      end
+
+      def deliver message, args = {}
+        recipient, _ = Orchestra::Util.extract_key_args args, :to
+        delivered[recipient] = message
+      end
+    end
+
+  end
+end

data/test/integration/multithreading_test.rb

@@ -0,0 +1,38 @@
+class MultithreadingTest < Minitest::Test
+  CustomError = Class.new StandardError
+
+  def setup
+    @operation = Orchestra.define do
+      node :map_thread_ids do
+        iterates_over :list
+        provides :thread_ids
+        perform do |item|
+          raise CustomError, "blow up" if item == :blow_up
+          Thread.current.object_id
+        end
+      end
+
+      self.result = :thread_ids
+    end
+
+    @conductor = Orchestra::Conductor.new
+    @conductor.thread_count = 5
+  end
+
+  def test_multithreading
+    list = (1..100).to_a
+
+    thread_ids = @conductor.perform @operation, :list => list
+
+    assert thread_ids.uniq.size > 2, "performance must be spread across threads"
+  end
+
+  def test_exception_during_multithreading
+    list = (1..100).to_a
+    list[23] = :blow_up
+
+    assert_raises CustomError do
+      @conductor.perform @operation, :list => list
+    end
+  end
+end

data/test/integration/recording_telemetry_test.rb

@@ -0,0 +1,86 @@
+class RecordingTelemetryTest < Minitest::Test
+  def test_recording_telemetry
+    output = StringIO.new
+    telemetry = {}
+
+    perform_with_telemetry telemetry, output
+
+    assert_equal_telemetry expected_telemetry, telemetry
+  end
+
+  private
+
+  def assert_equal_telemetry expected, actual
+    expected.keys.each do |key|
+      assert_equal expected[key], actual[key]
+    end
+  end
+
+  def expected_telemetry
+    {
+      :input => { :up_to => 16 },
+      :movements => {
+        :make_array => {
+          :input  => { :up_to => 16 },
+          :output => { :array => [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15] },
+        },
+        :apply_fizzbuzz => {
+          :input  => { :array => [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15] },
+          :output => {
+            :fizzbuzz => [
+              "1",
+              "2",
+              "Fizz",
+              "4",
+              "Buzz",
+              "Fizz",
+              "7",
+              "8",
+              "Fizz",
+              "Buzz",
+              "11",
+              "Fizz",
+              "13",
+              "14",
+              "FizzBuzz",
+            ],
+          },
+        },
+        :print => {
+          :input => {
+            :fizzbuzz => [
+              "1",
+              "2",
+              "Fizz",
+              "4",
+              "Buzz",
+              "Fizz",
+              "7",
+              "8",
+              "Fizz",
+              "Buzz",
+              "11",
+              "Fizz",
+              "13",
+              "14",
+              "FizzBuzz",
+            ],
+          },
+          :output => { :print => [] },
+        }
+      },
+      :output => [],
+    }
+  end
+
+  def perform_with_telemetry telemetry, io
+    conductor = Orchestra::Conductor.new :io => io
+
+    conductor.add_observer TelemetryRecorder.new telemetry
+
+    conductor.perform(
+      Examples::FizzBuzz,
+      :up_to => 16,
+    )
+  end
+end

data/test/integration/replayable_operation_test.rb

@@ -0,0 +1,53 @@
+class ReplayableOperationTest < Minitest::Test
+  include Examples::InvitationService::TestSetup
+
+  def test_replaying_an_operation_from_a_previous_recording
+    # Perform the operation against real services, saving a recording
+    recording = perform_for_real
+
+    # Write the recording out to a file. In this case, a StringIO is used for
+    # simplicity, and we serialize into JSON
+    file = StringIO.new
+    file.write JSON.dump recording
+    file.rewind
+
+    # Replay the operation, directing SMTP to an alternative service object
+    smtp_service = build_example_smtp
+    Orchestra.replay_recording(
+      Examples::InvitationService,
+      JSON.load(file.read),
+      :smtp => smtp_service,
+    )
+
+    # While replaying, the operation delivered all email using the alterantive
+    # SMTP service object we passed in
+    assert_equal(
+      ["captain_sheridan@babylon5.earth.gov"],
+      smtp_service.delivered.keys,
+    )
+  end
+
+  private
+
+  def perform_for_real
+    mock_smtp = build_example_smtp
+    db = build_example_database
+    stub_followers_request
+    stub_accounts_requests
+
+    conductor = Orchestra::Conductor.new(
+      :db   => db,
+      :http => Net::HTTP,
+      :smtp => mock_smtp,
+    )
+
+    recording = conductor.record(
+      Examples::InvitationService,
+      :account_name => 'realntl',
+    )
+
+    db.close
+
+    recording.to_h
+  end
+end

data/test/lib/console.rb

@@ -0,0 +1,103 @@
+module Console
+  extend self
+
+  def load
+    ENV['N'] ||= '4'
+    Bundler.require "debugger_#{RUBY_ENGINE}"
+    define_reload
+    define_rake
+    load_examples
+    clean_backtraces
+    puts <<-MESSAGE
+Debug console. Type `rake` to run the test suite. `bin/rake` also works for
+develompent environments that rely on binstubs.
+
+The example operations located in `test/examples` are loaded for you. To try one
+out:
+
+    [1] pry(Orchestra)> Orchestra.perform Examples::FizzBuzz, :up_to => 16, :io => $stdout
+
+See the README.md as well as the examples themselves for more information. You
+will need to reload the examples after raking (or invoking reload!)
+    MESSAGE
+  end
+
+  def load_examples
+    Dir["test/examples/**/*.rb"].each do |example_rb| Kernel.load example_rb end
+  end
+
+  def define_reload
+    Pry::Commands.block_command "reload!", "Reload gem code" do
+      puts "Reloading..."
+      Object.send :remove_const, :Examples if Object.const_defined? :Examples
+      Object.send :remove_const, :Orchestra if Object.const_defined? :Orchestra
+      Kernel.load "lib/orchestra.rb"
+      Console.load_examples
+      orchestra = Object.const_get :Orchestra
+      _pry_.binding_stack.push orchestra.__binding__
+      _pry_.binding_stack.shift
+    end
+  end
+
+  def define_rake
+    Pry::Commands.create_command %r{(?:bin/)?rake}, keep_retval: true do
+      description "Run the test suite"
+
+      def process
+        run "reload!"
+
+        status = TestRunner.run_in_subprocess
+
+        build_passed = status.exitstatus == 0
+        operator = args.shift
+        run_system_command(build_passed, operator, args) if operator
+        build_passed
+      end
+
+      def run_system_command build_passed, operator, system_args
+        if operator == '&&'
+          system *system_args if build_passed
+        elsif operator == '||' && !build_passed
+          system *system_args unless build_passed
+        else
+          raise ArgumentError, "Must supply either '&&' or '||' operators, followed by a shell command"
+        end
+      end
+    end
+  end
+
+  def clean_backtraces
+    if RUBY_ENGINE == "rbx"
+      Exception.class_eval do
+        def render_with_filtering *args
+          io = args[1] || STDERR
+          args[1] = BacktraceFiltering::Rubinius.new io
+          render_without_filtering *args
+        end
+        alias_method :render_without_filtering, :render
+        alias_method :render, :render_with_filtering
+      end
+    end
+  end
+
+  module BacktraceFiltering
+    REGEX = %r{(?:lib/orchestra|test)}i
+
+    class Rubinius
+      def initialize io
+        @io = io
+      end
+
+      def puts message = "\n"
+        dont_filter_yet = true
+        message.each_line do |line|
+          if line.match %r{ at }
+            next unless dont_filter_yet or line.match REGEX
+            dont_filter_yet = false
+          end
+          @io.puts line
+        end
+      end
+    end
+  end
+end

data/test/lib/test_runner.rb

@@ -0,0 +1,19 @@
+module TestRunner
+  extend self
+
+  def run_in_subprocess
+    pid = fork do run end
+    _, status = Process.wait2 pid
+    status
+  end
+
+  def run
+    Bundler.require :test
+    load 'test/test_helper.rb'
+    tests = Dir["test/**/*_test.rb"]
+    tests.select! do |test| test == ENV['TEST'] end if ENV['TEST']
+    tests.each &method(:load)
+    argv = (ENV['TESTOPTS'] || '').split %r{[[:space:]]+}
+    exit Minitest.run argv
+  end
+end

data/test/support/telemetry_recorder.rb

@@ -0,0 +1,49 @@
+class TelemetryRecorder
+  def initialize store
+    @store = store
+    @current_operation = nil
+    @embedded = false
+  end
+
+  def update message, *payload
+    method = "handle_#{message}"
+    public_send method, *payload if respond_to? method
+  end
+
+  def embedded?
+    @embedded
+  end
+
+  def handle_operation_entered operation_name, input
+    return if embedded?
+    @nodes = Hash.new do |hsh, key| hsh[key] = {} end
+    @store.update(
+      :input => input,
+      :movements => @nodes,
+      :performance_name => operation_name,
+      :service_calls => [],
+    )
+    @embedded = true
+  end
+
+  def handle_operation_exited operation_name, output
+    @store[:output] = output
+    @embedded = false
+  end
+
+  def handle_node_entered name, input
+    @nodes[name][:input] = input
+  end
+
+  def handle_node_exited name, output
+    @nodes[name][:output] = output
+  end
+
+  def handle_error_raised error
+    @store[:error] = error
+  end
+
+  def handle_service_accessed service_name, record
+    @store[:service_calls].<< record.merge :service => service_name
+  end
+end

data/test/test_helper.rb

@@ -0,0 +1,16 @@
+require "json"
+require "minitest/hell"
+require "minitest/mock"
+require "stringio"
+require "webmock/minitest"
+
+Dir['test/examples/**/*.rb'].each &method(:load) unless defined? Examples
+Dir['test/support/**/*.rb'].each &method(:load)
+
+WebMock.disable_net_connect!
+
+class Minitest::Test
+  def before_setup
+    Orchestra::Configuration.reset
+  end
+end

data/test/unit/conductor_test.rb

@@ -0,0 +1,25 @@
+class ConductorTest < Minitest::Test
+  def test_defaults_to_single_thread
+    conductor = Orchestra::Conductor.new
+
+    assert_equal 1, conductor.thread_count
+  end
+
+  def test_configuring_thread_pool_globally
+    Orchestra.configure do |defaults|
+      defaults.thread_count = 5
+    end
+
+    conductor = Orchestra::Conductor.new
+
+    assert_equal 5, conductor.thread_count
+  end
+
+  def test_configuring_thread_pool_on_an_instance
+    conductor = Orchestra::Conductor.new
+
+    conductor.thread_count = 5
+
+    assert_equal 5, conductor.thread_count
+  end
+end