catena 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: a115e71f17148d94f71b32657f92812dd5ce8f15
+  data.tar.gz: a9ff15140beffd48598c7d8a30ab8a97a56da7ff
+SHA512:
+  metadata.gz: 97e499b07b87f98476c024828be4d0cdcdb32b3e3612f6687e2f6196ba9880a8a258dac53a89f7eebf5c1215769d7aa6b568538a8136189783f770c740b9279b
+  data.tar.gz: 4867fe0ba6e70df362cc76a9a7fc168dc86b7abb325cb25cbf15031a0497ed612fa2c6d12869495d766c5ca6fb907128df314f06ee4d2c25d65fe6fc0a609b56

data/lib/catena/lang.rb

@@ -0,0 +1,133 @@
+require 'funkify'
+
+module Catena
+  module Lang
+    include Funkify
+
+    # we add the class methods to the base class so they don't have to.
+    def self.included(base_mod)
+      base_mod.extend ClassMethods
+    end
+
+    module ClassMethods
+      def def_task(task_name, &block)
+        self.class_eval do
+          # define the language func that creates bind task nodes
+          define_method(task_name) do |*args|
+            bind(__method__, *args)
+          end
+
+          callback_name = Lang.func_name_to_callback(task_name)
+          define_method(callback_name, &block)
+        end
+      end
+    end
+
+    # Helper functions
+
+    def self.callback_to_func_name(callback_name)
+      # strip the "__"
+      callback_name[2..-1]
+    end
+
+    def self.func_name_to_callback(func_name)
+      "__#{func_name}"
+    end
+
+    # basic tasks and their composition that return task nodes
+
+    def succeed(value)
+      {
+        "type" => "succeed",
+        "value" => value
+      }
+    end
+
+    def failure(error)
+      {
+        "type" => "failure",
+        "error" => error
+      }
+    end
+
+    # bind(callback_name, arg1, arg2, ...)
+    def bind(*args)
+      raise "Need at least callback_name" if args.length < 1
+      func_name = args[0]
+      func_args = args[1..-1] || []
+      {
+        "type" => "binding",
+        "callback_name" => Lang.func_name_to_callback(func_name),
+        "callback_args" => func_args,
+        "cancel" => nil,
+      }
+    end
+
+    auto_curry def and_then(bind_efx, efx_a)
+      binding_callback = bind_efx.is_a?(Symbol) ? bind(bind_efx) : bind_efx
+      raise "bind_efx needs to be a binding" if binding_callback["type"] != "binding"
+
+      {
+        "type" => "and_then",
+        "side_effect" => efx_a,
+        "binding_callback" => binding_callback,
+      }
+    end
+
+    auto_curry def on_error(bind_efx, efx_a)
+      binding_callback = bind_efx.is_a?(Symbol) ? bind(bind_efx) : bind_efx
+      raise "bind_efx needs to be a binding" if binding_callback["type"] != "binding"
+
+      {
+        "type" => "on_error",
+        "side_effect" => efx_a,
+        "binding_callback" => binding_callback,
+      }
+    end
+
+    # TODO We have #map2 to fan-in, but we don't have something that fans out
+    # example. We create an application space, but then need to upload two
+    # ml files to the space, and use the results to map into create_flask_app
+
+    auto_curry def map2(bind_efx, efx_b, efx_a)
+      binding_callback = bind_efx.is_a?(Symbol) ? bind(bind_efx) : bind_efx
+      raise "bind_efx needs to be a binding" if binding_callback["type"] != "binding"
+
+      pass(efx_a) >=
+        and_then(bind(:map2_a, binding_callback, efx_b))
+    end
+
+    # This only works because we're filling in all other args except val_a
+    auto_curry def __map2_a(bind_efx, efx_b, val_a, evaluator)
+      new_efx = pass(efx_b) >=
+        and_then(bind(:map2_b, bind_efx, val_a))
+
+      evaluator.call(new_efx)
+    end
+
+    auto_curry def __map2_b(bind_efx, val_a, val_b, evaluator)
+      # update binding to have val_a and val_b as arguments
+      func_name = Lang.callback_to_func_name(bind_efx["callback_name"])
+      args = bind_efx["callback_args"] + [val_a, val_b]
+      new_efx = bind(func_name, *args)
+
+      evaluator.call(new_efx)
+    end
+
+    auto_curry def smap(bind_efx, efxs)
+      self.send("map#{efxs.length}", bind_efx, *efxs.reverse)
+    end
+
+    # Unused. Sample usage
+    #  pass([store_to_cloud("arch.hdf5"), store_to_cloud("model.hdf5")]) >=
+    #    pmap(:tag_docker_image)
+    auto_curry def pmap(callback_name, efxs)
+      {
+        "type" => "pmap",
+        "side_effects" => efxs,
+        "serialized_callback" => "__#{callback_name}",
+      }
+    end
+
+  end
+end

data/lib/catena/scheduler.rb

@@ -0,0 +1,157 @@
+require_relative 'lang'
+require 'sidekiq'
+
+module Catena
+  class Scheduler
+    include Lang
+    include Sidekiq::Worker
+
+    MAX_STEPS = 10
+
+    def perform(efx, stack)
+      step(efx, stack, 0)
+    end
+
+    def step(efx, stack, steps)
+      if (steps > MAX_STEPS)
+        logger.warn "Exceeded MAX STEPS. Stowing efx #{efx}"
+        enqueue(efx, stack)
+        steps
+      else
+        # TODO if not use >= after pass, will have efx["type"] is nil, need error message
+        # TODO if has trailing "|" in chain, it'll subsume the evaluator call, and return nil
+        # TODO also need error message when argument length mismatches
+        logger.debug "EFX: #{efx.inspect}"
+        send("step_#{efx["type"]}", efx, stack, steps)
+      end
+    end
+
+    def step_succeed(efx, stack, steps)
+      logger.debug "Processing succeed: #{efx}. Stack.len = #{stack.length}"
+      new_stack = flush("on_error", stack)
+      logger.debug "  Flushed on_error. Stack.len = #{new_stack.length}"
+
+      if new_stack.empty?
+        return steps
+      else
+        and_then_node = new_stack.pop()
+        callback_node = and_then_node["binding_callback"] # the callback node is type binding
+        logger.debug "  Popped #{callback_node}. Stack.len = #{new_stack.length}"
+
+        new_efx = chain(callback_node, efx["value"])
+        raise "step_succeed new_efx is nil. succeed value: #{efx["value"]}" if new_efx.nil?
+        step(new_efx, new_stack, steps + 1)
+      end
+    end
+
+    def step_failure(efx, stack, steps)
+      logger.info "Processing failure: #{efx}. Stack.len = #{stack.length}"
+      new_stack = flush("and_then", stack)
+      #puts "  Flushed and_then. Stack.len = #{new_stack.length}"
+
+      if new_stack.empty?
+        return steps
+      else
+        on_error_node = new_stack.pop()
+        callback_node = on_error_node["binding_callback"]
+        logger.debug "  Popped #{callback_node}. Stack.len = #{new_stack.length}"
+
+        new_efx = chain(callback_node, efx["error"])
+        raise "step_failure new_efx is nil. failure value: #{efx["error"]}" if new_efx.nil?
+        step(new_efx, new_stack, steps + 1)
+      end
+    end
+
+    def step_binding(efx, stack, steps)
+      # TODO canceling the entire process should happen at binding
+      logger.info "Processing binding of #{efx["callback_name"]}"
+
+      # FIXME shouldn't need to know explicitly the tasks are on Deployment class
+
+      callback = find_callback(efx["callback_name"])
+      args = efx["callback_args"] + [evaluator(stack)]
+      logger.debug "  Calling '#{efx["callback_name"]}' with args: #{args.inspect}"
+
+      # FIXME check the arity and note if we're short?
+      # if we're at the end, and we're short on arguments, it'll happyly execute,
+      # return a lambda, and silently finish
+      callback.call(*args)
+
+      return steps + 1
+    end
+
+    def step_and_then(efx, stack, steps)
+      new_stack = stack.push(efx)
+      logger.info "Processing and_then. Stack.len = #{new_stack.length}"
+
+      raise "step_and_then new_efx is nil" if efx["side_effect"].nil?
+      step(efx["side_effect"], new_stack, steps + 1)
+    end
+
+    def step_on_error(efx, stack, steps)
+      logger.warn "Processing on_error...not implemented"
+      return steps
+    end
+
+    def step_pmap(efx, stack, steps)
+      logger.warn "Processing parallel map...not implemented"
+      return steps
+    end
+
+    #######################
+
+    private
+
+    # TODO should raise if callback isn't found
+    def find_callback(name)
+      mod_with_callback = Catena.config.modules.find do |mod|
+        mod = mod.is_a?(String) ? class_from_name(mod) : mod
+        mod.respond_to?(name)
+      end
+      return mod_with_callback.method(name)
+    end
+
+    def class_from_name()
+      mod_name.split("::").inject(Object) do |mod, class_name|
+        mod.const_get(class_name)
+      end
+    end
+
+    def enqueue(efx, stack)
+      logger.debug "Enqueued bound_efx: #{efx}"
+      Scheduler.perform_async(efx, stack)
+    end
+
+    def flush(node_type, stack)
+      stack.reject { |node| node["type"] == node_type }
+    end
+
+    # create another binding with the added efx["value"], and
+    # send it through a new step.
+    # while we can use funkify to partially apply, it doesn't work here, because
+    # we need to serialize it, and so we delay execution by creating another bind
+    # step through.
+    #
+    # same as calling the func that returns bind in Task
+    # TODO should use that so don't have to include Interpreter?
+    #      But then still have to solve the problem of knowing about Deployment in step_binding
+    # TODO maybe just destructively update binding_callbcak with new value in args?
+    def chain(callback_node, value_or_error)
+      func_name = Lang.callback_to_func_name(callback_node["callback_name"])
+      args = callback_node["callback_args"] + [value_or_error]
+      return bind(func_name, *args)
+    end
+
+    def evaluator(stack)
+      lambda { |result_efx|
+        # FIXME need to check if result_efx is nil and then throw
+        # - Do you have a trailing "|"?
+        # FIXME need to check if result_efx is actually an efx
+        logger.info "Enqueuing result for evaluation"
+        logger.debug "  result: #{result_efx}"
+        enqueue(result_efx, stack)
+      }
+    end
+
+  end
+end

data/lib/catena/task.rb

@@ -0,0 +1,14 @@
+
+module Catena
+
+  module Task
+    include Lang
+    include Funkify
+
+    auto_curry
+
+    # TODO move standard tasks here.
+
+  end
+
+end

data/lib/catena.rb

@@ -0,0 +1,28 @@
+require_relative 'catena/scheduler'
+require_relative 'catena/lang'
+require 'forwardable'
+
+module Catena
+  Configuration = Struct.new(:modules)
+
+  class << self
+    extend Forwardable
+
+    attr_reader :config
+
+    def configure(&block)
+      @config = Configuration.new if @config.nil?
+      block.call(@config)
+    end
+
+    def perform(task)
+      Catena::Scheduler.perform_async(task, [])
+    end
+
+    def perform_now(task)
+      # Need to use Scheduler's find_callback to then run it
+    end
+
+  end
+
+end

metadata

@@ -0,0 +1,48 @@
+--- !ruby/object:Gem::Specification
+name: catena
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Wil Chung
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2017-11-06 00:00:00.000000000 Z
+dependencies: []
+description: Catena lets you write and compose background tasks in a flexible way
+  to model business processes
+email: iamwil@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/catena.rb
+- lib/catena/lang.rb
+- lib/catena/scheduler.rb
+- lib/catena/task.rb
+homepage: https://github.com/iamwilhelm/catena
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.6.12
+signing_key: 
+specification_version: 4
+summary: Chainable background tasks
+test_files: []