catena 0.0.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/lib/catena/lang.rb +133 -0
- data/lib/catena/scheduler.rb +157 -0
- data/lib/catena/task.rb +14 -0
- data/lib/catena.rb +28 -0
- metadata +48 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: a115e71f17148d94f71b32657f92812dd5ce8f15
|
4
|
+
data.tar.gz: a9ff15140beffd48598c7d8a30ab8a97a56da7ff
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: 97e499b07b87f98476c024828be4d0cdcdb32b3e3612f6687e2f6196ba9880a8a258dac53a89f7eebf5c1215769d7aa6b568538a8136189783f770c740b9279b
|
7
|
+
data.tar.gz: 4867fe0ba6e70df362cc76a9a7fc168dc86b7abb325cb25cbf15031a0497ed612fa2c6d12869495d766c5ca6fb907128df314f06ee4d2c25d65fe6fc0a609b56
|
data/lib/catena/lang.rb
ADDED
@@ -0,0 +1,133 @@
|
|
1
|
+
require 'funkify'
|
2
|
+
|
3
|
+
module Catena
|
4
|
+
module Lang
|
5
|
+
include Funkify
|
6
|
+
|
7
|
+
# we add the class methods to the base class so they don't have to.
|
8
|
+
def self.included(base_mod)
|
9
|
+
base_mod.extend ClassMethods
|
10
|
+
end
|
11
|
+
|
12
|
+
module ClassMethods
|
13
|
+
def def_task(task_name, &block)
|
14
|
+
self.class_eval do
|
15
|
+
# define the language func that creates bind task nodes
|
16
|
+
define_method(task_name) do |*args|
|
17
|
+
bind(__method__, *args)
|
18
|
+
end
|
19
|
+
|
20
|
+
callback_name = Lang.func_name_to_callback(task_name)
|
21
|
+
define_method(callback_name, &block)
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
25
|
+
|
26
|
+
# Helper functions
|
27
|
+
|
28
|
+
def self.callback_to_func_name(callback_name)
|
29
|
+
# strip the "__"
|
30
|
+
callback_name[2..-1]
|
31
|
+
end
|
32
|
+
|
33
|
+
def self.func_name_to_callback(func_name)
|
34
|
+
"__#{func_name}"
|
35
|
+
end
|
36
|
+
|
37
|
+
# basic tasks and their composition that return task nodes
|
38
|
+
|
39
|
+
def succeed(value)
|
40
|
+
{
|
41
|
+
"type" => "succeed",
|
42
|
+
"value" => value
|
43
|
+
}
|
44
|
+
end
|
45
|
+
|
46
|
+
def failure(error)
|
47
|
+
{
|
48
|
+
"type" => "failure",
|
49
|
+
"error" => error
|
50
|
+
}
|
51
|
+
end
|
52
|
+
|
53
|
+
# bind(callback_name, arg1, arg2, ...)
|
54
|
+
def bind(*args)
|
55
|
+
raise "Need at least callback_name" if args.length < 1
|
56
|
+
func_name = args[0]
|
57
|
+
func_args = args[1..-1] || []
|
58
|
+
{
|
59
|
+
"type" => "binding",
|
60
|
+
"callback_name" => Lang.func_name_to_callback(func_name),
|
61
|
+
"callback_args" => func_args,
|
62
|
+
"cancel" => nil,
|
63
|
+
}
|
64
|
+
end
|
65
|
+
|
66
|
+
auto_curry def and_then(bind_efx, efx_a)
|
67
|
+
binding_callback = bind_efx.is_a?(Symbol) ? bind(bind_efx) : bind_efx
|
68
|
+
raise "bind_efx needs to be a binding" if binding_callback["type"] != "binding"
|
69
|
+
|
70
|
+
{
|
71
|
+
"type" => "and_then",
|
72
|
+
"side_effect" => efx_a,
|
73
|
+
"binding_callback" => binding_callback,
|
74
|
+
}
|
75
|
+
end
|
76
|
+
|
77
|
+
auto_curry def on_error(bind_efx, efx_a)
|
78
|
+
binding_callback = bind_efx.is_a?(Symbol) ? bind(bind_efx) : bind_efx
|
79
|
+
raise "bind_efx needs to be a binding" if binding_callback["type"] != "binding"
|
80
|
+
|
81
|
+
{
|
82
|
+
"type" => "on_error",
|
83
|
+
"side_effect" => efx_a,
|
84
|
+
"binding_callback" => binding_callback,
|
85
|
+
}
|
86
|
+
end
|
87
|
+
|
88
|
+
# TODO We have #map2 to fan-in, but we don't have something that fans out
|
89
|
+
# example. We create an application space, but then need to upload two
|
90
|
+
# ml files to the space, and use the results to map into create_flask_app
|
91
|
+
|
92
|
+
auto_curry def map2(bind_efx, efx_b, efx_a)
|
93
|
+
binding_callback = bind_efx.is_a?(Symbol) ? bind(bind_efx) : bind_efx
|
94
|
+
raise "bind_efx needs to be a binding" if binding_callback["type"] != "binding"
|
95
|
+
|
96
|
+
pass(efx_a) >=
|
97
|
+
and_then(bind(:map2_a, binding_callback, efx_b))
|
98
|
+
end
|
99
|
+
|
100
|
+
# This only works because we're filling in all other args except val_a
|
101
|
+
auto_curry def __map2_a(bind_efx, efx_b, val_a, evaluator)
|
102
|
+
new_efx = pass(efx_b) >=
|
103
|
+
and_then(bind(:map2_b, bind_efx, val_a))
|
104
|
+
|
105
|
+
evaluator.call(new_efx)
|
106
|
+
end
|
107
|
+
|
108
|
+
auto_curry def __map2_b(bind_efx, val_a, val_b, evaluator)
|
109
|
+
# update binding to have val_a and val_b as arguments
|
110
|
+
func_name = Lang.callback_to_func_name(bind_efx["callback_name"])
|
111
|
+
args = bind_efx["callback_args"] + [val_a, val_b]
|
112
|
+
new_efx = bind(func_name, *args)
|
113
|
+
|
114
|
+
evaluator.call(new_efx)
|
115
|
+
end
|
116
|
+
|
117
|
+
auto_curry def smap(bind_efx, efxs)
|
118
|
+
self.send("map#{efxs.length}", bind_efx, *efxs.reverse)
|
119
|
+
end
|
120
|
+
|
121
|
+
# Unused. Sample usage
|
122
|
+
# pass([store_to_cloud("arch.hdf5"), store_to_cloud("model.hdf5")]) >=
|
123
|
+
# pmap(:tag_docker_image)
|
124
|
+
auto_curry def pmap(callback_name, efxs)
|
125
|
+
{
|
126
|
+
"type" => "pmap",
|
127
|
+
"side_effects" => efxs,
|
128
|
+
"serialized_callback" => "__#{callback_name}",
|
129
|
+
}
|
130
|
+
end
|
131
|
+
|
132
|
+
end
|
133
|
+
end
|
@@ -0,0 +1,157 @@
|
|
1
|
+
require_relative 'lang'
|
2
|
+
require 'sidekiq'
|
3
|
+
|
4
|
+
module Catena
|
5
|
+
class Scheduler
|
6
|
+
include Lang
|
7
|
+
include Sidekiq::Worker
|
8
|
+
|
9
|
+
MAX_STEPS = 10
|
10
|
+
|
11
|
+
def perform(efx, stack)
|
12
|
+
step(efx, stack, 0)
|
13
|
+
end
|
14
|
+
|
15
|
+
def step(efx, stack, steps)
|
16
|
+
if (steps > MAX_STEPS)
|
17
|
+
logger.warn "Exceeded MAX STEPS. Stowing efx #{efx}"
|
18
|
+
enqueue(efx, stack)
|
19
|
+
steps
|
20
|
+
else
|
21
|
+
# TODO if not use >= after pass, will have efx["type"] is nil, need error message
|
22
|
+
# TODO if has trailing "|" in chain, it'll subsume the evaluator call, and return nil
|
23
|
+
# TODO also need error message when argument length mismatches
|
24
|
+
logger.debug "EFX: #{efx.inspect}"
|
25
|
+
send("step_#{efx["type"]}", efx, stack, steps)
|
26
|
+
end
|
27
|
+
end
|
28
|
+
|
29
|
+
def step_succeed(efx, stack, steps)
|
30
|
+
logger.debug "Processing succeed: #{efx}. Stack.len = #{stack.length}"
|
31
|
+
new_stack = flush("on_error", stack)
|
32
|
+
logger.debug " Flushed on_error. Stack.len = #{new_stack.length}"
|
33
|
+
|
34
|
+
if new_stack.empty?
|
35
|
+
return steps
|
36
|
+
else
|
37
|
+
and_then_node = new_stack.pop()
|
38
|
+
callback_node = and_then_node["binding_callback"] # the callback node is type binding
|
39
|
+
logger.debug " Popped #{callback_node}. Stack.len = #{new_stack.length}"
|
40
|
+
|
41
|
+
new_efx = chain(callback_node, efx["value"])
|
42
|
+
raise "step_succeed new_efx is nil. succeed value: #{efx["value"]}" if new_efx.nil?
|
43
|
+
step(new_efx, new_stack, steps + 1)
|
44
|
+
end
|
45
|
+
end
|
46
|
+
|
47
|
+
def step_failure(efx, stack, steps)
|
48
|
+
logger.info "Processing failure: #{efx}. Stack.len = #{stack.length}"
|
49
|
+
new_stack = flush("and_then", stack)
|
50
|
+
#puts " Flushed and_then. Stack.len = #{new_stack.length}"
|
51
|
+
|
52
|
+
if new_stack.empty?
|
53
|
+
return steps
|
54
|
+
else
|
55
|
+
on_error_node = new_stack.pop()
|
56
|
+
callback_node = on_error_node["binding_callback"]
|
57
|
+
logger.debug " Popped #{callback_node}. Stack.len = #{new_stack.length}"
|
58
|
+
|
59
|
+
new_efx = chain(callback_node, efx["error"])
|
60
|
+
raise "step_failure new_efx is nil. failure value: #{efx["error"]}" if new_efx.nil?
|
61
|
+
step(new_efx, new_stack, steps + 1)
|
62
|
+
end
|
63
|
+
end
|
64
|
+
|
65
|
+
def step_binding(efx, stack, steps)
|
66
|
+
# TODO canceling the entire process should happen at binding
|
67
|
+
logger.info "Processing binding of #{efx["callback_name"]}"
|
68
|
+
|
69
|
+
# FIXME shouldn't need to know explicitly the tasks are on Deployment class
|
70
|
+
|
71
|
+
callback = find_callback(efx["callback_name"])
|
72
|
+
args = efx["callback_args"] + [evaluator(stack)]
|
73
|
+
logger.debug " Calling '#{efx["callback_name"]}' with args: #{args.inspect}"
|
74
|
+
|
75
|
+
# FIXME check the arity and note if we're short?
|
76
|
+
# if we're at the end, and we're short on arguments, it'll happyly execute,
|
77
|
+
# return a lambda, and silently finish
|
78
|
+
callback.call(*args)
|
79
|
+
|
80
|
+
return steps + 1
|
81
|
+
end
|
82
|
+
|
83
|
+
def step_and_then(efx, stack, steps)
|
84
|
+
new_stack = stack.push(efx)
|
85
|
+
logger.info "Processing and_then. Stack.len = #{new_stack.length}"
|
86
|
+
|
87
|
+
raise "step_and_then new_efx is nil" if efx["side_effect"].nil?
|
88
|
+
step(efx["side_effect"], new_stack, steps + 1)
|
89
|
+
end
|
90
|
+
|
91
|
+
def step_on_error(efx, stack, steps)
|
92
|
+
logger.warn "Processing on_error...not implemented"
|
93
|
+
return steps
|
94
|
+
end
|
95
|
+
|
96
|
+
def step_pmap(efx, stack, steps)
|
97
|
+
logger.warn "Processing parallel map...not implemented"
|
98
|
+
return steps
|
99
|
+
end
|
100
|
+
|
101
|
+
#######################
|
102
|
+
|
103
|
+
private
|
104
|
+
|
105
|
+
# TODO should raise if callback isn't found
|
106
|
+
def find_callback(name)
|
107
|
+
mod_with_callback = Catena.config.modules.find do |mod|
|
108
|
+
mod = mod.is_a?(String) ? class_from_name(mod) : mod
|
109
|
+
mod.respond_to?(name)
|
110
|
+
end
|
111
|
+
return mod_with_callback.method(name)
|
112
|
+
end
|
113
|
+
|
114
|
+
def class_from_name()
|
115
|
+
mod_name.split("::").inject(Object) do |mod, class_name|
|
116
|
+
mod.const_get(class_name)
|
117
|
+
end
|
118
|
+
end
|
119
|
+
|
120
|
+
def enqueue(efx, stack)
|
121
|
+
logger.debug "Enqueued bound_efx: #{efx}"
|
122
|
+
Scheduler.perform_async(efx, stack)
|
123
|
+
end
|
124
|
+
|
125
|
+
def flush(node_type, stack)
|
126
|
+
stack.reject { |node| node["type"] == node_type }
|
127
|
+
end
|
128
|
+
|
129
|
+
# create another binding with the added efx["value"], and
|
130
|
+
# send it through a new step.
|
131
|
+
# while we can use funkify to partially apply, it doesn't work here, because
|
132
|
+
# we need to serialize it, and so we delay execution by creating another bind
|
133
|
+
# step through.
|
134
|
+
#
|
135
|
+
# same as calling the func that returns bind in Task
|
136
|
+
# TODO should use that so don't have to include Interpreter?
|
137
|
+
# But then still have to solve the problem of knowing about Deployment in step_binding
|
138
|
+
# TODO maybe just destructively update binding_callbcak with new value in args?
|
139
|
+
def chain(callback_node, value_or_error)
|
140
|
+
func_name = Lang.callback_to_func_name(callback_node["callback_name"])
|
141
|
+
args = callback_node["callback_args"] + [value_or_error]
|
142
|
+
return bind(func_name, *args)
|
143
|
+
end
|
144
|
+
|
145
|
+
def evaluator(stack)
|
146
|
+
lambda { |result_efx|
|
147
|
+
# FIXME need to check if result_efx is nil and then throw
|
148
|
+
# - Do you have a trailing "|"?
|
149
|
+
# FIXME need to check if result_efx is actually an efx
|
150
|
+
logger.info "Enqueuing result for evaluation"
|
151
|
+
logger.debug " result: #{result_efx}"
|
152
|
+
enqueue(result_efx, stack)
|
153
|
+
}
|
154
|
+
end
|
155
|
+
|
156
|
+
end
|
157
|
+
end
|
data/lib/catena/task.rb
ADDED
data/lib/catena.rb
ADDED
@@ -0,0 +1,28 @@
|
|
1
|
+
require_relative 'catena/scheduler'
|
2
|
+
require_relative 'catena/lang'
|
3
|
+
require 'forwardable'
|
4
|
+
|
5
|
+
module Catena
|
6
|
+
Configuration = Struct.new(:modules)
|
7
|
+
|
8
|
+
class << self
|
9
|
+
extend Forwardable
|
10
|
+
|
11
|
+
attr_reader :config
|
12
|
+
|
13
|
+
def configure(&block)
|
14
|
+
@config = Configuration.new if @config.nil?
|
15
|
+
block.call(@config)
|
16
|
+
end
|
17
|
+
|
18
|
+
def perform(task)
|
19
|
+
Catena::Scheduler.perform_async(task, [])
|
20
|
+
end
|
21
|
+
|
22
|
+
def perform_now(task)
|
23
|
+
# Need to use Scheduler's find_callback to then run it
|
24
|
+
end
|
25
|
+
|
26
|
+
end
|
27
|
+
|
28
|
+
end
|
metadata
ADDED
@@ -0,0 +1,48 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: catena
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.0.1
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Wil Chung
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
date: 2017-11-06 00:00:00.000000000 Z
|
12
|
+
dependencies: []
|
13
|
+
description: Catena lets you write and compose background tasks in a flexible way
|
14
|
+
to model business processes
|
15
|
+
email: iamwil@gmail.com
|
16
|
+
executables: []
|
17
|
+
extensions: []
|
18
|
+
extra_rdoc_files: []
|
19
|
+
files:
|
20
|
+
- lib/catena.rb
|
21
|
+
- lib/catena/lang.rb
|
22
|
+
- lib/catena/scheduler.rb
|
23
|
+
- lib/catena/task.rb
|
24
|
+
homepage: https://github.com/iamwilhelm/catena
|
25
|
+
licenses:
|
26
|
+
- MIT
|
27
|
+
metadata: {}
|
28
|
+
post_install_message:
|
29
|
+
rdoc_options: []
|
30
|
+
require_paths:
|
31
|
+
- lib
|
32
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
33
|
+
requirements:
|
34
|
+
- - ">="
|
35
|
+
- !ruby/object:Gem::Version
|
36
|
+
version: '0'
|
37
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
38
|
+
requirements:
|
39
|
+
- - ">="
|
40
|
+
- !ruby/object:Gem::Version
|
41
|
+
version: '0'
|
42
|
+
requirements: []
|
43
|
+
rubyforge_project:
|
44
|
+
rubygems_version: 2.6.12
|
45
|
+
signing_key:
|
46
|
+
specification_version: 4
|
47
|
+
summary: Chainable background tasks
|
48
|
+
test_files: []
|