funktor 0.2.18 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Gemfile.lock +23 -7
- data/funktor.gemspec +2 -0
- data/lib/active_job/queue_adapters/funktor_adapter.rb +45 -0
- data/lib/funktor.rb +7 -0
- data/lib/funktor/cli/bootstrap.rb +9 -1
- data/lib/funktor/cli/init.rb +17 -17
- data/lib/funktor/cli/templates/{config → funktor_config}/boot.rb +0 -0
- data/lib/funktor/cli/templates/{config → funktor_config}/environment.yml +2 -0
- data/lib/funktor/cli/templates/{config → funktor_config}/function_definitions/incoming_job_handler.yml +0 -0
- data/lib/funktor/cli/templates/{config → funktor_config}/function_definitions/work_queue_handler.yml +0 -0
- data/lib/funktor/cli/templates/{config → funktor_config}/funktor.yml +0 -0
- data/lib/funktor/cli/templates/{config → funktor_config}/iam_permissions/incoming_job_queue.yml +0 -0
- data/lib/funktor/cli/templates/{config → funktor_config}/iam_permissions/ssm.yml +0 -0
- data/lib/funktor/cli/templates/{config → funktor_config}/iam_permissions/work_queue.yml +0 -0
- data/lib/funktor/cli/templates/{config → funktor_config}/package.yml +1 -1
- data/lib/funktor/cli/templates/{config → funktor_config}/resources/cloudwatch_dashboard.yml +0 -0
- data/lib/funktor/cli/templates/{config → funktor_config}/resources/incoming_job_queue.yml +0 -0
- data/lib/funktor/cli/templates/{config → funktor_config}/resources/incoming_job_queue_user.yml +0 -0
- data/lib/funktor/cli/templates/{config → funktor_config}/resources/work_queue.yml +0 -0
- data/lib/funktor/cli/templates/{config → funktor_config}/ruby_layer.yml +0 -0
- data/lib/funktor/cli/templates/funktor_init.yml.tt +1 -1
- data/lib/funktor/cli/templates/lambda_event_handlers/incoming_job_handler.rb +1 -4
- data/lib/funktor/cli/templates/lambda_event_handlers/work_queue_handler.rb +1 -1
- data/lib/funktor/cli/templates/serverless.yml +4 -4
- data/lib/funktor/error_handler.rb +1 -1
- data/lib/funktor/fake_job_queue.rb +5 -2
- data/lib/funktor/job_pusher.rb +31 -0
- data/lib/funktor/rails.rb +19 -0
- data/lib/funktor/testing.rb +9 -7
- data/lib/funktor/version.rb +1 -1
- data/lib/funktor/worker.rb +8 -47
- data/lib/funktor/worker/funktor_options.rb +37 -0
- metadata +49 -17
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 25d6a36d1e21a09dda03678f7b81742dfec17ceb95c1220254a04690a3933d12
|
|
4
|
+
data.tar.gz: e43d8a4640ca2bced4c5d6e687a1f3b878016108790afeb7b502c0680e334363
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: c14ed21535c212873b4bddec9744467f6aa1ca25125aae1c4ea115a88e89ffc0dbc683916507400562c38c3041d15bcda9711265047bbcbc001195256dcf7543
|
|
7
|
+
data.tar.gz: 213dd13516d536fb9e253e6b2a59bee33771e5e1a90d1702f56de5c16c3aab5384641d0361df221f5c012bd25439e3bec5a26cc865c3321637604ffb0f86dd7a
|
data/Gemfile.lock
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
PATH
|
|
2
2
|
remote: .
|
|
3
3
|
specs:
|
|
4
|
-
funktor (0.
|
|
4
|
+
funktor (0.3.1)
|
|
5
5
|
activesupport
|
|
6
6
|
aws-sdk-sqs (~> 1.37)
|
|
7
7
|
thor
|
|
@@ -9,6 +9,9 @@ PATH
|
|
|
9
9
|
GEM
|
|
10
10
|
remote: https://rubygems.org/
|
|
11
11
|
specs:
|
|
12
|
+
activejob (6.1.4)
|
|
13
|
+
activesupport (= 6.1.4)
|
|
14
|
+
globalid (>= 0.3.6)
|
|
12
15
|
activesupport (6.1.4)
|
|
13
16
|
concurrent-ruby (~> 1.0, >= 1.0.2)
|
|
14
17
|
i18n (>= 1.6, < 2)
|
|
@@ -18,27 +21,38 @@ GEM
|
|
|
18
21
|
addressable (2.7.0)
|
|
19
22
|
public_suffix (>= 2.0.2, < 5.0)
|
|
20
23
|
aws-eventstream (1.1.1)
|
|
21
|
-
aws-partitions (1.
|
|
22
|
-
aws-sdk-core (3.
|
|
24
|
+
aws-partitions (1.481.0)
|
|
25
|
+
aws-sdk-core (3.118.0)
|
|
23
26
|
aws-eventstream (~> 1, >= 1.0.2)
|
|
24
27
|
aws-partitions (~> 1, >= 1.239.0)
|
|
25
28
|
aws-sigv4 (~> 1.1)
|
|
26
29
|
jmespath (~> 1.0)
|
|
27
|
-
aws-sdk-sqs (1.
|
|
28
|
-
aws-sdk-core (~> 3, >= 3.
|
|
30
|
+
aws-sdk-sqs (1.41.0)
|
|
31
|
+
aws-sdk-core (~> 3, >= 3.118.0)
|
|
29
32
|
aws-sigv4 (~> 1.1)
|
|
30
|
-
aws-sigv4 (1.2.
|
|
33
|
+
aws-sigv4 (1.2.4)
|
|
31
34
|
aws-eventstream (~> 1, >= 1.0.2)
|
|
35
|
+
byebug (11.1.3)
|
|
36
|
+
coderay (1.1.3)
|
|
32
37
|
concurrent-ruby (1.1.9)
|
|
33
38
|
crack (0.4.5)
|
|
34
39
|
rexml
|
|
35
40
|
diff-lcs (1.4.4)
|
|
36
41
|
docile (1.3.5)
|
|
42
|
+
globalid (0.5.1)
|
|
43
|
+
activesupport (>= 5.0)
|
|
37
44
|
hashdiff (1.0.1)
|
|
38
45
|
i18n (1.8.10)
|
|
39
46
|
concurrent-ruby (~> 1.0)
|
|
40
47
|
jmespath (1.4.0)
|
|
48
|
+
method_source (1.0.0)
|
|
41
49
|
minitest (5.14.4)
|
|
50
|
+
pry (0.13.1)
|
|
51
|
+
coderay (~> 1.1)
|
|
52
|
+
method_source (~> 1.0)
|
|
53
|
+
pry-byebug (3.9.0)
|
|
54
|
+
byebug (~> 11.0)
|
|
55
|
+
pry (~> 0.13.0)
|
|
42
56
|
public_suffix (4.0.6)
|
|
43
57
|
rake (12.3.3)
|
|
44
58
|
rexml (3.2.5)
|
|
@@ -74,11 +88,13 @@ PLATFORMS
|
|
|
74
88
|
ruby
|
|
75
89
|
|
|
76
90
|
DEPENDENCIES
|
|
91
|
+
activejob (>= 5.1.5)
|
|
77
92
|
funktor!
|
|
93
|
+
pry-byebug
|
|
78
94
|
rake (~> 12.0)
|
|
79
95
|
rspec (~> 3.0)
|
|
80
96
|
simplecov
|
|
81
97
|
webmock
|
|
82
98
|
|
|
83
99
|
BUNDLED WITH
|
|
84
|
-
2.
|
|
100
|
+
2.2.24
|
data/funktor.gemspec
CHANGED
|
@@ -33,6 +33,8 @@ Gem::Specification.new do |spec|
|
|
|
33
33
|
spec.add_dependency "activesupport" # TODO - Can we build our own verison of cattr_accessor to avoid this?
|
|
34
34
|
spec.add_dependency "thor" # Thor drives the CLI
|
|
35
35
|
|
|
36
|
+
spec.add_development_dependency 'activejob', '>= 5.1.5'
|
|
36
37
|
spec.add_development_dependency 'simplecov'
|
|
37
38
|
spec.add_development_dependency 'webmock'
|
|
39
|
+
spec.add_development_dependency 'pry-byebug'
|
|
38
40
|
end
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'active_job'
|
|
4
|
+
|
|
5
|
+
module ActiveJob
|
|
6
|
+
module QueueAdapters
|
|
7
|
+
# == Funktor adapter for Active Job
|
|
8
|
+
#
|
|
9
|
+
# To use Funktor set the queue_adapter config to +:funktor+.
|
|
10
|
+
#
|
|
11
|
+
# Rails.application.config.active_job.queue_adapter = :funktor
|
|
12
|
+
class FunktorAdapter
|
|
13
|
+
def enqueue(job) # :nodoc:
|
|
14
|
+
job.provider_job_id = Funktor.job_pusher.push({
|
|
15
|
+
"worker" => JobWrapper.to_s,
|
|
16
|
+
"wrapped" => job.class,
|
|
17
|
+
"queue" => job.class.work_queue,
|
|
18
|
+
"worker_params" => [ job.serialize ]
|
|
19
|
+
})
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
def enqueue_at(job, timestamp) # :nodoc:
|
|
23
|
+
job.provider_job_id = Funktor.job_pusher.push({
|
|
24
|
+
"worker" => JobWrapper.to_s,
|
|
25
|
+
"wrapped" => job.class,
|
|
26
|
+
"queue" => job.class.work_queue,
|
|
27
|
+
"worker_params" => [ job.serialize ],
|
|
28
|
+
"at" => timestamp
|
|
29
|
+
})
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
class JobWrapper #:nodoc:
|
|
33
|
+
include Funktor::Worker
|
|
34
|
+
|
|
35
|
+
def perform(job_data)
|
|
36
|
+
Base.execute job_data.first
|
|
37
|
+
end
|
|
38
|
+
end
|
|
39
|
+
end
|
|
40
|
+
end
|
|
41
|
+
|
|
42
|
+
class Base
|
|
43
|
+
include Funktor::Worker::FunktorOptions
|
|
44
|
+
end
|
|
45
|
+
end
|
data/lib/funktor.rb
CHANGED
|
@@ -3,6 +3,7 @@ require 'funktor/aws/sqs/event'
|
|
|
3
3
|
require 'funktor/aws/sqs/record'
|
|
4
4
|
require 'funktor/counter'
|
|
5
5
|
require 'funktor/job'
|
|
6
|
+
require 'funktor/job_pusher'
|
|
6
7
|
require 'funktor/logger'
|
|
7
8
|
require 'funktor/worker'
|
|
8
9
|
require 'funktor/middleware_chain'
|
|
@@ -22,6 +23,10 @@ module Funktor
|
|
|
22
23
|
yield self
|
|
23
24
|
end
|
|
24
25
|
|
|
26
|
+
def self.job_pusher
|
|
27
|
+
@job_pusher ||= JobPusher.new
|
|
28
|
+
end
|
|
29
|
+
|
|
25
30
|
def self.job_pusher_middleware
|
|
26
31
|
@job_pusher_chain ||= MiddlewareChain.new
|
|
27
32
|
yield @job_pusher_chain if block_given?
|
|
@@ -98,3 +103,5 @@ end
|
|
|
98
103
|
require 'funktor/middleware/metrics'
|
|
99
104
|
require 'funktor/error_handler'
|
|
100
105
|
require 'funktor/active_job_handler'
|
|
106
|
+
|
|
107
|
+
require 'funktor/rails' if defined?(::Rails::Engine)
|
|
@@ -9,6 +9,10 @@ module Funktor
|
|
|
9
9
|
:type => :string, :desc => "The bootstrap file to generate.",
|
|
10
10
|
:default => "funktor_init.yml"
|
|
11
11
|
|
|
12
|
+
class_option :directory, :aliases => "-d",
|
|
13
|
+
:type => :string, :desc => "The directory in which to place the bootstrap file.",
|
|
14
|
+
:default => nil
|
|
15
|
+
|
|
12
16
|
desc <<~DESC
|
|
13
17
|
Description:
|
|
14
18
|
Bootstrap a new funktor application by generating a funktor_init.yml file."
|
|
@@ -26,7 +30,11 @@ module Funktor
|
|
|
26
30
|
|
|
27
31
|
private
|
|
28
32
|
def funktor_file_target
|
|
29
|
-
File.join
|
|
33
|
+
File.join funktor_directory_target, options[:file]
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
def funktor_directory_target
|
|
37
|
+
options[:directory] || name
|
|
30
38
|
end
|
|
31
39
|
|
|
32
40
|
end
|
data/lib/funktor/cli/init.rb
CHANGED
|
@@ -32,11 +32,11 @@ module Funktor
|
|
|
32
32
|
|
|
33
33
|
def funktor_config_yml
|
|
34
34
|
#template "funktor_config.yml", File.join("funktor_config.yml")
|
|
35
|
-
template File.join("
|
|
36
|
-
template File.join("
|
|
37
|
-
template File.join("
|
|
38
|
-
template File.join("
|
|
39
|
-
template File.join("
|
|
35
|
+
template File.join("funktor_config", "funktor.yml"), File.join("funktor_config", "funktor.yml")
|
|
36
|
+
template File.join("funktor_config", "ruby_layer.yml"), File.join("funktor_config", "ruby_layer.yml")
|
|
37
|
+
template File.join("funktor_config", "package.yml"), File.join("funktor_config", "package.yml")
|
|
38
|
+
template File.join("funktor_config", "environment.yml"), File.join("funktor_config", "environment.yml")
|
|
39
|
+
template File.join("funktor_config", "boot.rb"), File.join("funktor_config", "boot.rb")
|
|
40
40
|
end
|
|
41
41
|
|
|
42
42
|
def package_json
|
|
@@ -56,33 +56,33 @@ module Funktor
|
|
|
56
56
|
end
|
|
57
57
|
|
|
58
58
|
def resources
|
|
59
|
-
template File.join("
|
|
60
|
-
template File.join("
|
|
59
|
+
template File.join("funktor_config", "resources", "incoming_job_queue.yml"), File.join("funktor_config", "resources", "incoming_job_queue.yml")
|
|
60
|
+
template File.join("funktor_config", "resources", "incoming_job_queue_user.yml"), File.join("funktor_config", "resources", "incoming_job_queue_user.yml")
|
|
61
61
|
# TODO - Figure out how to make the dashboard aware of various queues...
|
|
62
|
-
template File.join("
|
|
62
|
+
template File.join("funktor_config", "resources", "cloudwatch_dashboard.yml"), File.join("funktor_config", "resources", "cloudwatch_dashboard.yml")
|
|
63
63
|
queues.each do |queue_details|
|
|
64
64
|
@work_queue_name = queue_details.keys.first
|
|
65
65
|
@work_queue_config = queue_details.values.first
|
|
66
|
-
template File.join("
|
|
66
|
+
template File.join("funktor_config", "resources", "work_queue.yml"), File.join("funktor_config", "resources", "#{work_queue_name.underscore}_queue.yml")
|
|
67
67
|
end
|
|
68
68
|
end
|
|
69
69
|
|
|
70
70
|
def iam_permissions
|
|
71
|
-
template File.join("
|
|
72
|
-
template File.join("
|
|
71
|
+
template File.join("funktor_config", "iam_permissions", "ssm.yml"), File.join("funktor_config", "iam_permissions", "ssm.yml")
|
|
72
|
+
template File.join("funktor_config", "iam_permissions", "incoming_job_queue.yml"), File.join("funktor_config", "iam_permissions", "incoming_job_queue.yml")
|
|
73
73
|
queues.each do |queue_details|
|
|
74
74
|
@work_queue_name = queue_details.keys.first
|
|
75
75
|
@work_queue_config = queue_details.values.first
|
|
76
|
-
template File.join("
|
|
76
|
+
template File.join("funktor_config", "iam_permissions", "work_queue.yml"), File.join("funktor_config", "iam_permissions", "#{work_queue_name.underscore}_queue.yml")
|
|
77
77
|
end
|
|
78
78
|
end
|
|
79
79
|
|
|
80
80
|
def function_definitions
|
|
81
|
-
template File.join("
|
|
81
|
+
template File.join("funktor_config", "function_definitions", "incoming_job_handler.yml"), File.join("funktor_config", "function_definitions", "incoming_job_handler.yml")
|
|
82
82
|
queues.each do |queue_details|
|
|
83
83
|
@work_queue_name = queue_details.keys.first
|
|
84
84
|
@work_queue_config = queue_details.values.first
|
|
85
|
-
template File.join("
|
|
85
|
+
template File.join("funktor_config", "function_definitions", "work_queue_handler.yml"), File.join("funktor_config", "function_definitions", "#{work_queue_name.underscore}_queue_handler.yml")
|
|
86
86
|
end
|
|
87
87
|
end
|
|
88
88
|
|
|
@@ -112,15 +112,15 @@ module Funktor
|
|
|
112
112
|
end
|
|
113
113
|
|
|
114
114
|
def all_iam_permissions
|
|
115
|
-
Dir.glob(File.join('
|
|
115
|
+
Dir.glob(File.join('funktor_config', 'iam_permissions', '**.yml'))
|
|
116
116
|
end
|
|
117
117
|
|
|
118
118
|
def all_function_definitions
|
|
119
|
-
Dir.glob(File.join('
|
|
119
|
+
Dir.glob(File.join('funktor_config', 'function_definitions', '**.yml'))
|
|
120
120
|
end
|
|
121
121
|
|
|
122
122
|
def all_resources
|
|
123
|
-
Dir.glob(File.join('
|
|
123
|
+
Dir.glob(File.join('funktor_config', 'resources', '**.yml'))
|
|
124
124
|
end
|
|
125
125
|
|
|
126
126
|
def funktor_config
|
|
File without changes
|
|
File without changes
|
data/lib/funktor/cli/templates/{config → funktor_config}/function_definitions/work_queue_handler.yml
RENAMED
|
File without changes
|
|
File without changes
|
data/lib/funktor/cli/templates/{config → funktor_config}/iam_permissions/incoming_job_queue.yml
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
@@ -3,7 +3,7 @@ individually: false
|
|
|
3
3
|
include:
|
|
4
4
|
- Gemfile
|
|
5
5
|
- Gemfile.lock
|
|
6
|
-
-
|
|
6
|
+
- funktor_config/boot.rb
|
|
7
7
|
- app/**
|
|
8
8
|
# Evertyting is excluded by default with serverless-ruby-layer, but you could use
|
|
9
9
|
# the lines below to exlude files that are inside an include path.
|
|
File without changes
|
|
File without changes
|
data/lib/funktor/cli/templates/{config → funktor_config}/resources/incoming_job_queue_user.yml
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
@@ -1,7 +1,4 @@
|
|
|
1
|
-
|
|
2
|
-
# so instead of doing `require_relative '../config/boog'` we just manually require
|
|
3
|
-
# the one gem that we do need.
|
|
4
|
-
require 'funktor'
|
|
1
|
+
require_relative '../funktor_config/boot'
|
|
5
2
|
|
|
6
3
|
$handler = Funktor::IncomingJobHandler.new
|
|
7
4
|
|
|
@@ -24,7 +24,7 @@ provider:
|
|
|
24
24
|
name: aws
|
|
25
25
|
runtime: <%= runtime %>
|
|
26
26
|
lambdaHashingVersion: 20201221
|
|
27
|
-
environment: ${file(
|
|
27
|
+
environment: ${file(funktor_config/environment.yml)}
|
|
28
28
|
versionFunctions: false # Reduces the amount of storage used since all Lambdas together are limited to 75GB
|
|
29
29
|
iamRoleStatements:
|
|
30
30
|
<%- all_iam_permissions.each do |iam_permission| -%>
|
|
@@ -36,10 +36,10 @@ custom:
|
|
|
36
36
|
# Our stage is based on what is passed in when running serverless
|
|
37
37
|
# commands. Or fallsback to what we have set in the provider section.
|
|
38
38
|
stage: ${self:provider.stage, 'dev'}
|
|
39
|
-
funktor: ${file(
|
|
40
|
-
rubyLayer: ${file(
|
|
39
|
+
funktor: ${file(funktor_config/funktor.yml)}
|
|
40
|
+
rubyLayer: ${file(funktor_config/ruby_layer.yml)}
|
|
41
41
|
|
|
42
|
-
package: ${file(
|
|
42
|
+
package: ${file(funktor_config/package.yml)}
|
|
43
43
|
|
|
44
44
|
functions:
|
|
45
45
|
<%- all_function_definitions.each do |function_definition| -%>
|
|
@@ -2,7 +2,7 @@ module Funktor
|
|
|
2
2
|
module ErrorHandler
|
|
3
3
|
class Logger
|
|
4
4
|
def call(error, context)
|
|
5
|
-
Funktor.logger.warn(Funktor.dump_json(context))
|
|
5
|
+
Funktor.logger.warn(Funktor.dump_json(context)) if context
|
|
6
6
|
Funktor.logger.warn("#{error.class.name}: #{error.message}")
|
|
7
7
|
Funktor.logger.warn(error.backtrace.join("\n")) unless error.backtrace.nil?
|
|
8
8
|
end
|
|
@@ -1,7 +1,10 @@
|
|
|
1
|
+
require "active_support/core_ext/hash/indifferent_access"
|
|
2
|
+
|
|
1
3
|
module Funktor
|
|
2
4
|
module FakeJobQueue
|
|
3
|
-
def self.push(
|
|
4
|
-
|
|
5
|
+
def self.push(payload)
|
|
6
|
+
payload = payload.with_indifferent_access
|
|
7
|
+
jobs[payload["worker"].to_s].push(payload)
|
|
5
8
|
end
|
|
6
9
|
|
|
7
10
|
def self.jobs
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
require 'aws-sdk-sqs'
|
|
2
|
+
|
|
3
|
+
module Funktor
|
|
4
|
+
class JobPusher
|
|
5
|
+
|
|
6
|
+
def push(payload)
|
|
7
|
+
puts "payload ============"
|
|
8
|
+
pp payload
|
|
9
|
+
job_id = SecureRandom.uuid
|
|
10
|
+
payload[:job_id] = job_id
|
|
11
|
+
|
|
12
|
+
Funktor.job_pusher_middleware.invoke(payload) do
|
|
13
|
+
client.send_message({
|
|
14
|
+
queue_url: queue_url(payload),
|
|
15
|
+
message_body: Funktor.dump_json(payload)
|
|
16
|
+
})
|
|
17
|
+
return job_id
|
|
18
|
+
end
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
private
|
|
22
|
+
|
|
23
|
+
def client
|
|
24
|
+
@client ||= ::Aws::SQS::Client.new
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
def queue_url(payload)
|
|
28
|
+
payload[:incoming_job_queue_url] || ENV['FUNKTOR_INCOMING_JOB_QUEUE']
|
|
29
|
+
end
|
|
30
|
+
end
|
|
31
|
+
end
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
module Funktor
|
|
3
|
+
class Rails < ::Rails::Engine
|
|
4
|
+
# This hook happens after `Rails::Application` is inherited within
|
|
5
|
+
# config/application.rb and before config is touched, usually within the
|
|
6
|
+
# class block. Definitely before config/environments/*.rb and
|
|
7
|
+
# config/initializers/*.rb.
|
|
8
|
+
config.before_configuration do
|
|
9
|
+
if defined?(::ActiveJob)
|
|
10
|
+
require "sidekiq/worker"
|
|
11
|
+
require 'active_job/queue_adapters/funktor_adapter'
|
|
12
|
+
end
|
|
13
|
+
end
|
|
14
|
+
end if defined?(::Rails)
|
|
15
|
+
|
|
16
|
+
if defined?(::Rails) && ::Rails::VERSION::MAJOR < 5
|
|
17
|
+
raise "🚫 ERROR: Funktor does not support Rails versions under 5.x"
|
|
18
|
+
end
|
|
19
|
+
end
|
data/lib/funktor/testing.rb
CHANGED
|
@@ -17,9 +17,9 @@ module Funktor
|
|
|
17
17
|
|
|
18
18
|
def work_all_jobs
|
|
19
19
|
while jobs.any?
|
|
20
|
-
job_data = jobs.shift
|
|
21
|
-
worker = job_data[:worker]
|
|
22
|
-
worker_params = job_data[:
|
|
20
|
+
job_data = jobs.shift.with_indifferent_access
|
|
21
|
+
worker = Object.const_get job_data[:worker]
|
|
22
|
+
worker_params = job_data[:worker_params]
|
|
23
23
|
worker.new.perform(worker_params)
|
|
24
24
|
end
|
|
25
25
|
end
|
|
@@ -56,14 +56,16 @@ module Funktor
|
|
|
56
56
|
end
|
|
57
57
|
|
|
58
58
|
class InlineJobPusherMiddleware
|
|
59
|
-
def call(
|
|
60
|
-
|
|
59
|
+
def call(payload)
|
|
60
|
+
payload = payload.with_indifferent_access
|
|
61
|
+
worker = Object.const_get payload["worker"]
|
|
62
|
+
worker.new.perform(*payload["worker_params"])
|
|
61
63
|
end
|
|
62
64
|
end
|
|
63
65
|
|
|
64
66
|
class FakeJobPusherMiddleware
|
|
65
|
-
def call(
|
|
66
|
-
Funktor::FakeJobQueue.push(
|
|
67
|
+
def call(payload)
|
|
68
|
+
Funktor::FakeJobQueue.push(payload)
|
|
67
69
|
end
|
|
68
70
|
end
|
|
69
71
|
end
|
data/lib/funktor/version.rb
CHANGED
data/lib/funktor/worker.rb
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
require 'securerandom'
|
|
2
|
-
require 'aws-sdk-sqs'
|
|
3
2
|
require "active_support"
|
|
3
|
+
require 'funktor/worker/funktor_options'
|
|
4
4
|
|
|
5
5
|
module Funktor
|
|
6
6
|
class DelayTooLongError < StandardError; end
|
|
@@ -9,34 +9,10 @@ end
|
|
|
9
9
|
module Funktor::Worker
|
|
10
10
|
def self.included(base)
|
|
11
11
|
base.extend ClassMethods
|
|
12
|
-
base.
|
|
13
|
-
cattr_accessor :funktor_options_hash
|
|
14
|
-
#alias_method :perform_later, :perform_async
|
|
15
|
-
end
|
|
12
|
+
base.include(Funktor::Worker::FunktorOptions)
|
|
16
13
|
end
|
|
17
14
|
|
|
18
15
|
module ClassMethods
|
|
19
|
-
def funktor_options(options = {})
|
|
20
|
-
self.funktor_options_hash = options
|
|
21
|
-
end
|
|
22
|
-
|
|
23
|
-
def get_funktor_options
|
|
24
|
-
self.funktor_options_hash || {}
|
|
25
|
-
end
|
|
26
|
-
|
|
27
|
-
def custom_queue_url
|
|
28
|
-
get_funktor_options[:queue_url]
|
|
29
|
-
end
|
|
30
|
-
|
|
31
|
-
def custom_queue
|
|
32
|
-
get_funktor_options[:queue]
|
|
33
|
-
end
|
|
34
|
-
|
|
35
|
-
def queue_url
|
|
36
|
-
# TODO : Should this default to FUNKTOR_ACTIVE_JOB_QUEUE?
|
|
37
|
-
# Depends how e handle this in pro...?
|
|
38
|
-
custom_queue_url || ENV['FUNKTOR_INCOMING_JOB_QUEUE']
|
|
39
|
-
end
|
|
40
16
|
|
|
41
17
|
def perform_async(*worker_params)
|
|
42
18
|
self.perform_in(0, *worker_params)
|
|
@@ -54,39 +30,24 @@ module Funktor::Worker
|
|
|
54
30
|
if delay > max_delay
|
|
55
31
|
raise Funktor::DelayTooLongError.new("The delay can't be longer than #{max_delay} seconds. This is a limitation of SQS. Funktor Pro has mechanisms to work around this limitation.")
|
|
56
32
|
end
|
|
57
|
-
self.
|
|
33
|
+
self.push(delay, *worker_params)
|
|
58
34
|
end
|
|
59
35
|
|
|
60
|
-
def
|
|
61
|
-
|
|
62
|
-
payload
|
|
63
|
-
|
|
64
|
-
Funktor.job_pusher_middleware.invoke(self, payload) do
|
|
65
|
-
client.send_message({
|
|
66
|
-
queue_url: queue_url,
|
|
67
|
-
message_body: Funktor.dump_json(payload)
|
|
68
|
-
})
|
|
69
|
-
end
|
|
36
|
+
def push(delay, *worker_params)
|
|
37
|
+
payload = build_job_payload(delay, *worker_params)
|
|
38
|
+
Funktor.job_pusher.push(payload)
|
|
70
39
|
end
|
|
71
40
|
|
|
72
41
|
def max_delay
|
|
73
42
|
900
|
|
74
43
|
end
|
|
75
44
|
|
|
76
|
-
def
|
|
77
|
-
@client ||= Aws::SQS::Client.new
|
|
78
|
-
end
|
|
79
|
-
|
|
80
|
-
def work_queue
|
|
81
|
-
(self.custom_queue || 'default').to_s
|
|
82
|
-
end
|
|
83
|
-
|
|
84
|
-
def build_job_payload(job_id, delay, *worker_params)
|
|
45
|
+
def build_job_payload(delay, *worker_params)
|
|
85
46
|
{
|
|
86
47
|
worker: self.name,
|
|
87
48
|
worker_params: worker_params,
|
|
88
49
|
queue: self.work_queue,
|
|
89
|
-
|
|
50
|
+
incoming_job_queue_url: self.queue_url,
|
|
90
51
|
delay: delay,
|
|
91
52
|
funktor_options: get_funktor_options
|
|
92
53
|
}
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
module Funktor
|
|
2
|
+
module Worker
|
|
3
|
+
module FunktorOptions
|
|
4
|
+
def self.included(base)
|
|
5
|
+
base.extend ClassMethods
|
|
6
|
+
base.class_eval do
|
|
7
|
+
cattr_accessor :funktor_options_hash
|
|
8
|
+
end
|
|
9
|
+
end
|
|
10
|
+
module ClassMethods
|
|
11
|
+
def funktor_options(options = {})
|
|
12
|
+
self.funktor_options_hash = options
|
|
13
|
+
end
|
|
14
|
+
|
|
15
|
+
def get_funktor_options
|
|
16
|
+
self.funktor_options_hash || {}
|
|
17
|
+
end
|
|
18
|
+
|
|
19
|
+
def custom_queue_url
|
|
20
|
+
get_funktor_options[:queue_url]
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
def custom_queue
|
|
24
|
+
get_funktor_options[:queue]
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
def queue_url
|
|
28
|
+
custom_queue_url
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
def work_queue
|
|
32
|
+
(self.custom_queue || 'default').to_s
|
|
33
|
+
end
|
|
34
|
+
end
|
|
35
|
+
end
|
|
36
|
+
end
|
|
37
|
+
end
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: funktor
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.
|
|
4
|
+
version: 0.4.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Jeremy Green
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: exe
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2021-
|
|
11
|
+
date: 2021-08-02 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: aws-sdk-sqs
|
|
@@ -52,6 +52,20 @@ dependencies:
|
|
|
52
52
|
- - ">="
|
|
53
53
|
- !ruby/object:Gem::Version
|
|
54
54
|
version: '0'
|
|
55
|
+
- !ruby/object:Gem::Dependency
|
|
56
|
+
name: activejob
|
|
57
|
+
requirement: !ruby/object:Gem::Requirement
|
|
58
|
+
requirements:
|
|
59
|
+
- - ">="
|
|
60
|
+
- !ruby/object:Gem::Version
|
|
61
|
+
version: 5.1.5
|
|
62
|
+
type: :development
|
|
63
|
+
prerelease: false
|
|
64
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
65
|
+
requirements:
|
|
66
|
+
- - ">="
|
|
67
|
+
- !ruby/object:Gem::Version
|
|
68
|
+
version: 5.1.5
|
|
55
69
|
- !ruby/object:Gem::Dependency
|
|
56
70
|
name: simplecov
|
|
57
71
|
requirement: !ruby/object:Gem::Requirement
|
|
@@ -80,6 +94,20 @@ dependencies:
|
|
|
80
94
|
- - ">="
|
|
81
95
|
- !ruby/object:Gem::Version
|
|
82
96
|
version: '0'
|
|
97
|
+
- !ruby/object:Gem::Dependency
|
|
98
|
+
name: pry-byebug
|
|
99
|
+
requirement: !ruby/object:Gem::Requirement
|
|
100
|
+
requirements:
|
|
101
|
+
- - ">="
|
|
102
|
+
- !ruby/object:Gem::Version
|
|
103
|
+
version: '0'
|
|
104
|
+
type: :development
|
|
105
|
+
prerelease: false
|
|
106
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
107
|
+
requirements:
|
|
108
|
+
- - ">="
|
|
109
|
+
- !ruby/object:Gem::Version
|
|
110
|
+
version: '0'
|
|
83
111
|
description: Background processing in AWS Lambda.
|
|
84
112
|
email:
|
|
85
113
|
- jeremy@octolabs.com
|
|
@@ -104,6 +132,7 @@ files:
|
|
|
104
132
|
- exe/funktor
|
|
105
133
|
- exe/funktor-deploy
|
|
106
134
|
- funktor.gemspec
|
|
135
|
+
- lib/active_job/queue_adapters/funktor_adapter.rb
|
|
107
136
|
- lib/funktor.rb
|
|
108
137
|
- lib/funktor/active_job_handler.rb
|
|
109
138
|
- lib/funktor/aws/sqs/event.rb
|
|
@@ -116,20 +145,20 @@ files:
|
|
|
116
145
|
- lib/funktor/cli/init.rb
|
|
117
146
|
- lib/funktor/cli/templates/Gemfile
|
|
118
147
|
- lib/funktor/cli/templates/app/workers/hello_worker.rb
|
|
119
|
-
- lib/funktor/cli/templates/
|
|
120
|
-
- lib/funktor/cli/templates/
|
|
121
|
-
- lib/funktor/cli/templates/
|
|
122
|
-
- lib/funktor/cli/templates/
|
|
123
|
-
- lib/funktor/cli/templates/
|
|
124
|
-
- lib/funktor/cli/templates/
|
|
125
|
-
- lib/funktor/cli/templates/
|
|
126
|
-
- lib/funktor/cli/templates/
|
|
127
|
-
- lib/funktor/cli/templates/
|
|
128
|
-
- lib/funktor/cli/templates/
|
|
129
|
-
- lib/funktor/cli/templates/
|
|
130
|
-
- lib/funktor/cli/templates/
|
|
131
|
-
- lib/funktor/cli/templates/
|
|
132
|
-
- lib/funktor/cli/templates/
|
|
148
|
+
- lib/funktor/cli/templates/funktor_config/boot.rb
|
|
149
|
+
- lib/funktor/cli/templates/funktor_config/environment.yml
|
|
150
|
+
- lib/funktor/cli/templates/funktor_config/function_definitions/incoming_job_handler.yml
|
|
151
|
+
- lib/funktor/cli/templates/funktor_config/function_definitions/work_queue_handler.yml
|
|
152
|
+
- lib/funktor/cli/templates/funktor_config/funktor.yml
|
|
153
|
+
- lib/funktor/cli/templates/funktor_config/iam_permissions/incoming_job_queue.yml
|
|
154
|
+
- lib/funktor/cli/templates/funktor_config/iam_permissions/ssm.yml
|
|
155
|
+
- lib/funktor/cli/templates/funktor_config/iam_permissions/work_queue.yml
|
|
156
|
+
- lib/funktor/cli/templates/funktor_config/package.yml
|
|
157
|
+
- lib/funktor/cli/templates/funktor_config/resources/cloudwatch_dashboard.yml
|
|
158
|
+
- lib/funktor/cli/templates/funktor_config/resources/incoming_job_queue.yml
|
|
159
|
+
- lib/funktor/cli/templates/funktor_config/resources/incoming_job_queue_user.yml
|
|
160
|
+
- lib/funktor/cli/templates/funktor_config/resources/work_queue.yml
|
|
161
|
+
- lib/funktor/cli/templates/funktor_config/ruby_layer.yml
|
|
133
162
|
- lib/funktor/cli/templates/funktor_init.yml.tt
|
|
134
163
|
- lib/funktor/cli/templates/gitignore
|
|
135
164
|
- lib/funktor/cli/templates/lambda_event_handlers/incoming_job_handler.rb
|
|
@@ -141,12 +170,15 @@ files:
|
|
|
141
170
|
- lib/funktor/fake_job_queue.rb
|
|
142
171
|
- lib/funktor/incoming_job_handler.rb
|
|
143
172
|
- lib/funktor/job.rb
|
|
173
|
+
- lib/funktor/job_pusher.rb
|
|
144
174
|
- lib/funktor/logger.rb
|
|
145
175
|
- lib/funktor/middleware/metrics.rb
|
|
146
176
|
- lib/funktor/middleware_chain.rb
|
|
177
|
+
- lib/funktor/rails.rb
|
|
147
178
|
- lib/funktor/testing.rb
|
|
148
179
|
- lib/funktor/version.rb
|
|
149
180
|
- lib/funktor/worker.rb
|
|
181
|
+
- lib/funktor/worker/funktor_options.rb
|
|
150
182
|
homepage: https://github.com/Octo-Labs/funktor
|
|
151
183
|
licenses:
|
|
152
184
|
- MIT
|
|
@@ -168,7 +200,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
|
168
200
|
- !ruby/object:Gem::Version
|
|
169
201
|
version: '0'
|
|
170
202
|
requirements: []
|
|
171
|
-
rubygems_version: 3.1.
|
|
203
|
+
rubygems_version: 3.1.6
|
|
172
204
|
signing_key:
|
|
173
205
|
specification_version: 4
|
|
174
206
|
summary: Background processing in AWS Lambda.
|