funktor 0.3.1 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Gemfile.lock +22 -6
- data/funktor.gemspec +2 -0
- data/lib/active_job/queue_adapters/funktor_adapter.rb +45 -0
- data/lib/funktor.rb +7 -0
- data/lib/funktor/cli/templates/funktor_config/environment.yml +1 -0
- data/lib/funktor/fake_job_queue.rb +5 -2
- data/lib/funktor/job_pusher.rb +31 -0
- data/lib/funktor/rails.rb +19 -0
- data/lib/funktor/testing.rb +9 -7
- data/lib/funktor/version.rb +1 -1
- data/lib/funktor/worker.rb +8 -47
- data/lib/funktor/worker/funktor_options.rb +37 -0
- metadata +35 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 25d6a36d1e21a09dda03678f7b81742dfec17ceb95c1220254a04690a3933d12
|
4
|
+
data.tar.gz: e43d8a4640ca2bced4c5d6e687a1f3b878016108790afeb7b502c0680e334363
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: c14ed21535c212873b4bddec9744467f6aa1ca25125aae1c4ea115a88e89ffc0dbc683916507400562c38c3041d15bcda9711265047bbcbc001195256dcf7543
|
7
|
+
data.tar.gz: 213dd13516d536fb9e253e6b2a59bee33771e5e1a90d1702f56de5c16c3aab5384641d0361df221f5c012bd25439e3bec5a26cc865c3321637604ffb0f86dd7a
|
data/Gemfile.lock
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
PATH
|
2
2
|
remote: .
|
3
3
|
specs:
|
4
|
-
funktor (0.3.
|
4
|
+
funktor (0.3.1)
|
5
5
|
activesupport
|
6
6
|
aws-sdk-sqs (~> 1.37)
|
7
7
|
thor
|
@@ -9,6 +9,9 @@ PATH
|
|
9
9
|
GEM
|
10
10
|
remote: https://rubygems.org/
|
11
11
|
specs:
|
12
|
+
activejob (6.1.4)
|
13
|
+
activesupport (= 6.1.4)
|
14
|
+
globalid (>= 0.3.6)
|
12
15
|
activesupport (6.1.4)
|
13
16
|
concurrent-ruby (~> 1.0, >= 1.0.2)
|
14
17
|
i18n (>= 1.6, < 2)
|
@@ -18,27 +21,38 @@ GEM
|
|
18
21
|
addressable (2.7.0)
|
19
22
|
public_suffix (>= 2.0.2, < 5.0)
|
20
23
|
aws-eventstream (1.1.1)
|
21
|
-
aws-partitions (1.
|
22
|
-
aws-sdk-core (3.
|
24
|
+
aws-partitions (1.481.0)
|
25
|
+
aws-sdk-core (3.118.0)
|
23
26
|
aws-eventstream (~> 1, >= 1.0.2)
|
24
27
|
aws-partitions (~> 1, >= 1.239.0)
|
25
28
|
aws-sigv4 (~> 1.1)
|
26
29
|
jmespath (~> 1.0)
|
27
|
-
aws-sdk-sqs (1.
|
28
|
-
aws-sdk-core (~> 3, >= 3.
|
30
|
+
aws-sdk-sqs (1.41.0)
|
31
|
+
aws-sdk-core (~> 3, >= 3.118.0)
|
29
32
|
aws-sigv4 (~> 1.1)
|
30
|
-
aws-sigv4 (1.2.
|
33
|
+
aws-sigv4 (1.2.4)
|
31
34
|
aws-eventstream (~> 1, >= 1.0.2)
|
35
|
+
byebug (11.1.3)
|
36
|
+
coderay (1.1.3)
|
32
37
|
concurrent-ruby (1.1.9)
|
33
38
|
crack (0.4.5)
|
34
39
|
rexml
|
35
40
|
diff-lcs (1.4.4)
|
36
41
|
docile (1.3.5)
|
42
|
+
globalid (0.5.1)
|
43
|
+
activesupport (>= 5.0)
|
37
44
|
hashdiff (1.0.1)
|
38
45
|
i18n (1.8.10)
|
39
46
|
concurrent-ruby (~> 1.0)
|
40
47
|
jmespath (1.4.0)
|
48
|
+
method_source (1.0.0)
|
41
49
|
minitest (5.14.4)
|
50
|
+
pry (0.13.1)
|
51
|
+
coderay (~> 1.1)
|
52
|
+
method_source (~> 1.0)
|
53
|
+
pry-byebug (3.9.0)
|
54
|
+
byebug (~> 11.0)
|
55
|
+
pry (~> 0.13.0)
|
42
56
|
public_suffix (4.0.6)
|
43
57
|
rake (12.3.3)
|
44
58
|
rexml (3.2.5)
|
@@ -74,7 +88,9 @@ PLATFORMS
|
|
74
88
|
ruby
|
75
89
|
|
76
90
|
DEPENDENCIES
|
91
|
+
activejob (>= 5.1.5)
|
77
92
|
funktor!
|
93
|
+
pry-byebug
|
78
94
|
rake (~> 12.0)
|
79
95
|
rspec (~> 3.0)
|
80
96
|
simplecov
|
data/funktor.gemspec
CHANGED
@@ -33,6 +33,8 @@ Gem::Specification.new do |spec|
|
|
33
33
|
spec.add_dependency "activesupport" # TODO - Can we build our own verison of cattr_accessor to avoid this?
|
34
34
|
spec.add_dependency "thor" # Thor drives the CLI
|
35
35
|
|
36
|
+
spec.add_development_dependency 'activejob', '>= 5.1.5'
|
36
37
|
spec.add_development_dependency 'simplecov'
|
37
38
|
spec.add_development_dependency 'webmock'
|
39
|
+
spec.add_development_dependency 'pry-byebug'
|
38
40
|
end
|
@@ -0,0 +1,45 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'active_job'
|
4
|
+
|
5
|
+
module ActiveJob
|
6
|
+
module QueueAdapters
|
7
|
+
# == Funktor adapter for Active Job
|
8
|
+
#
|
9
|
+
# To use Funktor set the queue_adapter config to +:funktor+.
|
10
|
+
#
|
11
|
+
# Rails.application.config.active_job.queue_adapter = :funktor
|
12
|
+
class FunktorAdapter
|
13
|
+
def enqueue(job) # :nodoc:
|
14
|
+
job.provider_job_id = Funktor.job_pusher.push({
|
15
|
+
"worker" => JobWrapper.to_s,
|
16
|
+
"wrapped" => job.class,
|
17
|
+
"queue" => job.class.work_queue,
|
18
|
+
"worker_params" => [ job.serialize ]
|
19
|
+
})
|
20
|
+
end
|
21
|
+
|
22
|
+
def enqueue_at(job, timestamp) # :nodoc:
|
23
|
+
job.provider_job_id = Funktor.job_pusher.push({
|
24
|
+
"worker" => JobWrapper.to_s,
|
25
|
+
"wrapped" => job.class,
|
26
|
+
"queue" => job.class.work_queue,
|
27
|
+
"worker_params" => [ job.serialize ],
|
28
|
+
"at" => timestamp
|
29
|
+
})
|
30
|
+
end
|
31
|
+
|
32
|
+
class JobWrapper #:nodoc:
|
33
|
+
include Funktor::Worker
|
34
|
+
|
35
|
+
def perform(job_data)
|
36
|
+
Base.execute job_data.first
|
37
|
+
end
|
38
|
+
end
|
39
|
+
end
|
40
|
+
end
|
41
|
+
|
42
|
+
class Base
|
43
|
+
include Funktor::Worker::FunktorOptions
|
44
|
+
end
|
45
|
+
end
|
data/lib/funktor.rb
CHANGED
@@ -3,6 +3,7 @@ require 'funktor/aws/sqs/event'
|
|
3
3
|
require 'funktor/aws/sqs/record'
|
4
4
|
require 'funktor/counter'
|
5
5
|
require 'funktor/job'
|
6
|
+
require 'funktor/job_pusher'
|
6
7
|
require 'funktor/logger'
|
7
8
|
require 'funktor/worker'
|
8
9
|
require 'funktor/middleware_chain'
|
@@ -22,6 +23,10 @@ module Funktor
|
|
22
23
|
yield self
|
23
24
|
end
|
24
25
|
|
26
|
+
def self.job_pusher
|
27
|
+
@job_pusher ||= JobPusher.new
|
28
|
+
end
|
29
|
+
|
25
30
|
def self.job_pusher_middleware
|
26
31
|
@job_pusher_chain ||= MiddlewareChain.new
|
27
32
|
yield @job_pusher_chain if block_given?
|
@@ -98,3 +103,5 @@ end
|
|
98
103
|
require 'funktor/middleware/metrics'
|
99
104
|
require 'funktor/error_handler'
|
100
105
|
require 'funktor/active_job_handler'
|
106
|
+
|
107
|
+
require 'funktor/rails' if defined?(::Rails::Engine)
|
@@ -1,7 +1,10 @@
|
|
1
|
+
require "active_support/core_ext/hash/indifferent_access"
|
2
|
+
|
1
3
|
module Funktor
|
2
4
|
module FakeJobQueue
|
3
|
-
def self.push(
|
4
|
-
|
5
|
+
def self.push(payload)
|
6
|
+
payload = payload.with_indifferent_access
|
7
|
+
jobs[payload["worker"].to_s].push(payload)
|
5
8
|
end
|
6
9
|
|
7
10
|
def self.jobs
|
@@ -0,0 +1,31 @@
|
|
1
|
+
require 'aws-sdk-sqs'
|
2
|
+
|
3
|
+
module Funktor
|
4
|
+
class JobPusher
|
5
|
+
|
6
|
+
def push(payload)
|
7
|
+
puts "payload ============"
|
8
|
+
pp payload
|
9
|
+
job_id = SecureRandom.uuid
|
10
|
+
payload[:job_id] = job_id
|
11
|
+
|
12
|
+
Funktor.job_pusher_middleware.invoke(payload) do
|
13
|
+
client.send_message({
|
14
|
+
queue_url: queue_url(payload),
|
15
|
+
message_body: Funktor.dump_json(payload)
|
16
|
+
})
|
17
|
+
return job_id
|
18
|
+
end
|
19
|
+
end
|
20
|
+
|
21
|
+
private
|
22
|
+
|
23
|
+
def client
|
24
|
+
@client ||= ::Aws::SQS::Client.new
|
25
|
+
end
|
26
|
+
|
27
|
+
def queue_url(payload)
|
28
|
+
payload[:incoming_job_queue_url] || ENV['FUNKTOR_INCOMING_JOB_QUEUE']
|
29
|
+
end
|
30
|
+
end
|
31
|
+
end
|
@@ -0,0 +1,19 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
module Funktor
|
3
|
+
class Rails < ::Rails::Engine
|
4
|
+
# This hook happens after `Rails::Application` is inherited within
|
5
|
+
# config/application.rb and before config is touched, usually within the
|
6
|
+
# class block. Definitely before config/environments/*.rb and
|
7
|
+
# config/initializers/*.rb.
|
8
|
+
config.before_configuration do
|
9
|
+
if defined?(::ActiveJob)
|
10
|
+
require "sidekiq/worker"
|
11
|
+
require 'active_job/queue_adapters/funktor_adapter'
|
12
|
+
end
|
13
|
+
end
|
14
|
+
end if defined?(::Rails)
|
15
|
+
|
16
|
+
if defined?(::Rails) && ::Rails::VERSION::MAJOR < 5
|
17
|
+
raise "🚫 ERROR: Funktor does not support Rails versions under 5.x"
|
18
|
+
end
|
19
|
+
end
|
data/lib/funktor/testing.rb
CHANGED
@@ -17,9 +17,9 @@ module Funktor
|
|
17
17
|
|
18
18
|
def work_all_jobs
|
19
19
|
while jobs.any?
|
20
|
-
job_data = jobs.shift
|
21
|
-
worker = job_data[:worker]
|
22
|
-
worker_params = job_data[:
|
20
|
+
job_data = jobs.shift.with_indifferent_access
|
21
|
+
worker = Object.const_get job_data[:worker]
|
22
|
+
worker_params = job_data[:worker_params]
|
23
23
|
worker.new.perform(worker_params)
|
24
24
|
end
|
25
25
|
end
|
@@ -56,14 +56,16 @@ module Funktor
|
|
56
56
|
end
|
57
57
|
|
58
58
|
class InlineJobPusherMiddleware
|
59
|
-
def call(
|
60
|
-
|
59
|
+
def call(payload)
|
60
|
+
payload = payload.with_indifferent_access
|
61
|
+
worker = Object.const_get payload["worker"]
|
62
|
+
worker.new.perform(*payload["worker_params"])
|
61
63
|
end
|
62
64
|
end
|
63
65
|
|
64
66
|
class FakeJobPusherMiddleware
|
65
|
-
def call(
|
66
|
-
Funktor::FakeJobQueue.push(
|
67
|
+
def call(payload)
|
68
|
+
Funktor::FakeJobQueue.push(payload)
|
67
69
|
end
|
68
70
|
end
|
69
71
|
end
|
data/lib/funktor/version.rb
CHANGED
data/lib/funktor/worker.rb
CHANGED
@@ -1,6 +1,6 @@
|
|
1
1
|
require 'securerandom'
|
2
|
-
require 'aws-sdk-sqs'
|
3
2
|
require "active_support"
|
3
|
+
require 'funktor/worker/funktor_options'
|
4
4
|
|
5
5
|
module Funktor
|
6
6
|
class DelayTooLongError < StandardError; end
|
@@ -9,34 +9,10 @@ end
|
|
9
9
|
module Funktor::Worker
|
10
10
|
def self.included(base)
|
11
11
|
base.extend ClassMethods
|
12
|
-
base.
|
13
|
-
cattr_accessor :funktor_options_hash
|
14
|
-
#alias_method :perform_later, :perform_async
|
15
|
-
end
|
12
|
+
base.include(Funktor::Worker::FunktorOptions)
|
16
13
|
end
|
17
14
|
|
18
15
|
module ClassMethods
|
19
|
-
def funktor_options(options = {})
|
20
|
-
self.funktor_options_hash = options
|
21
|
-
end
|
22
|
-
|
23
|
-
def get_funktor_options
|
24
|
-
self.funktor_options_hash || {}
|
25
|
-
end
|
26
|
-
|
27
|
-
def custom_queue_url
|
28
|
-
get_funktor_options[:queue_url]
|
29
|
-
end
|
30
|
-
|
31
|
-
def custom_queue
|
32
|
-
get_funktor_options[:queue]
|
33
|
-
end
|
34
|
-
|
35
|
-
def queue_url
|
36
|
-
# TODO : Should this default to FUNKTOR_ACTIVE_JOB_QUEUE?
|
37
|
-
# Depends how e handle this in pro...?
|
38
|
-
custom_queue_url || ENV['FUNKTOR_INCOMING_JOB_QUEUE']
|
39
|
-
end
|
40
16
|
|
41
17
|
def perform_async(*worker_params)
|
42
18
|
self.perform_in(0, *worker_params)
|
@@ -54,39 +30,24 @@ module Funktor::Worker
|
|
54
30
|
if delay > max_delay
|
55
31
|
raise Funktor::DelayTooLongError.new("The delay can't be longer than #{max_delay} seconds. This is a limitation of SQS. Funktor Pro has mechanisms to work around this limitation.")
|
56
32
|
end
|
57
|
-
self.
|
33
|
+
self.push(delay, *worker_params)
|
58
34
|
end
|
59
35
|
|
60
|
-
def
|
61
|
-
|
62
|
-
payload
|
63
|
-
|
64
|
-
Funktor.job_pusher_middleware.invoke(self, payload) do
|
65
|
-
client.send_message({
|
66
|
-
queue_url: queue_url,
|
67
|
-
message_body: Funktor.dump_json(payload)
|
68
|
-
})
|
69
|
-
end
|
36
|
+
def push(delay, *worker_params)
|
37
|
+
payload = build_job_payload(delay, *worker_params)
|
38
|
+
Funktor.job_pusher.push(payload)
|
70
39
|
end
|
71
40
|
|
72
41
|
def max_delay
|
73
42
|
900
|
74
43
|
end
|
75
44
|
|
76
|
-
def
|
77
|
-
@client ||= Aws::SQS::Client.new
|
78
|
-
end
|
79
|
-
|
80
|
-
def work_queue
|
81
|
-
(self.custom_queue || 'default').to_s
|
82
|
-
end
|
83
|
-
|
84
|
-
def build_job_payload(job_id, delay, *worker_params)
|
45
|
+
def build_job_payload(delay, *worker_params)
|
85
46
|
{
|
86
47
|
worker: self.name,
|
87
48
|
worker_params: worker_params,
|
88
49
|
queue: self.work_queue,
|
89
|
-
|
50
|
+
incoming_job_queue_url: self.queue_url,
|
90
51
|
delay: delay,
|
91
52
|
funktor_options: get_funktor_options
|
92
53
|
}
|
@@ -0,0 +1,37 @@
|
|
1
|
+
module Funktor
|
2
|
+
module Worker
|
3
|
+
module FunktorOptions
|
4
|
+
def self.included(base)
|
5
|
+
base.extend ClassMethods
|
6
|
+
base.class_eval do
|
7
|
+
cattr_accessor :funktor_options_hash
|
8
|
+
end
|
9
|
+
end
|
10
|
+
module ClassMethods
|
11
|
+
def funktor_options(options = {})
|
12
|
+
self.funktor_options_hash = options
|
13
|
+
end
|
14
|
+
|
15
|
+
def get_funktor_options
|
16
|
+
self.funktor_options_hash || {}
|
17
|
+
end
|
18
|
+
|
19
|
+
def custom_queue_url
|
20
|
+
get_funktor_options[:queue_url]
|
21
|
+
end
|
22
|
+
|
23
|
+
def custom_queue
|
24
|
+
get_funktor_options[:queue]
|
25
|
+
end
|
26
|
+
|
27
|
+
def queue_url
|
28
|
+
custom_queue_url
|
29
|
+
end
|
30
|
+
|
31
|
+
def work_queue
|
32
|
+
(self.custom_queue || 'default').to_s
|
33
|
+
end
|
34
|
+
end
|
35
|
+
end
|
36
|
+
end
|
37
|
+
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: funktor
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.4.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Jeremy Green
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2021-
|
11
|
+
date: 2021-08-02 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: aws-sdk-sqs
|
@@ -52,6 +52,20 @@ dependencies:
|
|
52
52
|
- - ">="
|
53
53
|
- !ruby/object:Gem::Version
|
54
54
|
version: '0'
|
55
|
+
- !ruby/object:Gem::Dependency
|
56
|
+
name: activejob
|
57
|
+
requirement: !ruby/object:Gem::Requirement
|
58
|
+
requirements:
|
59
|
+
- - ">="
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
version: 5.1.5
|
62
|
+
type: :development
|
63
|
+
prerelease: false
|
64
|
+
version_requirements: !ruby/object:Gem::Requirement
|
65
|
+
requirements:
|
66
|
+
- - ">="
|
67
|
+
- !ruby/object:Gem::Version
|
68
|
+
version: 5.1.5
|
55
69
|
- !ruby/object:Gem::Dependency
|
56
70
|
name: simplecov
|
57
71
|
requirement: !ruby/object:Gem::Requirement
|
@@ -80,6 +94,20 @@ dependencies:
|
|
80
94
|
- - ">="
|
81
95
|
- !ruby/object:Gem::Version
|
82
96
|
version: '0'
|
97
|
+
- !ruby/object:Gem::Dependency
|
98
|
+
name: pry-byebug
|
99
|
+
requirement: !ruby/object:Gem::Requirement
|
100
|
+
requirements:
|
101
|
+
- - ">="
|
102
|
+
- !ruby/object:Gem::Version
|
103
|
+
version: '0'
|
104
|
+
type: :development
|
105
|
+
prerelease: false
|
106
|
+
version_requirements: !ruby/object:Gem::Requirement
|
107
|
+
requirements:
|
108
|
+
- - ">="
|
109
|
+
- !ruby/object:Gem::Version
|
110
|
+
version: '0'
|
83
111
|
description: Background processing in AWS Lambda.
|
84
112
|
email:
|
85
113
|
- jeremy@octolabs.com
|
@@ -104,6 +132,7 @@ files:
|
|
104
132
|
- exe/funktor
|
105
133
|
- exe/funktor-deploy
|
106
134
|
- funktor.gemspec
|
135
|
+
- lib/active_job/queue_adapters/funktor_adapter.rb
|
107
136
|
- lib/funktor.rb
|
108
137
|
- lib/funktor/active_job_handler.rb
|
109
138
|
- lib/funktor/aws/sqs/event.rb
|
@@ -141,12 +170,15 @@ files:
|
|
141
170
|
- lib/funktor/fake_job_queue.rb
|
142
171
|
- lib/funktor/incoming_job_handler.rb
|
143
172
|
- lib/funktor/job.rb
|
173
|
+
- lib/funktor/job_pusher.rb
|
144
174
|
- lib/funktor/logger.rb
|
145
175
|
- lib/funktor/middleware/metrics.rb
|
146
176
|
- lib/funktor/middleware_chain.rb
|
177
|
+
- lib/funktor/rails.rb
|
147
178
|
- lib/funktor/testing.rb
|
148
179
|
- lib/funktor/version.rb
|
149
180
|
- lib/funktor/worker.rb
|
181
|
+
- lib/funktor/worker/funktor_options.rb
|
150
182
|
homepage: https://github.com/Octo-Labs/funktor
|
151
183
|
licenses:
|
152
184
|
- MIT
|
@@ -168,7 +200,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
168
200
|
- !ruby/object:Gem::Version
|
169
201
|
version: '0'
|
170
202
|
requirements: []
|
171
|
-
rubygems_version: 3.
|
203
|
+
rubygems_version: 3.1.6
|
172
204
|
signing_key:
|
173
205
|
specification_version: 4
|
174
206
|
summary: Background processing in AWS Lambda.
|