strum-pipeline 0.1.0 → 0.1.4
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +19 -0
- data/Gemfile.lock +18 -11
- data/README.md +61 -0
- data/lib/strum/pipeline/storage.rb +4 -7
- data/lib/strum/pipeline/version.rb +1 -1
- data/lib/strum/pipeline.rb +12 -5
- data/strum-pipeline.gemspec +5 -5
- metadata +10 -9
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 5934d11a9020a020b55ae39e223df484555dcacbdb1d3c5b49aa89d49a93ae95
|
4
|
+
data.tar.gz: 71e27af119c3f788ba668a7458a8aad88b132e9e249550174ccebc1c5c3ff6c1
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: e70550e1118932b81da15c99da3505358f01da2bfc1c6aab74f3f2bfd49f7cd274b381adb09034ca7f9d2d3526235ea8210de822e104337057e2d411f3b720a5
|
7
|
+
data.tar.gz: 91675d7db033165fbe449fa28ac53fd19bcf1423cd1df2b930049e5d8d37310f8b27745bc42ebe3f484cbc75629b5e401a50bd79a01796e45aa9281ab6a3a94b
|
data/CHANGELOG.md
ADDED
@@ -0,0 +1,19 @@
|
|
1
|
+
|
2
|
+
# Changelog
|
3
|
+
All notable changes to this project will be documented in this file.
|
4
|
+
|
5
|
+
## [0.1.4] - 2021-01-11
|
6
|
+
### Changed
|
7
|
+
- `strum-esb` dependency version to `~> 0.2` by [@valeriia.kolisnyk].
|
8
|
+
- pipeline name format to snake case by [@valeriia.kolisnyk].
|
9
|
+
|
10
|
+
## [0.1.2] - 2021-07-01
|
11
|
+
### Added
|
12
|
+
- Redis connection using env var or config by [@serhiy.nazarov].
|
13
|
+
|
14
|
+
## [0.1.1] - 2021-06-02
|
15
|
+
### Fixed
|
16
|
+
- custom handlers for resource name with `-` by [@anton.klyzhka].
|
17
|
+
|
18
|
+
### Changed
|
19
|
+
- `strum-esb` dependency version to `~> 0.1.1` by [@anton.klyzhka].
|
data/Gemfile.lock
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
PATH
|
2
2
|
remote: .
|
3
3
|
specs:
|
4
|
-
strum-pipeline (0.1.
|
4
|
+
strum-pipeline (0.1.4)
|
5
5
|
bunny (~> 2.15)
|
6
6
|
connection_pool (~> 2.2)
|
7
7
|
dry-configurable (~> 0.11)
|
@@ -10,16 +10,17 @@ PATH
|
|
10
10
|
redis-objects (~> 1.5)
|
11
11
|
singleton (~> 0.1)
|
12
12
|
sneakers (~> 2.12)
|
13
|
-
strum-esb (~> 0.
|
13
|
+
strum-esb (~> 0.2)
|
14
14
|
|
15
15
|
GEM
|
16
16
|
remote: https://rubygems.org/
|
17
17
|
specs:
|
18
18
|
amq-protocol (2.3.2)
|
19
19
|
ast (2.4.2)
|
20
|
-
bunny (2.
|
20
|
+
bunny (2.19.0)
|
21
21
|
amq-protocol (~> 2.3, >= 2.3.1)
|
22
|
-
|
22
|
+
sorted_set (~> 1, >= 1.0.2)
|
23
|
+
concurrent-ruby (1.1.9)
|
23
24
|
connection_pool (2.2.5)
|
24
25
|
debase (0.2.4.1)
|
25
26
|
debase-ruby_core_source (>= 0.10.2)
|
@@ -28,18 +29,19 @@ GEM
|
|
28
29
|
dry-configurable (0.12.1)
|
29
30
|
concurrent-ruby (~> 1.0)
|
30
31
|
dry-core (~> 0.5, >= 0.5.0)
|
31
|
-
dry-core (0.
|
32
|
+
dry-core (0.7.1)
|
32
33
|
concurrent-ruby (~> 1.0)
|
33
|
-
dry-inflector (0.2.
|
34
|
-
json (2.
|
34
|
+
dry-inflector (0.2.1)
|
35
|
+
json (2.6.1)
|
35
36
|
parallel (1.20.1)
|
36
37
|
parser (3.0.1.0)
|
37
38
|
ast (~> 2.4.1)
|
38
39
|
rainbow (3.0.0)
|
39
40
|
rake (12.3.3)
|
40
|
-
|
41
|
-
redis
|
42
|
-
|
41
|
+
rbtree (0.4.4)
|
42
|
+
redis (4.5.1)
|
43
|
+
redis-objects (1.5.1)
|
44
|
+
redis (~> 4.2)
|
43
45
|
regexp_parser (2.1.1)
|
44
46
|
rexml (3.2.5)
|
45
47
|
rspec (3.10.0)
|
@@ -71,6 +73,7 @@ GEM
|
|
71
73
|
ruby-progressbar (1.11.0)
|
72
74
|
serverengine (2.1.1)
|
73
75
|
sigdump (~> 0.2.2)
|
76
|
+
set (1.0.2)
|
74
77
|
sigdump (0.2.4)
|
75
78
|
singleton (0.1.1)
|
76
79
|
sneakers (2.12.0)
|
@@ -79,9 +82,13 @@ GEM
|
|
79
82
|
rake (~> 12.3)
|
80
83
|
serverengine (~> 2.1.0)
|
81
84
|
thor
|
82
|
-
|
85
|
+
sorted_set (1.0.3)
|
86
|
+
rbtree
|
87
|
+
set (~> 1.0)
|
88
|
+
strum-esb (0.2.0)
|
83
89
|
bunny (~> 2.15)
|
84
90
|
connection_pool (~> 2.2.2)
|
91
|
+
dry-configurable (~> 0.12.1)
|
85
92
|
json (~> 2.3)
|
86
93
|
sneakers (~> 2.12)
|
87
94
|
thor (1.1.0)
|
data/README.md
CHANGED
@@ -34,3 +34,64 @@ To install this gem onto your local machine, run `bundle exec rake install`. To
|
|
34
34
|
|
35
35
|
Bug reports and pull requests are welcome on GitHub at https://github.com/[USERNAME]/strum-pipeline.
|
36
36
|
|
37
|
+
## Example
|
38
|
+
|
39
|
+
```ruby
|
40
|
+
# frozen_string_literal: true
|
41
|
+
|
42
|
+
require "sneakers/runner"
|
43
|
+
require "strum/esb"
|
44
|
+
require "strum/pipeline"
|
45
|
+
|
46
|
+
# demo class
|
47
|
+
class DemoPipeline
|
48
|
+
include Strum::Pipeline
|
49
|
+
|
50
|
+
# changing default exchangers
|
51
|
+
Strum::Esb.config.info_exchange = "demo.info"
|
52
|
+
Strum::Esb.config.event_exchange = "demo.events"
|
53
|
+
Strum::Esb.config.action_exchange = "demo.actions"
|
54
|
+
Strum::Esb.config.notice_exchange = "demo.notice"
|
55
|
+
|
56
|
+
# changing default redis connection
|
57
|
+
Strum::Pipeline.config.redis_connection = { url: ENV.fetch("DEMO_REDIS_URL") }
|
58
|
+
|
59
|
+
ttl 60
|
60
|
+
init :event, "identity/create"
|
61
|
+
init :event, "identity/update"
|
62
|
+
step :action, "get/user", :demo_custom_handler
|
63
|
+
step :action, "get/company"
|
64
|
+
step :info, "user"
|
65
|
+
|
66
|
+
def event_identity_create(_payload)
|
67
|
+
storage[:counter1] = 1
|
68
|
+
storage[:counter] = 1
|
69
|
+
Strum::Esb::Action.call({}, :get, :user)
|
70
|
+
# Strum::Esb::Action.({}, :get, :company)
|
71
|
+
end
|
72
|
+
|
73
|
+
def demo_custom_handler(_payload)
|
74
|
+
Strum::Esb::Info.call({}, :user)
|
75
|
+
storage[:counter2] = 2
|
76
|
+
Strum::Pipeline::Storage[:counter] = 2
|
77
|
+
Strum::Pipeline::Storage.list << 2
|
78
|
+
end
|
79
|
+
|
80
|
+
def info_user(payload)
|
81
|
+
logger.debug payload
|
82
|
+
storage[:counter3] = 3
|
83
|
+
storage[:counter] = 3
|
84
|
+
end
|
85
|
+
end
|
86
|
+
|
87
|
+
r = Sneakers::Runner.new([DemoPipeline])
|
88
|
+
r.run
|
89
|
+
```
|
90
|
+
|
91
|
+
Also Redis connection can be set up by env var:
|
92
|
+
```
|
93
|
+
PIPELINE_REDIS_HOST=127.0.0.1
|
94
|
+
PIPELINE_REDIS_PORT=6379
|
95
|
+
PIPELINE_REDIS_DB=0
|
96
|
+
```
|
97
|
+
|
@@ -1,6 +1,6 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
require "redis
|
3
|
+
require "redis/objects"
|
4
4
|
require "singleton"
|
5
5
|
require "dry/inflector"
|
6
6
|
|
@@ -22,11 +22,10 @@ module Strum
|
|
22
22
|
that = self
|
23
23
|
Class.new do
|
24
24
|
define_singleton_method :[] do |key|
|
25
|
-
|
26
|
-
redis_entity.expire(that.ttl)
|
27
|
-
redis_entity
|
25
|
+
Redis.const_get(Dry::Inflector.new.camelize(method)).new("#{that.pipeline}::#{key}", expiration: that.ttl)
|
28
26
|
end
|
29
27
|
end
|
28
|
+
|
30
29
|
rescue StandardError
|
31
30
|
super
|
32
31
|
end
|
@@ -42,9 +41,7 @@ module Strum
|
|
42
41
|
end
|
43
42
|
|
44
43
|
def []=(key, value)
|
45
|
-
|
46
|
-
redis_value.value = value
|
47
|
-
redis_value.expire(ttl)
|
44
|
+
Redis::Value.new("#{pipeline}::#{key}", expiration: ttl).value = value
|
48
45
|
end
|
49
46
|
end
|
50
47
|
end
|
data/lib/strum/pipeline.rb
CHANGED
@@ -1,6 +1,8 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
3
|
require "strum/pipeline/version"
|
4
|
+
require "redis/objects"
|
5
|
+
require 'connection_pool'
|
4
6
|
require "strum/esb"
|
5
7
|
require "strum/pipeline/storage"
|
6
8
|
require "dry/configurable"
|
@@ -11,11 +13,15 @@ module Strum
|
|
11
13
|
extend Dry::Configurable
|
12
14
|
|
13
15
|
setting :ttl, 15 * 60 # default ttl is 15 mins
|
14
|
-
setting :redis_connection, {
|
15
|
-
|
16
|
+
setting :redis_connection, {
|
17
|
+
host: ENV.fetch("PIPELINE_REDIS_HOST", "localhost"),
|
18
|
+
port: ENV.fetch("PIPELINE_REDIS_PORT", "6379"),
|
19
|
+
db: ENV.fetch("PIPELINE_REDIS_DB", "0")
|
20
|
+
}
|
21
|
+
# setting :redis_connection_pool
|
16
22
|
|
17
23
|
Strum::Esb.config.after_fork_hooks << proc do
|
18
|
-
|
24
|
+
Redis::Objects.redis = ConnectionPool.new(size: 5, timeout: 5) do
|
19
25
|
Redis.new(Strum::Pipeline.config.redis_connection)
|
20
26
|
end
|
21
27
|
end
|
@@ -57,13 +63,14 @@ module Strum
|
|
57
63
|
from_queue pipeline_name, bindings: bindings
|
58
64
|
_, *msg = Strum::Esb::Functions.public_send("#{message_type}_explain", message_binding)
|
59
65
|
@handlers ||= {}
|
60
|
-
|
66
|
+
params = msg.map{ |param| param&.to_s.gsub(/[^a-zA-Z0-9]/, "_")&.downcase }
|
67
|
+
handler_key = ([message_type] + params).join("-")
|
61
68
|
@handlers[handler_key] = handler.to_s if handler
|
62
69
|
end
|
63
70
|
|
64
71
|
def pipeline_name
|
65
72
|
word = name.dup
|
66
|
-
word.gsub!(/::/, "
|
73
|
+
word.gsub!(/::/, "_")
|
67
74
|
word.gsub!(/([A-Z]+)([A-Z][a-z])/, '\1_\2')
|
68
75
|
word.gsub!(/([a-z\d])([A-Z])/, '\1_\2')
|
69
76
|
word.tr!("-", "_")
|
data/strum-pipeline.gemspec
CHANGED
@@ -10,15 +10,15 @@ Gem::Specification.new do |spec|
|
|
10
10
|
|
11
11
|
spec.summary = "Defining data processing Pipeline"
|
12
12
|
# spec.description = "Defining data processing Pipeline"
|
13
|
-
spec.homepage = "https://
|
13
|
+
spec.homepage = "https://gitlab.com/strum-rb/strum-pipeline"
|
14
14
|
spec.required_ruby_version = Gem::Requirement.new(">= 2.3.0")
|
15
15
|
|
16
16
|
spec.metadata["allowed_push_host"] = "https://rubygems.org"
|
17
17
|
|
18
18
|
spec.metadata["homepage_uri"] = spec.homepage
|
19
|
-
spec.metadata["source_code_uri"] = "https://
|
20
|
-
spec.metadata["readme_uri"] = "https://
|
21
|
-
spec.metadata["changelog_uri"] = "https://
|
19
|
+
spec.metadata["source_code_uri"] = "https://gitlab.com/strum-rb/strum-pipeline"
|
20
|
+
spec.metadata["readme_uri"] = "https://gitlab.com/strum-rb/strum-pipeline/-/blob/master/README.md"
|
21
|
+
spec.metadata["changelog_uri"] = "https://gitlab.com/strum-rb/strum-pipeline/-/blob/master/CHANGELOG.md"
|
22
22
|
|
23
23
|
# Specify which files should be added to the gem when it is released.
|
24
24
|
# The `git ls-files -z` loads the files in the RubyGem that have been added into git.
|
@@ -38,5 +38,5 @@ Gem::Specification.new do |spec|
|
|
38
38
|
spec.add_dependency "singleton", "~> 0.1"
|
39
39
|
spec.add_dependency "sneakers", "~> 2.12"
|
40
40
|
|
41
|
-
spec.add_dependency "strum-esb", "~> 0.
|
41
|
+
spec.add_dependency "strum-esb", "~> 0.2"
|
42
42
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: strum-pipeline
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.
|
4
|
+
version: 0.1.4
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Serhiy Nazarov
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2022-01-11 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bunny
|
@@ -128,14 +128,14 @@ dependencies:
|
|
128
128
|
requirements:
|
129
129
|
- - "~>"
|
130
130
|
- !ruby/object:Gem::Version
|
131
|
-
version: 0.
|
131
|
+
version: '0.2'
|
132
132
|
type: :runtime
|
133
133
|
prerelease: false
|
134
134
|
version_requirements: !ruby/object:Gem::Requirement
|
135
135
|
requirements:
|
136
136
|
- - "~>"
|
137
137
|
- !ruby/object:Gem::Version
|
138
|
-
version: 0.
|
138
|
+
version: '0.2'
|
139
139
|
description:
|
140
140
|
email:
|
141
141
|
- sn@nazarov.com.ua
|
@@ -149,6 +149,7 @@ files:
|
|
149
149
|
- ".rubocop.yml"
|
150
150
|
- ".ruby-version"
|
151
151
|
- ".travis.yml"
|
152
|
+
- CHANGELOG.md
|
152
153
|
- Gemfile
|
153
154
|
- Gemfile.lock
|
154
155
|
- README.md
|
@@ -159,14 +160,14 @@ files:
|
|
159
160
|
- lib/strum/pipeline/storage.rb
|
160
161
|
- lib/strum/pipeline/version.rb
|
161
162
|
- strum-pipeline.gemspec
|
162
|
-
homepage: https://
|
163
|
+
homepage: https://gitlab.com/strum-rb/strum-pipeline
|
163
164
|
licenses: []
|
164
165
|
metadata:
|
165
166
|
allowed_push_host: https://rubygems.org
|
166
|
-
homepage_uri: https://
|
167
|
-
source_code_uri: https://
|
168
|
-
readme_uri: https://
|
169
|
-
changelog_uri: https://
|
167
|
+
homepage_uri: https://gitlab.com/strum-rb/strum-pipeline
|
168
|
+
source_code_uri: https://gitlab.com/strum-rb/strum-pipeline
|
169
|
+
readme_uri: https://gitlab.com/strum-rb/strum-pipeline/-/blob/master/README.md
|
170
|
+
changelog_uri: https://gitlab.com/strum-rb/strum-pipeline/-/blob/master/CHANGELOG.md
|
170
171
|
post_install_message:
|
171
172
|
rdoc_options: []
|
172
173
|
require_paths:
|