magic_pipe 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.gitignore +11 -0
- data/.rspec +3 -0
- data/.travis.yml +8 -0
- data/CHANGELOG.md +10 -0
- data/Gemfile +6 -0
- data/Gemfile.lock +92 -0
- data/LICENSE.txt +19 -0
- data/README.md +241 -0
- data/Rakefile +6 -0
- data/bin/console +16 -0
- data/bin/setup +8 -0
- data/lib/magic_pipe.rb +45 -0
- data/lib/magic_pipe/client.rb +41 -0
- data/lib/magic_pipe/codecs.rb +24 -0
- data/lib/magic_pipe/codecs/base.rb +26 -0
- data/lib/magic_pipe/codecs/json.rb +38 -0
- data/lib/magic_pipe/codecs/message_pack.rb +81 -0
- data/lib/magic_pipe/codecs/thrift.rb +17 -0
- data/lib/magic_pipe/codecs/yaml.rb +18 -0
- data/lib/magic_pipe/config.rb +103 -0
- data/lib/magic_pipe/envelope.rb +29 -0
- data/lib/magic_pipe/errors.rb +14 -0
- data/lib/magic_pipe/loaders.rb +22 -0
- data/lib/magic_pipe/loaders/simple_active_record.rb +54 -0
- data/lib/magic_pipe/metrics.rb +54 -0
- data/lib/magic_pipe/senders.rb +23 -0
- data/lib/magic_pipe/senders/async.rb +76 -0
- data/lib/magic_pipe/senders/base.rb +24 -0
- data/lib/magic_pipe/senders/metrics_mixin.rb +20 -0
- data/lib/magic_pipe/senders/sync.rb +38 -0
- data/lib/magic_pipe/transports.rb +26 -0
- data/lib/magic_pipe/transports/base.rb +17 -0
- data/lib/magic_pipe/transports/debug.rb +17 -0
- data/lib/magic_pipe/transports/https.rb +80 -0
- data/lib/magic_pipe/transports/kafka.rb +8 -0
- data/lib/magic_pipe/transports/log.rb +15 -0
- data/lib/magic_pipe/transports/multi.rb +44 -0
- data/lib/magic_pipe/transports/sqs.rb +68 -0
- data/lib/magic_pipe/version.rb +3 -0
- data/magic_pipe.gemspec +44 -0
- metadata +257 -0
data/lib/magic_pipe.rb
ADDED
@@ -0,0 +1,45 @@
|
|
1
|
+
require "magic_pipe/version"
|
2
|
+
require "magic_pipe/errors"
|
3
|
+
|
4
|
+
require "magic_pipe/config"
|
5
|
+
require "magic_pipe/metrics"
|
6
|
+
|
7
|
+
require "magic_pipe/envelope"
|
8
|
+
|
9
|
+
require "magic_pipe/loaders"
|
10
|
+
require "magic_pipe/codecs"
|
11
|
+
require "magic_pipe/senders"
|
12
|
+
require "magic_pipe/transports"
|
13
|
+
|
14
|
+
require "magic_pipe/client"
|
15
|
+
|
16
|
+
module MagicPipe
|
17
|
+
class << self
|
18
|
+
def lookup_client(name)
|
19
|
+
@store[name.to_sym]
|
20
|
+
end
|
21
|
+
|
22
|
+
# All this should be loaded before Sidekiq
|
23
|
+
# or Puma start forking threads.
|
24
|
+
#
|
25
|
+
def store_client(client)
|
26
|
+
@store ||= {}
|
27
|
+
@store[client.name.to_sym] = client
|
28
|
+
end
|
29
|
+
|
30
|
+
def clear_clients
|
31
|
+
@store = {}
|
32
|
+
end
|
33
|
+
|
34
|
+
def build(&block)
|
35
|
+
unless block_given?
|
36
|
+
raise ConfigurationError, "No configuration block provided."
|
37
|
+
end
|
38
|
+
|
39
|
+
config = Config.new(&block)
|
40
|
+
client = Client.new(config)
|
41
|
+
store_client(client)
|
42
|
+
client
|
43
|
+
end
|
44
|
+
end
|
45
|
+
end
|
@@ -0,0 +1,41 @@
|
|
1
|
+
module MagicPipe
|
2
|
+
class Client
|
3
|
+
def initialize(config)
|
4
|
+
@config = config
|
5
|
+
@name = config.client_name
|
6
|
+
|
7
|
+
@metrics = Metrics.new(@config)
|
8
|
+
|
9
|
+
@transport = build_transport
|
10
|
+
|
11
|
+
@codec = Codecs.lookup(config.codec)
|
12
|
+
@sender = Senders.lookup(config.sender)
|
13
|
+
|
14
|
+
@loader = Loaders.lookup(config.loader)
|
15
|
+
end
|
16
|
+
|
17
|
+
attr_reader :name, :config, :codec, :transport, :sender, :loader, :metrics
|
18
|
+
|
19
|
+
def send_data(object:, topic:, wrapper: nil, time: Time.now.utc)
|
20
|
+
sender.new(
|
21
|
+
object,
|
22
|
+
topic,
|
23
|
+
wrapper,
|
24
|
+
time,
|
25
|
+
codec,
|
26
|
+
transport,
|
27
|
+
@config,
|
28
|
+
@metrics
|
29
|
+
).call
|
30
|
+
true
|
31
|
+
end
|
32
|
+
|
33
|
+
|
34
|
+
private
|
35
|
+
|
36
|
+
def build_transport
|
37
|
+
klass = Transports.lookup(@config.transport)
|
38
|
+
klass.new(@config, @metrics)
|
39
|
+
end
|
40
|
+
end
|
41
|
+
end
|
@@ -0,0 +1,24 @@
|
|
1
|
+
module MagicPipe
|
2
|
+
module Codecs
|
3
|
+
def self.lookup(type)
|
4
|
+
case type
|
5
|
+
when :json then Json
|
6
|
+
when :thrift then Thrift
|
7
|
+
when :msgpack, :message_pack then MessagePack
|
8
|
+
when :yaml then Yaml
|
9
|
+
when Class then type
|
10
|
+
else
|
11
|
+
raise ConfigurationError, "Unknown MagicPipe::Codecs type: '#{type}'."
|
12
|
+
end
|
13
|
+
end
|
14
|
+
end
|
15
|
+
end
|
16
|
+
|
17
|
+
files = File.expand_path("../codecs/**/*.rb", __FILE__)
|
18
|
+
Dir[files].each do |f|
|
19
|
+
begin
|
20
|
+
require f
|
21
|
+
rescue LoadError
|
22
|
+
# Some components have extra dependencies
|
23
|
+
end
|
24
|
+
end
|
@@ -0,0 +1,26 @@
|
|
1
|
+
module MagicPipe
|
2
|
+
module Codecs
|
3
|
+
class Base
|
4
|
+
TYPE = "none"
|
5
|
+
|
6
|
+
# object should be something similar
|
7
|
+
# to an ActiveModel::Serializer or
|
8
|
+
# ActiveRecord object.
|
9
|
+
#
|
10
|
+
def initialize(object)
|
11
|
+
@object = object
|
12
|
+
end
|
13
|
+
|
14
|
+
attr_reader :object
|
15
|
+
alias_method :o, :object
|
16
|
+
|
17
|
+
def encode
|
18
|
+
raise NotImplementedError
|
19
|
+
end
|
20
|
+
|
21
|
+
def type
|
22
|
+
self.class.const_get(:TYPE)
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
@@ -0,0 +1,38 @@
|
|
1
|
+
require "magic_pipe/codecs/base"
|
2
|
+
|
3
|
+
module MagicPipe
|
4
|
+
module Codecs
|
5
|
+
class Json < Base
|
6
|
+
TYPE = "application/json"
|
7
|
+
|
8
|
+
def encode
|
9
|
+
if o.respond_to?(:to_json)
|
10
|
+
o.to_json
|
11
|
+
elsif o.respond_to?(:as_json)
|
12
|
+
json_dump(o.as_json)
|
13
|
+
else
|
14
|
+
json_dump(o)
|
15
|
+
end
|
16
|
+
end
|
17
|
+
|
18
|
+
|
19
|
+
private
|
20
|
+
|
21
|
+
|
22
|
+
begin
|
23
|
+
require "oj"
|
24
|
+
|
25
|
+
def json_dump(data)
|
26
|
+
Oj.dump(data)
|
27
|
+
end
|
28
|
+
rescue LoadError
|
29
|
+
puts "[#{self.to_s}] The oj gem is not available. Using json from the stdlib."
|
30
|
+
require "json"
|
31
|
+
|
32
|
+
def json_dump(data)
|
33
|
+
JSON.dump(data)
|
34
|
+
end
|
35
|
+
end
|
36
|
+
end
|
37
|
+
end
|
38
|
+
end
|
@@ -0,0 +1,81 @@
|
|
1
|
+
require "magic_pipe/codecs/base"
|
2
|
+
|
3
|
+
begin
|
4
|
+
require "msgpack"
|
5
|
+
|
6
|
+
module MagicPipe
|
7
|
+
module Codecs
|
8
|
+
class MessagePack < Base
|
9
|
+
TYPE = "application/x-msgpack"
|
10
|
+
|
11
|
+
def encode
|
12
|
+
case o
|
13
|
+
when Hash
|
14
|
+
o.to_msgpack
|
15
|
+
else
|
16
|
+
o.as_json.to_msgpack
|
17
|
+
end
|
18
|
+
end
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
22
|
+
|
23
|
+
# Extensions required to serialize time values
|
24
|
+
|
25
|
+
begin
|
26
|
+
# If used in Rails, most timestamps will be
|
27
|
+
# ActiveSupport::TimeWithZone instances.
|
28
|
+
|
29
|
+
require "active_support/time_with_zone"
|
30
|
+
|
31
|
+
ActiveSupport::TimeWithZone.class_eval do
|
32
|
+
def to_msgpack_ext
|
33
|
+
("TWZ[" + self.to_s + "]").to_msgpack
|
34
|
+
end
|
35
|
+
|
36
|
+
def self.from_msgpack_ext(data)
|
37
|
+
Time.zone.parse(data)
|
38
|
+
end
|
39
|
+
end
|
40
|
+
MessagePack::DefaultFactory.register_type 0x42, ActiveSupport::TimeWithZone
|
41
|
+
rescue LoadError
|
42
|
+
end
|
43
|
+
|
44
|
+
require "time"
|
45
|
+
require "date"
|
46
|
+
|
47
|
+
Time.class_eval do
|
48
|
+
def to_msgpack_ext
|
49
|
+
to_i.to_msgpack
|
50
|
+
end
|
51
|
+
|
52
|
+
def self.from_msgpack_ext(data)
|
53
|
+
n = MessagePack.unpack(data)
|
54
|
+
Time.at(n)
|
55
|
+
end
|
56
|
+
end
|
57
|
+
MessagePack::DefaultFactory.register_type 0x43, Time
|
58
|
+
|
59
|
+
Date.class_eval do
|
60
|
+
def to_msgpack_ext
|
61
|
+
("D[" + to_s + "]").to_msgpack
|
62
|
+
end
|
63
|
+
|
64
|
+
def self.from_msgpack_ext(data)
|
65
|
+
Date.parse(MessagePack.unpack(data))
|
66
|
+
end
|
67
|
+
end
|
68
|
+
MessagePack::DefaultFactory.register_type 0x44, Date
|
69
|
+
|
70
|
+
DateTime.class_eval do
|
71
|
+
def to_msgpack_ext
|
72
|
+
("DT[" + to_s + "]").to_msgpack
|
73
|
+
end
|
74
|
+
|
75
|
+
def self.from_msgpack_ext(data)
|
76
|
+
DateTime.parse(MessagePack.unpack(data))
|
77
|
+
end
|
78
|
+
end
|
79
|
+
MessagePack::DefaultFactory.register_type 0x45, DateTime
|
80
|
+
rescue LoadError
|
81
|
+
end
|
@@ -0,0 +1,17 @@
|
|
1
|
+
require "magic_pipe/codecs/base"
|
2
|
+
|
3
|
+
module MagicPipe
|
4
|
+
module Codecs
|
5
|
+
class Thrift < Base
|
6
|
+
# application/vnd.apache.thrift.binary
|
7
|
+
# application/vnd.apache.thrift.compact
|
8
|
+
# application/vnd.apache.thrift.json
|
9
|
+
|
10
|
+
TYPE = "application/vnd.apache.thrift.binary"
|
11
|
+
|
12
|
+
def encode
|
13
|
+
"not implemented"
|
14
|
+
end
|
15
|
+
end
|
16
|
+
end
|
17
|
+
end
|
@@ -0,0 +1,18 @@
|
|
1
|
+
require "magic_pipe/codecs/base"
|
2
|
+
require "yaml"
|
3
|
+
|
4
|
+
module MagicPipe
|
5
|
+
module Codecs
|
6
|
+
class Yaml < Base
|
7
|
+
# text/vnd.yaml
|
8
|
+
# text/x-yaml
|
9
|
+
# application/x-yaml
|
10
|
+
|
11
|
+
TYPE = "application/x-yaml"
|
12
|
+
|
13
|
+
def encode
|
14
|
+
::YAML.dump(o)
|
15
|
+
end
|
16
|
+
end
|
17
|
+
end
|
18
|
+
end
|
@@ -0,0 +1,103 @@
|
|
1
|
+
require "logger"
|
2
|
+
require "singleton"
|
3
|
+
|
4
|
+
module MagicPipe
|
5
|
+
class Config
|
6
|
+
FIELDS = [
|
7
|
+
:client_name, # the name of this client
|
8
|
+
:producer_name,
|
9
|
+
:logger, # A Logger
|
10
|
+
:metrics_client, # Statsd compatible object
|
11
|
+
|
12
|
+
:loader,
|
13
|
+
:codec,
|
14
|
+
:transport,
|
15
|
+
:sender,
|
16
|
+
|
17
|
+
:https_transport_options,
|
18
|
+
:sqs_transport_options,
|
19
|
+
:async_transport_options,
|
20
|
+
]
|
21
|
+
|
22
|
+
attr_accessor *FIELDS
|
23
|
+
alias_method :transports=, :transport=
|
24
|
+
alias_method :transports, :transport
|
25
|
+
|
26
|
+
|
27
|
+
def initialize
|
28
|
+
yield self if block_given?
|
29
|
+
set_defaults
|
30
|
+
end
|
31
|
+
|
32
|
+
|
33
|
+
private
|
34
|
+
|
35
|
+
|
36
|
+
def set_defaults
|
37
|
+
@client_name ||= "magic_pipe"
|
38
|
+
@producer_name ||= "Anonymous Piper"
|
39
|
+
@logger ||= Logger.new($stdout)
|
40
|
+
@metrics_client ||= dummy_metrics_object
|
41
|
+
|
42
|
+
@loader ||= :simple_active_record
|
43
|
+
@sender ||= :sync
|
44
|
+
@codec ||= :yaml
|
45
|
+
@transport ||= :log
|
46
|
+
|
47
|
+
set_https_defaults
|
48
|
+
set_sqs_defaults
|
49
|
+
set_async_defaults
|
50
|
+
end
|
51
|
+
|
52
|
+
|
53
|
+
def set_https_defaults
|
54
|
+
return unless @https_transport_options
|
55
|
+
|
56
|
+
defaults = {
|
57
|
+
url: "https://localhost:8080/foo",
|
58
|
+
auth_token: "missing",
|
59
|
+
timeout: 2,
|
60
|
+
open_timeout: 3,
|
61
|
+
}
|
62
|
+
@https_transport_options = defaults.merge(@https_transport_options)
|
63
|
+
end
|
64
|
+
|
65
|
+
|
66
|
+
def set_sqs_defaults
|
67
|
+
@sqs_transport_options ||= {}
|
68
|
+
defaults = {
|
69
|
+
queue: "magic_pipe",
|
70
|
+
}
|
71
|
+
@sqs_transport_options = defaults.merge(@sqs_transport_options)
|
72
|
+
end
|
73
|
+
|
74
|
+
|
75
|
+
# Since Sidekiq is the go-to sender for production, this
|
76
|
+
# should always be defined.
|
77
|
+
#
|
78
|
+
def set_async_defaults
|
79
|
+
@async_transport_options ||= {}
|
80
|
+
defaults = {
|
81
|
+
queue: "magic_pipe"
|
82
|
+
}
|
83
|
+
@async_transport_options = defaults.merge(@async_transport_options)
|
84
|
+
end
|
85
|
+
|
86
|
+
|
87
|
+
def dummy_metrics_object
|
88
|
+
Class.new do
|
89
|
+
def initialize(logger)
|
90
|
+
@out = logger
|
91
|
+
end
|
92
|
+
def method_missing(name, *args, &block)
|
93
|
+
@out.debug("[metrics] #{name}: #{args}")
|
94
|
+
# Uncomment this to create a black hole
|
95
|
+
# self.class.new(@out)
|
96
|
+
end
|
97
|
+
def respond_to_missing?(*)
|
98
|
+
true
|
99
|
+
end
|
100
|
+
end.new(@logger)
|
101
|
+
end
|
102
|
+
end
|
103
|
+
end
|
@@ -0,0 +1,29 @@
|
|
1
|
+
module MagicPipe
|
2
|
+
class Envelope
|
3
|
+
def initialize(body:, topic:, producer:, time:, mime:)
|
4
|
+
@body = body
|
5
|
+
@topic = topic
|
6
|
+
@producer = producer
|
7
|
+
@time = time.to_i
|
8
|
+
@mime = mime
|
9
|
+
end
|
10
|
+
|
11
|
+
attr_accessor :body
|
12
|
+
|
13
|
+
|
14
|
+
def as_json(*)
|
15
|
+
{
|
16
|
+
body: @body.as_json,
|
17
|
+
topic: @topic,
|
18
|
+
producer: @producer,
|
19
|
+
time: @time,
|
20
|
+
mime: @mime,
|
21
|
+
}
|
22
|
+
end
|
23
|
+
|
24
|
+
|
25
|
+
def ==(other)
|
26
|
+
as_json == other.as_json
|
27
|
+
end
|
28
|
+
end
|
29
|
+
end
|
@@ -0,0 +1,14 @@
|
|
1
|
+
module MagicPipe
|
2
|
+
class Error < StandardError
|
3
|
+
end
|
4
|
+
|
5
|
+
class ConfigurationError < Error
|
6
|
+
end
|
7
|
+
|
8
|
+
class LoaderError < Error
|
9
|
+
def initialize(class_name, context)
|
10
|
+
@message = "Can't resolve class name '#{class_name}' (#{context})"
|
11
|
+
end
|
12
|
+
attr_reader :message
|
13
|
+
end
|
14
|
+
end
|