magic_pipe 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (42) hide show
  1. checksums.yaml +7 -0
  2. data/.gitignore +11 -0
  3. data/.rspec +3 -0
  4. data/.travis.yml +8 -0
  5. data/CHANGELOG.md +10 -0
  6. data/Gemfile +6 -0
  7. data/Gemfile.lock +92 -0
  8. data/LICENSE.txt +19 -0
  9. data/README.md +241 -0
  10. data/Rakefile +6 -0
  11. data/bin/console +16 -0
  12. data/bin/setup +8 -0
  13. data/lib/magic_pipe.rb +45 -0
  14. data/lib/magic_pipe/client.rb +41 -0
  15. data/lib/magic_pipe/codecs.rb +24 -0
  16. data/lib/magic_pipe/codecs/base.rb +26 -0
  17. data/lib/magic_pipe/codecs/json.rb +38 -0
  18. data/lib/magic_pipe/codecs/message_pack.rb +81 -0
  19. data/lib/magic_pipe/codecs/thrift.rb +17 -0
  20. data/lib/magic_pipe/codecs/yaml.rb +18 -0
  21. data/lib/magic_pipe/config.rb +103 -0
  22. data/lib/magic_pipe/envelope.rb +29 -0
  23. data/lib/magic_pipe/errors.rb +14 -0
  24. data/lib/magic_pipe/loaders.rb +22 -0
  25. data/lib/magic_pipe/loaders/simple_active_record.rb +54 -0
  26. data/lib/magic_pipe/metrics.rb +54 -0
  27. data/lib/magic_pipe/senders.rb +23 -0
  28. data/lib/magic_pipe/senders/async.rb +76 -0
  29. data/lib/magic_pipe/senders/base.rb +24 -0
  30. data/lib/magic_pipe/senders/metrics_mixin.rb +20 -0
  31. data/lib/magic_pipe/senders/sync.rb +38 -0
  32. data/lib/magic_pipe/transports.rb +26 -0
  33. data/lib/magic_pipe/transports/base.rb +17 -0
  34. data/lib/magic_pipe/transports/debug.rb +17 -0
  35. data/lib/magic_pipe/transports/https.rb +80 -0
  36. data/lib/magic_pipe/transports/kafka.rb +8 -0
  37. data/lib/magic_pipe/transports/log.rb +15 -0
  38. data/lib/magic_pipe/transports/multi.rb +44 -0
  39. data/lib/magic_pipe/transports/sqs.rb +68 -0
  40. data/lib/magic_pipe/version.rb +3 -0
  41. data/magic_pipe.gemspec +44 -0
  42. metadata +257 -0
data/lib/magic_pipe.rb ADDED
@@ -0,0 +1,45 @@
1
+ require "magic_pipe/version"
2
+ require "magic_pipe/errors"
3
+
4
+ require "magic_pipe/config"
5
+ require "magic_pipe/metrics"
6
+
7
+ require "magic_pipe/envelope"
8
+
9
+ require "magic_pipe/loaders"
10
+ require "magic_pipe/codecs"
11
+ require "magic_pipe/senders"
12
+ require "magic_pipe/transports"
13
+
14
+ require "magic_pipe/client"
15
+
16
+ module MagicPipe
17
+ class << self
18
+ def lookup_client(name)
19
+ @store[name.to_sym]
20
+ end
21
+
22
+ # All this should be loaded before Sidekiq
23
+ # or Puma start forking threads.
24
+ #
25
+ def store_client(client)
26
+ @store ||= {}
27
+ @store[client.name.to_sym] = client
28
+ end
29
+
30
+ def clear_clients
31
+ @store = {}
32
+ end
33
+
34
+ def build(&block)
35
+ unless block_given?
36
+ raise ConfigurationError, "No configuration block provided."
37
+ end
38
+
39
+ config = Config.new(&block)
40
+ client = Client.new(config)
41
+ store_client(client)
42
+ client
43
+ end
44
+ end
45
+ end
@@ -0,0 +1,41 @@
1
+ module MagicPipe
2
+ class Client
3
+ def initialize(config)
4
+ @config = config
5
+ @name = config.client_name
6
+
7
+ @metrics = Metrics.new(@config)
8
+
9
+ @transport = build_transport
10
+
11
+ @codec = Codecs.lookup(config.codec)
12
+ @sender = Senders.lookup(config.sender)
13
+
14
+ @loader = Loaders.lookup(config.loader)
15
+ end
16
+
17
+ attr_reader :name, :config, :codec, :transport, :sender, :loader, :metrics
18
+
19
+ def send_data(object:, topic:, wrapper: nil, time: Time.now.utc)
20
+ sender.new(
21
+ object,
22
+ topic,
23
+ wrapper,
24
+ time,
25
+ codec,
26
+ transport,
27
+ @config,
28
+ @metrics
29
+ ).call
30
+ true
31
+ end
32
+
33
+
34
+ private
35
+
36
+ def build_transport
37
+ klass = Transports.lookup(@config.transport)
38
+ klass.new(@config, @metrics)
39
+ end
40
+ end
41
+ end
@@ -0,0 +1,24 @@
1
+ module MagicPipe
2
+ module Codecs
3
+ def self.lookup(type)
4
+ case type
5
+ when :json then Json
6
+ when :thrift then Thrift
7
+ when :msgpack, :message_pack then MessagePack
8
+ when :yaml then Yaml
9
+ when Class then type
10
+ else
11
+ raise ConfigurationError, "Unknown MagicPipe::Codecs type: '#{type}'."
12
+ end
13
+ end
14
+ end
15
+ end
16
+
17
+ files = File.expand_path("../codecs/**/*.rb", __FILE__)
18
+ Dir[files].each do |f|
19
+ begin
20
+ require f
21
+ rescue LoadError
22
+ # Some components have extra dependencies
23
+ end
24
+ end
@@ -0,0 +1,26 @@
1
+ module MagicPipe
2
+ module Codecs
3
+ class Base
4
+ TYPE = "none"
5
+
6
+ # object should be something similar
7
+ # to an ActiveModel::Serializer or
8
+ # ActiveRecord object.
9
+ #
10
+ def initialize(object)
11
+ @object = object
12
+ end
13
+
14
+ attr_reader :object
15
+ alias_method :o, :object
16
+
17
+ def encode
18
+ raise NotImplementedError
19
+ end
20
+
21
+ def type
22
+ self.class.const_get(:TYPE)
23
+ end
24
+ end
25
+ end
26
+ end
@@ -0,0 +1,38 @@
1
+ require "magic_pipe/codecs/base"
2
+
3
+ module MagicPipe
4
+ module Codecs
5
+ class Json < Base
6
+ TYPE = "application/json"
7
+
8
+ def encode
9
+ if o.respond_to?(:to_json)
10
+ o.to_json
11
+ elsif o.respond_to?(:as_json)
12
+ json_dump(o.as_json)
13
+ else
14
+ json_dump(o)
15
+ end
16
+ end
17
+
18
+
19
+ private
20
+
21
+
22
+ begin
23
+ require "oj"
24
+
25
+ def json_dump(data)
26
+ Oj.dump(data)
27
+ end
28
+ rescue LoadError
29
+ puts "[#{self.to_s}] The oj gem is not available. Using json from the stdlib."
30
+ require "json"
31
+
32
+ def json_dump(data)
33
+ JSON.dump(data)
34
+ end
35
+ end
36
+ end
37
+ end
38
+ end
@@ -0,0 +1,81 @@
1
+ require "magic_pipe/codecs/base"
2
+
3
+ begin
4
+ require "msgpack"
5
+
6
+ module MagicPipe
7
+ module Codecs
8
+ class MessagePack < Base
9
+ TYPE = "application/x-msgpack"
10
+
11
+ def encode
12
+ case o
13
+ when Hash
14
+ o.to_msgpack
15
+ else
16
+ o.as_json.to_msgpack
17
+ end
18
+ end
19
+ end
20
+ end
21
+ end
22
+
23
+ # Extensions required to serialize time values
24
+
25
+ begin
26
+ # If used in Rails, most timestamps will be
27
+ # ActiveSupport::TimeWithZone instances.
28
+
29
+ require "active_support/time_with_zone"
30
+
31
+ ActiveSupport::TimeWithZone.class_eval do
32
+ def to_msgpack_ext
33
+ ("TWZ[" + self.to_s + "]").to_msgpack
34
+ end
35
+
36
+ def self.from_msgpack_ext(data)
37
+ Time.zone.parse(data)
38
+ end
39
+ end
40
+ MessagePack::DefaultFactory.register_type 0x42, ActiveSupport::TimeWithZone
41
+ rescue LoadError
42
+ end
43
+
44
+ require "time"
45
+ require "date"
46
+
47
+ Time.class_eval do
48
+ def to_msgpack_ext
49
+ to_i.to_msgpack
50
+ end
51
+
52
+ def self.from_msgpack_ext(data)
53
+ n = MessagePack.unpack(data)
54
+ Time.at(n)
55
+ end
56
+ end
57
+ MessagePack::DefaultFactory.register_type 0x43, Time
58
+
59
+ Date.class_eval do
60
+ def to_msgpack_ext
61
+ ("D[" + to_s + "]").to_msgpack
62
+ end
63
+
64
+ def self.from_msgpack_ext(data)
65
+ Date.parse(MessagePack.unpack(data))
66
+ end
67
+ end
68
+ MessagePack::DefaultFactory.register_type 0x44, Date
69
+
70
+ DateTime.class_eval do
71
+ def to_msgpack_ext
72
+ ("DT[" + to_s + "]").to_msgpack
73
+ end
74
+
75
+ def self.from_msgpack_ext(data)
76
+ DateTime.parse(MessagePack.unpack(data))
77
+ end
78
+ end
79
+ MessagePack::DefaultFactory.register_type 0x45, DateTime
80
+ rescue LoadError
81
+ end
@@ -0,0 +1,17 @@
1
+ require "magic_pipe/codecs/base"
2
+
3
+ module MagicPipe
4
+ module Codecs
5
+ class Thrift < Base
6
+ # application/vnd.apache.thrift.binary
7
+ # application/vnd.apache.thrift.compact
8
+ # application/vnd.apache.thrift.json
9
+
10
+ TYPE = "application/vnd.apache.thrift.binary"
11
+
12
+ def encode
13
+ "not implemented"
14
+ end
15
+ end
16
+ end
17
+ end
@@ -0,0 +1,18 @@
1
+ require "magic_pipe/codecs/base"
2
+ require "yaml"
3
+
4
+ module MagicPipe
5
+ module Codecs
6
+ class Yaml < Base
7
+ # text/vnd.yaml
8
+ # text/x-yaml
9
+ # application/x-yaml
10
+
11
+ TYPE = "application/x-yaml"
12
+
13
+ def encode
14
+ ::YAML.dump(o)
15
+ end
16
+ end
17
+ end
18
+ end
@@ -0,0 +1,103 @@
1
+ require "logger"
2
+ require "singleton"
3
+
4
+ module MagicPipe
5
+ class Config
6
+ FIELDS = [
7
+ :client_name, # the name of this client
8
+ :producer_name,
9
+ :logger, # A Logger
10
+ :metrics_client, # Statsd compatible object
11
+
12
+ :loader,
13
+ :codec,
14
+ :transport,
15
+ :sender,
16
+
17
+ :https_transport_options,
18
+ :sqs_transport_options,
19
+ :async_transport_options,
20
+ ]
21
+
22
+ attr_accessor *FIELDS
23
+ alias_method :transports=, :transport=
24
+ alias_method :transports, :transport
25
+
26
+
27
+ def initialize
28
+ yield self if block_given?
29
+ set_defaults
30
+ end
31
+
32
+
33
+ private
34
+
35
+
36
+ def set_defaults
37
+ @client_name ||= "magic_pipe"
38
+ @producer_name ||= "Anonymous Piper"
39
+ @logger ||= Logger.new($stdout)
40
+ @metrics_client ||= dummy_metrics_object
41
+
42
+ @loader ||= :simple_active_record
43
+ @sender ||= :sync
44
+ @codec ||= :yaml
45
+ @transport ||= :log
46
+
47
+ set_https_defaults
48
+ set_sqs_defaults
49
+ set_async_defaults
50
+ end
51
+
52
+
53
+ def set_https_defaults
54
+ return unless @https_transport_options
55
+
56
+ defaults = {
57
+ url: "https://localhost:8080/foo",
58
+ auth_token: "missing",
59
+ timeout: 2,
60
+ open_timeout: 3,
61
+ }
62
+ @https_transport_options = defaults.merge(@https_transport_options)
63
+ end
64
+
65
+
66
+ def set_sqs_defaults
67
+ @sqs_transport_options ||= {}
68
+ defaults = {
69
+ queue: "magic_pipe",
70
+ }
71
+ @sqs_transport_options = defaults.merge(@sqs_transport_options)
72
+ end
73
+
74
+
75
+ # Since Sidekiq is the go-to sender for production, this
76
+ # should always be defined.
77
+ #
78
+ def set_async_defaults
79
+ @async_transport_options ||= {}
80
+ defaults = {
81
+ queue: "magic_pipe"
82
+ }
83
+ @async_transport_options = defaults.merge(@async_transport_options)
84
+ end
85
+
86
+
87
+ def dummy_metrics_object
88
+ Class.new do
89
+ def initialize(logger)
90
+ @out = logger
91
+ end
92
+ def method_missing(name, *args, &block)
93
+ @out.debug("[metrics] #{name}: #{args}")
94
+ # Uncomment this to create a black hole
95
+ # self.class.new(@out)
96
+ end
97
+ def respond_to_missing?(*)
98
+ true
99
+ end
100
+ end.new(@logger)
101
+ end
102
+ end
103
+ end
@@ -0,0 +1,29 @@
1
+ module MagicPipe
2
+ class Envelope
3
+ def initialize(body:, topic:, producer:, time:, mime:)
4
+ @body = body
5
+ @topic = topic
6
+ @producer = producer
7
+ @time = time.to_i
8
+ @mime = mime
9
+ end
10
+
11
+ attr_accessor :body
12
+
13
+
14
+ def as_json(*)
15
+ {
16
+ body: @body.as_json,
17
+ topic: @topic,
18
+ producer: @producer,
19
+ time: @time,
20
+ mime: @mime,
21
+ }
22
+ end
23
+
24
+
25
+ def ==(other)
26
+ as_json == other.as_json
27
+ end
28
+ end
29
+ end
@@ -0,0 +1,14 @@
1
+ module MagicPipe
2
+ class Error < StandardError
3
+ end
4
+
5
+ class ConfigurationError < Error
6
+ end
7
+
8
+ class LoaderError < Error
9
+ def initialize(class_name, context)
10
+ @message = "Can't resolve class name '#{class_name}' (#{context})"
11
+ end
12
+ attr_reader :message
13
+ end
14
+ end