magic_pipe 0.1.0

data/lib/magic_pipe/transports/kafka.rb

@@ -0,0 +1,8 @@
+require "magic_pipe/transports/base"
+
+module MagicPipe
+  module Transports
+    class Kafka < Base
+    end
+  end
+end

data/lib/magic_pipe/transports/log.rb

@@ -0,0 +1,15 @@
+require "magic_pipe/transports/base"
+
+module MagicPipe
+  module Transports
+    class Log < Base
+      def initialize(config, metrics)
+        super(config, metrics)
+      end
+
+      def submit(payload, _metadata)
+        @logger.info "[Trasport#submit]: ↩️\n#{payload}\n"
+      end
+    end
+  end
+end

data/lib/magic_pipe/transports/multi.rb

@@ -0,0 +1,44 @@
+require "magic_pipe/transports/base"
+
+module MagicPipe
+  module Transports
+    class Multi < Base
+      def initialize(config, metrics)
+        super(config, metrics)
+        @transports = build_nested_transports
+      end
+
+
+      def submit(payload, metadata)
+        @transports.map do |transport|
+          begin
+            transport.submit(payload, metadata)
+          rescue => e
+            log_error(e, transport)
+          end
+        end
+      end
+
+
+      private
+
+
+      def build_nested_transports
+        @config.transport.map do |transport|
+          klass = MagicPipe::Transports.lookup(transport)
+          klass.new(@config, @metrics)
+        end
+      end
+
+
+      def log_error(e, transport)
+        @logger.error(
+          "[MagicPipe] Transports::Multi, error submitting with %{t} (%{e})." % {
+            t: transport.class,
+            e: e
+          }
+        )
+      end
+    end
+  end
+end

data/lib/magic_pipe/transports/sqs.rb

@@ -0,0 +1,68 @@
+require "magic_pipe/transports/base"
+
+require "aws-sdk-sqs"
+
+# https://docs.aws.amazon.com/sdk-for-ruby/v3/api/Aws/SQS.html
+# https://docs.aws.amazon.com/sdk-for-ruby/v3/api/Aws/SQS/Client.html
+
+module MagicPipe
+  module Transports
+    class Sqs < Base
+      def initialize(config, metrics)
+        super(config, metrics)
+        @options = @config.sqs_transport_options
+        @client = Aws::SQS::Client.new
+      end
+
+
+      def submit(payload, metadata)
+        send_message(payload, metadata)
+      end
+
+
+      private
+
+
+      def queue_name
+        @options.fetch(:queue)
+      end
+
+
+      def queue_url
+        @queue_url ||= @client.get_queue_url(queue_name: queue_name).queue_url
+      end
+
+
+      def send_message(payload, metadata)
+        @client.send_message({
+          queue_url: queue_url, # required
+          message_body: payload, # required
+          delay_seconds: 0,
+          message_attributes: meta_attributes(metadata)
+        })
+      end
+
+
+      def meta_attributes(metadata)
+        {
+          "topic" => {
+            string_value: metadata[:topic],
+            data_type: "String", # required
+          },
+          "producer" => {
+            string_value: metadata[:producer],
+            data_type: "String", # required
+          },
+          "sent_at" => {
+            string_value: metadata[:time].to_s,
+            data_type: "Number", # required
+          },
+          "mime" => {
+            string_value: metadata[:mime],
+            data_type: "String", # required
+          },
+        }
+      end
+    end
+  end
+end

data/lib/magic_pipe/version.rb

@@ -0,0 +1,3 @@
+module MagicPipe
+  VERSION = "0.1.0"
+end

data/magic_pipe.gemspec

@@ -0,0 +1,44 @@
+lib = File.expand_path("../lib", __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require "magic_pipe/version"
+
+Gem::Specification.new do |spec|
+  spec.name          = "magic_pipe"
+  spec.version       = MagicPipe::VERSION
+  spec.authors       = ["Tommaso Pavese"]
+  spec.email         = ["tommaso@pavese.me"]
+
+  spec.licenses      = ['MIT']
+
+  spec.summary       = %q{A Magic Pipe to send data in arbitrary formats to configurable backends, with topics.}
+
+  spec.description   = %q{A Magic Pipe to send data in arbitrary formats to configurable backends, with topics.} +
+                       %q{ It features a modular design that allows to configure and extend how the payloads are} +
+                       %q{ fetched, serialized, encoded, submitted, and most importantly where: SQS, HTTPS} +
+                       %q{ endpoints, etc.}
+
+  spec.homepage      = "https://github.com/tompave/magic_pipe"
+
+  spec.files         = `git ls-files -z`.split("\x0").reject do |f|
+    f.match(%r{^(test|spec|features)/})
+  end
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.16"
+  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "pry", '~> 0.11'
+
+  spec.add_development_dependency "rspec", "~> 3.0"
+  spec.add_development_dependency 'webmock', '~> 3.3'
+  spec.add_development_dependency 'timecop', '~> 0.9'
+
+  # Optional dependencies for the adapters
+  spec.add_development_dependency 'sidekiq'
+  spec.add_development_dependency 'oj'
+  spec.add_development_dependency 'msgpack'
+  spec.add_development_dependency 'typhoeus'
+  spec.add_development_dependency 'faraday'
+  spec.add_development_dependency 'aws-sdk-sqs', '~> 1.3'
+end

metadata

@@ -0,0 +1,257 @@
+--- !ruby/object:Gem::Specification
+name: magic_pipe
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Tommaso Pavese
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2018-02-04 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.16'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.16'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.11'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.11'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: webmock
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.3'
+- !ruby/object:Gem::Dependency
+  name: timecop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.9'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.9'
+- !ruby/object:Gem::Dependency
+  name: sidekiq
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: oj
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: msgpack
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: typhoeus
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: faraday
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: aws-sdk-sqs
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.3'
+description: 'A Magic Pipe to send data in arbitrary formats to configurable backends,
+  with topics. It features a modular design that allows to configure and extend how
+  the payloads are fetched, serialized, encoded, submitted, and most importantly where:
+  SQS, HTTPS endpoints, etc.'
+email:
+- tommaso@pavese.me
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rspec"
+- ".travis.yml"
+- CHANGELOG.md
+- Gemfile
+- Gemfile.lock
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/console
+- bin/setup
+- lib/magic_pipe.rb
+- lib/magic_pipe/client.rb
+- lib/magic_pipe/codecs.rb
+- lib/magic_pipe/codecs/base.rb
+- lib/magic_pipe/codecs/json.rb
+- lib/magic_pipe/codecs/message_pack.rb
+- lib/magic_pipe/codecs/thrift.rb
+- lib/magic_pipe/codecs/yaml.rb
+- lib/magic_pipe/config.rb
+- lib/magic_pipe/envelope.rb
+- lib/magic_pipe/errors.rb
+- lib/magic_pipe/loaders.rb
+- lib/magic_pipe/loaders/simple_active_record.rb
+- lib/magic_pipe/metrics.rb
+- lib/magic_pipe/senders.rb
+- lib/magic_pipe/senders/async.rb
+- lib/magic_pipe/senders/base.rb
+- lib/magic_pipe/senders/metrics_mixin.rb
+- lib/magic_pipe/senders/sync.rb
+- lib/magic_pipe/transports.rb
+- lib/magic_pipe/transports/base.rb
+- lib/magic_pipe/transports/debug.rb
+- lib/magic_pipe/transports/https.rb
+- lib/magic_pipe/transports/kafka.rb
+- lib/magic_pipe/transports/log.rb
+- lib/magic_pipe/transports/multi.rb
+- lib/magic_pipe/transports/sqs.rb
+- lib/magic_pipe/version.rb
+- magic_pipe.gemspec
+homepage: https://github.com/tompave/magic_pipe
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.6.14
+signing_key: 
+specification_version: 4
+summary: A Magic Pipe to send data in arbitrary formats to configurable backends,
+  with topics.
+test_files: []
+has_rdoc: