arthropod 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 402aefa5ebb2883aa9660285c850995454a8a5dd87dc5cf4b1fc3131c66a25cc
+  data.tar.gz: fb12c70b9088b22e81cac21de40edd01e23fb50b668b8f775010ed80cfdff31b
+SHA512:
+  metadata.gz: 2bf3436d18735c33198c163fc950aaca20f663828c6e5cdc918000df26bcf109a1c6abac885c1a96dec22fc42c8a4d0ea64c745165974d3b0a75d4738ed81364
+  data.tar.gz: 52a1c6f6b0455633e97414eae5eeadf359b4436aae49653714031f9aba8f023538845c77fc9c9536426a5c749073c99aae67db1225345ee793b3916eb23467d1

data/.gitignore

@@ -0,0 +1 @@
+.env

data/.rspec

@@ -0,0 +1 @@
+--require spec_helper

data/Gemfile

@@ -0,0 +1,3 @@
+source "https://rubygems.org"
+
+gemspec

data/Gemfile.lock

@@ -0,0 +1,49 @@
+PATH
+  remote: .
+  specs:
+    arthropod (0.0.1)
+      aws-sdk-sqs
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    aws-eventstream (1.0.3)
+    aws-partitions (1.251.0)
+    aws-sdk-core (3.84.0)
+      aws-eventstream (~> 1.0, >= 1.0.2)
+      aws-partitions (~> 1, >= 1.239.0)
+      aws-sigv4 (~> 1.1)
+      jmespath (~> 1.0)
+    aws-sdk-sqs (1.23.1)
+      aws-sdk-core (~> 3, >= 3.71.0)
+      aws-sigv4 (~> 1.1)
+    aws-sigv4 (1.1.0)
+      aws-eventstream (~> 1.0, >= 1.0.2)
+    diff-lcs (1.3)
+    jmespath (1.4.0)
+    rake (13.0.1)
+    rspec (3.9.0)
+      rspec-core (~> 3.9.0)
+      rspec-expectations (~> 3.9.0)
+      rspec-mocks (~> 3.9.0)
+    rspec-core (3.9.0)
+      rspec-support (~> 3.9.0)
+    rspec-expectations (3.9.0)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.9.0)
+    rspec-mocks (3.9.0)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.9.0)
+    rspec-support (3.9.0)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  arthropod!
+  bundler
+  rake
+  rspec
+
+BUNDLED WITH
+   2.0.2

data/README.md

@@ -0,0 +1,92 @@
+# Arthropod
+
+Arthropod is a easy way to run remote ruby code synchronously, using Amazon SQS.
+
+*Do not use it yet, the API isn't stable at all and it wasn't tested enough on production*
+
+## Usage
+
+A simple use case first, let's say you want to push a video encoding task to another server:
+
+```ruby
+url_of_the_video = "https://my_storage.my_service.com/my_video_file.mp4"
+response = Arthropod::Client.push(queue_name: "video_encoding", body: { url: url_of_the_video })
+
+puts response.body
+# => "https://my_storage.my_service.com/my_reencoded_video_file.mp4"
+```
+
+On the "server" side:
+
+```ruby
+Arthropod::Server.pull(queue_name: "video_encoding") do |request|
+  video_url = request.body.url
+
+  # Do the encoding stuff
+  encoded_video_url = VideoEncoder.encode(video_url)
+
+  encoded_video_url # Everything evaluated here will be sent back to the client
+end
+```
+
+As you see, it's all synchronous and, since SQS will save your messages until they are consumed, your server doesn't even have to be listening right when you push the task (more on that later).
+
+It is also possible to push updates from the server:
+
+```ruby
+Arthropod::Server.pull(queue_name: "video_encoding") do |request|
+  video_url = request.body.url
+
+  # Do the encoding stuff but this time the VideoEncoder class will give you a percentage of completion
+  VideoEncoder.encode(video_url) do |percentage_of_completion|
+    request.respond { percentage_of_completion: percentage_of_completion }
+  end
+
+  encoded_video_url # Everything evaluated here will be sent back to the client
+end
+```
+
+And on the client side:
+
+```ruby
+url_of_the_video = "https://my_storage.my_service.com/my_video_file.mp4"
+response = Arthropod::Client.push(queue_name: "video_encoding", body: { url: url_of_the_video }) do |response|
+  puts response.body.percentage_of_completion # => 10, 20, 30, etc
+end
+
+puts response.body
+# => "https://my_storage.my_service.com/my_reencoded_video_file.mp4"
+```
+
+## API
+
+```ruby
+response = Arthropod::Client.push(queue_name: "video_encoding", body: { url: url_of_the_video }) do |response|
+  puts response.body
+end
+```
+
+This method pushes a job to the SQS queue `queue_name` and waits for the job completion, a block can be optionally provided if you expect the server to send you some updates along the way. The return value is the last value evaluated in the server block.
+
+```ruby
+Arthropod::Server.pull(queue_name: "video_encoding") do |request|
+  request.respond "some_update"
+
+  "final_result"
+end
+```
+
+This method will take a job from the queue and give it to the block, if no job are available the method will return immediately, it's your responsiblity to put this call in the loop if you want to. The last value from the block will be sent back to the client.
+
+## Why would you do take an asynchronous thing and make it synchronous?
+
+This library is here to solve a few real use-cases we encoutered, most of them involves running heavy tasks on remote servers or on remote computers that are not accessible through the internet. For example:
+
+* running some CUDA-related stuff from a cheap server, sometimes it's way cheaper to have a pretty beefy computers in house and run your heavy tasks on them instead of renting one for several hundred dollars each month.
+* sometimes you need to access some data that are only accessible locally, thing about 3D rendering where your assets, cache, etc are all stored locally for better performance. Now your local computer can pull tasks from the SQS queue, run them and push the results.
+
+Of course you can also achieve that by simply using SQS or any kind of message system, what Arthropod does is just to make it easier, however it's your responsibilty to run it in an asynchronous environment, think about an ActiveJob task for example. At its core Arthropod is just a thin layer around SQS.
+
+## Example: the poor man's video encoding service
+
+If you're not concerned about latency, you can for example push some heavy video encoding task from and ActiveJob job in your Rails task and run a little cron job every minute on your uber-CUDA-powered computer at home to pull those jobs and reencode your videos. It should be reliable enough and it may be even be way faster than doing it with the CPU off a regular server.

data/Rakefile

@@ -0,0 +1,11 @@
+require "bundler"
+Bundler.setup
+Bundler::GemHelper.install_tasks
+
+require "rspec/core/rake_task"
+require "rspec/core/version"
+
+desc "Run all examples"
+RSpec::Core::RakeTask.new :spec
+
+task default: [:spec]

data/arthropod.gemspec

@@ -0,0 +1,27 @@
+# -*- encoding: utf-8 -*-
+
+lib = File.expand_path("../lib", __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require "arthropod/version"
+
+Gem::Specification.new do |gem|
+  gem.name          = "arthropod"
+  gem.version       = Arthropod::VERSION
+  gem.authors       = ["Victor Goya"]
+  gem.email         = ["goya.victor@gmail.com"]
+  gem.description   = "Easy remote task execution with SQS"
+  gem.summary       = "Execute ruby task on another server synchronously using Amazon SQS"
+
+  gem.files         = `git ls-files -z`.split("\x0")
+  gem.require_paths = ["lib"]
+
+  gem.licenses      = ["MIT"]
+
+  gem.add_dependency 'aws-sdk-sqs'
+
+  gem.required_ruby_version = "~> 2.0"
+
+  gem.add_development_dependency 'bundler'
+  gem.add_development_dependency 'rspec'
+  gem.add_development_dependency 'rake'
+end

data/examples/client.rb

@@ -0,0 +1,8 @@
+require 'arthropod'
+
+loop do
+  response = Arthropod::Client.push(queue_name: "my_little_queue", body: { text: gets }) do |response|
+    puts "Downcase: #{response.body}"
+  end
+  puts "Upcase: #{response.body}"
+end

data/examples/server.rb

@@ -0,0 +1,9 @@
+require 'arthropod'
+
+loop do
+  Arthropod::Server.pull(queue_name: "my_little_queue") do |request|
+    request.respond request.body["text"].downcase
+
+    request.body["text"].upcase
+  end
+end

data/lib/arthropod.rb

@@ -0,0 +1,4 @@
+require 'arthropod/client'
+require 'arthropod/server'
+require 'arthropod/request'
+require 'arthropod/response'

data/lib/arthropod/client.rb

@@ -0,0 +1,34 @@
+require 'aws-sdk-sqs'
+require 'securerandom'
+
+module Arthropod
+  module Client
+    def self.push(queue_name:, body:, client: nil)
+      client ||= Aws::SQS::Client.new
+
+      sender_queue = client.create_queue(queue_name: queue_name)
+      return_queue = client.create_queue(queue_name: SecureRandom.uuid.gsub("-", "_"))
+
+      # Send our order with a return queue so we can get responses
+      client.send_message(queue_url: sender_queue.queue_url, message_body: JSON.dump({ return_queue_url: return_queue.queue_url, body: body }))
+
+      loop do
+        response = client.receive_message(queue_url: return_queue.queue_url, max_number_of_messages: 1, wait_time_seconds: 1)
+        response.messages.each do |message|
+          response = Arthropod::Response.new(client: client, message: message)
+          begin
+            if response.state == "close"
+              return response
+            else
+              yield response if block_given?
+            end
+          ensure
+            client.delete_message(queue_url: return_queue.queue_url, receipt_handle: message.receipt_handle)
+          end
+        end
+      end
+    ensure
+      client.delete_queue(queue_url: return_queue.queue_url) if return_queue
+    end
+  end
+end

data/lib/arthropod/request.rb

@@ -0,0 +1,43 @@
+module Arthropod
+  class Request
+    attr_reader :message, :client
+
+    def initialize(client:, message:)
+      @client = client
+      @message = message
+    end
+
+    def body
+      parsed_message_body["body"]
+    end
+
+    def return_queue_url
+      parsed_message_body["return_queue_url"]
+    end
+
+    def state
+      parsed_message_body["state"]
+    end
+
+    def respond(body = nil)
+      send_message({ state: "open", body: body })
+    end
+
+    def close(body = nil)
+      send_message({ state: "close", body: body })
+    end
+
+    private
+
+    def send_message(message_body)
+      client.send_message({
+        queue_url: return_queue_url,
+        message_body: JSON.dump(message_body)
+      })
+    end
+
+    def parsed_message_body
+      @parsed_message_body ||= JSON.parse(message.body)
+    end
+  end
+end

data/lib/arthropod/response.rb

@@ -0,0 +1,24 @@
+module Arthropod
+  class Response
+    attr_reader :message, :client
+
+    def initialize(client:, message:)
+      @client = client
+      @message = message
+    end
+
+    def body
+      parsed_message_body["body"]
+    end
+
+    def state
+      parsed_message_body["state"]
+    end
+
+    private
+
+    def parsed_message_body
+      @parsed_message_body ||= JSON.parse(message.body)
+    end
+  end
+end

data/lib/arthropod/server.rb

@@ -0,0 +1,20 @@
+require 'aws-sdk-sqs'
+
+module Arthropod
+  module Server
+    def self.pull(queue_name:, client: nil)
+      client ||= Aws::SQS::Client.new
+
+      sender_queue = client.create_queue(queue_name: queue_name)
+      response = client.receive_message(queue_url: sender_queue.queue_url, max_number_of_messages: 1, wait_time_seconds: 1)
+      response.messages.each do |message|
+        request = Arthropod::Request.new(client: client, message: message)
+        begin
+          request.close(yield request)
+        ensure
+          client.delete_message(queue_url: sender_queue.queue_url, receipt_handle: message.receipt_handle)
+        end
+      end
+    end
+  end
+end

data/lib/arthropod/version.rb

@@ -0,0 +1,3 @@
+module Arthropod
+  VERSION = "0.0.1"
+end

data/spec/client_spec.rb

@@ -0,0 +1,67 @@
+require 'arthropod'
+
+RSpec.describe(Arthropod::Client) do
+  context "#push" do
+    let(:client) do
+      Aws::SQS::Client.new(stub_responses: true)
+    end
+
+    let(:sender_queue_url) do
+      "https://aws.com/sender_queue_url"
+    end
+
+    let(:return_queue_url) do
+      "https://aws.com/return_queue_url"
+    end
+
+    let(:body) do
+      { my_key: "my_value" }
+    end
+
+    let(:uuid) do
+      "444444444444-4444-4444-8888-88888888"
+    end
+
+    before(:each) do
+      expect(SecureRandom).to(
+        receive(:uuid)
+          .and_return(uuid)
+      )
+      expect(client).to(
+        receive(:create_queue)
+          .with({ queue_name: "test" })
+          .and_return(OpenStruct.new(queue_url: sender_queue_url))
+      )
+      expect(client).to(
+        receive(:create_queue)
+          .with({ queue_name: uuid.gsub("-", "_") })
+          .and_return(OpenStruct.new(queue_url: return_queue_url))
+      )
+      expect(client).to(
+        receive(:send_message)
+        .with({ queue_url: sender_queue_url, message_body: JSON.dump({ return_queue_url: return_queue_url, body: body }) })
+      )
+      expect(client).to(
+        receive(:receive_message)
+        .with({ queue_url: return_queue_url, max_number_of_messages: 1, wait_time_seconds: 1 })
+        .and_return(OpenStruct.new(messages: [ OpenStruct.new({ "body" => JSON.dump({ "state" => "open", "body" => "update" }), receipt_handle: "receipt_handle" }) ]))
+      )
+      expect(client).to(
+        receive(:receive_message)
+        .with({ queue_url: return_queue_url, max_number_of_messages: 1, wait_time_seconds: 1 })
+        .and_return(OpenStruct.new(messages: [ OpenStruct.new({ "body" => JSON.dump({ "state" => "close", "body" => "payload" }), receipt_handle: "receipt_handle" }) ]))
+      )
+      expect(client).to(
+        receive(:delete_queue)
+        .with({ queue_url: return_queue_url })
+      )
+    end
+
+    it "works" do
+      response = Arthropod::Client.push(client: client, queue_name: "test", body: body) do |response|
+        expect(response.body).to eq("update")
+      end
+      expect(response.body).to eq("payload")
+    end
+  end
+end

data/spec/server_spec.rb

@@ -0,0 +1,51 @@
+require 'arthropod'
+
+RSpec.describe(Arthropod::Server) do
+  context "#pull" do
+    let(:client) do
+      Aws::SQS::Client.new(stub_responses: true)
+    end
+
+    let(:sender_queue_url) do
+      "https://aws.com/sender_queue_url"
+    end
+
+    let(:return_queue_url) do
+      "https://aws.com/return_queue_url"
+    end
+
+    let(:uuid) do
+      "444444444444-4444-4444-8888-88888888"
+    end
+
+    before(:each) do
+      expect(client).to(
+        receive(:create_queue)
+          .with({ queue_name: "test" })
+          .and_return(OpenStruct.new(queue_url: sender_queue_url))
+      )
+      expect(client).to(
+        receive(:receive_message)
+        .with({ queue_url: sender_queue_url, max_number_of_messages: 1, wait_time_seconds: 1 })
+        .and_return(OpenStruct.new(messages: [ OpenStruct.new({ "body" => JSON.dump({ "body" => "request", "return_queue_url" => return_queue_url }), receipt_handle: "receipt_handle" }) ]))
+      )
+      expect(client).to(
+        receive(:send_message)
+        .with({ queue_url: return_queue_url, message_body: JSON.dump({ state: "open", body: "response" }) })
+      )
+      expect(client).to(
+        receive(:send_message)
+        .with({ queue_url: return_queue_url, message_body: JSON.dump({ state: "close", body: "final_response" }) })
+      )
+    end
+
+    it "works" do
+      Arthropod::Server.pull(client: client, queue_name: "test") do |request|
+        expect(request.body).to eq("request")
+        request.respond "response"
+
+        "final_response"
+      end
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,100 @@
+# This file was generated by the `rspec --init` command. Conventionally, all
+# specs live under a `spec` directory, which RSpec adds to the `$LOAD_PATH`.
+# The generated `.rspec` file contains `--require spec_helper` which will cause
+# this file to always be loaded, without a need to explicitly require it in any
+# files.
+#
+# Given that it is always loaded, you are encouraged to keep this file as
+# light-weight as possible. Requiring heavyweight dependencies from this file
+# will add to the boot time of your test suite on EVERY test run, even for an
+# individual file that may not need all of that loaded. Instead, consider making
+# a separate helper file that requires the additional dependencies and performs
+# the additional setup, and require it from the spec files that actually need
+# it.
+#
+# See http://rubydoc.info/gems/rspec-core/RSpec/Core/Configuration
+RSpec.configure do |config|
+  # rspec-expectations config goes here. You can use an alternate
+  # assertion/expectation library such as wrong or the stdlib/minitest
+  # assertions if you prefer.
+  config.expect_with :rspec do |expectations|
+    # This option will default to `true` in RSpec 4. It makes the `description`
+    # and `failure_message` of custom matchers include text for helper methods
+    # defined using `chain`, e.g.:
+    #     be_bigger_than(2).and_smaller_than(4).description
+    #     # => "be bigger than 2 and smaller than 4"
+    # ...rather than:
+    #     # => "be bigger than 2"
+    expectations.include_chain_clauses_in_custom_matcher_descriptions = true
+  end
+
+  # rspec-mocks config goes here. You can use an alternate test double
+  # library (such as bogus or mocha) by changing the `mock_with` option here.
+  config.mock_with :rspec do |mocks|
+    # Prevents you from mocking or stubbing a method that does not exist on
+    # a real object. This is generally recommended, and will default to
+    # `true` in RSpec 4.
+    mocks.verify_partial_doubles = true
+  end
+
+  # This option will default to `:apply_to_host_groups` in RSpec 4 (and will
+  # have no way to turn it off -- the option exists only for backwards
+  # compatibility in RSpec 3). It causes shared context metadata to be
+  # inherited by the metadata hash of host groups and examples, rather than
+  # triggering implicit auto-inclusion in groups with matching metadata.
+  config.shared_context_metadata_behavior = :apply_to_host_groups
+
+# The settings below are suggested to provide a good initial experience
+# with RSpec, but feel free to customize to your heart's content.
+=begin
+  # This allows you to limit a spec run to individual examples or groups
+  # you care about by tagging them with `:focus` metadata. When nothing
+  # is tagged with `:focus`, all examples get run. RSpec also provides
+  # aliases for `it`, `describe`, and `context` that include `:focus`
+  # metadata: `fit`, `fdescribe` and `fcontext`, respectively.
+  config.filter_run_when_matching :focus
+
+  # Allows RSpec to persist some state between runs in order to support
+  # the `--only-failures` and `--next-failure` CLI options. We recommend
+  # you configure your source control system to ignore this file.
+  config.example_status_persistence_file_path = "spec/examples.txt"
+
+  # Limits the available syntax to the non-monkey patched syntax that is
+  # recommended. For more details, see:
+  #   - http://rspec.info/blog/2012/06/rspecs-new-expectation-syntax/
+  #   - http://www.teaisaweso.me/blog/2013/05/27/rspecs-new-message-expectation-syntax/
+  #   - http://rspec.info/blog/2014/05/notable-changes-in-rspec-3/#zero-monkey-patching-mode
+  config.disable_monkey_patching!
+
+  # This setting enables warnings. It's recommended, but in some cases may
+  # be too noisy due to issues in dependencies.
+  config.warnings = true
+
+  # Many RSpec users commonly either run the entire suite or an individual
+  # file, and it's useful to allow more verbose output when running an
+  # individual spec file.
+  if config.files_to_run.one?
+    # Use the documentation formatter for detailed output,
+    # unless a formatter has already been configured
+    # (e.g. via a command-line flag).
+    config.default_formatter = "doc"
+  end
+
+  # Print the 10 slowest examples and example groups at the
+  # end of the spec run, to help surface which specs are running
+  # particularly slow.
+  config.profile_examples = 10
+
+  # Run specs in random order to surface order dependencies. If you find an
+  # order dependency and want to debug it, you can fix the order by providing
+  # the seed, which is printed after each run.
+  #     --seed 1234
+  config.order = :random
+
+  # Seed global randomization in this process using the `--seed` CLI option.
+  # Setting this allows you to use `--seed` to deterministically reproduce
+  # test failures related to randomization by passing the same `--seed` value
+  # as the one that triggered the failure.
+  Kernel.srand config.seed
+=end
+end

metadata

@@ -0,0 +1,117 @@
+--- !ruby/object:Gem::Specification
+name: arthropod
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Victor Goya
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2019-12-07 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: aws-sdk-sqs
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Easy remote task execution with SQS
+email:
+- goya.victor@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rspec"
+- Gemfile
+- Gemfile.lock
+- README.md
+- Rakefile
+- arthropod.gemspec
+- examples/client.rb
+- examples/server.rb
+- lib/arthropod.rb
+- lib/arthropod/client.rb
+- lib/arthropod/request.rb
+- lib/arthropod/response.rb
+- lib/arthropod/server.rb
+- lib/arthropod/version.rb
+- spec/client_spec.rb
+- spec/server_spec.rb
+- spec/spec_helper.rb
+homepage: 
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - "~>"
+    - !ruby/object:Gem::Version
+      version: '2.0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.0.3
+signing_key: 
+specification_version: 4
+summary: Execute ruby task on another server synchronously using Amazon SQS
+test_files: []