web_fetch 0.1.0

data/lib/web_fetch/event_machine_helpers.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+module WebFetch
+  # EventMachine layer-specific helpers
+  module EventMachineHelpers
+    def wait_for_response(deferred, response)
+      deferred[:http].callback do
+        Logger.debug("HTTP fetch complete for uid: #{deferred[:uid]}")
+        deferred[:succeeded] = true
+      end
+
+      deferred[:http].errback do
+        Logger.debug("HTTP fetch failed for uid: #{deferred[:uid]}")
+        deferred[:failed] = true
+      end
+
+      tick_loop(deferred, response)
+    end
+
+    def tick_loop(deferred, response)
+      # XXX There may be a much nicer way to wait for an async task to complete
+      # before returning a response but I couldn't figure it out, so I used
+      # EM.tick_loop which effectively does the same as a Twisted deferred
+      # callback chain, just much more explicitly.
+      EM.tick_loop do
+        if deferred[:succeeded]
+          succeed(deferred, response)
+          :stop
+        elsif deferred[:failed]
+          fail_(deferred, response)
+          :stop
+        end
+      end
+    end
+  end
+end

data/lib/web_fetch/gatherer.rb

@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+module WebFetch
+  # Handles requests to gather URLs and delegates to the EventMachine web
+  # server
+  class Gatherer
+    include Validatable
+
+    HASHABLE_KEYS = %i[url query_string headers method].freeze
+
+    def initialize(server, params)
+      @requests = params[:requests]
+      @server = server
+    end
+
+    def start
+      tagged = { requests: tag_requests }
+      @server.gather(tagged[:requests])
+      tagged
+    end
+
+    private
+
+    def validate
+      error(:requests_missing) if requests_missing?
+      error(:requests_not_array) if requests_not_array?
+      error(:requests_empty) if requests_empty?
+      error(:missing_url) if missing_url?
+    end
+
+    def requests_missing?
+      @requests.nil?
+    end
+
+    def requests_not_array?
+      !@requests.nil? && !@requests.is_a?(Array)
+    end
+
+    def requests_empty?
+      @requests.is_a?(Array) && @requests.length.zero?
+    end
+
+    def missing_url?
+      @requests.is_a?(Array) && @requests.any? { |req| req[:url].nil? }
+    end
+
+    def tag_requests
+      @requests.map do |request|
+        { request: request, hash: hash(request), uid: uid }
+      end
+    end
+
+    def hash(obj)
+      string = JSON.dump(obj.select { |key| HASHABLE_KEYS.include?(key) })
+      Digest.hexencode(Digest::SHA1.new.digest(string))
+    end
+
+    def uid
+      SecureRandom.uuid
+    end
+  end
+end

data/lib/web_fetch/helpers.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module WebFetch
+  # Shared code used throughout the application
+  module Helpers
+    def symbolize(obj)
+      # >:)
+      JSON.parse(JSON.dump(obj), symbolize_names: true)
+    end
+  end
+end

data/lib/web_fetch/http_helpers.rb

@@ -0,0 +1,71 @@
+# frozen_string_literal: true
+
+module WebFetch
+  # Convenience methods for WebFetch HTTP layer
+  module HTTPHelpers
+    def respond_immediately(result, response)
+      response.status = result[:status]
+      response.content = compress(result[:payload].to_json)
+      response.send_response
+    end
+
+    def compress(string)
+      ActiveSupport::Gzip.compress(string)
+    end
+
+    def default_headers(response)
+      response.headers['Content-Type'] = 'application/json; charset=utf-8'
+      response.headers['Cache-Control'] = 'max-age=0, private, must-revalidate'
+      response.headers['Content-Encoding'] = 'gzip'
+      response.headers['Vary'] = 'Accept-Encoding'
+    end
+
+    def request_params
+      { method: @http_request_method,
+        query_string: @http_query_string,
+        post_data: post_data,
+        server: self }
+    end
+
+    def post_data
+      return nil unless @http_post_content
+
+      JSON.parse(@http_post_content, symbolize_names: true)
+    end
+
+    def succeed(deferred, response)
+      response.status = 200
+      response.content = compress(JSON.dump(success(deferred)))
+      response.send_response
+    end
+
+    def success(deferred)
+      result = deferred[:http]
+      { response: {
+        success: true,
+        body: result.response,
+        headers: result.headers,
+        status: result.response_header.status
+      },
+        uid: deferred[:uid] }
+    end
+
+    def fail_(deferred, response)
+      response.status = 200
+      response.content = compress(JSON.dump(failure(deferred)))
+      response.send_response
+    end
+
+    def failure(deferred)
+      result = deferred[:http]
+      { response: {
+        success: false,
+        body: result.response,
+        headers: result.headers,
+        status: result.response_header.status,
+        error: (result.error&.inspect)
+      },
+        uid: deferred[:uid] }
+    end
+  end
+end

data/lib/web_fetch/logger.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+require 'forwardable'
+
+module WebFetch
+  # EventMachine-friendly Logger
+  class Logger
+    extend SingleForwardable
+
+    def self.logger(path)
+      @logger ||= EM::Logger.new(::Logger.new(log_file(path)))
+    end
+
+    def_delegators :@logger, :debug, :info, :warn, :error, :fatal
+
+    class << self
+      private
+
+      def log_file(path)
+        return STDOUT if STDOUT.isatty && path.nil?
+        return File.open(File::NULL, 'w') if path.nil?
+
+        log = File.open(path, 'a')
+        log.sync = true # Prevent buffering
+        log
+      end
+    end
+  end
+end

data/lib/web_fetch/resources.rb

@@ -0,0 +1,59 @@
+# frozen_string_literal: true
+
+module WebFetch
+  # Glue between the router and the guts of the application; calls the relevant
+  # code and builds responses
+  class Resources
+    class << self
+      def root(_server, _params)
+        { status: status(:ok), payload: { application: 'WebFetch' } }
+      end
+
+      def gather(server, params)
+        gatherer = Gatherer.new(server, params)
+        if gatherer.valid?
+          { status: status(:ok), payload: gatherer.start }
+        else
+          { status: status(:unprocessable),
+            payload: { error: gatherer.errors } }
+        end
+      end
+
+      def retrieve(server, params)
+        retriever = Retriever.new(server, params)
+        unless retriever.valid?
+          return { status: status(:unprocessable),
+                   payload: { error: retriever.errors } }
+        end
+        defer_if_found(retriever)
+      end
+
+      private
+
+      def status(name)
+        {
+          ok: 200,
+          unprocessable: 422,
+          not_found: 404
+        }.fetch(name)
+      end
+
+      def not_found(retriever)
+        {
+          status: status(:not_found),
+          payload: { error: retriever.not_found_error }
+        }
+      end
+
+      def defer_if_found(retriever)
+        found = retriever.find
+        if found.nil?
+          { status: status(:not_found),
+            payload: { error: retriever.not_found_error } }
+        else
+          { deferred: found }
+        end
+      end
+    end
+  end
+end

data/lib/web_fetch/retriever.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+module WebFetch
+  # Retrieves a gathered HTTP request
+  class Retriever
+    include Validatable
+
+    attr_reader :not_found_error
+
+    def initialize(server, params)
+      @uid = params[:uid]
+      @hash = params[:hash]
+      @server = server
+    end
+
+    def find
+      stored = @server.storage.fetch(@uid)
+      return not_found if stored.nil?
+
+      stored
+    end
+
+    private
+
+    def validate
+      error(:hash_or_uid_but_not_both) if !@uid.nil? && !@hash.nil?
+      error(:missing_hash_and_uid) if @uid.nil? && @hash.nil?
+    end
+
+    def not_found
+      @not_found_error = if !@uid.nil?
+                           I18n.t(:uid_not_found)
+                         elsif !@hash.nil?
+                           I18n.t(:hash_not_found)
+                         end
+      nil
+    end
+  end
+end

data/lib/web_fetch/router.rb

@@ -0,0 +1,71 @@
+# frozen_string_literal: true
+
+require 'hanami/router'
+require 'rack/utils'
+
+module WebFetch
+  # Dispatches requests to correct resource
+  class Router
+    include Helpers
+
+    def initialize
+      @router = setup
+    end
+
+    def route(url, options = {})
+      @server = options.delete(:server)
+      options = { query_string: nil, method: 'GET' }.merge(options)
+      method = options[:method].downcase.to_sym
+      Logger.info("#{url}: #{options}")
+      begin
+        params = build_params(options)
+      rescue JSON::ParserError
+        return { status: 400, payload: I18n.t(:bad_json) }
+      end
+      @router.recognize(url, method: method).call(params)
+    end
+
+    private
+
+    # rubocop:disable Metrics/MethodLength
+    def setup
+      resource_finder = lambda do |name, env|
+        Resources.public_send(name, @server, env)
+      end
+
+      Hanami::Router.new do
+        get '/', to: lambda { |params|
+          resource_finder.call(:root, params)
+        }
+
+        post '/gather', to: lambda { |params|
+          resource_finder.call(:gather, params)
+        }
+
+        get '/retrieve', to: lambda { |params|
+          resource_finder.call(:retrieve, params)
+        }
+      end
+    end
+    # rubocop:enable Metrics/MethodLength
+
+    def build_params(options)
+      params = Rack::Utils.parse_nested_query(options[:query_string])
+      merge_json!(params)
+      params = symbolize(params)
+      params.merge!(options[:post_data] || {})
+      params
+    end
+
+    def merge_json(params)
+      params.merge(
+        JSON.parse(params.delete('json') || '{}',
+                   symbolize_names: true)
+      )
+    end
+
+    def merge_json!(params)
+      params.merge!(merge_json(params))
+    end
+  end
+end

data/lib/web_fetch/server.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+module WebFetch
+  # Web server that accepts requests to gather and retrieve external HTTP
+  # requests
+  class Server < EM::Connection
+    attr_reader :storage
+
+    include EM::HttpServer
+    include HTTPHelpers
+    include EventMachineHelpers
+
+    def post_init
+      super
+      @router = Router.new
+      @storage = Storage
+      no_environment_strings
+    end
+
+    def process_http_request
+      result = @router.route(@http_request_uri, request_params)
+      response = EM::DelegatedHttpResponse.new(self)
+
+      default_headers(response)
+
+      if result[:deferred].nil?
+        respond_immediately(result, response)
+      else
+        wait_for_response(result[:deferred], response)
+      end
+    end
+
+    # Note that #gather is called by WebFetch itself to asynchronously gather
+    # the required HTTP objects. All public API requests go via
+    # #process_http_request and subsequently WebFetch::Router#route
+    def gather(targets)
+      targets.each do |target|
+        request = target[:request]
+        async_request = EM::HttpRequest.new(request[:url])
+        method = request.fetch(:method, 'GET').downcase.to_sym
+        http = async_request.public_send(method,
+                                         head: request[:headers],
+                                         query: request.fetch(:query, {}),
+                                         body: request.fetch(:body, nil))
+        @storage.store(target[:uid], uid: target[:uid], http: http)
+      end
+    end
+  end
+end

data/lib/web_fetch/storage.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+module WebFetch
+  # Rudimentary global storage for responses
+  class Storage
+    @_storage = {}
+
+    def self.store(key, obj)
+      @_storage[key] = obj
+    end
+
+    def self.fetch(key)
+      @_storage.delete(key)
+    end
+  end
+end

data/lib/web_fetch/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module WebFetch
+  VERSION = '0.1.0'
+end

data/lib/web_fetch.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+require 'eventmachine'
+require 'evma_httpserver'
+require 'em-http'
+require 'em-logger'
+require 'i18n'
+require 'logger'
+require 'json'
+require 'digest'
+require 'securerandom'
+require 'faraday'
+require 'childprocess'
+require 'active_support/gzip'
+
+locales_path = File.expand_path('../config/locales/*.yml', __dir__)
+
+I18n.load_path += Dir[locales_path]
+
+# Avoid i18n conflicts when using as a gem in a Rails application
+unless Gem.loaded_specs.key?('rails')
+  I18n.load_path += Dir[locales_path]
+  I18n.backend.load_translations
+  I18n.config.available_locales = :en
+end
+
+require 'web_fetch/logger'
+require 'web_fetch/helpers'
+require 'web_fetch/event_machine_helpers'
+require 'web_fetch/http_helpers'
+require 'web_fetch/concerns/validatable'
+require 'web_fetch/concerns/http_helpers'
+require 'web_fetch/storage'
+require 'web_fetch/server'
+require 'web_fetch/router'
+require 'web_fetch/resources'
+require 'web_fetch/gatherer'
+require 'web_fetch/retriever'
+require 'web_fetch/client'
+require 'web_fetch/version'

data/spec/client_spec.rb

@@ -0,0 +1,63 @@
+# frozen_string_literal: true
+
+describe WebFetch::Client do
+  let(:client) { described_class.new('localhost', 8089, log: File::NULL) }
+
+  before(:each) do
+    stub_request(:any, 'http://blah.blah/success')
+      .to_return(body: 'hello, everybody')
+  end
+
+  it 'can be instantiated with host and port params' do
+    client
+  end
+
+  describe '#alive?' do
+    it 'confirms server is alive and accepting requests' do
+      expect(client.alive?).to be true
+    end
+  end
+
+  describe '#gather' do
+    it 'makes `gather` requests to a running server' do
+      result = client.gather([{ url: 'http://blah.blah/success' }])
+      expect(result.first[:uid]).to_not be_nil
+    end
+  end
+
+  describe '#retrieve_by_uid' do
+    it 'retrieves a gathered item' do
+      result = client.gather([{ url: 'http://blah.blah/success' }])
+      uid = result.first[:uid]
+
+      retrieved = client.retrieve_by_uid(uid)
+      expect(retrieved[:response][:status]).to eql 200
+      expect(retrieved[:response][:body]).to eql 'hello, everybody'
+      expect(retrieved[:uid]).to eql uid
+    end
+
+    it 'returns nil for non-requested items' do
+      client.gather([{ url: 'http://blah.blah/success' }])
+
+      retrieved = client.retrieve_by_uid('lalalala')
+      expect(retrieved).to be_nil
+    end
+  end
+
+  describe '#create' do
+    it 'spawns a server and returns a client able to connect' do
+      client = described_class.create('localhost', 8077, log: File::NULL)
+      expect(client.alive?).to be true
+      client.stop
+    end
+  end
+
+  describe '#stop' do
+    it 'can spawn a server and stop the process when needed' do
+      client = described_class.create('localhost', 8077, log: File::NULL)
+      expect(client.alive?).to be true
+      client.stop
+      expect(client.alive?).to be false
+    end
+  end
+end

data/spec/concerns/validatable_spec.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+describe WebFetch::Validatable do
+  class ConcernedInvalid
+    include ::WebFetch::Validatable
+    def validate
+      error(:bad_json)
+      error(:missing_url)
+      error(:requests_empty, 'Hello there')
+    end
+  end
+
+  class ConcernedValid
+    include ::WebFetch::Validatable
+    def validate; end
+  end
+
+  class ConcernedNotOverridden
+    include ::WebFetch::Validatable
+  end
+
+  describe '#valid?' do
+    context 'invalid' do
+      subject { ConcernedInvalid.new }
+
+      it 'runs validations and provides errors including supplementary text' do
+        expect(subject.valid?).to be false
+        expect(subject.errors).to include I18n.t(:bad_json)
+        expect(subject.errors).to include I18n.t(:missing_url)
+        expect(subject.errors.last).to include 'Hello there'
+      end
+    end
+
+    context 'valid' do
+      subject { ConcernedValid.new }
+
+      it 'runs validations and provides (empty) errors' do
+        expect(subject.valid?).to be true
+        expect(subject.errors).to be_empty
+      end
+    end
+
+    context '#validate not overridden' do
+      subject { ConcernedNotOverridden.new }
+
+      it 'raises NotImplementedError when #valid? called' do
+        expect do
+          subject.valid?
+        end.to raise_error(NotImplementedError)
+      end
+    end
+  end
+end