web_fetch 0.4.0 → 0.5.0

data/lib/web_fetch/promise.rb

@@ -6,15 +6,15 @@ module WebFetch
 
     def initialize(client, options = {})
       @client = client
-      @uid = options[:uid]
-      @request = Request.from_hash(options[:request])
+      @uid = options.fetch(:uid)
+      @request = Request.from_hash(options.fetch(:request))
     end
 
     def fetch(options = {})
       return @response if complete?
 
       wait = options.fetch(:wait, true)
-      (@response = @client.fetch(@uid, wait: wait))
+      @response = @client.fetch(@uid, wait: wait)
     end
 
     def custom

data/lib/web_fetch/request.rb

@@ -2,19 +2,16 @@
 
 module WebFetch
   class Request
-    attr_writer :url, :query, :headers, :body, :custom
+    attr_writer :url, :query, :headers, :body, :custom, :method
     attr_reader :url, :query, :headers, :body, :custom
 
     def initialize
+      @method = 'GET'
       yield self
     end
 
-    def method=(val)
-      @method = val.downcase.to_sym
-    end
-
     def method
-      @method ||= :get
+      @method.downcase.to_sym
     end
 
     def to_h
@@ -38,27 +35,23 @@ module WebFetch
       eql?(other)
     end
 
-    def self.from_hash(hash)
+    def self.from_hash(hash, options = {})
       hash_copy = hash.dup
       request = build_request(hash_copy)
+      return request unless options.fetch(:validate, true)
       raise ArgumentError, "Unrecognized keys: #{hash}" unless hash_copy.empty?
 
       request
     end
 
     class << self
-      # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity
       def build_request(hash)
         Request.new do |request|
-          request.url = hash.delete(:url) if hash.key?(:url)
-          request.query = hash.delete(:query) if hash.key?(:query)
-          request.headers = hash.delete(:headers) if hash.key?(:headers)
-          request.body = hash.delete(:body) if hash.key?(:body)
-          request.method = hash.delete(:method) if hash.key?(:method)
-          request.custom = hash.delete(:custom) if hash.key?(:custom)
+          %i[url query headers body method custom].each do |key|
+            request.send("#{key}=", hash.delete(key))
+          end
         end
       end
-      # rubocop:enable Metrics/AbcSize, Metrics/CyclomaticComplexity
     end
   end
 end

data/lib/web_fetch/resources.rb

@@ -6,23 +6,27 @@ module WebFetch
   class Resources
     class << self
       def root(_server, _params)
-        { status: status(:ok), payload: { application: 'WebFetch' } }
+        {
+          status: status(:ok),
+          command: 'root',
+          payload: { application: 'WebFetch' }
+        }
       end
 
       def gather(server, params)
-        gatherer = Gatherer.new(server, params)
+        gatherer = Gatherer.new(server.storage, params)
         if gatherer.valid?
-          { status: status(:ok), payload: gatherer.start }
+          { status: status(:ok), payload: gatherer.start, command: 'gather' }
         else
           { status: status(:unprocessable),
-            payload: { error: gatherer.errors } }
+            payload: { error: gatherer.errors }, command: 'gather' }
         end
       end
 
       def retrieve(server, params, options = {})
-        retriever = Retriever.new(server, params, options)
+        retriever = Retriever.new(server.storage, params, options)
         unless retriever.valid?
-          return { status: status(:unprocessable),
+          return { status: status(:unprocessable), command: 'retrieve',
                    payload: { error: retriever.errors } }
         end
         defer_if_found(retriever)
@@ -37,26 +41,12 @@ module WebFetch
       def status(name)
         {
           ok: 200,
-          unprocessable: 422,
-          not_found: 404
+          unprocessable: 422
         }.fetch(name)
       end
 
-      def not_found(retriever)
-        {
-          status: status(:not_found),
-          payload: { error: retriever.not_found_error }
-        }
-      end
-
       def defer_if_found(retriever)
-        found = retriever.find
-        if found.nil?
-          { status: status(:not_found),
-            payload: { error: retriever.not_found_error } }
-        else
-          { request: found }
-        end
+        { command: 'retrieve', request: retriever.find }
       end
     end
   end

data/lib/web_fetch/response.rb

@@ -4,18 +4,23 @@ module WebFetch
   class Response
     attr_reader :request, :body, :headers, :status, :error, :uid, :response_time
 
-    def initialize(options = {})
-      @pending = options.fetch(:pending, false)
+    def initialize(response)
+      @pending = response.fetch(:pending, false)
       return if pending?
 
-      @body = options.fetch(:body)
-      @headers = options.fetch(:headers)
-      @status = options.fetch(:status)
-      @request = Request.from_hash(options.fetch(:request))
-      @success = options.fetch(:success)
-      @error = options.fetch(:error)
-      @uid = options.fetch(:uid)
-      @response_time = options.fetch(:response_time)
+      outcome = response.fetch(:request)
+      @uid = outcome.fetch(:uid)
+      @response_time = outcome.fetch(:response_time, nil)
+      @request = Request.from_hash(outcome.fetch(:request), validate: false)
+      initialize_response(outcome.fetch(:response))
+    end
+
+    def initialize_response(response)
+      @body = Base64.decode64(response.fetch(:body))
+      @headers = response.fetch(:headers)
+      @status = response.fetch(:status)
+      @success = response.fetch(:success)
+      @error = response.fetch(:error, nil)
     end
 
     def pending?

data/lib/web_fetch/retriever.rb

@@ -5,19 +5,16 @@ module WebFetch
   class Retriever
     include Validatable
 
-    attr_reader :not_found_error
-
-    def initialize(server, params, options)
+    def initialize(storage, params, options)
       @uid = params[:uid]
       @hash = params[:hash]
-      @server = server
+      @storage = storage
       @block = options.fetch(:block, true)
     end
 
     def find
-      request = @server.storage.fetch(@uid)
-      return not_found if request.nil?
-      return request.merge(pending: true) if pending?(request)
+      request = @storage.fetch(@uid) unless @uid.nil?
+      return pending if request.nil?
 
       request
     end
@@ -29,24 +26,8 @@ module WebFetch
       error(:missing_hash_and_uid) if @uid.nil? && @hash.nil?
     end
 
-    def not_found
-      @not_found_error = if !@uid.nil?
-                           I18n.t(:uid_not_found)
-                         elsif !@hash.nil?
-                           I18n.t(:hash_not_found)
-                         end
-      nil
-    end
-
-    def pending?(request)
-      return false if request.nil?
-      return false if request[:succeeded]
-      return false if request[:failed]
-      # User requested blocking operation so we will wait until item is ready
-      # rather than return a `pending` status
-      return false if @block
-
-      true
+    def pending
+      { uid: @uid, pending: true }
     end
   end
 end

data/lib/web_fetch/server.rb

@@ -8,52 +8,39 @@ module WebFetch
 
     include EM::HttpServer
     include HTTPHelpers
-    include EventMachineHelpers
 
     def post_init
       super
       @router = Router.new
-      @storage = Storage
+      @storage = WebFetch::Storage.create
+
       no_environment_strings
     end
 
     def process_http_request
-      result = @router.route(@http_request_uri, request_params)
+      resource = @router.route(@http_request_uri, request_params)
       response = EM::DelegatedHttpResponse.new(self)
       default_headers(response)
-      outcome(result, response)
-    end
 
-    # Note that #gather is called by WebFetch itself to asynchronously gather
-    # the required HTTP objects. All public API requests go via
-    # #process_http_request and subsequently WebFetch::Router#route
-    def gather(targets)
-      targets.each do |target|
-        http = request_async(target)
-        request = { uid: target[:uid],
-                    start_time: target[:start_time],
-                    request: target[:request],
-                    deferred: http }
-        apply_callbacks(request)
-        @storage.store(target[:uid], request)
-      end
+      outcome(resource, response)
     end
 
     private
 
-    def outcome(result, response)
-      # User requested an unrecognised ID
-      return respond_immediately(result, response) if result[:request].nil?
+    def immediate?(command)
+      %w[gather root].include?(command)
+    end
 
-      # Fetch has already completed
-      return succeed(result, response) if result[:succeeded]
-      return fail_(result, response) if result[:failed]
+    def outcome(resource, response)
+      command = resource[:command]
+      Logger.debug(command)
+      return respond_immediately(resource, response) if immediate?(command)
+      return pending(resource, response) if resource[:request][:pending]
 
-      # User requested non-blocking call
-      return pending(result, response) if result[:request][:pending]
+      succeeded = resource[:request][:response][:success]
+      return succeed(resource, response) if succeeded
 
-      # User requested blocking call
-      wait_for_response(result[:request], response)
+      fail_(resource, response)
     end
   end
 end

data/lib/web_fetch/storage.rb

@@ -1,21 +1,25 @@
 # frozen_string_literal: true
 
 module WebFetch
-  # Rudimentary global storage for responses. The intention is that this will
-  # one day prescribe an interface to e.g. memcached
-  class Storage
-    @storage = {}
+  module Storage
+    class << self
+      def create
+        {
+          'memory' => Memory,
+          'memcached' => Memcached,
+          'redis' => Redis
+        }.fetch(backend).new
+      end
 
-    def self.store(key, obj)
-      @storage[key] = obj
-    end
-
-    def self.fetch(key)
-      @storage.fetch(key, nil)
-    end
+      private
 
-    def self.delete(key)
-      @storage.delete(key)
+      def backend
+        ENV.fetch('WEB_FETCH_BACK_END', 'memory')
+      end
     end
   end
 end
+
+require 'web_fetch/storage/memcached'
+require 'web_fetch/storage/memory'
+require 'web_fetch/storage/redis'

data/lib/web_fetch/storage/memcached.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+module WebFetch
+  module Storage
+    class Memcached
+      def initialize(client = nil)
+        require 'dalli' if client.nil?
+        @client = client || Dalli::Client
+        @config = {
+          host: ENV.fetch('WEB_FETCH_MEMCACHED_HOST', 'localhost'),
+          port: ENV.fetch('WEB_FETCH_MEMCACHED_PORT', '11211')
+        }
+      end
+
+      def store(key, obj)
+        storage.set(key, obj.to_json)
+      end
+
+      def fetch(key)
+        result = storage.get(key)
+        return JSON.parse(result, symbolize_names: true) unless result.nil?
+
+        nil
+      end
+
+      def delete(key)
+        storage.delete(key)
+      end
+
+      private
+
+      def storage
+        @storage ||= begin
+          host = @config.fetch(:host)
+          port = @config.fetch(:port)
+          @client.new("#{host}:#{port}", expires_in: ttl)
+        end
+      end
+
+      def ttl
+        @ttl ||= ENV.fetch('WEB_FETCH_MEMCACHED_TTL', '60').to_i
+      end
+    end
+  end
+end

data/lib/web_fetch/storage/memory.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+module WebFetch
+  module Storage
+    class Memory
+      @storage = {}
+
+      class << self
+        attr_reader :storage
+      end
+
+      def clear
+        self.class.storage.clear
+      end
+
+      def store(key, obj)
+        storage[key] = obj
+      end
+
+      def fetch(key)
+        storage.fetch(key, nil)
+      end
+
+      def delete(key)
+        storage.delete(key)
+      end
+
+      private
+
+      def storage
+        self.class.storage
+      end
+    end
+  end
+end

data/lib/web_fetch/storage/redis.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+module WebFetch
+  module Storage
+    class Redis
+      def initialize(client = nil)
+        require 'redis' if client.nil?
+        @client = client || ::Redis
+        @config = {
+          host: ENV.fetch('WEB_FETCH_REDIS_HOST', 'localhost'),
+          port: ENV.fetch('WEB_FETCH_REDIS_PORT', '6379')
+        }
+      end
+
+      def store(key, obj)
+        storage.set(key, obj.to_json, ex: ttl)
+      end
+
+      def fetch(key)
+        result = storage.get(key)
+        return JSON.parse(result, symbolize_names: true) unless result.nil?
+
+        nil
+      end
+
+      def delete(key)
+        storage.del(key)
+      end
+
+      private
+
+      def storage
+        @storage ||= begin
+          host = @config.fetch(:host)
+          port = @config.fetch(:port)
+          @client.new(url: "redis://#{host}:#{port}")
+        end
+      end
+
+      def ttl
+        @ttl ||= ENV.fetch('WEB_FETCH_REDIS_TTL', '60').to_i
+      end
+    end
+  end
+end