web_fetch 0.4.0 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 19518942fe1e37355b4cf571590fb7d844c0eded61b035c036bb923a114e0aa4
-  data.tar.gz: ba143ee2c1244b5c37459f1057beaba210945f310c4d8174b0253d9cec90d0f0
+  metadata.gz: 3ffbcc60483ca651ca9e7ca3a4ebf945bebb26e7e464ad0dbb6fde65aadd9772
+  data.tar.gz: 809d8962eb85f784844d38e8f57f30fd33e21c393bc71765ad36181f6ebd0d04
 SHA512:
-  metadata.gz: f37dcd9cca372a63f47737ff619a996cfa68494038d1746d3f28445f7ad7fd8c9651e43f284e2ece92696b9883a28295bedc9e8d55789fe91aeb4d60032a342e
-  data.tar.gz: 0542a7735cda7d6eddaee2a94df7b5143934184a4902d3c21c2546dd2361d74226916efd783f635828db73d0eec49e10400d50594cab82aa094faf83a63a6231
+  metadata.gz: 62186119c071a4345ad28dae952084b982327de94b5dbbaa8e8f01982e5e3a88278c877780afd4385b827a6c316d280a939476adb34e43a003d0d64844dd7cb8
+  data.tar.gz: 2f116096a7b94477abfefdf2dfa3ad64ae69898791f58a6010689c248d98298c4511b23ea252ff5a0a9833bb445bb5bd1cd739a7564bc7163623120b678e17d7

data/.gitignore

@@ -19,3 +19,5 @@ web_fetch-*.gem
 Gemfile.lock
 
 docker/.build
+
+run-dev

data/.strong_versions.yml

@@ -0,0 +1,6 @@
+on_failure: warn
+ignore:
+  - rails
+  - rack
+  - i18n
+  - activesupport

data/Makefile

@@ -1,6 +1,15 @@
-.PHONY: docker
+.PHONY: docker manifest
 
+manifest:
+	git ls-files > manifest
+
+test:
+	bin/rspec
+	bin/rubocop
+	bin/strong_versions
+
+docker: version := $(shell bundle exec ruby -e "require 'web_fetch'; puts WebFetch::VERSION")
 docker:
 	mkdir -p docker/.build
 	git archive --format tar.gz -o docker/.build/web_fetch.tar.gz master
-	docker build -t webfetch/webfetch docker
+	docker build -t webfetch/webfetch:${version} docker

data/README.md

@@ -18,7 +18,7 @@ This permits issuing multiple HTTP requests in parallel, in a fully encapsulated
 In your `Gemfile`, add:
 
 ``` ruby
-gem 'web_fetch'
+gem 'web_fetch', '~> 0.5.0'
 ```
 
 and update your bundle:
@@ -33,10 +33,31 @@ Require WebFetch in your application:
 require 'web_fetch'
 ```
 
+### Memcached
+
+It is highly recommended to use _Memcached_ or _Redis_ as a back end to _WebFetch_ in production.
+
+The following environment variables can be used to select and configure the back end:
+
+```bash
+# Memcached
+WEB_FETCH_BACK_END=memcached
+WEB_FETCH_MEMCACHED_HOST=localhost
+WEB_FETCH_MEMCACHED_PORT=11211
+WEB_FETCH_MEMCACHED_TTL=60
+
+# Redis
+WEB_FETCH_BACK_END=redis
+WEB_FETCH_REDIS_HOST=localhost
+WEB_FETCH_REDIS_PORT=6379
+WEB_FETCH_REDIS_TTL=60
+```
+
+Note that _WebFetch_ is intended to function as a fast proxy server so a low TTL (default 60 seconds) is recommended. With a _Memcached_ back end multiple instances of _WebFetch_ can operate on the same memory store. This provides good options for containerisation, scaling, and high availability.
+
 ### Launch or connect to a server
 
-Launch the server from your application (recommended for familiarising yourself
-with WebFetch):
+Launch the server from your application (recommended for familiarising yourself with WebFetch):
 
 ``` ruby
 client = WebFetch::Client.create('localhost', 8077)
@@ -105,7 +126,7 @@ response.response_time
 response.request # The original request, provided as a `WebFetch::Request` object
 ```
 
-Note that `WebFech::Promise#fetch` will block until the response is complete by default. If you want to continue executing other code if the response is not ready (e.g. to see if any other responses are ready), you can pass `wait: false`
+Note that `WebFetch::Promise#fetch` will block until the response is complete by default. If you want to continue executing other code if the response is not ready (e.g. to see if any other responses are ready), you can pass `wait: false`
 
 ``` ruby
 response = promises.first.fetch(wait: false)
@@ -198,8 +219,6 @@ web_fetch_control run -- --port 8000 --host 0.0.0.0
 
 No pid file will be created unless the `--pidfile` parameter is passed. It is recommended to use a process monitoring tool (e.g. `monit` or `systemd`) to monitor the WebFetch process.
 
-When running as a daemon, WebFetch will log to the null device so it is advised to always pass `--log` in this case.
-
 ## Docker
 
 To use WebFetch in Docker you can either use the provided [`Dockerfile`](docker/Dockerfile) or the public image [`web_fetch/web_fetch`](https://hub.docker.com/r/webfetch/webfetch/)

data/Rakefile

@@ -1,6 +1,8 @@
-require "bundler/gem_tasks"
-require "rspec/core/rake_task"
+# frozen_string_literal: true
+
+require 'bundler/gem_tasks'
+require 'rspec/core/rake_task'
 
 RSpec::Core::RakeTask.new(:spec)
 
-task :default => :spec
+task default: :spec

data/bin/strong_versions

@@ -0,0 +1,29 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+#
+# This file was generated by Bundler.
+#
+# The application 'strong_versions' is installed as part of a gem, and
+# this file is here to facilitate running it.
+#
+
+require "pathname"
+ENV["BUNDLE_GEMFILE"] ||= File.expand_path("../../Gemfile",
+  Pathname.new(__FILE__).realpath)
+
+bundle_binstub = File.expand_path("../bundle", __FILE__)
+
+if File.file?(bundle_binstub)
+  if File.read(bundle_binstub, 300) =~ /This file was generated by Bundler/
+    load(bundle_binstub)
+  else
+    abort("Your `bin/bundle` was not generated by Bundler, so this binstub cannot run.
+Replace `bin/bundle` by running `bundle binstubs bundler --force`, then run this command again.")
+  end
+end
+
+require "rubygems"
+require "bundler/setup"
+
+load Gem.bin_path("strong_versions", "strong_versions")

data/config/locales/en.yml

@@ -8,9 +8,6 @@ en:
   hash_or_uid_but_not_both: "Cannot retrieve by both `uid` and `hash`"
   missing_hash_and_uid: "Must pass either `uid` or `hash` to retrieve"
 
-  uid_not_found: "Provided `uid` has not yet been requested"
-  hash_not_found: "Provided `hash` has not yet been requested"
-
   pending: "Your request is still being processed"
 
   no_request: "No active request found for UID: %{uid}"

data/docker/Dockerfile

@@ -1,5 +1,5 @@
-FROM library/ruby
+FROM library/ruby:2.5.3
 ADD .build/web_fetch.tar.gz .
 WORKDIR web_fetch
 RUN bundle && bundle exec rake install
-CMD ["web_fetch_control", "run", "--", "--port", "8077", "--host", "0.0.0.0"]
+CMD ["bundle", "exec", "web_fetch_control", "run", "--", "--port", "8077", "--host", "0.0.0.0"]

data/lib/web_fetch.rb

@@ -6,6 +6,7 @@ require 'em-http'
 require 'em-logger'
 require 'i18n'
 require 'logger'
+require 'base64'
 require 'json'
 require 'digest'
 require 'securerandom'
@@ -24,9 +25,11 @@ unless Gem.loaded_specs.key?('rails')
   I18n.config.available_locales = :en
 end
 
+module WebFetch
+end
+
 require 'web_fetch/logger'
 require 'web_fetch/helpers'
-require 'web_fetch/concerns/event_machine_helpers'
 require 'web_fetch/concerns/http_helpers'
 require 'web_fetch/concerns/validatable'
 require 'web_fetch/concerns/client_http'

data/lib/web_fetch/client.rb

@@ -48,30 +48,25 @@ module WebFetch
       handle_error(JSON.parse(response.body)['error']) unless response.success?
 
       requests = JSON.parse(response.body, symbolize_names: true)[:requests]
+
       promises(requests)
     end
 
     def fetch(uid, options = {})
       block = options.fetch(:wait, true)
 
-      outcome = block ? retrieve_by_uid(uid) : find_by_uid(uid)
+      outcome = block ? fetch_blocking(uid) : fetch_nonblocking(uid)
       no_request_error(uid) if outcome.nil?
 
-      new_response(outcome)
+      Response.new(outcome.merge(uid: uid))
     end
 
     def retrieve_by_uid(uid)
-      response = get("retrieve/#{uid}")
-      return nil unless response.success?
-
-      JSON.parse(response.body, symbolize_names: true)
+      fetch_blocking(uid)[:request]
     end
 
     def find_by_uid(uid)
-      response = get("find/#{uid}")
-      return nil unless response.success?
-
-      JSON.parse(response.body, symbolize_names: true)
+      fetch_nonblocking(uid)[:request]
     end
 
     class << self
@@ -94,6 +89,29 @@ module WebFetch
 
     private
 
+    def decode_response(response)
+      return response unless response[:command] == 'retrieve'
+
+      response[:request][:response][:body] = Base64.decode64(
+        response[:request][:response][:body]
+      )
+      response
+    end
+
+    def fetch_blocking(uid)
+      response = get("retrieve/#{uid}")
+      return nil unless response.success?
+
+      decode_response(JSON.parse(response.body, symbolize_names: true))
+    end
+
+    def fetch_nonblocking(uid)
+      response = get("find/#{uid}")
+      return nil unless response.success?
+
+      decode_response(JSON.parse(response.body, symbolize_names: true))
+    end
+
     def handle_error(error)
       raise WebFetch::ClientError, error
     end
@@ -102,25 +120,6 @@ module WebFetch
       raise RequestNotFoundError, [I18n.t('no_request', uid: uid)]
     end
 
-    def new_response(outcome)
-      response = outcome[:response] || {}
-      # FIXME: This is sort-of duplicated from `Promise#new_response` but we
-      # build it very slightly differently. This means we have to update in
-      # both places if we change the structure. Not quite sure how to unify
-      # this and ensure the same structure in both places.
-      Response.new(
-        pending: outcome[:pending],
-        body: response[:body],
-        headers: response[:headers],
-        status: response[:status],
-        success: response[:success],
-        error: response[:error],
-        uid: outcome[:uid],
-        response_time: response[:response_time],
-        request: outcome[:request]
-      )
-    end
-
     def promises(requests)
       requests.map do |request|
         Promise.new(self, uid: request[:uid], request: request[:request])

data/lib/web_fetch/concerns/http_helpers.rb

@@ -9,10 +9,10 @@ module WebFetch
       response.send_response
     end
 
-    def pending(result, response)
+    def pending(uid, response)
       respond_immediately({
                             payload: {
-                              uid: result[:request][:uid],
+                              uid: uid,
                               pending: true,
                               message: I18n.t(:pending)
                             }
@@ -45,45 +45,18 @@ module WebFetch
       JSON.parse(@http_post_content, symbolize_names: true)
     end
 
-    def succeed(request, response)
+    def succeed(resource, response)
       response.status = 200
-      response.content = compress(JSON.dump(success(request)))
+      response.content = compress(JSON.dump(resource))
+      storage.delete(resource[:request][:uid])
       response.send_response
-      storage.delete(request[:uid])
     end
 
-    def success(request)
-      result = request[:deferred]
-      { response: {
-        success: true,
-        body: result.response,
-        headers: result.headers,
-        status: result.response_header.status,
-        response_time: request[:response_time]
-      },
-        request: request[:request],
-        uid: request[:uid] }
-    end
-
-    def fail_(request, response)
+    def fail_(resource, response)
       response.status = 200
-      response.content = compress(JSON.dump(failure(request)))
+      response.content = compress(JSON.dump(resource))
+      storage.delete(resource[:request][:uid])
       response.send_response
-      storage.delete(request[:uid])
-    end
-
-    def failure(request)
-      result = request[:deferred]
-      { response: {
-        success: false,
-        body: result.response,
-        headers: result.headers,
-        status: result.response_header.status,
-        response_time: request[:response_time],
-        error: (result.error&.inspect)
-      },
-        request: request[:request],
-        uid: request[:uid] }
     end
 
     def accept_gzip?

data/lib/web_fetch/gatherer.rb

@@ -8,19 +8,59 @@ module WebFetch
 
     HASHABLE_KEYS = %i[url query_string headers method].freeze
 
-    def initialize(server, params)
+    def initialize(storage, params, logger = Logger, http = EM::HttpRequest)
       @requests = params[:requests]
-      @server = server
+      @storage = storage
+      @logger = logger
+      @http = http
     end
 
     def start
       tagged = { requests: tag_requests }
-      @server.gather(tagged[:requests])
+      gather(tagged[:requests])
       tagged
     end
 
     private
 
+    def gather(targets)
+      targets.each do |target|
+        uid = target[:uid]
+        @logger.debug("Initialising async for uid: #{uid}")
+        deferred = request_async(target)
+        request = { uid: uid, start_time: target[:start_time],
+                    request: target[:request] }
+        apply_callbacks(request, deferred)
+      end
+    end
+
+    def apply_callbacks(request, deferred)
+      uid = request[:uid]
+      deferred.callback do
+        @logger.debug("HTTP fetch successful for uid: #{uid}")
+        @storage.store(uid, response(request, deferred, success: true))
+      end
+
+      deferred.errback do
+        @logger.debug("HTTP fetch failure for uid: #{uid}")
+        @storage.store(uid, response(request, deferred, success: false))
+      end
+
+      @logger.debug("HTTP fetch started for uid: #{uid}")
+    end
+
+    def request_async(target)
+      request = target[:request]
+      target[:start_time] = Time.now.utc
+      async_request = @http.new(request[:url])
+      method = request.fetch(:method, 'GET').downcase.to_sym
+      async_request.public_send(
+        method, head: request[:headers],
+                query: request.fetch(:query, {}),
+                body: request.fetch(:body, nil)
+      )
+    end
+
     def validate
       error(:requests_missing) if requests_missing?
       error(:requests_not_array) if requests_not_array?
@@ -58,5 +98,27 @@ module WebFetch
     def uid
       SecureRandom.uuid
     end
+
+    def response_time(request)
+      Time.now.utc - request[:start_time]
+    end
+
+    def response(request, result, options = {})
+      {
+        response: result(request, result, options.fetch(:success)),
+        request: request,
+        uid: request[:uid]
+      }
+    end
+
+    def result(request, result, success)
+      {
+        success: success,
+        body: Base64.encode64(result.response),
+        headers: result.headers,
+        status: result.response_header.status,
+        response_time: response_time(request)
+      }.merge(success ? {} : { error: (result.error&.inspect) })
+    end
   end
 end

data/lib/web_fetch/logger.rb

@@ -17,8 +17,7 @@ module WebFetch
       private
 
       def log_file(path)
-        return STDOUT if STDOUT.isatty && path.nil?
-        return File.open(File::NULL, 'w') if path.nil?
+        return STDOUT if path.nil?
 
         log = File.open(path, 'a')
         log.sync = true # Prevent buffering