web_fetch 0.2.1 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 22d65c1496314f4ab5ce0f6393e6c59209ccc3dff7e13657eb574111187c9674
-  data.tar.gz: 172a59a513ac5073b2071cc82c6a20b8143a426e77139c12bf015d40ea558b94
+  metadata.gz: aab3d826db4d54d4ecd8f935086af80ee34edf38b80c62cfeaed980a5e045f31
+  data.tar.gz: dd4818dd4dcfb3949b37ae124370871906558e185f573ca6f9f9c0a65e1c33a4
 SHA512:
-  metadata.gz: 8c57ff4c160cd76b68a84e16ddd8cc7976041b9e6c6d675ad08d616c060577e680491a30fbc5ab1fb6215bc017dafccb6d84e0c679d5bacea871b4fbfd11f063
-  data.tar.gz: 61481a84084e8837d1194199ccd3454cfe659d789e1eaff559779d5286ec446ae6417baa09cae10d66f2493612fb80c90c7ec75a8f9215ce1461a8684aaf233d
+  metadata.gz: f525afda585cc0f785db99477a0f6cda72cca602cc0a84bc6726c72214c76a8074069a9d034c9c050cc4a1815f73ca164104755c8325b86361314d1a9eeb05bf
+  data.tar.gz: 9f5678f845dee9af389ff24b46f96189c06ca6255dd8ba88da1b265a4bad1732736b8a9cfc547500f3c6e157c3250fcf4c84ab93558232149251be9f61ac5fdf

data/README.md

@@ -96,6 +96,7 @@ result.headers
 result.status # HTTP status code
 result.success? # False if a network error (not HTTP error) occurred
 result.error # Underlying network error if applicable
+result.response_time
 ```
 
 Note that `WebFech::Promise#fetch` will block until the result is complete by default. If you want to continue executing other code if the result is not ready (e.g. to see if any other results are ready), you can pass `wait: false`

data/lib/web_fetch.rb

@@ -10,7 +10,7 @@ require 'json'
 require 'digest'
 require 'securerandom'
 require 'faraday'
-require 'childprocess'
+require 'subprocess'
 require 'active_support/gzip'
 
 locales_path = File.expand_path('../config/locales/*.yml', __dir__)
@@ -26,10 +26,9 @@ end
 
 require 'web_fetch/logger'
 require 'web_fetch/helpers'
-require 'web_fetch/event_machine_helpers'
-require 'web_fetch/http_helpers'
-require 'web_fetch/concerns/validatable'
+require 'web_fetch/concerns/event_machine_helpers'
 require 'web_fetch/concerns/http_helpers'
+require 'web_fetch/concerns/validatable'
 require 'web_fetch/concerns/client_http'
 require 'web_fetch/storage'
 require 'web_fetch/server'

data/lib/web_fetch/client.rb

@@ -16,17 +16,17 @@ module WebFetch
 
     def self.create(host, port, options = {})
       # Will block until process is responsive
-      process = spawn(host, port, options)
+      process = build_process(host, port, options)
       client = new(host, port, process: process)
       sleep 0.1 until client.alive?
       client
     end
 
     def stop
-      # Will block until process dies
       return if @process.nil?
 
-      @process.stop
+      @process.terminate
+      # Will block until process dies
       @process.wait
     end
 
@@ -77,22 +77,16 @@ module WebFetch
     end
 
     class << self
-      def spawn(host, port, options)
-        process = build_process(host, port, options)
-        process.cwd = File.join(File.dirname(__dir__), '..')
-        process.io.inherit!
-        process.start
-        process
-      end
-
-      private
-
       def build_process(host, port, options)
         command = options.fetch(:start_command, standard_start_command)
         args = ['--host', host, '--port', port.to_s]
         args += ['--log', options[:log]] unless options[:log].nil?
         args.push('--daemonize') if options[:daemonize]
-        ChildProcess.build(*command, *args)
+        Subprocess.popen(command + args, cwd: cwd)
+      end
+
+      def cwd
+        File.join(File.dirname(__dir__), '..')
       end
 
       def standard_start_command
@@ -112,13 +106,18 @@ module WebFetch
 
     def new_result(outcome)
       response = outcome[:response]
+      # FIXME: This is sort-of duplicated from `Promise#new_result` but we
+      # build it very slightly differently. This means we have to update in
+      # both places if we change the structure. Not quite sure how to unify
+      # this and ensure the same structure in both places.
       Result.new(
         body: response[:body],
         headers: response[:headers],
         status: response[:status],
         success: response[:success],
         error: response[:error],
-        uid: outcome[:uid]
+        uid: outcome[:uid],
+        response_time: response[:response_time]
       )
     end
 

data/lib/web_fetch/{event_machine_helpers.rb → concerns/event_machine_helpers.rb}

@@ -3,7 +3,9 @@
 module WebFetch
   # EventMachine layer-specific helpers
   module EventMachineHelpers
-    def request_async(request)
+    def request_async(target)
+      request = target[:request]
+      target[:start_time] = Time.now.utc
       async_request = EM::HttpRequest.new(request[:url])
       method = request.fetch(:method, 'GET').downcase.to_sym
       async_request.public_send(
@@ -17,11 +19,13 @@ module WebFetch
     def apply_callbacks(request)
       request[:deferred].callback do
         Logger.debug("HTTP fetch complete for uid: #{request[:uid]}")
+        save_response_time(request)
         request[:succeeded] = true
       end
 
       request[:deferred].errback do
         Logger.debug("HTTP fetch failed for uid: #{request[:uid]}")
+        save_response_time(request)
         request[:failed] = true
       end
     end
@@ -45,5 +49,9 @@ module WebFetch
         end
       end
     end
+
+    def save_response_time(request)
+      request[:response_time] = Time.now.utc - request[:start_time]
+    end
   end
 end

data/lib/web_fetch/concerns/http_helpers.rb

@@ -1,15 +1,34 @@
 # frozen_string_literal: true
 
 module WebFetch
-  module HttpHelpers
+  # Convenience methods for WebFetch HTTP layer
+  module HTTPHelpers
+    def respond_immediately(result, response)
+      response.status = result[:status]
+      response.content = compress(result[:payload].to_json)
+      response.send_response
+    end
+
+    def pending(result, response)
+      respond_immediately({
+                            payload: {
+                              uid: result[:request][:uid],
+                              pending: true,
+                              message: I18n.t(:pending)
+                            }
+                          }, response)
+    end
+
     def compress(string)
+      return string unless accept_gzip?
+
       ActiveSupport::Gzip.compress(string)
     end
 
     def default_headers(response)
       response.headers['Content-Type'] = 'application/json; charset=utf-8'
       response.headers['Cache-Control'] = 'max-age=0, private, must-revalidate'
-      response.headers['Content-Encoding'] = 'gzip'
+      response.headers['Content-Encoding'] = 'gzip' if accept_gzip?
       response.headers['Vary'] = 'Accept-Encoding'
     end
 
@@ -26,39 +45,48 @@ module WebFetch
       JSON.parse(@http_post_content, symbolize_names: true)
     end
 
-    def succeed(deferred, response)
+    def succeed(request, response)
       response.status = 200
-      response.content = compress(JSON.dump(success(deferred)))
+      response.content = compress(JSON.dump(success(request)))
       response.send_response
+      storage.delete(request[:uid])
     end
 
-    def success(deferred)
-      result = deferred[:http]
+    def success(request)
+      result = request[:deferred]
       { response: {
         success: true,
         body: result.response,
         headers: result.headers,
-        status: result.response_header.status
+        status: result.response_header.status,
+        response_time: request[:response_time]
       },
-        uid: deferred[:uid] }
+        uid: request[:uid] }
     end
 
-    def fail_(deferred, response)
+    def fail_(request, response)
       response.status = 200
-      response.content = compress(JSON.dump(failure(deferred)))
+      response.content = compress(JSON.dump(failure(request)))
       response.send_response
+      storage.delete(request[:uid])
     end
 
-    def failure(deferred)
-      result = deferred[:http]
+    def failure(request)
+      result = request[:deferred]
       { response: {
         success: false,
         body: result.response,
         headers: result.headers,
         status: result.response_header.status,
+        response_time: request[:response_time],
         error: (result.error&.inspect)
       },
-        uid: deferred[:uid] }
+        uid: request[:uid] }
+    end
+
+    def accept_gzip?
+      # em-http-request doesn't do us any favours with parsing the HTTP headers
+      @http_headers.downcase.include?('accept-encoding: gzip')
     end
   end
 end

data/lib/web_fetch/promise.rb

@@ -62,13 +62,16 @@ module WebFetch
     end
 
     def new_result(response)
+      # XXX: Any changes to this structure need to be reflected by
+      # `Client#new_result`
       Result.new(
         body: response[:body],
         headers: response[:headers],
         status: response[:status],
         success: @raw_result[:response][:success],
         error: @raw_result[:response][:error],
-        uid: @uid
+        uid: @uid,
+        response_time: @raw_result[:response_time]
       )
     end
   end

data/lib/web_fetch/result.rb

@@ -2,7 +2,7 @@
 
 module WebFetch
   class Result
-    attr_reader :body, :headers, :status, :error, :uid
+    attr_reader :body, :headers, :status, :error, :uid, :response_time
 
     def initialize(options = {})
       @pending = options.fetch(:pending, false)
@@ -14,6 +14,7 @@ module WebFetch
       @success = options.fetch(:success)
       @error = options.fetch(:error)
       @uid = options.fetch(:uid)
+      @response_time = options.fetch(:response_time)
     end
 
     def pending?

data/lib/web_fetch/retriever.rb

@@ -17,7 +17,6 @@ module WebFetch
     def find
       request = @server.storage.fetch(@uid)
       return not_found if request.nil?
-      return not_found if request.nil?
       return request.merge(pending: true) if pending?(request)
 
       request
@@ -42,6 +41,7 @@ module WebFetch
     def pending?(request)
       return false if request.nil?
       return false if request[:succeeded]
+      return false if request[:failed]
       # User requested blocking operation so we will wait until item is ready
       # rather than return a `pending` status
       return false if @block

data/lib/web_fetch/server.rb

@@ -29,8 +29,10 @@ module WebFetch
     # #process_http_request and subsequently WebFetch::Router#route
     def gather(targets)
       targets.each do |target|
-        http = request_async(target[:request])
-        request = { uid: target[:uid], deferred: http }
+        http = request_async(target)
+        request = { uid: target[:uid],
+                    start_time: target[:start_time],
+                    deferred: http }
         apply_callbacks(request)
         @storage.store(target[:uid], request)
       end

data/lib/web_fetch/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module WebFetch
-  VERSION = '0.2.1'
+  VERSION = '0.3.0'
 end

data/spec/client_spec.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 describe WebFetch::Client do
-  let(:client) { described_class.new('localhost', 8089, log: File::NULL) }
+  let(:client) { described_class.new('localhost', 60_085, log: File::NULL) }
 
   before(:each) do
     stub_request(:any, 'http://blah.blah/success')
@@ -63,6 +63,7 @@ describe WebFetch::Client do
     subject { client.fetch(responses.first.uid) }
 
     it { is_expected.to be_a WebFetch::Result }
+
     context 'no matching request found' do
       subject { proc { client.fetch('not-found') } }
       it { is_expected.to raise_error WebFetch::RequestNotFoundError }
@@ -80,6 +81,7 @@ describe WebFetch::Client do
       retrieved = client.retrieve_by_uid(uid)
       expect(retrieved[:response][:status]).to eql 200
       expect(retrieved[:response][:body]).to eql 'hello, everybody'
+      expect(retrieved[:response][:response_time]).to be_a Float
       expect(retrieved[:uid]).to eql uid
     end
 
@@ -111,7 +113,7 @@ describe WebFetch::Client do
 
   describe '#create' do
     it 'spawns a server and returns a client able to connect' do
-      client = described_class.create('localhost', 8077, log: File::NULL)
+      client = described_class.create('localhost', 60_085, log: File::NULL)
       expect(client.alive?).to be true
       client.stop
     end
@@ -119,7 +121,12 @@ describe WebFetch::Client do
 
   describe '#stop' do
     it 'can spawn a server and stop the process when needed' do
-      client = described_class.create('localhost', 8077, log: File::NULL)
+      pending <<-PENDING.gsub(/\s+/, ' ')
+      I can't quite figure out what's going on here but the parent process
+      seems to be holding on to the child process' FDs and keeping the server
+      alive. `Client#stop` definitely works though ..."
+      PENDING
+      client = described_class.create('localhost', 60_085, log: File::NULL)
       expect(client.alive?).to be true
       client.stop
       expect(client.alive?).to be false

data/spec/gatherer_spec.rb

@@ -5,7 +5,7 @@ describe WebFetch::Gatherer do
 
   let(:valid_params) do
     { requests: [
-      { url: 'http://localhost:8089' },
+      { url: 'http://localhost:60085' },
       { url: 'http://remotehost:8089' }
     ] }
   end

data/spec/result_spec.rb

@@ -9,7 +9,8 @@ RSpec.describe WebFetch::Result do
       pending: false,
       success: false,
       error: 'foo error happened',
-      uid: 'uid123'
+      uid: 'uid123',
+      response_time: 123.45
     )
   end
 
@@ -24,4 +25,5 @@ RSpec.describe WebFetch::Result do
   its(:success?) { is_expected.to be false }
   its(:error) { is_expected.to eql 'foo error happened' }
   its(:uid) { is_expected.to eql 'uid123' }
+  its(:response_time) { is_expected.to eql 123.45 }
 end

data/spec/server_spec.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 describe WebFetch::Server do
-  let(:port) { 8089 }
+  let(:port) { 60_085 }
   let(:host) { 'localhost' }
   let(:host_uri) { "http://#{host}:#{port}" }
 

data/spec/spec_helper.rb

@@ -14,7 +14,7 @@ WebFetch::Logger.logger(File::NULL)
 
 Thread.new do
   EM.run do
-    EM.start_server 'localhost', 8089, WebFetch::Server
+    EM.start_server 'localhost', 60_085, WebFetch::Server
   end
 end
 waiting = true

data/swagger.yaml

@@ -117,6 +117,8 @@ definitions:
             type: object
           status:
             type: integer
+          response_time:
+            type: float
   Found:
     type: object
     properties:
@@ -134,6 +136,8 @@ definitions:
             type: integer
           pending:
             type: boolean
+          response_time:
+            type: float
 
   GatherResponse:
     type: array

data/web_fetch.gemspec

@@ -21,7 +21,6 @@ Gem::Specification.new do |s|
   s.executables << 'web_fetch_control'
 
   s.add_dependency 'activesupport', '>= 4.0'
-  s.add_dependency 'childprocess', '~> 0.5'
   s.add_dependency 'daemons', '~> 1.2'
   s.add_dependency 'em-http-request', '~> 1.1'
   s.add_dependency 'em-logger', '~> 0.1'
@@ -32,6 +31,7 @@ Gem::Specification.new do |s|
   s.add_dependency 'hanami-utils', '~> 1.0'
   s.add_dependency 'i18n', '>= 0.7'
   s.add_dependency 'rack', '>= 1.6'
+  s.add_dependency 'subprocess', '~> 1.3'
 
   s.add_development_dependency 'byebug', '~> 9.0'
   s.add_development_dependency 'rspec', '~> 3.5'

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: web_fetch
 version: !ruby/object:Gem::Version
-  version: 0.2.1
+  version: 0.3.0
 platform: ruby
 authors:
 - Bob Farrell
@@ -24,20 +24,6 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '4.0'
-- !ruby/object:Gem::Dependency
-  name: childprocess
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '0.5'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '0.5'
 - !ruby/object:Gem::Dependency
   name: daemons
   requirement: !ruby/object:Gem::Requirement
@@ -178,6 +164,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '1.6'
+- !ruby/object:Gem::Dependency
+  name: subprocess
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.3'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.3'
 - !ruby/object:Gem::Dependency
   name: byebug
   requirement: !ruby/object:Gem::Requirement
@@ -277,13 +277,12 @@ files:
 - lib/web_fetch.rb
 - lib/web_fetch/client.rb
 - lib/web_fetch/concerns/client_http.rb
+- lib/web_fetch/concerns/event_machine_helpers.rb
 - lib/web_fetch/concerns/http_helpers.rb
 - lib/web_fetch/concerns/validatable.rb
 - lib/web_fetch/errors.rb
-- lib/web_fetch/event_machine_helpers.rb
 - lib/web_fetch/gatherer.rb
 - lib/web_fetch/helpers.rb
-- lib/web_fetch/http_helpers.rb
 - lib/web_fetch/logger.rb
 - lib/web_fetch/promise.rb
 - lib/web_fetch/request.rb

data/lib/web_fetch/http_helpers.rb

@@ -1,90 +0,0 @@
-# frozen_string_literal: true
-
-module WebFetch
-  # Convenience methods for WebFetch HTTP layer
-  module HTTPHelpers
-    def respond_immediately(result, response)
-      response.status = result[:status]
-      response.content = compress(result[:payload].to_json)
-      response.send_response
-    end
-
-    def pending(result, response)
-      respond_immediately({
-                            payload: {
-                              uid: result[:request][:uid],
-                              pending: true,
-                              message: I18n.t(:pending)
-                            }
-                          }, response)
-    end
-
-    def compress(string)
-      return string unless accept_gzip?
-
-      ActiveSupport::Gzip.compress(string)
-    end
-
-    def default_headers(response)
-      response.headers['Content-Type'] = 'application/json; charset=utf-8'
-      response.headers['Cache-Control'] = 'max-age=0, private, must-revalidate'
-      response.headers['Content-Encoding'] = 'gzip' if accept_gzip?
-      response.headers['Vary'] = 'Accept-Encoding'
-    end
-
-    def request_params
-      { method: @http_request_method,
-        query_string: @http_query_string,
-        post_data: post_data,
-        server: self }
-    end
-
-    def post_data
-      return nil unless @http_post_content
-
-      JSON.parse(@http_post_content, symbolize_names: true)
-    end
-
-    def succeed(request, response)
-      response.status = 200
-      response.content = compress(JSON.dump(success(request)))
-      response.send_response
-      storage.delete(request[:uid])
-    end
-
-    def success(request)
-      result = request[:deferred]
-      { response: {
-        success: true,
-        body: result.response,
-        headers: result.headers,
-        status: result.response_header.status
-      },
-        uid: request[:uid] }
-    end
-
-    def fail_(request, response)
-      response.status = 200
-      response.content = compress(JSON.dump(failure(request)))
-      response.send_response
-      storage.delete(request[:uid])
-    end
-
-    def failure(request)
-      result = request[:deferred]
-      { response: {
-        success: false,
-        body: result.response,
-        headers: result.headers,
-        status: result.response_header.status,
-        error: (result.error&.inspect)
-      },
-        uid: request[:uid] }
-    end
-
-    def accept_gzip?
-      # em-http-request doesn't do us any favours with parsing the HTTP headers
-      @http_headers.downcase.include?('accept-encoding: gzip')
-    end
-  end
-end