web_fetch 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: dc2d09b5ad2253bdc730796a48ce5c6d08adacf91bee689aeb6dc50602199c63
+  data.tar.gz: 9ec7709c430d9a9b64f1de87301ee674f85f91412f2f2d108fba5a6179d031c8
+SHA512:
+  metadata.gz: 7120f22d17fda9952b486b894315068504a99124e6a048908a0fc233c309edca4a4edaad5980e03166bf01f59bcaac415fda56b461f467164fc216272fbbfc7e
+  data.tar.gz: 2f9c1cdee57f1c9ccf219c619b78f4fa8d3fd6dbb058d9e8374fd41c5b7b74c10a99e173d6bda8373e5e181694793f8eab763c378ff6fc15cb25728115c77782

data/.gitignore

@@ -0,0 +1,15 @@
+.byebug_history
+
+# Mac finder artefacts
+.DS_Store
+
+# vim artefacts
+**.swp
+**.swo
+
+# Emacs stuff
+*~
+#*#
+*#
+#*
+.#*

data/.rspec

@@ -0,0 +1,3 @@
+--color
+--format documentation
+--require spec_helper

data/.rubocop.yml

@@ -0,0 +1,10 @@
+Style/Documentation:
+  Enabled: false
+
+Metrics/BlockLength:
+  Exclude:
+    - 'spec/**/*'
+
+AllCops:
+  Exclude:
+    - 'bin/**/*'

data/.ruby-version

@@ -0,0 +1 @@
+2.5.0

data/Gemfile

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+source 'https://rubygems.org'
+
+gemspec

data/Gemfile.lock

@@ -0,0 +1,120 @@
+PATH
+  remote: .
+  specs:
+    web_fetch (0.1.0)
+      activesupport (~> 4.0)
+      childprocess (~> 0.5)
+      daemons (~> 1.2)
+      em-http-request (~> 1.1)
+      em-logger (~> 0.1)
+      eventmachine (~> 1.0)
+      eventmachine_httpserver (~> 0.2)
+      faraday (~> 0.9)
+      hanami-router (~> 0.7)
+      hanami-utils (= 0.8.0)
+      i18n (~> 0.7)
+      rack (~> 1.6)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    activesupport (4.2.10)
+      i18n (~> 0.7)
+      minitest (~> 5.1)
+      thread_safe (~> 0.3, >= 0.3.4)
+      tzinfo (~> 1.1)
+    addressable (2.5.2)
+      public_suffix (>= 2.0.2, < 4.0)
+    ast (2.4.0)
+    byebug (9.1.0)
+    childprocess (0.9.0)
+      ffi (~> 1.0, >= 1.0.11)
+    concurrent-ruby (1.0.5)
+    cookiejar (0.3.3)
+    crack (0.4.3)
+      safe_yaml (~> 1.0.0)
+    daemons (1.2.6)
+    diff-lcs (1.3)
+    em-http-request (1.1.5)
+      addressable (>= 2.3.4)
+      cookiejar (!= 0.3.1)
+      em-socksify (>= 0.3)
+      eventmachine (>= 1.0.3)
+      http_parser.rb (>= 0.6.0)
+    em-logger (0.1.0)
+      eventmachine (>= 0.12.10)
+    em-socksify (0.3.2)
+      eventmachine (>= 1.0.0.beta.4)
+    eventmachine (1.2.7)
+    eventmachine_httpserver (0.2.1)
+    faraday (0.15.3)
+      multipart-post (>= 1.2, < 3)
+    ffi (1.9.25)
+    hanami-router (0.7.0)
+      hanami-utils (~> 0.8)
+      http_router (~> 0.11)
+      rack (~> 1.6)
+    hanami-utils (0.8.0)
+    hashdiff (0.3.7)
+    http_parser.rb (0.6.0)
+    http_router (0.11.2)
+      rack (>= 1.0.0)
+      url_mount (~> 0.2.1)
+    i18n (0.9.5)
+      concurrent-ruby (~> 1.0)
+    jaro_winkler (1.5.1)
+    minitest (5.11.3)
+    multipart-post (2.0.0)
+    parallel (1.12.1)
+    parser (2.5.1.2)
+      ast (~> 2.4.0)
+    powerpack (0.1.2)
+    public_suffix (3.0.3)
+    rack (1.6.10)
+    rainbow (3.0.0)
+    rspec (3.8.0)
+      rspec-core (~> 3.8.0)
+      rspec-expectations (~> 3.8.0)
+      rspec-mocks (~> 3.8.0)
+    rspec-core (3.8.0)
+      rspec-support (~> 3.8.0)
+    rspec-expectations (3.8.1)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.8.0)
+    rspec-mocks (3.8.0)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.8.0)
+    rspec-support (3.8.0)
+    rubocop (0.59.2)
+      jaro_winkler (~> 1.5.1)
+      parallel (~> 1.10)
+      parser (>= 2.5, != 2.5.1.1)
+      powerpack (~> 0.1)
+      rainbow (>= 2.2.2, < 4.0)
+      ruby-progressbar (~> 1.7)
+      unicode-display_width (~> 1.0, >= 1.0.1)
+    ruby-progressbar (1.10.0)
+    safe_yaml (1.0.4)
+    thread_safe (0.3.6)
+    tzinfo (1.2.5)
+      thread_safe (~> 0.1)
+    unicode-display_width (1.4.0)
+    url_mount (0.2.1)
+      rack
+    webmock (3.4.2)
+      addressable (>= 2.3.6)
+      crack (>= 0.3.2)
+      hashdiff
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  byebug (~> 9.0)
+  rspec (~> 3.5)
+  rubocop (~> 0.59.2)
+  web_fetch!
+  webmock (~> 3.4)
+
+BUNDLED WITH
+   1.16.5

data/LICENSE

@@ -0,0 +1,7 @@
+Copyright 2018 Robert Farrell
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,149 @@
+# WebFetch
+
+## Overview
+
+WebFetch is an asynchronous HTTP proxy server that accepts multiple requests for HTTP retrieval, immediately returning a token for each request, and then allowing that token to be redeemed later when the entity has fully responded.
+
+This permits issuing multiple HTTP requests in parallel, in a fully encapsulated and external process, without having to resort to multi-threading, multi-processing, or complex non-blocking IO implementations.  [EventMachine][1] is used to handle the heavy lifting.
+
+![WebFetch architecture][2]
+
+## Getting Started
+
+Although WebFetch runs as a web server and provides all functionality over a RESTful API (see below), the simplest way to use it is with its Ruby client implementation, which wraps the HTTP API for you, using [Faraday][3]. This also serves as a [reference][4] for writing WebFetch clients in other languages.
+
+In your `Gemfile`, add:
+
+``` ruby
+gem 'web_fetch', git: 'https://github.com/bobf/web_fetch.git'
+```
+
+and update your bundle:
+
+``` ruby
+bundle install
+```
+
+Create, connect to, and wrap a Ruby client object around a new WebFetch server instance, listening as `localhost` on port `8077`:
+
+``` ruby
+require 'web_fetch'
+client = WebFetch::Client.create('localhost', 8077)
+```
+
+Issue some requests [asynchronously]:
+
+``` ruby
+requests = [{ url: 'http://foobar.baz/' },
+            { url: 'http://barfoo.baz/foobar',
+              headers: { 'User-Agent' => 'Foo Browser' } },
+              query: { foo: 'what is foo', bar: 'what is baz' } ]
+jobs = client.gather(requests)
+```
+
+Retrieve the responses [synchronously - *any result that has not yet arrived will block until it has arrived while other requests continue to run in parallel*]:
+
+``` ruby
+responses = []
+jobs.each do |job|
+  response = client.retrieve_by_uid(job[:uid])
+  responses.push(response)
+end
+```
+
+See [a working example][5]
+
+## HTTP API
+
+If you need to use the WebFetch server's HTTP API directly refer to the [Swagger API Reference][6]
+
+## Managing the WebFetch process yourself
+
+You may want to run the WebFetch server yourself rather than instantiate it via the client. For this case, the executable `bin/web_fetch_control` is provided.
+
+WebFetch can be started in the terminal with output going to STDOUT or as a daemon.
+
+Run the server as a daemon:
+
+```
+$ bundle exec bin/web_fetch_control start -- --log /tmp/web_fetch.log
+```
+
+**Note that you should always pass `--log` when running as a daemon otherwise all output will go to the null device.**
+
+Run the server in the terminal:
+
+```
+$ bundle exec bin/web_fetch_control run -- --port 8080
+```
+
+It is further recommended to use a process management tool to monitor the pidfile (pass `--pidfile /path/to/file.pid` to specify an explicit location).
+
+To connect to an existing process, use `WebFetch::Client.new` rather than `WebFetch::Client.create`. For example:
+
+``` ruby
+WebFetch::Client.new('localhost', 8087)
+```
+
+## WebFetch Client request options
+
+`WebFetch::Client#gather` accepts an array of hashes which may contain the following parameters:
+
+  * `url`: The target URL [string]
+  * `headers`: HTTP headers [hash]
+  * `query`: Query parameters [hash]
+  * `method`: HTTP method (default: `"GET"`) [string]
+  * `body`: HTTP body [string]
+
+These parameters will all be used (where provided) when initiating the HTTP request on the target.
+
+Arbitrary parameters can also be passed and will be returned by `#gather` (though they will not be used to construct the HTTP request). This allows tagging requests with arbitrary information if you need to identify them in a particular way. For example, you may want to generate your own unique identifier for a request, in which case you could do:
+
+``` ruby
+client.gather([{ url: 'http://foobar.baz', my_unique_id: '123-456-789' }])
+# [{:request=>{:url=>"http://foobar.baz", :my_unique_id=>"123-456-789"}, :hash=>"7c511911d16e1072363fa1653bdd93df65208901", :uid=>"1fb4ee7a-9fc0-4896-9af2-7cbdf234a468"}]
+```
+
+## Logging
+
+WebFetch logs to STDOUT by default. An alternative log file can be set either
+by passing `--log /path/to/logfile` to the command line server, or by passing
+`log: '/path/to/logfile'` to `WebFetch::Client.create`:
+
+```
+$ bundle exec bin/web_fetch_server --log /tmp/web_fetch.log
+```
+
+```
+client = WebFetch::Client.create('localhost', 8077, log: '/tmp/web_fetch.log')
+```
+
+## Contributing
+
+WebFetch uses `rspec` for testing:
+
+```
+bin/rspec
+```
+
+Rubocop is used for code style governance:
+
+```
+bin/rubocop
+```
+
+Make sure that any new code you write has an appropriate test and that all Rubocop checks pass.
+
+Feel free to fork and create a pull request if you would like to make any changes.
+
+## License
+
+WebFetch is licensed under the [MIT License][7]. You are encouraged to re-use the code in any way you see fit as long as you give credit to the original author. If you do use the code for any other projects then feel free to let me know but, of course, this is not required.
+
+[1]: https://github.com/eventmachine/eventmachine
+[2]: doc/web_fetch_architecture.png
+[3]: https://github.com/lostisland/faraday
+[4]: lib/web_fetch/client.rb
+[5]: doc/client_example.rb
+[6]: swagger.yaml
+[7]: LICENSE

data/bin/rspec

@@ -0,0 +1,29 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+#
+# This file was generated by Bundler.
+#
+# The application 'rspec' is installed as part of a gem, and
+# this file is here to facilitate running it.
+#
+
+require "pathname"
+ENV["BUNDLE_GEMFILE"] ||= File.expand_path("../../Gemfile",
+  Pathname.new(__FILE__).realpath)
+
+bundle_binstub = File.expand_path("../bundle", __FILE__)
+
+if File.file?(bundle_binstub)
+  if File.read(bundle_binstub, 300) =~ /This file was generated by Bundler/
+    load(bundle_binstub)
+  else
+    abort("Your `bin/bundle` was not generated by Bundler, so this binstub cannot run.
+Replace `bin/bundle` by running `bundle binstubs bundler --force`, then run this command again.")
+  end
+end
+
+require "rubygems"
+require "bundler/setup"
+
+load Gem.bin_path("rspec-core", "rspec")

data/bin/rubocop

@@ -0,0 +1,29 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+#
+# This file was generated by Bundler.
+#
+# The application 'rubocop' is installed as part of a gem, and
+# this file is here to facilitate running it.
+#
+
+require "pathname"
+ENV["BUNDLE_GEMFILE"] ||= File.expand_path("../../Gemfile",
+  Pathname.new(__FILE__).realpath)
+
+bundle_binstub = File.expand_path("../bundle", __FILE__)
+
+if File.file?(bundle_binstub)
+  if File.read(bundle_binstub, 300) =~ /This file was generated by Bundler/
+    load(bundle_binstub)
+  else
+    abort("Your `bin/bundle` was not generated by Bundler, so this binstub cannot run.
+Replace `bin/bundle` by running `bundle binstubs bundler --force`, then run this command again.")
+  end
+end
+
+require "rubygems"
+require "bundler/setup"
+
+load Gem.bin_path("rubocop", "rubocop")

data/bin/web_fetch_control

@@ -0,0 +1,6 @@
+#!/usr/bin/env ruby
+
+require 'daemons'
+
+server = File.expand_path(File.join(File.dirname(__FILE__), 'web_fetch_server'))
+Daemons.run(server, multiple: true)

data/bin/web_fetch_server

@@ -0,0 +1,30 @@
+#!/usr/bin/env ruby
+
+require 'optparse'
+
+options = { host: 'localhost', port: '60089' }
+OptionParser.new do |opt|
+  opt.on('--host localhost') { |o| options[:host] = o }
+  opt.on('--port 60089') { |o| options[:port] = o }
+  opt.on('--pidfile tmp/web_fetch.pid') { |o| options[:pidfile] = o }
+  opt.on('--log web_fetch.log') { |o| options[:log] = o }
+end.parse!
+
+require 'web_fetch'
+WebFetch::Logger.logger(options[:log])
+WebFetch::Logger.info("Loading WebFetch server: #{options}")
+
+pid_path = options[:pidfile].nil? ? nil : File.expand_path(options[:pidfile])
+if pid_path
+  pid = Process.pid
+  File.open(pid_path, 'w+') do |f|
+    f.write(pid)
+  end
+  WebFetch::Logger.info("Pidfile created: #{pid_path} (#{pid})")
+end
+
+EM.run do
+  EM.start_server(options[:host],
+                  options[:port].to_i,
+                  WebFetch::Server)
+end

data/config/locales/en.yml

@@ -0,0 +1,12 @@
+en:
+  requests_missing: "`requests` parameter missing"
+  requests_not_array: "`requests` parameter must be an array"
+  requests_empty: "`requests` parameter must not be empty"
+  missing_url: "All `requests` parameters must contain a URL"
+  bad_json: "`json` parameter could not be parsed as valid JSON"
+
+  hash_or_uid_but_not_both: "Cannot retrieve by both `uid` and `hash`"
+  missing_hash_and_uid: "Must pass either `uid` or `hash` to retrieve"
+
+  uid_not_found: "Provided `uid` has not yet been requested"
+  hash_not_found: "Provided `hash` has not yet been requested"

data/doc/client_example.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+# Run me with:
+# bundle exec ruby doc/client_example.rb
+require 'web_fetch'
+begin
+  cli = WebFetch::Client.create('localhost', 8077)
+  results = cli.gather([
+                         { url: 'http://localhost:8077/' },
+                         { url: 'http://yahoo.com/' },
+                         { url: 'http://lycos.com/' },
+                         { url: 'http://google.com/' }
+                       ])
+  results.each do |res|
+    p cli.retrieve_by_uid(res[:uid])
+  end
+ensure
+  cli.stop
+end

data/lib/web_fetch/client.rb

@@ -0,0 +1,101 @@
+# frozen_string_literal: true
+
+module WebFetch
+  # Client to be used in application code. Capable of spawning a server and
+  # interacting with it to gather requests and retrieve them
+  class Client
+    def initialize(host, port, options = {})
+      @host = host
+      @port = port
+      @process = options[:process]
+    end
+
+    def self.create(host, port, options = {})
+      # Will block until process is responsive
+      process = spawn(host, port, options)
+      client = new(host, port, process: process)
+      sleep 0.1 until client.alive?
+      client
+    end
+
+    def stop
+      # Will block until process dies
+      return if @process.nil?
+
+      @process.stop
+      @process.wait
+    end
+
+    def alive?
+      begin
+        response = get('')
+      rescue Faraday::ConnectionFailed
+        return false
+      end
+      return false unless response.success?
+
+      JSON.parse(response.body)['application'] == 'WebFetch'
+    end
+
+    def gather(requests)
+      json = JSON.dump(requests: requests)
+      response = post('gather', json)
+      return nil unless response.success?
+
+      JSON.parse(response.body, symbolize_names: true)[:requests]
+    end
+
+    def retrieve_by_uid(uid)
+      response = get('retrieve', uid: uid)
+      return nil unless response.success?
+
+      JSON.parse(response.body, symbolize_names: true)
+    end
+
+    class << self
+      def spawn(host, port, options)
+        process = build_process(host, port, options)
+        process.cwd = File.join(File.dirname(__dir__), '..')
+        process.io.inherit!
+        process.start
+        process
+      end
+
+      private
+
+      def build_process(host, port, options)
+        command = options.fetch(:start_command, standard_start_command)
+        args = ['--host', host, '--port', port.to_s]
+        args += ['--log', options[:log]] unless options[:log].nil?
+        args.push('--daemonize') if options[:daemonize]
+        ChildProcess.build(*command, *args)
+      end
+
+      def standard_start_command
+        %w[bundle exec ./bin/web_fetch_control run --]
+      end
+    end
+
+    private
+
+    def base_uri
+      "http://#{@host}:#{@port}"
+    end
+
+    def get(endpoint, params = {})
+      conn = Faraday.new(url: base_uri)
+      conn.get do |request|
+        request.url "/#{endpoint}"
+        request.params.merge!(params)
+      end
+    end
+
+    def post(endpoint, body)
+      conn = Faraday.new(url: base_uri)
+      conn.post do |request|
+        request.url "/#{endpoint}"
+        request.body = body
+      end
+    end
+  end
+end

data/lib/web_fetch/concerns/http_helpers.rb

@@ -0,0 +1,64 @@
+# frozen_string_literal: true
+
+module WebFetch
+  module HttpHelpers
+    def compress(string)
+      ActiveSupport::Gzip.compress(string)
+    end
+
+    def default_headers(response)
+      response.headers['Content-Type'] = 'application/json; charset=utf-8'
+      response.headers['Cache-Control'] = 'max-age=0, private, must-revalidate'
+      response.headers['Content-Encoding'] = 'gzip'
+      response.headers['Vary'] = 'Accept-Encoding'
+    end
+
+    def request_params
+      { method: @http_request_method,
+        query_string: @http_query_string,
+        post_data: post_data,
+        server: self }
+    end
+
+    def post_data
+      return nil unless @http_post_content
+
+      JSON.parse(@http_post_content, symbolize_names: true)
+    end
+
+    def succeed(deferred, response)
+      response.status = 200
+      response.content = compress(JSON.dump(success(deferred)))
+      response.send_response
+    end
+
+    def success(deferred)
+      result = deferred[:http]
+      { response: {
+        success: true,
+        body: result.response,
+        headers: result.headers,
+        status: result.response_header.status
+      },
+        uid: deferred[:uid] }
+    end
+
+    def fail_(deferred, response)
+      response.status = 200
+      response.content = compress(JSON.dump(failure(deferred)))
+      response.send_response
+    end
+
+    def failure(deferred)
+      result = deferred[:http]
+      { response: {
+        success: false,
+        body: result.response,
+        headers: result.headers,
+        status: result.response_header.status,
+        error: (result.error&.inspect)
+      },
+        uid: deferred[:uid] }
+    end
+  end
+end

data/lib/web_fetch/concerns/validatable.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module WebFetch
+  # Provides boilerplate for a validatable model
+  module Validatable
+    attr_reader :errors
+
+    def valid?
+      @errors = []
+      validate
+      @errors.empty?
+    end
+
+    private
+
+    def validate
+      error = <<-MSG.gsub(/\s+/, ' ')
+        Override and call `error(:i18n_key)` as many times as required for each
+        validation failure.
+        Supplementary text can be added to the error by passing as the second
+        parameter to `error`
+      MSG
+      raise NotImplementedError, error
+    end
+
+    def error(name, aux = '')
+      aux = ' ' + aux unless aux.empty?
+      @errors.push(I18n.t(name) + aux)
+    end
+  end
+end