commute 0.2.0.rc.2 → 0.3.0.pre

data/.todo

@@ -1,15 +1,31 @@
-conditional sequencer (body), routing on constants
-conditional processors instead of only sequencers
 easy transforms, reactivity? When not to add a transform
-disabling
-groups of commuters (Group and delaying)
-queueing
-batching/grouping
 net/http
 stubbing
-gzip
-resourcing (commute-resourcing)
-Api::Metal
-caching
-calling processor from within commuter
-simpler sequencer internal working
+request error handling (timeouts...)
+
+layers:
+OAuth with refreshing (and on_refresh callback)
+Redirect following
+Caching
+Batching
+Page Crusher
+
+api method on context to select context
+complete method for quick layer (eg github gist star).
+why not use stream, run, request in layers or in complete system.
+
+HOOKS
+
+stream flattening [[1,2,3]] => [1,2,3] (flat_map?)
+stream::lazy; enumerable?
+
+New relic instrumentation
+Separate Parse/Render
+
+move layer specific config to using istead of with
+using [:render, :parse] => {
+  template: 'person'
+}
+
+param :reload # transform { |r, param| r.query[param] = c[param] }
+cache param blocks

data/README.md

@@ -1,6 +1,5 @@
 # Commute 
 [![Build Status](https://secure.travis-ci.org/challengee/commute.png)](http://travis-ci.org/challengee/commute)
-[![Dependency Status](https://gemnasium.com/challengee/commute.png?travis)](https://gemnasium.com/challengee/commute)
 
 Commute helps you to:
 

data/commute.gemspec

@@ -14,17 +14,16 @@ Gem::Specification.new do |s|
   s.require_paths = ["lib"]
   s.version       = Commute::VERSION
 
-  s.add_dependency 'typhoeus', '0.5.0.rc'
-  s.add_dependency 'em-http-request'
+  s.add_development_dependency 'typhoeus'
+  s.add_development_dependency 'em-http-request'
+  s.add_development_dependency 'yajl-ruby'
 
   s.add_development_dependency 'rake'
   s.add_development_dependency 'mocha'
+  s.add_development_dependency 'rb-fsevent'
   s.add_development_dependency 'guard'
   s.add_development_dependency 'guard-minitest'
   s.add_development_dependency 'yard'
   s.add_development_dependency 'simplecov'
   s.add_development_dependency 'webmock'
-
-  # For the examples.
-  s.add_development_dependency 'yajl-ruby'
 end

data/lib/commute/common/basic_auth.rb

@@ -4,16 +4,17 @@ module Commute
   module Common
 
     class BasicAuth
-      @id = :auth
+      @name = :auth
 
-      def call commuter, options = {}
-        commuter.change do |request|
-          # Ruby's Base64 puts newlines at the end (and every 60 chars)...
-          # This breaks HTTP. So strip them!
-          authorization = Base64.encode64("#{options[:username]}:#{options[:password]}}").gsub "\n", ''
-          request.headers['Authorization'] = "Basic #{authorization}"
-          request
-        end
+      def call router, request, options = {}
+        # Create the Authorization header.
+        authorization = Base64.strict_encode64 \
+          "#{options[:username]}:#{options[:password]}}"
+        # Set the header on the http request.
+        request.http.headers['Authorization'] = "Basic #{authorization}"
+
+        # Send the request, pipe the data, and forward the response.
+        request.pipe router.call(request.http, &request.responder)
       end
     end
   end

data/lib/commute/common/caching.rb

@@ -0,0 +1,208 @@
+require 'commute/core/http'
+
+module Commute
+  module Common
+
+    # Internal: Fetches cached responses to make requests faster.
+    #
+    # Any GET requests that comes in will be fetched in the cache
+    # based on its URL.
+    #
+    # When nothing is found in the cache, a request is made as
+    # planned, and the result is stored in the cache (when it is a GET request).
+    #
+    # When it is found in the cache, there are actually two possibilities.
+    # One is if validation is on (:validate option). When it is on, it will
+    # detect if the resource has already changed on the server before returning
+    # from the cache (using etags).
+    # If validation is off, the result from the cache is returned.
+    #
+    class Caching
+      @name = :caching
+
+      # Internal: Special status that wraps a http status and
+      # adds cache indications.
+      #
+      class CacheStatus < Status
+
+        # Public: The underlaying http status.
+        attr_accessor :http
+
+        # Public: Whether the response was cached.
+        attr_accessor :cached
+
+        # Public: Whether the cache was updated on this request.
+        attr_accessor :updated
+
+        # Public: Creates a new CacheStatus.
+        #
+        # http        - Underlaying http status, can be nil.
+        # cached      - Whether the response was cached.
+        # updated     - Whether the cache was updated on this request.
+        #
+        def initialize http
+          @http = http
+          @updated = false
+        end
+
+        # Public: Success or not?
+        def success?
+          (http? && http.success?) || cached
+        end
+
+        # Public: Whether there was an http request performed.
+        def http?
+          !http.nil?
+        end
+      end
+
+      # Internal: One entry in the cache, contains the data of a
+      # resource and the etag it corresponds with.
+      class CacheEntry
+
+        # Public: The value present in the cache.
+        attr_reader :value
+
+        # Public: The key of this cache value.
+        attr_reader :key
+
+        # Initialize a new Entry.
+        #
+        # cache - The cache to work in.
+        # key   - The key for this cache entry.
+        # value - The contents for this cache entry.
+        #
+        def initialize cache, key, value, exists = false
+          @cache = cache
+          @key = key
+          @value = value
+          @exists = exists
+        end
+
+        # Search for a CacheEntry in a cache.
+        #
+        # cache - The cache to look in.
+        # key   - The key to look for.
+        #
+        # Returns a CacheEntry or nil if none was found.
+        def self.lookup cache, key
+          value = cache.get(key)
+          CacheEntry.new(cache, key, value, !value.nil?)
+        end
+
+        # Stores the entry in the cache.
+        def store
+          @cache.set @key, @value
+          @exists = true
+          nil
+        end
+
+        # Updates the entry to a new value
+        def update value
+          @value = value
+          self.store
+        end
+
+        def exists?
+          @exists
+        end
+      end
+
+      # The caching logic itself.
+      #
+      # options - Hash of options for caching (default: {}):
+      #           :cache    - A cache (#get, #set).
+      #           :validate - Whether to check if the cache
+      #                       entry is still valid (default: false).
+      #
+      def call router, request, options = {}
+        # Fetch some options.
+        cache, validate = options[:cache], options[:validate] || false
+        # Cannot proceed if there is no cache.
+        raise 'You need to configure a cache first' unless cache
+
+        # This cache only works for get request, proceed if something else.
+        router.call request and return unless request.http.method == :get
+
+        # Get the cache entry.
+        entry = CacheEntry.lookup(cache, request.http.uri.to_s)
+
+        # To do when we have something to show.
+        on_response = proc do |response, status|
+          # Just pipe the response from the net/cache to the response for this request.
+          response.pipe request.respond(response, status)
+        end
+
+        # When validation is on, or we have no cache entry, get the latest (request or cache)
+        if validate || !entry.exists?
+          respond_latest router, entry, request, &on_response
+        # When we have something in the cache and we do not want to validate it,
+        # Then just return our cache entry.
+        else
+          respond_from_cache entry, request, &on_response
+        end
+      end
+
+      private
+
+      # Responds with the latest version for a given remote resource. Either from the
+      # cache or from the internets. Checks based on etags.
+      # Warning: Could modify the given request.
+      #
+      # stack   - The Commute stack.
+      # entry   - The cache entry.
+      # request - The request for the remote resource.
+      #
+      # Yields a response like you would get from a stack call.
+      #
+      def respond_latest router, entry, request, &callback
+        # Modify the request to be a head request of nothing was found in the cache.
+        if entry.exists?
+          request.http.headers['If-None-Match'] = entry.value[:etag]
+        end
+        # Fire the conditional request.
+        validate_request = router.call request.http do |response, http_status|
+          # If it was not modified, the it is still valid.
+          if http_status.not_modified?
+            # Respond from cache.
+            respond_from_cache entry, request do |response, status|
+              callback.call response, status.tap { |s| s.http = http_status }
+            end
+
+          # If it was modified and a success, respond and update the cache.
+          elsif http_status.success?
+            # Update the cache.
+            response.once(:data) do |body|
+              entry.update data: body, etag: response.http.etag
+            end
+            # Respond with wrapped status marked as invalidated.
+            callback.call response, CacheStatus.new(http_status).tap { |status|
+              status.updated = true
+              status.cached  = entry.exists?
+            }
+
+          # The response was not a success, that sucks, respond with it anyway.
+          else
+            callback.call response, CacheStatus.new(http_status).mark(cached: !entry.nil?)
+          end
+        end
+        validate_request.end
+      end
+
+      # Responds with a cached version.
+      # Raises an error when the entry is nil.
+      #
+      # entry - The cache entry.
+      #
+      # Yields a response like you would get from a stack call.
+      #
+      def respond_from_cache entry, request, &callback
+        raise ArgumentError, "Entry is nil" unless entry
+        response = Layer::Response.new Http::Response.new(request)
+        callback.call response, CacheStatus.new(nil).tap { |s| s.cached = true }
+        # Stream the cached data in one big chunk.
+        response.end entry.value[:data]
+      end
+    end
+  end
+end

data/lib/commute/common/chemicals.rb

@@ -2,36 +2,59 @@ require 'chemicals'
 
 module Commute
   module Common
-    class Chemicals
-      @id = :chemicals
 
-      def initialize root = nil
-        @root = root
-        @cache = {}
-      end
+    module Chemicals
+
+      module Chemicals::Base
+        TEMPLATE_CACHE = {}
+
+        def initialize template_root = './'
+          @template_root = template_root
+        end
 
-      def call commuter, path
-        # Build the path to the template.
-        path = if @root
-          "#{@root}/#{path}.xml"
-        else
-          "#{path}.xml"
+        def template path
+          file = File.expand_path "#{path}.xml", @template_root
+
+          if TEMPLATE_CACHE.has_key? file
+            TEMPLATE_CACHE[file]
+          else
+            TEMPLATE_CACHE[file] = ::Chemicals::Template.new IO.read(file)
+          end
         end
-        # Create/Cache the Template.
-        template = if @cache[path]
-          @cache[path]
-        else
-          ::Chemicals::Template.new(IO.read(path)).tap do |t|
-            @cache[path] = t
+      end
+
+      class Parser
+        include Chemicals::Base
+        @name = :parse
+
+        def call router, request, options
+          path = options[:path]
+
+          _request = router.call(request.http) do |_response, status|
+            response = request.respond _response.http, status
+
+            _response.inject('', &:<<).map { |body|
+              template(path).parse body
+            }.pipe response
           end
+          request.pipe _request
         end
-        # Use the template to parse/render.
-        if commuter.get.body
-          if commuter.get.body.kind_of?(String) && !commuter.get.body.strip.empty?
-            commuter.get.body = template.parse(commuter.get.body)
-          elsif !commuter.get.body.kind_of?(String)
-            commuter.get.body = template.render(commuter.get.body).to_s
+      end
+
+      class Renderer
+        include Chemicals::Base
+        @name = :render
+
+        def call router, request, options
+          path = options[:path]
+
+          _request = router.call(request.http) do |_response, status|
+            _response.pipe request.respond(_response.http, status)
           end
+
+          request.map { |body|
+            template(path).render(body).to_s
+          }.pipe _request
         end
       end
     end

data/lib/commute/common/eventmachine.rb

@@ -0,0 +1,68 @@
+require 'em-http-request'
+
+require 'commute/core/http'
+
+module Commute
+  module Common
+
+    # Public: Adapter that uses em-http-request, an Eventmachine based
+    # HTTP client, as a underlaying commute engine.
+    #
+    # This requires all the commute requests to be made within the
+    # Eventmachine loop.
+    #
+    class Eventmachine
+      @name = :adapter
+
+      # Internal: Make a request through Eventmachine.
+      def call router, request, options = {}
+        # Buffer the request (only streaming response).
+        request.buffer.on(:data) do |body|
+          response = nil
+          # Create a native em-http request.
+          em_request = to_request request.http, body.join
+
+          em_request.headers do
+            # Create a Http response.
+            http_response = to_response request, em_request.response_header
+            # Create a Http::Status (determined from response code).
+            status = Http::Status.new(http_response.code)
+            # Respond.
+            response = request.respond(http_response, status)
+          end
+
+          # Set the data callback.
+          em_request.stream do |chunk|
+            response.write chunk
+          end
+
+          # Set the end callback.
+          em_request.callback do
+            response.end
+          end
+        end
+      end
+
+      protected
+
+      # Internal: Converts a Commute requests into an em-http request.
+      def to_request request, body
+        EventMachine::HttpRequest.new(request.uri.to_s).send request.method, \
+          query: request.query,
+          head: {
+            'Accept-Encoding' => 'gzip, deflate'
+          }.merge!(request.headers),
+          body: body
+      end
+
+      # Internal: Converts an em-http response into a Commute response.
+      # Note: trims whitespace bodies.
+      def to_response request, eventmachine_response
+        Http::Response.new(request).tap do |response|
+          response.code = eventmachine_response.status
+          response.headers = eventmachine_response.raw
+        end
+      end
+    end
+  end
+end

data/lib/commute/common/synchrony.rb

@@ -0,0 +1,42 @@
+require "em-synchrony"
+
+require 'commute/common/eventmachine'
+
+module Commute
+  module Common
+
+    # Public: Adapter that uses em-synchrony (with em-http-request).
+    #
+    # Requires use of em-synchrony and thus Eventmachine.
+    #
+    class Synchrony < Eventmachine
+      @name = :adapter
+
+      # Internal: Make a request through em-synchrony.
+      def call router, request, options = {}
+        # Get the current fiber.
+        body = nil
+        request.on(:data) { |_body| body = _body }
+        # Buffer the request (only streaming response).
+        request.on(:end) do |body|
+          response = nil
+          # Create a native em-http request.
+          em_request = to_request request.http, body
+
+          # Create a Http response.
+          http_response = to_response request, em_request.response_header
+          # Create a Http::Status (determined from response code).
+          status = Http::Status.new(http_response.code)
+          # Respond.
+          response = request.respond(http_response, status)
+
+          response.write em_request.response unless \
+            em_request.response.strip.empty?
+
+          # Request is finished, end the response.
+          response.end
+        end
+      end
+    end
+  end
+end

data/lib/commute/common/typhoeus.rb

@@ -0,0 +1,64 @@
+require 'typhoeus'
+
+require 'commute/core/http'
+
+module Commute
+  module Common
+
+    # Internal: Adapter that uses Typhoeus to make requests.
+    #
+    # Request bodies are entirely buffered before sending.
+    # Responses are also completely buffered before they
+    # are sent out in one big chunk from this layer.
+    #
+    class Typhoeus
+      @name = :adapter
+
+      def call router, request, options = {}
+        # Here we can only send when the request is buffered (body is needed).
+        body = []
+        request.buffer.on(:data) { |_body| body = _body }
+        request.on(:end) do
+          # Build a Typhoeus request from this request.
+          # Body should be a String.
+          typhoeus_request = to_request(request.http, body.join)
+          # Run the request with Hydra.
+          hydra = ::Typhoeus::Hydra.new
+          hydra.queue typhoeus_request
+          hydra.run
+          # Build a Http response.
+          http_response = to_response(request.http, typhoeus_request.response)
+          # Create a Http::Status (determined from response code).
+          status = Http::Status.new(http_response.code)
+          # Respond with that response and send the data.
+          response = request.respond(http_response, status) do |r|
+            # Write the response body if there is something besides whitespace.
+            r.write typhoeus_request.response.body unless \
+              typhoeus_request.response.body.strip.empty? || status.fail?
+          end
+        end
+      end
+
+      protected
+
+      # Internal: Creates a Typhoeus::Request from a Http::Request
+      # and body.
+      def to_request request, body
+        ::Typhoeus::Request.new request.uri.to_s, \
+            method: request.method,
+            params: request.query || {},
+            headers: request.headers,
+            body: body,
+            accept_encoding: ''
+      end
+
+      # Internal: Creates a Http::Response from a Typhoeus::Response.
+      def to_response request, typhoeus_response
+        Http::Response.new(request).tap do |response|
+          response.code    = typhoeus_response.code
+          response.headers = typhoeus_response.headers
+        end
+      end
+    end
+  end
+end

data/lib/commute/core/api.rb

@@ -1,10 +1,5 @@
 require 'commute/core/context'
 
-require 'commute/core/processors/hook'
-require 'commute/core/processors/request_builder'
-require 'commute/core/processors/sequencer'
-require 'commute/core/processors/code_status_processor'
-
 module Commute
 
   # Public: An API is a context that already provides some standard
@@ -45,6 +40,7 @@ module Commute
   class Api < Builder
     class << self
       include Buildable
+      include Forwardable
 
       # The Builder methods should only be called in the Class itself.
       # Otherwise, something like TestApi.with ... would modify
@@ -55,13 +51,38 @@ module Commute
       private *Builder::METHODS
 
       # Internal: Memoized base builder.
+      #
+      # The builder is actually an instance of the Api itself.
+      # This means that all methods available on the Api can
+      # be used in the base builder to create a base Api context.
       def builder
-        @builder ||= Builder.new Context.new(Stack.new {})
+        @builder ||= self.new Context.new(Stack.new {})
       end
 
       # Internal: When an Api inherits another Api, it inherits its context.
       def inherited klass
-        klass.instance_variable_set :@builder, Builder.new(builder.context)
+        klass.instance_variable_set :@builder, klass.new(builder.context)
+      end
+
+      # Internal: On include, define forwarding methods to the builder.
+      #
+      # Whenever some extension is included, it works for Api instances.
+      # However we want to define a default context in the Api class, so
+      # the extension methods need to be present the class as wel (and be
+      # forwarded to the builder). We define these forwarding methods here.
+      #
+      def include mod
+        # Do a normal include.
+        super
+        # Define a forwarding method for each extension method.
+        mod.const_get(:METHODS).each do |method|
+          # Define the method.
+          self.instance_eval %{
+            def #{method} *args, &block
+              builder.#{method} *args, &block
+            end
+          }
+        end
       end
     end
 
@@ -79,28 +100,12 @@ module Commute
     #
     # Also provides basic on_request, on_response and
     # on_complete hooks.
-    using do |stack, main|
-      main.append RequestBuilder.new
-      main.append Sequencer.new(:request), as: :request
-      main.append Sequencer.new(:execution), as: :execution
-      main.append Sequencer.new(:response), as: :response
-      main.append CodeStatusProcessor.new, as: :status
-      main.append Hook.new, as: :on_complete
-
-      stack.sequence(:execution) do
-        append Proc.new { |c|
-          Configuration.adapter.call c
-        }
-      end
-
-      # sequence(:request) do
-      #   append Scope.new(Sequencer.new(:request_body), :body), as: :body
-      # end
-
-      # sequence(:response) do
-      #   append Hook.new, as: :on_response
-      #   append Scope.new(Sequencer.new(:response_body), :body), as: :body
-      # end
+    using do
+      sequence.append sequence(:request)
+      sequence.append sequence(:response)
+      sequence.append proc { |router, request, options = {}|
+        Configuration.adapter.call router, request, options
+      }
     end
 
     # Public: Creates a new Api Builder, starting to build from
@@ -112,5 +117,13 @@ module Commute
     def initialize context = self.class.builder.context
       super
     end
+
+    # Public: Executes a method only if it exists. Otherwise just
+    # returns self.
+    #
+    # Returns the output of the existing method or self.
+    def try method, *args, &block
+      self.respond_to?(method) ? self.send(method, *args, &block) : self
+    end
   end
 end