batch_api2 0.3.0

data/lib/batch_api/operation/rails.rb

@@ -0,0 +1,42 @@
+require 'batch_api/operation/rack'
+
+module BatchApi
+  # Public: an individual batch operation.
+  module Operation
+    class Rails < Operation::Rack
+      # Public: create a new Rails Operation.  It does all that Rack does
+      # and also some additional Rails-specific processing.
+      def initialize(op, base_env, app)
+        super
+        @params = params_with_path_components
+      end
+
+      # Internal: customize the request environment.  This is currently done
+      # manually and feels clunky and brittle, but is mostly likely fine, though
+      # there are one or two environment parameters not yet adjusted.
+      def process_env
+        # parameters
+        super
+        @env["action_dispatch.request.parameters"] = @params
+        @env["action_dispatch.request.request_parameters"] = @params
+      end
+
+      private
+
+      # Internal: process the params the Rails way, merging in the
+      # path_parameters.  If the route can't be recognized, it will
+      # leave the params unchanged.
+      #
+      # Returns the updated params.
+      def params_with_path_components
+        begin
+          path_params = ::Rails.application.routes.recognize_path(@url, @op)
+          @params.merge(path_params)
+        rescue
+          @params
+        end
+      end
+    end
+  end
+end
+

data/lib/batch_api/processor.rb

@@ -0,0 +1,113 @@
+require 'batch_api/processor/sequential'
+require 'batch_api/operation'
+
+module BatchApi
+  class Processor
+    attr_reader :ops, :options, :app
+
+    # Public: create a new Processor.
+    #
+    # env - a Rack environment hash
+    # app - a Rack application
+    #
+    # Raises OperationLimitExceeded if more operations are requested than
+    # allowed by the BatchApi configuration.
+    # Raises Errors::BadOptionError if other provided options are invalid.
+    # Raises ArgumentError if no operations are provided (nil or []).
+    #
+    # Returns the new Processor instance.
+    def initialize(request, app)
+      @app = app
+      @request = request
+      @env = request.env
+      @ops = self.process_ops
+      @options = self.process_options
+    end
+
+    # Public: the processing strategy to use, based on the options
+    # provided in BatchApi setup and the request.
+    # Currently only Sequential is supported.
+    def strategy
+      BatchApi::Processor::Sequential
+    end
+
+    # Public: run the batch operations according to the appropriate strategy.
+    #
+    # Returns a set of BatchResponses
+    def execute!
+      stack = InternalMiddleware.batch_stack(self)
+      format_response(stack.call(middleware_env))
+    end
+
+    protected
+
+    def middleware_env
+      {
+        ops: @ops,
+        rack_env: @env,
+        rack_app: @app,
+        options: @options
+      }
+    end
+
+    # Internal: format the result of the operations, and include
+    # any other appropriate information (such as timestamp).
+    #
+    # result - the array of batch operations
+    #
+    # Returns a hash ready to go to the user
+    def format_response(operation_results)
+      {
+        "results" => operation_results
+      }
+    end
+
+    # Internal: Validate that an allowable number of operations have been
+    # provided, and turn them into BatchApi::Operation objects.
+    #
+    # ops - a series of operations
+    #
+    # Raises Errors::OperationLimitExceeded if more operations are requested than
+    # allowed by the BatchApi configuration.
+    # Raises Errors::NoOperationsError if no operations are provided.
+    #
+    # Returns an array of BatchApi::Operation objects
+    def process_ops
+      ops = @request.params.delete("ops")
+      if !ops || ops.empty?
+        raise Errors::NoOperationsError, "No operations provided"
+      elsif ops.length > BatchApi.config.limit
+        raise Errors::OperationLimitExceeded,
+          "Only #{BatchApi.config.limit} operations can be submitted at once, " +
+          "#{ops.length} were provided"
+      else
+        ops.map do |op|
+          self.class.operation_klass.new(op, @env, @app)
+        end
+      end
+    end
+
+    # Internal: which operation class to used.
+    #
+    # Returns Batch::Operation::(Rack|Rails) depending on the environment
+    def self.operation_klass
+      BatchApi.rails? ? Operation::Rails : Operation::Rack
+    end
+
+    # Internal: Processes any other provided options for validity.
+    # Currently, the :sequential option is REQUIRED (until parallel
+    # implementation is created).
+    #
+    # options - an options hash
+    #
+    # Raises Errors::BadOptionError if sequential is not provided.
+    #
+    # Returns the valid options hash.
+    def process_options
+      unless @request.params["sequential"]
+        raise Errors::BadOptionError, "Sequential flag is currently required"
+      end
+      @request.params
+    end
+  end
+end

data/lib/batch_api/processor/executor.rb

@@ -0,0 +1,18 @@
+module BatchApi
+  class Processor
+    # Public: a simple middleware that lives at the end of the internal chain
+    # and simply executes each batch operation.
+    class Executor
+
+      # Public: initialize the middleware.
+      def initialize(app)
+        @app = app
+      end
+
+      # Public: execute the batch operation.
+      def call(env)
+        env[:op].execute
+      end
+    end
+  end
+end

data/lib/batch_api/processor/sequential.rb

@@ -0,0 +1,29 @@
+module BatchApi
+  class Processor
+    class Sequential
+      # Public: initialize with the app.
+      def initialize(app)
+        @app = app
+      end
+
+      # Public: execute all operations sequentially.
+      #
+      # ops - a set of BatchApi::Operations
+      # options - a set of options
+      #
+      # Returns an array of BatchApi::Response objects.
+      def call(env)
+        env[:ops].collect do |op|
+          # set the current op
+          env[:op] = op
+
+          # execute the individual request inside the operation-specific
+          # middeware, then clear out the current op afterward
+          middleware = InternalMiddleware.operation_stack
+          middleware.call(env).tap {|r| env.delete(:op) }
+        end
+      end
+    end
+  end
+end
+

data/lib/batch_api/rack_middleware.rb

@@ -0,0 +1,37 @@
+module BatchApi
+  class RackMiddleware
+    def initialize(app, &block)
+      @app = app
+      yield BatchApi.config if block
+    end
+
+    def call(env)
+      if batch_request?(env)
+        begin
+          request = request_klass.new(env)
+          result = BatchApi::Processor.new(request, @app).execute!
+          [200, self.class.content_type, [result.to_json]]
+        rescue => err
+          ErrorWrapper.new(err).render
+        end
+      else
+        @app.call(env)
+      end
+    end
+
+    def self.content_type
+      {"Content-Type" => "application/json"}
+    end
+
+    private
+
+    def batch_request?(env)
+      env["PATH_INFO"] == BatchApi.config.endpoint &&
+        env["REQUEST_METHOD"] == BatchApi.config.verb.to_s.upcase
+    end
+
+    def request_klass
+      defined?(ActionDispatch) ? ActionDispatch::Request : Rack::Request
+    end
+  end
+end

data/lib/batch_api/response.rb

@@ -0,0 +1,38 @@
+module BatchApi
+  # Public: a response from an internal operation in the Batch API.
+  # It contains all the details that are needed to describe the call's
+  # outcome.
+  class Response
+    # Public: the attributes of the HTTP response.
+    attr_accessor :status, :body, :headers
+
+    # Public: create a new response representation from a Rack-compatible
+    # response (e.g. [status, headers, response_object]).
+    def initialize(response)
+      @status, @headers = *response
+      @body = process_body(response[2])
+    end
+
+    # Public: convert the response to JSON.  nil values are ignored.
+    def as_json(options = {})
+      {}.tap do |result|
+        result[:body] = @body unless @body.nil?
+        result[:headers] = @headers unless @headers.nil?
+        result[:status] = @status unless @status.nil?
+      end
+    end
+
+    private
+
+    def process_body(body_pieces)
+      # bodies have to respond to .each, but may otherwise
+      # not be suitable for JSON serialization
+      # (I'm looking at you, ActionDispatch::Response)
+      # so turn it into a string
+      base_body = ""
+      body_pieces.each {|str| base_body << str}
+      base_body
+    end
+  end
+end
+

data/lib/batch_api/utils.rb

@@ -0,0 +1,17 @@
+module BatchApi
+  module Utils
+
+    def self.deep_dup(object)
+      if object.is_a?(Hash)
+        duplicate = object.dup
+        duplicate.each_pair do |k,v|
+          tv = duplicate[k]
+          duplicate[k] = tv.is_a?(Hash) && v.is_a?(Hash) ? deep_dup(tv) : v
+        end
+        duplicate
+      else
+        object
+      end
+    end
+  end
+end

data/lib/batch_api/version.rb

@@ -0,0 +1,3 @@
+module BatchApi
+  VERSION = "0.3.0"
+end

data/lib/tasks/batch_api_tasks.rake

@@ -0,0 +1,4 @@
+# desc "Explaining what the task does"
+# task :batch_api do
+#   # Task goes here
+# end

data/readme.md

@@ -0,0 +1,243 @@
+[![Build Status](https://travis-ci.org/arsduo/batch_api.svg?branch=master)](http://travis-ci.org/arsduo/batch_api)
+
+## What's this?
+
+A gem that provides a RESTful Batch API for Rails and other Rack applications.
+In this system, batch requests are simply collections of regular REST calls,
+whose results are returned as an equivalent collection of regular REST results.
+
+This is heavily inspired by [Facebook's Batch API](https://developers.facebook.com/docs/graph-api/making-multiple-requests).
+
+## A Quick Example
+
+Making a batch request:
+
+```
+# POST /batch
+# Content-Type: application/json
+
+{
+  ops: [
+    {method: "get",    url: "/patrons"},
+    {method: "post",   url: "/orders/new",  params: {dish_id: 123}},
+    {method: "get",    url: "/oh/no/error", headers: {break: "fast"}},
+    {method: "delete", url: "/patrons/456"}
+  ],
+  sequential: true
+}
+```
+
+Reading the response:
+
+```
+{
+  results: [
+    {status: 200, body: [{id: 1, name: "Jim-Bob"}, ...], headers: {}},
+    {status: 201, body: {id: 4, dish_name: "Spicy Crab Legs"}, headers: {}},
+    {status: 500, body: {error: {oh: "noes!"}}, headers: {Problem: "woops"}},
+    {status: 200, body: null, headers: {}}}
+  ]
+}
+```
+
+### How It Works
+
+#### Requests
+
+As you can see from the example above, each request in the batch (an
+"operation", in batch parlance) describes the same features any HTTP request
+would include:
+
+* _url_ - the API endpoint to hit, formatted exactly as you would for a regular
+REST API request, leading / and all. (required)
+* _method_ - what type of request to make -- GET, POST, PUT, etc.  If no method
+is supplied, GET is assumed. (optional)
+* _args_ - a hash of arguments to the API. This can be used for both GET and
+PUT/POST/PATCH requests. (optional)
+* _headers_ - a hash of request-specific headers. The headers sent in the
+request will be included as well, with operation-specific headers taking
+precendence. (optional)
+* _silent_ - whether to return a response for this request. You can save on
+transfer if, for instance, you're making several PUT/POST requests, then
+executing a GET at the end.
+
+These individual operations are supplied as the "ops" parameter in the
+overall request.  Other options include:
+
+* _sequential_ - execute all operations sequentially, rather than in parallel.
+*This parameter is currently REQUIRED and must be set to true.* (In the future
+the Batch API will offer parallel processing for thread-safe apps, and hence
+this parameter must be supplied in order to explicitly preserve expected
+behavior.)
+
+Other options may be provided in the future for both the global request
+and individual operations.
+
+### Responses
+
+The Batch API will always return a 200, with a JSON body containing the
+individual responses under the "results" key.  Those responses, in turn,
+contain the same main components of any HTTP response:
+
+* _status_ - the HTTP status (200, 201, 400, etc.)
+* _body_ - the rendered body
+* _headers_ - any response headers
+
+### Errors
+
+Errors in individual Batch API requests will be returned inline, with the
+same status code and body they would return as individual requests.
+
+If the Batch API itself returns a non-200 status code, that indicates a global
+problem.
+
+## Installation
+
+Setting up the Batch API is simple.  Just add the gem to your middlewares:
+
+```ruby
+# in application.rb
+config.middleware.use BatchApi::RackMiddleware do |batch_config|
+  # you can set various configuration options:
+  batch_config.verb = :put # default :post
+  batch_config.endpoint = "/batchapi" # default /batch
+  batch_config.limit = 100 # how many operations max per request, default 50
+
+  # default middleware stack run for each batch request
+  batch_config.batch_middleware = Proc.new { }
+  # default middleware stack run for each individual operation
+  batch_config.operation_middleware = Proc.new { }
+end
+```
+
+That's it!  Just fire up your curl, hit your endpoint with the right verb and a properly formatted request, and enjoy some batch API action.
+
+## Why a Batch API?
+
+Batch APIs, though unRESTful, are useful for reducing HTTP overhead
+by combining requests; this is particularly valuable for mobile clients,
+which may generate groups of offline actions and which desire to
+reduce battery consumption while connected by making fewer, better-compressed
+requests.
+
+### Why not HTTP Pipelining?
+
+HTTP pipelining is an awesome and promising technology, and would provide a
+simple and effortless way to parallel process many requests; however, using
+pipelining raised several issues for us, one of which was a blocker:
+
+* [Lack of browser
+support](http://en.wikipedia.org/wiki/HTTP_pipelining#Implementation_in_web_browsers):
+a number of key browsers do not yet support HTTP pipelining (or have it
+disabled by default).  This will of course change in time,
+but for now this takes pipelining out of consideration.  (There a similar but
+more minor issue
+with [many web
+proxies](http://en.wikipedia.org/wiki/HTTP_pipelining#Implementation_in_web_proxies).)
+* The HTTP pipelining specification states that non-idempotent requests (e.g.
+[POST](http://en.wikipedia.org/wiki/HTTP_pipelining) and
+[in some
+descriptions](http://www-archive.mozilla.org/projects/netlib/http/pipelining-faq.html) PUT)
+shouldn't be made via pipelining.  Though I have heard that some server
+implementations do support POST requests (putting all subsequent requests on
+hold until it's done), for applications that submit a lot of POSTs this raised
+concerns as well.
+
+Given this state of affairs -- and my desire to hack up a Batch API gem :P --,
+we decided to implement an API-based solution.
+
+### Why this Approach?
+
+There are two main approaches to writing batch APIs:
+
+* A limited, specialized batch endpoint (or endpoints), which usually handles
+  updates and creates.  DHH sketched out such a bulk update/create endpoint
+  for Rails 3.2 [in a gist](https://gist.github.com/981520) last year.
+* A general-purpose RESTful API that can handle anything in your application,
+  a la the Facebook Batch API.
+
+The second approach, IMO, minimizes code duplication and complexity. Rather
+than have two systems that manage resources (or a more complicated one that
+can handle both batch and individual requests), we simply route requests as we
+always would.
+
+This solution has several specific benefits:
+
+* Less complexity - non-batch endpoints don't need any extra code, which means
+  less to maintain on your end.
+* Complete flexibility - as you add new features to your application,
+  they become immediately and automatically available via the Batch API.
+* More RESTful - as individual operations are simply actions on RESTful
+  resources, you preserve an important characteristic of your API.
+
+As well as the general benefits of all batch operations:
+
+* Reuse of state - user authentication, request stack processing, and
+  similar processing only needs to be done once.
+* Better for clients - clients need to make fewer requests, as described above.
+* Parallelizable - in the future, we could run requests in parallel (if
+  our app is thread-safe).  Clients would be able to explicitly specify
+  dependencies between operations (or simply run all sequentially).  This
+  should make for some fun experimentation :)
+
+There's only one downside I can think of to this approach as opposed to a
+specialized endpoint:
+
+* Reduced ability to optimize - unlike a specialized API endpoint, each request
+  will be treated in isolation, which makes it harder to optimize the
+  underlying database queries via more efficient (read: complicated) SQL logic.
+  (Better identity maps would help with this, and since the main pain point
+  this approach addresses is at the HTTP connection layer, I submit we can
+  accept this.)
+
+## Implementation
+
+The Batch API is implemented as a Rack middleware.  Here's how it works:
+
+First, if the request isn't a batch request (as defined by the endpoint and
+method in BatchApi.config), it gets processed normally by your app.
+
+If it is a batch request, we:
+* Read and validate the parameters for the request, constructing a
+  representation of the operation.
+* Compile a customized Rack environment hash with the appropriate parameters,
+  so that your app interprets the request as being for the appropriate action.
+  (This is requires a bit of extra processing for Rails.)
+* Send each request up the middleware stack as normal, collecting the results.
+  Errors are caught and recorded appropriately.
+* Send you back the results.
+
+At both the batch level (processing all requests) and the individual operation
+request, there is an internal, customizable midleware stack that you can
+customize to insert additional custom behavior, such as handling authentication
+or decoding JSON bodies for individual requests (this latter comes
+pre-included).  Check out the lib/batch_api/internal_middleware.rb for more
+information.
+
+## To Do
+
+The core of the Batch API is complete and solid, and so ready to go that it's
+in use at 6Wunderkinder already :P
+
+Here are some immediate tasks:
+
+* Test against additional frameworks (beyond Rails and Sinatra)
+* Write more usage docs / create a wiki.
+* Add additional features inspired by the Facebook API, such as the ability to
+  surpress output for individual requests, etc.
+* Add RDoc to the spec task and ensure all methods are documented.
+* Research and implement parallelization and dependency management.
+
+## Thanks
+
+To 6Wunderkinder, for all their support for this open-source project, and their
+general awesomeness.
+
+To Facebook, for providing inspiration and a great implementation in this and
+many other things.
+
+To [JT Archie](http://github.com/jtarchie) for his help and feedback.
+
+## Issues? Questions? Ideas?
+
+Open a ticket or send a pull request!