berkshelf-api 0.2.0 → 1.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 213f69bfd58c7200d8d779788f6774828087cf1b
-  data.tar.gz: 8ebeaff3fae7fca72a3e5d1f6a88690c80beb7fd
+  metadata.gz: f238c1aec4a689f0461aaea5792d0588501e4d8c
+  data.tar.gz: f99f8dac7ac5aa4133ac4bef0420e74d128cd01f
 SHA512:
-  metadata.gz: 7a30703cd2c06d4b43c6e0bf80c4846d74120e13fa105ee5d7573c6818aac8d1bc1d8d71ea9c39e85fe7f48a50955989d13c4131fb9b9d945d1d27f6b1474133
-  data.tar.gz: d3cd8e1029ddd4a73291e50feb24c3d635195a24f0531445250b73eb6a7b45a699dfc00a2f2863c6ba0a49028d95ae65c118dc3e15e0e95300b969a9e4325865
+  metadata.gz: 0910bcf676f10ea3f0c7061a685400ee232350b39fe08866ed89558e6d3dc649cb6a6b0288041a59924c010825a23510fada8adc043466a37b56374b74ad1543
+  data.tar.gz: de9395b533f039629f2eef14bd583de33839afac67e225844616c5cdc22a6d3038b3852839e2f110b4f19e2e227ec946284d58bfc0f9dfd04792a645b966d1aa

data/CHANGELOG.md

@@ -0,0 +1,4 @@
+# 1.0.0
+
+* Prioritize Cache Builders so their entries do not clobber one another
+* An entire config file is checksumed so that when it changes, the cache is rebuilt

data/README.md

@@ -2,6 +2,7 @@
 [![Gem Version](https://badge.fury.io/rb/berkshelf-api.png)](http://badge.fury.io/rb/berkshelf-api)
 [![Build Status](https://secure.travis-ci.org/RiotGames/berkshelf-api.png?branch=master)](http://travis-ci.org/RiotGames/berkshelf-api)
 [![Dependency Status](https://gemnasium.com/RiotGames/berkshelf-api.png?travis)](https://gemnasium.com/RiotGames/berkshelf-api)
+[![Code Climate](https://codeclimate.com/github/RiotGames/berkshelf-api.png)](https://codeclimate.com/github/RiotGames/berkshelf-api)
 
 A server which indexes cookbooks from various sources and hosts it over a REST API
 
@@ -23,35 +24,39 @@ Ruby 1.9.1 and 1.9.2 are not officially supported. If you encounter problems, pl
 
 ## Configuring Endpoints
 
-Which endpoints to index can be configured by editing the JSON configuration file (by default at: `~/.berkshelf/api-server/config.json`).
+You may configure the endpoints to index by editing the JSON configuration file (default: `~/.berkshelf/api-server/config.json`).
 
 ### Opscode Community Site
 
-  {
-    "endpoints": [
-      {
-        type: "opscode",
-        options: {
-          url: 'http://cookbooks.opscode.com/api/v1'
-        }
+````json
+{
+  "endpoints": [
+    {
+      "type": "opscode",
+      "options": {
+        "url": "http://cookbooks.opscode.com/api/v1"
       }
-    ]
-  }
+    }
+  ]
+}
+```
 
 ### Chef Server
 
-  {
-    "endpoints": [
-      {
-        "type": "chef_server",
-        "options": {
-          "url": "https://api.opscode.com/organizations/vialstudios",
-          "client_name": "berkshelf",
-          "client_key": "/etc/berkshelf/api-server/client.pem"
-        }
+```json
+{
+  "endpoints": [
+    {
+      "type": "chef_server",
+      "options": {
+        "url": "https://api.opscode.com/organizations/vialstudios",
+        "client_name": "berkshelf",
+        "client_key": "/etc/berkshelf/api-server/client.pem"
       }
-    ]
-  }
+    }
+  ]
+}
+```
 
 ## Contributing
 

data/berkshelf-api.gemspec

@@ -19,9 +19,9 @@ Gem::Specification.new do |spec|
   spec.require_paths = ["lib"]
   spec.required_ruby_version = ">= 1.9.3"
 
-  spec.add_dependency 'ridley', '~> 1.3.0'
-  spec.add_dependency 'celluloid', '~> 0.14.0'
-  spec.add_dependency 'reel', '= 0.4.0.pre'
+  spec.add_dependency 'ridley', '~> 1.6'
+  spec.add_dependency 'celluloid', '~> 0.15'
+  spec.add_dependency 'reel', '>= 0.4.0'
   spec.add_dependency 'grape', '~> 0.5.0'
   spec.add_dependency 'hashie', '>= 2.0.4'
   spec.add_dependency 'faraday'

data/lib/berkshelf/api/application.rb

@@ -116,7 +116,10 @@ module Berkshelf::API
         loop do
           supervisor = run!(options)
 
-          sleep 0.1 while supervisor.alive?
+          while supervisor.alive?
+            sleep 0.1
+            instance.terminate if @shutdown
+          end
 
           break if @shutdown
 
@@ -153,7 +156,6 @@ module Berkshelf::API
       #   if there is no running instance
       def shutdown
         @shutdown = true
-        instance.terminate
       end
     end
   end

data/lib/berkshelf/api/cache_builder/worker/chef_server.rb

@@ -19,7 +19,7 @@ module Berkshelf::API
             connection.cookbook.all.each do |cookbook, versions|
               versions.each do |version|
                 cookbook_versions << RemoteCookbook.new(cookbook, version, self.class.worker_type,
-                  @connection.server_url)
+                  @connection.server_url, priority)
               end
             end
           end

data/lib/berkshelf/api/cache_builder/worker/opscode.rb

@@ -19,7 +19,7 @@ module Berkshelf::API
               [ cookbook, connection.future(:versions, cookbook) ]
             end.each do |cookbook, versions|
               versions.value.each do |version|
-                cookbook_versions << RemoteCookbook.new(cookbook, version, self.class.worker_type, @connection.api_uri)
+                cookbook_versions << RemoteCookbook.new(cookbook, version, self.class.worker_type, @connection.api_uri, priority)
               end
             end
           end

data/lib/berkshelf/api/cache_builder/worker.rb

@@ -16,8 +16,11 @@ module Berkshelf::API
         include Berkshelf::API::Mixin::Services
 
         attr_reader :options
+        attr_reader :priority
 
-        def initialize(options = {}); end
+        def initialize(options = {})
+          @priority = options[:priority]
+        end
 
         # @abstract
         #
@@ -36,52 +39,6 @@ module Berkshelf::API
           raise RuntimeError, "must be implemented"
         end
 
-        def build
-          log.info "#{self} building..."
-          log.info "#{self} determining if the cache is stale..."
-          if stale?
-            log.info "#{self} cache is stale."
-            update_cache
-          else
-            log.info "#{self} cache is up to date."
-          end
-
-          log.info "clearing diff"
-          clear_diff
-        end
-
-        # @return [Array<Array<RemoteCookbook>, Array<RemoteCookbook>>]
-        def diff
-          @diff ||= cache_manager.diff(cookbooks)
-        end
-
-        def update_cache
-          created_cookbooks, deleted_cookbooks = diff
-
-          log.info "#{self} adding (#{created_cookbooks.length}) items..."
-          created_cookbooks.collect do |remote|
-            [ remote, future(:metadata, remote) ]
-          end.each do |remote, metadata|
-            cache_manager.add(remote, metadata.value)
-          end
-
-          log.info "#{self} removing (#{deleted_cookbooks.length}) items..."
-          deleted_cookbooks.each { |remote| cache_manager.remove(remote.name, remote.version) }
-
-          log.info "#{self} cache updated."
-          cache_manager.save
-        end
-
-        def stale?
-          created_cookbooks, deleted_cookbooks = diff
-          created_cookbooks.any? || deleted_cookbooks.any?
-        end
-
-        private
-
-          def clear_diff
-            @diff = nil
-          end
       end
 
       class << self

data/lib/berkshelf/api/cache_builder.rb

@@ -8,6 +8,7 @@ module Berkshelf::API
 
     include Berkshelf::API::GenericServer
     include Berkshelf::API::Logging
+    include Berkshelf::API::Mixin::Services
 
     server_name :cache_builder
     finalizer :finalize_callback
@@ -18,21 +19,14 @@ module Berkshelf::API
       @worker_supervisor = WorkerSupervisor.new(@worker_registry)
       @building          = false
 
-      Application.config.endpoints.each do |endpoint|
+      Application.config.endpoints.each_with_index do |endpoint, index|
         endpoint_options = endpoint.options.to_hash.deep_symbolize_keys
-        @worker_supervisor.supervise(CacheBuilder::Worker[endpoint.type], endpoint_options)
+        @worker_supervisor.supervise(CacheBuilder::Worker[endpoint.type], endpoint_options.merge(priority: index))
       end
     end
 
-    # Issue a single build command to all workers
-    #
-    # @return [Array]
     def build
-      workers.collect { |actor| actor.future(:build) }.map do |f|
-        begin
-          f.value
-        rescue; end
-      end
+      cache_manager.process_workers(workers)
     end
 
     # Issue a build command to all workers at the scheduled interval
@@ -44,7 +38,7 @@ module Berkshelf::API
       loop do
         @building = true
         build
-        sleep BUILD_INTERVAL
+        sleep interval
       end
     end
 

data/lib/berkshelf/api/cache_manager.rb

@@ -10,12 +10,16 @@ module Berkshelf::API
     include Berkshelf::API::GenericServer
     include Berkshelf::API::Logging
 
+    extend Forwardable
+    def_delegators :@cache, :warmed?, :set_warmed, :clear
+
     SAVE_INTERVAL = 30.0
 
     server_name :cache_manager
     finalizer :finalize_callback
-    exclusive :add, :clear, :remove, :save
+    exclusive :merge, :add, :remove
 
+    # @return [DependencyCache]
     attr_reader :cache
 
     def initialize
@@ -30,14 +34,77 @@ module Berkshelf::API
     #
     # @return [Hash]
     def add(cookbook, metadata)
-      @cache.add(cookbook, metadata)
+      log.debug "#{self} adding (#{cookbook.name}, #{cookbook.version})"
+      cache.add(cookbook, metadata)
     end
 
-    # Clear any items added to the cache
+    # Remove the cached item matching the given name and version
     #
-    # @return [Hash]
-    def clear
-      @cache.clear
+    # @param [#to_s] name
+    # @param [#to_s] version
+    #
+    # @return [DependencyCache]
+    def remove(name, version)
+      log.debug "#{self} removing (#{name}, #{version})"
+      cache.remove(name, version)
+    end
+
+    # Loops through a list of workers and merges their cookbook sets into the cache
+    #
+    # @param [Array<CacheBuilder::Worker::Base>] workers
+    #   The workers for this cache
+    #
+    # @return [Boolean]
+    def process_workers(workers)
+      # If the cache has been warmed already, we want to spawn
+      # workers for all the endpoints concurrently. However, if the
+      # cache is cold we want to run sequentially, so higher priority
+      # endpoints can work before lower priority, avoiding duplicate
+      # downloads.
+      # We don't want crashing workers to crash the CacheManager.
+      # Crashes are logged so just ignore the exceptions
+      if warmed?
+        Array(workers).flatten.collect do |worker|
+          self.future(:process_worker, worker)
+        end.each do |f|
+          f.value rescue nil
+        end
+      else
+        Array(workers).flatten.each do |worker|
+          process_worker(worker) rescue nil
+        end
+      end
+      self.set_warmed
+    end
+
+    # @param [CacheBuilder::Worker::Base] worker
+    def process_worker(worker)
+      log.info "processing #{worker}"
+      remote_cookbooks = worker.cookbooks
+      log.info "found #{remote_cookbooks.size} cookbooks from #{worker}"
+      created_cookbooks, deleted_cookbooks = diff(remote_cookbooks, worker.priority)
+      log.debug "#{created_cookbooks.size} cookbooks to be added to the cache from #{worker}"
+      log.debug "#{deleted_cookbooks.size} cookbooks to be removed from the cache from #{worker}"
+
+      # Process metadata in chunks - Ridley cookbook resource uses a
+      # task_class TaskThread, which means each future gets its own
+      # thread. If we have many (>2000) cookbooks we can easily
+      # exhaust the available threads on the system.
+      created_cookbooks_with_metadata = []
+      until created_cookbooks.empty?
+        work = created_cookbooks.slice!(0,500)
+        log.info "processing metadata for #{work.size} cookbooks with #{created_cookbooks.size} remaining on #{worker}"
+        work.map! do |remote|
+          [ remote, worker.future(:metadata, remote) ]
+        end.map! do |remote, metadata|
+          [remote, metadata.value]
+        end
+        created_cookbooks_with_metadata += work
+      end
+
+      log.info "about to merge cookbooks"
+      merge(created_cookbooks_with_metadata, deleted_cookbooks)
+      log.info "#{self} cache updated."
     end
 
     # Check if the cache knows about the given cookbook version
@@ -51,44 +118,53 @@ module Berkshelf::API
     end
 
     def load_save
+      log.info "Loading save from #{self.class.cache_file}"
       @cache = DependencyCache.from_file(self.class.cache_file)
+      log.info "Cache contains #{@cache.cookbooks.size} items"
     end
 
-    # Remove the cached item matching the given name and version
-    #
-    # @param [#to_s] name
-    # @param [#to_s] version
-    #
-    # @return [DependencyCache]
-    def remove(name, version)
-      @cache.remove(name, version)
-    end
+    private
 
-    def save
-      log.info "Saving the cache to: #{self.class.cache_file}"
-      cache.save(self.class.cache_file)
-      log.info "Cache saved!"
-    end
+      def merge(created_cookbooks, deleted_cookbooks)
+        log.info "#{self} adding (#{created_cookbooks.length}) items..."
+        created_cookbooks.each do |remote_with_metadata|
+          remote, metadata = remote_with_metadata
+          add(remote, metadata)
+        end
 
-    # @param [Array<RemoteCookbook>] cookbooks
-    #   An array of RemoteCookbooks representing all the cookbooks on the indexed site
-    #
-    # @return [Array<Array<RemoteCookbook>, Array<RemoteCookbook>>]
-    #   A tuple of Arrays of RemoteCookbooks
-    #   The first array contains items not in the cache
-    #   The second array contains items in the cache, but not in the cookbooks parameter
-    def diff(cookbooks)
-      known_cookbooks   = cache.cookbooks
-      created_cookbooks = cookbooks - known_cookbooks
-      deleted_cookbooks = known_cookbooks - cookbooks
-      [ created_cookbooks, deleted_cookbooks ]
-    end
+        log.info "#{self} removing (#{deleted_cookbooks.length}) items..."
+        deleted_cookbooks.each { |remote| remove(remote.name, remote.version) }
 
-    private
+        log.info "#{self} cache updated."
+        save
+      end
+
+      def save
+        if warmed?
+          log.info "Saving the cache to: #{self.class.cache_file}"
+          cache.save(self.class.cache_file)
+          log.info "Cache saved!"
+        end
+      end
+
+      # @param [Array<RemoteCookbook>] cookbooks
+      #   An array of RemoteCookbooks representing all the cookbooks on the indexed site
+      # @param [Integer] worker_priority
+      #   The priority/ID of the endpoint that is running
+      # @return [Array(Array<RemoteCookbook>, Array<RemoteCookbook>)]
+      #   A tuple of Arrays of RemoteCookbooks
+      #   The first array contains items not in the cache
+      #   The second array contains items in the cache, but not in the cookbooks parameter
+      def diff(cookbooks, worker_priority)
+        known_cookbooks   = cache.cookbooks.select { |c| c.priority <= worker_priority }
+        created_cookbooks = cookbooks - known_cookbooks
+        deleted_cookbooks = (known_cookbooks - cookbooks).select { |c| c.priority == worker_priority }
+        [ created_cookbooks, deleted_cookbooks ]
+      end
 
       def finalize_callback
         log.info "Cache Manager shutting down..."
-        self.save
+        save
       end
   end
 end

data/lib/berkshelf/api/config.rb

@@ -1,4 +1,5 @@
 require 'buff/config/json'
+require 'digest/sha1'
 
 module Berkshelf::API
   class Config < Buff::Config::JSON
@@ -24,5 +25,9 @@ module Berkshelf::API
           }
         }
       ]
+
+    def endpoints_checksum
+      Digest::SHA1.hexdigest(endpoints.collect {|x| x.to_hash }.to_s)
+    end
   end
 end

data/lib/berkshelf/api/dependency_cache.rb

@@ -6,6 +6,7 @@ module Berkshelf::API
   #   {
   #     "cookbook_name" => {
   #       "x.y.z" => {
+  #         :endpoint_priority => 1,
   #         :dependencies => { "cookbook_name" => "constraint" },
   #         :platforms => { "platform" => "constraint" }
   #       }
@@ -32,12 +33,19 @@ module Berkshelf::API
       end
     end
 
+    include Berkshelf::API::Logging
     extend Forwardable
     def_delegators :@cache, :[], :[]=
 
     # @param [Hash] contents
     def initialize(contents = {})
-      @cache = Hash[contents]
+      @warmed = false
+      @cache  = Hash[contents].with_indifferent_access
+      if @cache['endpoints_checksum'] && (@cache['endpoints_checksum'] != Application.config.endpoints_checksum)
+        log.warn "Endpoints in config have changed - invalidating cache"
+        @cache.clear
+      end
+      @cache.delete('endpoints_checksum')
     end
 
     # @param [RemoteCookbook] cookbook
@@ -49,6 +57,7 @@ module Berkshelf::API
       dependencies = metadata.dependencies || Hash.new
       @cache[cookbook.name.to_s] ||= Hash.new
       @cache[cookbook.name.to_s][cookbook.version.to_s] = {
+        endpoint_priority: cookbook.priority,
         platforms: platforms,
         dependencies: dependencies,
         location_type: cookbook.location_type,
@@ -99,9 +108,13 @@ module Berkshelf::API
       @cache.to_hash
     end
 
+    # @param [Hash] options
+    #
     # @return [String]
     def to_json(options = {})
-      JSON.generate(to_hash, options)
+      output = to_hash
+      output['endpoints_checksum'] = Application.config.endpoints_checksum if options[:saving]
+      JSON.generate(output, options)
     end
 
     # @return [Array<RemoteCookbook>]
@@ -109,7 +122,7 @@ module Berkshelf::API
       [].tap do |remote_cookbooks|
         @cache.each_pair do |name, versions|
           versions.each do |version, metadata|
-            remote_cookbooks << RemoteCookbook.new(name, version, metadata[:location_type], metadata[:location_path])
+            remote_cookbooks << RemoteCookbook.new(name, version, metadata[:location_type], metadata[:location_path], metadata[:endpoint_priority])
           end
         end
       end
@@ -117,7 +130,15 @@ module Berkshelf::API
 
     def save(path)
       FileUtils.mkdir_p(File.dirname(path))
-      File.open(path, 'w+') { |f| f.write(self.to_json) }
+      File.open(path, 'w+') { |f| f.write(self.to_json(saving: true)) }
+    end
+
+    def warmed?
+      @warmed
+    end
+
+    def set_warmed
+      @warmed = true
     end
   end
 end

data/lib/berkshelf/api/endpoint/v1.rb

@@ -12,7 +12,12 @@ module Berkshelf::API
 
       desc "list all known cookbooks"
       get 'universe' do
-        cache_manager.cache
+        if cache_manager.warmed?
+          cache_manager.cache
+        else
+          header "Retry-After", 600
+          status 503
+        end
       end
     end
   end

data/lib/berkshelf/api/remote_cookbook.rb

@@ -1,3 +1,11 @@
 module Berkshelf::API
-  class RemoteCookbook < Struct.new(:name, :version, :location_type, :location_path); end
+  class RemoteCookbook < Struct.new(:name, :version, :location_type, :location_path, :priority)
+    def hash
+      "#{name}|#{version}".hash
+    end
+
+    def eql?(other)
+      self.hash == other.hash
+    end
+  end
 end

data/lib/berkshelf/api/rest_gateway.rb

@@ -13,50 +13,144 @@ module Berkshelf::API
       workers: 10
     }.freeze
 
+    INITIAL_BODY = ''
+
+    CONTENT_TYPE_ORIG   = 'Content-Type'.freeze
+    CONTENT_LENGTH_ORIG = 'Content-Length'.freeze
+    CONTENT_TYPE        = 'CONTENT_TYPE'.freeze
+    CONTENT_LENGTH      = 'CONTENT_LENGTH'.freeze
+
+    SERVER_SOFTWARE     = 'SERVER_SOFTWARE'.freeze
+    SERVER_NAME         = 'SERVER_NAME'.freeze
+    SERVER_PORT         = 'SERVER_PORT'.freeze
+    SERVER_PROTOCOL     = 'SERVER_PROTOCOL'.freeze
+    GATEWAY_INTERFACE   = "GATEWAY_INTERFACE".freeze
+    LOCALHOST           = 'localhost'.freeze
+    HTTP_VERSION        = 'HTTP_VERSION'.freeze
+    CGI_1_1             = 'CGI/1.1'.freeze
+    REMOTE_ADDR         = 'REMOTE_ADDR'.freeze
+    CONNECTION          = 'HTTP_CONNECTION'.freeze
+    SCRIPT_NAME         = 'SCRIPT_NAME'.freeze
+    PATH_INFO           = 'PATH_INFO'.freeze
+    REQUEST_METHOD      = 'REQUEST_METHOD'.freeze
+    QUERY_STRING        = 'QUERY_STRING'.freeze
+    HTTP_1_0            = 'HTTP/1.0'.freeze
+    HTTP_1_1            = 'HTTP/1.1'.freeze
+    HTTP_               = 'HTTP_'.freeze
+    HOST                = 'Host'.freeze
+
+    RACK_INPUT          = 'rack.input'.freeze
+    RACK_LOGGER         = 'rack.logger'.freeze
+    RACK_VERSION        = 'rack.version'.freeze
+    RACK_ERRORS         = 'rack.errors'.freeze
+    RACK_MULTITHREAD    = 'rack.multithread'.freeze
+    RACK_MULTIPROCESS   = 'rack.multiprocess'.freeze
+    RACK_RUN_ONCE       = 'rack.run_once'.freeze
+    RACK_URL_SCHEME     = 'rack.url_scheme'.freeze
+    RACK_WEBSOCKET      = 'rack.websocket'.freeze
+
+    PROTO_RACK_ENV = {
+      RACK_VERSION      => ::Rack::VERSION,
+      RACK_ERRORS       => STDERR,
+      RACK_MULTITHREAD  => true,
+      RACK_MULTIPROCESS => false,
+      RACK_RUN_ONCE     => false,
+      RACK_URL_SCHEME   => "http".freeze,
+      SCRIPT_NAME       => ENV[SCRIPT_NAME] || "",
+      SERVER_PROTOCOL   => HTTP_1_1,
+      SERVER_SOFTWARE   => "berkshelf-api/#{Berkshelf::API::VERSION}".freeze,
+      GATEWAY_INTERFACE => CGI_1_1
+    }.freeze
+
     # @return [String]
     attr_reader :host
+
     # @return [Integer]
     attr_reader :port
+
     # @return [Integer]
     attr_reader :workers
 
-    def_delegator :handler, :rack_app
+    # @return [Berkshelf::API::RackApp]
+    attr_reader :app
 
     server_name :rest_gateway
-    finalizer :finalize_callback
 
     # @option options [String] :host ('0.0.0.0')
     # @option options [Integer] :port (26200)
     # @option options [Boolean] :quiet (false)
     # @option options [Integer] :workers (10)
     def initialize(options = {})
-      options       = DEFAULT_OPTIONS.merge(options)
-      options[:app] = Berkshelf::API::RackApp.new
-
-      @host    = options[:host]
-      @port    = options[:port]
-      @workers = options[:workers]
-      @handler = ::Rack::Handler::Reel.new(options)
-      @pool    = ::Reel::RackWorker.pool(size: @workers, args: [ @handler ])
+      options = DEFAULT_OPTIONS.merge(options)
+      @host   = options[:host]
+      @port   = options[:port]
 
       log.info "REST Gateway listening on #{@host}:#{@port}"
       super(@host, @port, &method(:on_connect))
+      @app = Berkshelf::API::RackApp.new
     end
 
-    # @param [Reel::Connection] connection
     def on_connect(connection)
-      pool.handle(connection.detach)
+      while request = connection.request
+        case request
+        when request.websocket?
+          request.respond(:bad_request, "WebSockets not supported")
+        else
+          route_request(connection, request)
+        end
+      end
+    end
+
+    def route_request(connection, request)
+      status, headers, body_parts = app.call(request_env(request, connection))
+      body, is_stream             = response_body(body_parts)
+
+      response_klass = is_stream ? Reel::StreamResponse : Reel::Response
+      response       = response_klass.new(status, headers, body)
+      connection.respond(response)
+    end
+
+    def request_env(request, connection)
+      env = env(request)
+      env[REMOTE_ADDR] = connection.remote_ip
+      env
     end
 
     private
 
-      # @return [Reel::RackWorker]
-      attr_reader :pool
-      # @return [Rack::Handler::Reel]
-      attr_reader :handler
+      def env(request)
+        env = Hash[PROTO_RACK_ENV]
+
+        env[RACK_INPUT] = StringIO.new(request.body.to_s || INITIAL_BODY)
+        env[RACK_INPUT].set_encoding(Encoding::BINARY) if env[RACK_INPUT].respond_to?(:set_encoding)
+        env[SERVER_NAME], env[SERVER_PORT] = (request[HOST]||'').split(':', 2)
+        env[SERVER_PORT] ||= port.to_s
+        env[HTTP_VERSION]   = request.version || env[SERVER_PROTOCOL]
+        env[REQUEST_METHOD] = request.method
+        env[PATH_INFO]      = request.path
+        env[QUERY_STRING]   = request.query_string || ''
+
+        (_ = request.headers.delete CONTENT_TYPE_ORIG) && (env[CONTENT_TYPE] = _)
+        (_ = request.headers.delete CONTENT_LENGTH_ORIG) && (env[CONTENT_LENGTH] = _)
+
+        request.headers.each_pair do |key, val|
+          env[HTTP_ + key.gsub('-', '_').upcase] = val
+        end
+        env
+      end
 
-      def finalize_callback
-        pool.terminate if pool && pool.alive?
+      def response_body(body_parts)
+        if body_parts.respond_to?(:to_path)
+          ::File.new(body_parts.to_path)
+        else
+          body = ''
+          body_parts.each do |c|
+            return [c, true] if c.is_a?(Reel::Stream)
+            body << c
+          end
+          body_parts.close if body_parts.respond_to?(:close)
+          body
+        end
       end
   end
 end

data/lib/berkshelf/api/rspec/server.rb

@@ -18,7 +18,10 @@ module Berkshelf::API::RSpec
       def start(options = {})
         options = options.reverse_merge(port: 26210, log_location: "/dev/null", endpoints: [])
         Berkshelf::API::Application.config.endpoints = options[:endpoints]
-        Berkshelf::API::Application.run!(options) unless running?
+        unless running?
+          Berkshelf::API::Application.run!(options)
+          cache_builder.build
+        end
       end
 
       def stop

data/lib/berkshelf/api/site_connector/opscode.rb

@@ -92,7 +92,7 @@ module Berkshelf::API
       #
       # @return [String, nil]
       def download(name, version, destination = Dir.mktmpdir)
-        log.info "downloading #{name}(#{version})"
+        log.debug "downloading #{name}(#{version})"
         if uri = download_uri(name, version)
           archive = stream(uri)
           Archive.extract(archive.path, destination)

data/lib/berkshelf/api/version.rb

@@ -1,5 +1,5 @@
 module Berkshelf
   module API
-    VERSION = "0.2.0"
+    VERSION = "1.0.0"
   end
 end

data/lib/berkshelf/api.rb

@@ -1,3 +1,4 @@
+require 'berkshelf/api/version'
 require 'celluloid'
 require 'hashie'
 require 'ridley'

data/spec/unit/berkshelf/api/cache_builder/worker/chef_server_spec.rb

@@ -35,25 +35,4 @@ describe Berkshelf::API::CacheBuilder::Worker::ChefServer do
       end
     end
   end
-
-  describe "#build" do
-    before do
-      Berkshelf::API::CacheManager.start
-      chef_cookbook("ruby", "1.0.0")
-      chef_cookbook("ruby", "2.0.0")
-      chef_cookbook("elixir", "3.0.0")
-      chef_cookbook("elixir", "3.0.1")
-    end
-
-    let(:cache) { Berkshelf::API::CacheManager.instance.cache }
-
-    it "adds each item to the cache" do
-      subject.build
-      expect(cache).to have_cookbook("ruby", "1.0.0")
-      expect(cache).to have_cookbook("ruby", "2.0.0")
-      expect(cache).to have_cookbook("elixir", "3.0.0")
-      expect(cache).to have_cookbook("elixir", "3.0.1")
-      expect(cache.cookbooks).to have(4).items
-    end
-  end
 end

data/spec/unit/berkshelf/api/cache_builder/worker_spec.rb

@@ -37,44 +37,4 @@ describe Berkshelf::API::CacheBuilder::Worker::Base do
       end
     end
   end
-
-  let(:cache_manager) { double(:diff => :chicken) }
-  subject { described_class.new }
-
-  describe "#diff" do
-    it "should delegate to the cache_manager to calculate the diff" do
-      subject.should_receive(:cache_manager).and_return(cache_manager)
-      subject.should_receive(:cookbooks).and_return(:cookbooks)
-
-      expect(subject.diff).to eql(:chicken)
-    end
-
-    it "should memoize the diff to prevent recalculating" do
-      subject.should_receive(:cache_manager).exactly(1).times.and_return(cache_manager)
-      subject.should_receive(:cookbooks).and_return(:cookbooks)
-
-      subject.diff
-      subject.diff
-    end
-  end
-
-  describe "#clear_diff" do
-    it "should set the diff to nil" do
-      subject.should_receive(:cache_manager).and_return(cache_manager)
-      subject.should_receive(:cookbooks).and_return(:cookbooks)
-
-      subject.diff
-      expect(subject.instance_variable_get(:@diff)).to eql(:chicken)
-      subject.send(:clear_diff)
-      expect(subject.instance_variable_get(:@diff)).to eql(nil)
-    end
-
-    it "memoizes the diff to prevent recalculating" do
-      subject.should_receive(:cache_manager).exactly(1).times.and_return(cache_manager)
-      subject.should_receive(:cookbooks).and_return(:cookbooks)
-
-      subject.diff
-      subject.diff
-    end
-  end
 end

data/spec/unit/berkshelf/api/cache_builder_spec.rb

@@ -1,17 +1,21 @@
 require 'spec_helper'
 
 describe Berkshelf::API::CacheBuilder do
+
+  before { Berkshelf::API::CacheManager.start }
   let(:instance) { described_class.new }
 
   describe "#build" do
     subject(:build) { instance.build }
     let(:workers) { [ double('worker') ] }
     let(:future) { double('future', value: nil) }
+    let(:cache_manager) { double('cache_manager') }
 
     before { instance.stub(workers: workers) }
 
-    it "sends a #build message to each worker" do
-      workers.each { |worker| worker.should_receive(:future).with(:build).and_return(future) }
+    it "asks the cache_manager to process all of its actors" do
+      instance.stub(:cache_manager).and_return(cache_manager)
+      cache_manager.should_receive(:process_workers).with(instance.workers).and_return(future)
       build
     end
   end
@@ -26,12 +30,51 @@ describe Berkshelf::API::CacheBuilder do
       end
     end
 
-    it "has one worker started by default" do
-      expect(workers).to have(1).item
+    context "when no workers are explicitly configured" do
+      it "has one worker started by default" do
+        expect(workers).to have(1).item
+      end
+
+      it "has an opscode worker started by default" do
+        expect(workers.first).to be_a(described_class::Worker::Opscode)
+      end
     end
 
-    it "has an opscode worker started by default" do
-      expect(workers.first).to be_a(described_class::Worker::Opscode)
+    context "when there are multiple workers" do
+      let(:endpoint_array) { [ first_worker, second_worker ] }
+      let(:first_worker) { double(options: endpoint_options.dup.merge(priority: 0), type: 'chicken') }
+      let(:second_worker) { double(options: endpoint_options.dup.merge(priority: 1), type: 'tuna') }
+      let(:endpoint_options) do
+        {
+          "url" => "www.fake.com",
+          "client_name" => "fake",
+          "client_key" => "/path/to/fake.key"
+        }
+      end
+      let(:dummy_endpoint_klass) do
+        Class.new do
+          attr_reader :options
+          include Celluloid
+
+          def initialize(options = {})
+            @options = options
+          end
+        end
+      end
+
+      before do
+        Berkshelf::API::Application.config.stub(:endpoints).and_return(endpoint_array)
+        Berkshelf::API::CacheBuilder::Worker.stub(:[]).and_return(dummy_endpoint_klass, dummy_endpoint_klass)
+      end
+
+      it "has two workers" do
+        expect(workers).to have(2).items
+      end
+
+      it "keeps the ordering" do
+        expect(workers.first.options[:priority]).to be(0)
+        expect(workers.last.options[:priority]).to be(1)
+      end
     end
   end
 end

data/spec/unit/berkshelf/api/cache_manager_spec.rb

@@ -67,23 +67,48 @@ describe Berkshelf::API::CacheManager do
   end
 
   describe "#diff" do
-    let(:cookbook_one) { Berkshelf::API::RemoteCookbook.new("ruby", "1.2.3", "opscode") }
-    let(:cookbook_two) { Berkshelf::API::RemoteCookbook.new("elixir", "2.0.0", "opscode") }
+    let(:cookbook_one) { Berkshelf::API::RemoteCookbook.new("ruby", "1.2.3", "opscode", nil, 1) }
+    let(:cookbook_two) { Berkshelf::API::RemoteCookbook.new("elixir", "2.0.0", "opscode", nil, 1) }
     let(:comparison) { Array.new }
 
     before do
-      subject.add(cookbook_one, double(dependencies: nil, platforms: nil))
-      subject.add(cookbook_two, double(dependencies: nil, platforms: nil))
+      subject.send(:add, cookbook_one, double(dependencies: nil, platforms: nil))
+      subject.send(:add, cookbook_two, double(dependencies: nil, platforms: nil))
 
-      @created, @deleted = @diff = subject.diff(comparison)
+      @created, @deleted = @diff = subject.send(:diff, comparison, 1)
     end
 
     it "returns two items" do
       expect(@diff).to have(2).items
     end
 
+    context "when there are more than one worker endpoints" do
+      let(:new_cookbook) { Berkshelf::API::RemoteCookbook.new("ruby", "3.0.0", "opscode", nil, 2) }
+      let(:comparison) { [ cookbook_one, cookbook_two, new_cookbook ] }
+
+      before do
+        @created, @deleted = @diff = subject.send(:diff, comparison, 2)
+      end
+
+      it "only creates cookbooks that have the same or lower priority" do
+        expect(@created).to eql([new_cookbook])
+      end
+
+      context "when the cookbook has been deleted" do
+        let(:comparison) { [cookbook_one] }
+
+        before do
+         @created, @deleted = @diff = subject.send(:diff, comparison, 1) 
+        end
+
+        it "only deletes cookbooks at the same priority" do
+          expect(@deleted).to eql([cookbook_two])
+        end
+      end
+    end
+
     context "when there are created and deleted cookbooks" do
-      let(:new_cookbook) { Berkshelf::API::RemoteCookbook.new("ruby", "3.0.0", "opscode") }
+      let(:new_cookbook) { Berkshelf::API::RemoteCookbook.new("ruby", "3.0.0", "opscode", nil, 1) }
       let(:comparison) { [ cookbook_one, new_cookbook ] }
 
       it "should return created and deleted cookbooks" do
@@ -93,7 +118,7 @@ describe Berkshelf::API::CacheManager do
     end
 
     context "when there are only created cookbooks" do
-      let(:new_cookbook) { Berkshelf::API::RemoteCookbook.new("ruby", "3.0.0", "opscode") }
+      let(:new_cookbook) { Berkshelf::API::RemoteCookbook.new("ruby", "3.0.0", "opscode", nil, 1) }
       let(:comparison) { [ cookbook_one, cookbook_two, new_cookbook ] }
 
       it "should return only created cookbooks" do
@@ -105,6 +130,10 @@ describe Berkshelf::API::CacheManager do
     context "when there are only deleted cookbooks" do
       let(:comparison) { [ cookbook_one ] }
 
+      before do
+        @created, @deleted = @diff = subject.send(:diff, comparison, 1)
+      end
+
       it "should return only deleted cookbooks" do
         expect(@created).to be_empty
         expect(@deleted).to eql([cookbook_two])

data/spec/unit/berkshelf/api/dependency_cache_spec.rb

@@ -67,6 +67,13 @@ describe Berkshelf::API::DependencyCache do
 
   subject { described_class.new(contents) }
 
+  context "when a new DependencyCache is created" do
+    it "should allow indifferent access to items in the cache" do
+      expect(subject[:chicken]).to be_a(Hash)
+      expect(subject[:chicken][:'1.0'][:dependencies]).to be_a(Hash)
+    end
+  end
+
   describe "#cookbooks" do
     it "should return a list of RemoteCookbooks" do
       expected_value = [

data/spec/unit/berkshelf/api/endpoint/v1_spec.rb

@@ -8,11 +8,24 @@ describe Berkshelf::API::Endpoint::V1 do
   let(:app) { described_class.new }
 
   describe "GET /universe" do
-    before  { get '/universe' }
-    subject { last_response }
-    let(:app_cache) { cache_manager.cache }
+    context "the cache has been warmed" do
+      before { cache_manager.set_warmed; get '/universe' }
 
-    its(:status) { should be(200) }
-    its(:body) { should eq(app_cache.to_json) }
+      subject { last_response }
+      let(:app_cache) { cache_manager.cache }
+
+      its(:status) { should be(200) }
+      its(:body) { should eq(app_cache.to_json) }
+    end
+
+    context "the cache is still warming" do
+      before { get '/universe' }
+
+      subject { last_response }
+      let(:app_cache) { cache_manager.cache }
+
+      its(:status) { should be(503) }
+      its(:headers) { should have_key("Retry-After") }
+    end
   end
 end

data/spec/unit/berkshelf/api/rest_gateway_spec.rb

@@ -9,18 +9,12 @@ describe Berkshelf::API::RESTGateway do
 
       its(:host) { should eql(described_class::DEFAULT_OPTIONS[:host]) }
       its(:port) { should eql(described_class::DEFAULT_OPTIONS[:port]) }
-      its(:workers) { should eql(described_class::DEFAULT_OPTIONS[:workers]) }
-      its(:rack_app) { should be_a(Berkshelf::API::RackApp) }
+      its(:app)  { should be_a(Berkshelf::API::RackApp) }
 
       context "given a different port" do
         before { options[:port] = 26210 }
         its(:port) { should eql(26210) }
       end
-
-      context "given a different amount of workers" do
-        before { options[:workers] = 20 }
-        its(:workers) { should eql(20) }
-      end
     end
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: berkshelf-api
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 1.0.0
 platform: ruby
 authors:
 - Jamie Winsor
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-07-17 00:00:00.000000000 Z
+date: 2013-10-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ridley
@@ -17,42 +17,42 @@ dependencies:
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 1.3.0
+        version: '1.6'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 1.3.0
+        version: '1.6'
 - !ruby/object:Gem::Dependency
   name: celluloid
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 0.14.0
+        version: '0.15'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 0.14.0
+        version: '0.15'
 - !ruby/object:Gem::Dependency
   name: reel
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '='
+    - - '>='
       - !ruby/object:Gem::Version
-        version: 0.4.0.pre
+        version: 0.4.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '='
+    - - '>='
       - !ruby/object:Gem::Version
-        version: 0.4.0.pre
+        version: 0.4.0
 - !ruby/object:Gem::Dependency
   name: grape
   requirement: !ruby/object:Gem::Requirement
@@ -191,6 +191,7 @@ files:
 - .gitignore
 - .ruby-version
 - .travis.yml
+- CHANGELOG.md
 - CONTRIBUTING.md
 - Gemfile
 - Guardfile