berkshelf-api 0.1.0

data/bin/berks-api

@@ -0,0 +1,5 @@
+#!/usr/bin/env ruby
+$:.push File.expand_path("../../lib", __FILE__)
+require 'berkshelf/api/srv_ctl'
+
+Berkshelf::API::SrvCtl.run(ARGV, File.basename(__FILE__))

data/lib/berkshelf-api.rb

@@ -0,0 +1 @@
+require 'berkshelf/api'

data/lib/berkshelf/api.rb

@@ -0,0 +1,25 @@
+require 'celluloid'
+require 'hashie'
+require 'ridley'
+require 'faraday'
+require 'json'
+
+module Berkshelf
+  module API
+    require_relative 'api/errors'
+    require_relative 'api/logging'
+    require_relative 'api/mixin'
+    require_relative 'api/generic_server'
+
+    require_relative 'api/application'
+    require_relative 'api/cache_builder'
+    require_relative 'api/cache_manager'
+    require_relative 'api/config'
+    require_relative 'api/dependency_cache'
+    require_relative 'api/endpoint'
+    require_relative 'api/rack_app'
+    require_relative 'api/remote_cookbook'
+    require_relative 'api/site_connector'
+    require_relative 'api/srv_ctl'
+  end
+end

data/lib/berkshelf/api/application.rb

@@ -0,0 +1,114 @@
+trap 'INT' do
+  Berkshelf::API::Application.shutdown
+end
+
+trap 'TERM' do
+  Berkshelf::API::Application.shutdown
+end
+
+module Berkshelf::API
+  class ApplicationSupervisor < Celluloid::SupervisionGroup
+    # @option options [Boolean] :disable_http (false)
+    #   run the application without the rest gateway
+    def initialize(registry, options = {})
+      super(registry)
+      supervise_as(:cache_manager, Berkshelf::API::CacheManager)
+      supervise_as(:cache_builder, Berkshelf::API::CacheBuilder)
+
+      unless options[:disable_http]
+        require_relative 'rest_gateway'
+        supervise_as(:rest_gateway, Berkshelf::API::RESTGateway, options)
+      end
+    end
+  end
+
+  module Application
+    class << self
+      extend Forwardable
+      include Berkshelf::API::Logging
+      include Berkshelf::API::Mixin::Services
+
+      def_delegators :registry, :[], :[]=
+
+      def config
+        @config ||= begin
+          Berkshelf::API::Config.from_file(Berkshelf::API::Config.default_path)
+        rescue
+          Berkshelf::API::Config.new
+        end
+      end
+
+      # @option options [String, Fixnum] :log_location (STDOUT)
+      # @option options [String, nil] :log_level ("INFO")
+      #   - "DEBUG
+      #   - "INFO"
+      #   - "WARN"
+      #   - "ERROR"
+      #   - "FATAL"
+      def configure_logger(options = {})
+        Logging.init(level: options[:log_level], location: options[:log_location])
+      end
+
+      def instance
+        return @instance if @instance
+
+        raise NotStartedError, "application not running"
+      end
+
+      # The Actor registry for Berkshelf::API.
+      #
+      # @note Berkshelf::API uses it's own registry instead of Celluloid::Registry.root to
+      #   avoid conflicts in the larger namespace. Use Berkshelf::API::Application[] to access Berkshelf::API
+      #   actors instead of Celluloid::Actor[].
+      #
+      # @return [Celluloid::Registry]
+      def registry
+        @registry ||= Celluloid::Registry.new
+      end
+
+      # Run the application in the foreground (sleep on main thread)
+      #
+      # @option options [Boolean] :disable_http (false)
+      #   run the application without the rest gateway
+      def run(options = {})
+        loop do
+          supervisor = run!(options)
+
+          sleep 0.1 while supervisor.alive?
+
+          break if @shutdown
+
+          log.error "!!! #{self} crashed. Restarting..."
+        end
+      end
+
+      # Run the application in the background
+      #
+      # @option options [Boolean] :disable_http (false)
+      #   run the application without the rest gateway
+      # @option options [Boolean] :eager_build (false)
+      #   automatically begin and loop all cache builders
+      #
+      # @return [Berkshelf::API::Application]
+      def run!(options = {})
+        options = { disable_http: false, eager_build: false }.merge(options)
+        configure_logger(options)
+        @instance = ApplicationSupervisor.new(registry, options)
+        cache_builder.async(:build_loop) if options[:eager_build]
+        @instance
+      end
+
+      # @return [Boolean]
+      def running?
+        instance.alive?
+      rescue NotStartedError
+        false
+      end
+
+      def shutdown
+        @shutdown = true
+        instance.terminate
+      end
+    end
+  end
+end

data/lib/berkshelf/api/cache_builder.rb

@@ -0,0 +1,60 @@
+module Berkshelf::API
+  class CacheBuilder
+    require_relative 'cache_builder/worker'
+
+    class WorkerSupervisor < Celluloid::SupervisionGroup; end
+
+    BUILD_INTERVAL = 5.0
+
+    include Berkshelf::API::GenericServer
+    include Berkshelf::API::Logging
+
+    server_name :cache_builder
+    finalizer :finalize_callback
+
+    def initialize
+      log.info "Cache Builder starting..."
+      @worker_registry   = Celluloid::Registry.new
+      @worker_supervisor = WorkerSupervisor.new(@worker_registry)
+      @building          = false
+
+      Application.config.endpoints.each do |endpoint|
+        @worker_supervisor.supervise(CacheBuilder::Worker[endpoint.type], endpoint.options)
+      end
+    end
+
+    # Issue a single build command to all workers
+    #
+    # @return [Array]
+    def build
+      workers.collect { |actor| actor.future(:build) }.map(&:value)
+    end
+
+    # Issue a build command to all workers at the scheduled interval
+    #
+    # @param [Fixnum, Float] interval
+    def build_loop(interval = BUILD_INTERVAL)
+      return if @building
+
+      loop do
+        @building = true
+        build
+        sleep BUILD_INTERVAL
+      end
+    end
+
+    # Return the list of running workers
+    #
+    # @return [Array<CacheBuilder::Worker::Base>]
+    def workers
+      @worker_supervisor.actors
+    end
+
+    private
+
+      def finalize_callback
+        log.info "Cache Builder shutting down..."
+        @worker_supervisor.terminate if @worker_supervisor && @worker_supervisor.alive?
+      end
+  end
+end

data/lib/berkshelf/api/cache_builder/worker.rb

@@ -0,0 +1,116 @@
+module Berkshelf::API
+  class CacheBuilder
+    module Worker
+      class Base
+        class << self
+          # @param [#to_s, nil] type
+          def worker_type(type = nil)
+            return @worker_type if @worker_type
+            @worker_type = type.to_s
+            Worker.register(@worker_type, self)
+          end
+        end
+
+        include Celluloid
+        include Berkshelf::API::Logging
+        include Berkshelf::API::Mixin::Services
+
+        attr_reader :options
+
+        def initialize(options = {}); end
+
+        # @abstract
+        #
+        # @param [RemoteCookbook] remote
+        #
+        # @return [Ridley::Chef::Cookbook::Metadata]
+        def metadata(remote)
+          raise RuntimeError, "must be implemented"
+        end
+
+        # @abstract
+        #
+        # @return [Array<RemoteCookbook>]
+        #  The list of cookbooks this builder can find
+        def cookbooks
+          raise RuntimeError, "must be implemented"
+        end
+
+        def build
+          log.info "#{self} building..."
+          log.info "#{self} determining if the cache is stale..."
+          if stale?
+            log.info "#{self} cache is stale."
+            update_cache
+          else
+            log.info "#{self} cache is up to date."
+          end
+
+          log.info "clearing diff"
+          clear_diff
+        end
+
+        # @return [Array<Array<RemoteCookbook>, Array<RemoteCookbook>>]
+        def diff
+          @diff ||= cache_manager.diff(cookbooks)
+        end
+
+        def update_cache
+          created_cookbooks, deleted_cookbooks = diff
+
+          log.info "#{self} adding (#{created_cookbooks.length}) items..."
+          created_cookbooks.collect do |remote|
+            [ remote, future(:metadata, remote) ]
+          end.each do |remote, metadata|
+            cache_manager.add(remote, metadata.value)
+          end
+
+          log.info "#{self} removing (#{deleted_cookbooks.length}) items..."
+          deleted_cookbooks.each { |remote| cache_manager.remove(remote.name, remote.version) }
+
+          log.info "#{self} cache updated."
+          cache_manager.save
+        end
+
+        def stale?
+          created_cookbooks, deleted_cookbooks = diff
+          created_cookbooks.any? || deleted_cookbooks.any?
+        end
+
+        private
+
+          def clear_diff
+            @diff = nil
+          end
+      end
+
+      class << self
+        # @param [#to_s] name
+        #
+        # @return [Worker::Base]
+        def [](name)
+          types[name.to_s]
+        end
+
+        # @param [#to_s] name
+        # @param [Worker::Base] klass
+        def register(name, klass)
+          name = name.to_s
+          if types.has_key?(name)
+            raise RuntimeError, "worker already registered with the name '#{name}'"
+          end
+          types[name] = klass
+        end
+
+        # @return [Hash]
+        def types
+          @types ||= Hash.new
+        end
+      end
+    end
+  end
+end
+
+Dir["#{File.dirname(__FILE__)}/worker/*.rb"].sort.each do |path|
+  require_relative "worker/#{File.basename(path, '.rb')}"
+end

data/lib/berkshelf/api/cache_builder/worker/chef_server.rb

@@ -0,0 +1,46 @@
+module Berkshelf::API
+  class CacheBuilder
+    module Worker
+      class ChefServer < Worker::Base
+        worker_type "chef_server"
+
+        finalizer :finalize_callback
+
+        def initialize(options = {})
+          @connection = Ridley::Client.new_link(server_url: options[:url], client_key: options[:client_key],
+            client_name: options[:client_name], ssl: { verify: options[:ssl_verify] })
+          super
+        end
+
+        # @return [Array<RemoteCookbook>]
+        #  The list of cookbooks this builder can find
+        def cookbooks
+          [].tap do |cookbook_versions|
+            connection.cookbook.all.each do |cookbook, versions|
+              versions.each do |version|
+                cookbook_versions << RemoteCookbook.new(cookbook, version, self.class.worker_type,
+                  @connection.server_url)
+              end
+            end
+          end
+        end
+
+        # @param [RemoteCookbook] remote
+        #
+        # @return [Ridley::Chef::Cookbook::Metadata]
+        def metadata(remote)
+          metadata_hash = connection.cookbook.find(remote.name, remote.version).metadata
+          Ridley::Chef::Cookbook::Metadata.from_hash(metadata_hash)
+        end
+
+        private
+
+          attr_reader :connection
+
+          def finalize_callback
+            connection.terminate if connection && connection.alive?
+          end
+      end
+    end
+  end
+end

data/lib/berkshelf/api/cache_builder/worker/opscode.rb

@@ -0,0 +1,59 @@
+module Berkshelf::API
+  class CacheBuilder
+    module Worker
+      class Opscode < Worker::Base
+        worker_type "opscode"
+
+        finalizer :finalize_callback
+
+        def initialize(options = {})
+          @connection = Berkshelf::API::SiteConnector::Opscode.pool_link(size: 25)
+          super
+        end
+
+        # @return [Array<RemoteCookbook>]
+        #  The list of cookbooks this builder can find
+        def cookbooks
+          [].tap do |cookbook_versions|
+            connection.cookbooks.collect do |cookbook|
+              [ cookbook, connection.future(:versions, cookbook) ]
+            end.each do |cookbook, versions|
+              versions.value.each do |version|
+                cookbook_versions << RemoteCookbook.new(cookbook, version, self.class.worker_type, @connection.api_uri)
+              end
+            end
+          end
+        end
+
+        # @param [RemoteCookbook] remote
+        #
+        # @return [Ridley::Chef::Cookbook::Metadata]
+        def metadata(remote)
+          Dir.mktmpdir do |destination|
+            connection.download(remote.name, remote.version, destination)
+            load_metadata(destination, remote.name)
+          end
+        end
+
+        private
+
+          attr_accessor :connection
+
+          def finalize_callback
+            connection.terminate if connection && connection.alive?
+          end
+
+          def load_metadata(directory, cookbook)
+            # The community site does not enforce the name of the cookbook contained in the archive
+            # downloaded and extracted. This will just find the first metadata.json and load it.
+            file     = Dir["#{directory}/**/*/metadata.json"].first
+            metadata = File.read(file)
+            Ridley::Chef::Cookbook::Metadata.from_json(metadata)
+          rescue JSON::ParserError => ex
+            log.warn "Error loading metadata for #{cookbook} from: #{file}"
+            abort MetadataLoadError.new(ex)
+          end
+      end
+    end
+  end
+end

data/lib/berkshelf/api/cache_manager.rb

@@ -0,0 +1,96 @@
+module Berkshelf::API
+  class CacheManager
+    class << self
+      attr_writer :cache_file
+
+      # @return [String]
+      def cache_file
+        @cache_file ||= File.expand_path("~/.berkshelf/api-server/cerch")
+      end
+    end
+
+    include Berkshelf::API::GenericServer
+    include Berkshelf::API::Logging
+
+    SAVE_INTERVAL = 30.0
+
+    server_name :cache_manager
+    finalizer :finalize_callback
+    exclusive :add, :clear, :remove, :save
+
+    attr_reader :cache
+
+    def initialize
+      log.info "Cache Manager starting..."
+      @cache = DependencyCache.new
+      load_save if File.exist?(self.class.cache_file)
+      every(SAVE_INTERVAL) { save }
+    end
+
+    # @param [RemoteCookbook] cookbook
+    # @param [Ridley::Chef::Cookbook::Metadata] metadata
+    #
+    # @return [Hash]
+    def add(cookbook, metadata)
+      @cache.add(cookbook, metadata)
+    end
+
+    # Clear any items added to the cache
+    #
+    # @return [Hash]
+    def clear
+      @cache.clear
+    end
+
+    # Check if the cache knows about the given cookbook version
+    #
+    # @param [#to_s] name
+    # @param [#to_s] version
+    #
+    # @return [Boolean]
+    def has_cookbook?(name, version)
+      @cache.has_cookbook?(name, version)
+    end
+
+    def load_save
+      @cache = DependencyCache.from_file(self.class.cache_file)
+    end
+
+    # Remove the cached item matching the given name and version
+    #
+    # @param [#to_s] name
+    # @param [#to_s] version
+    #
+    # @return [DependencyCache]
+    def remove(name, version)
+      @cache.remove(name, version)
+    end
+
+    def save
+      log.info "Saving the cache to: #{self.class.cache_file}"
+      cache.save(self.class.cache_file)
+      log.info "Cache saved!"
+    end
+
+    # @param [Array<RemoteCookbook>] cookbooks
+    #   An array of RemoteCookbooks representing all the cookbooks on the indexed site
+    #
+    # @return [Array<Array<RemoteCookbook>, Array<RemoteCookbook>>]
+    #   A tuple of Arrays of RemoteCookbooks
+    #   The first array contains items not in the cache
+    #   The second array contains items in the cache, but not in the cookbooks parameter
+    def diff(cookbooks)
+      known_cookbooks   = cache.cookbooks
+      created_cookbooks = cookbooks - known_cookbooks
+      deleted_cookbooks = known_cookbooks - cookbooks
+      [ created_cookbooks, deleted_cookbooks ]
+    end
+
+    private
+
+      def finalize_callback
+        log.info "Cache Manager shutting down..."
+        self.save
+      end
+  end
+end