berkshelf-api 1.0.0 → 1.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: f238c1aec4a689f0461aaea5792d0588501e4d8c
-  data.tar.gz: f99f8dac7ac5aa4133ac4bef0420e74d128cd01f
+  metadata.gz: 00fbbdefda724c9caf02b454baeddf35e2da8744
+  data.tar.gz: b71657d7e5e315e63c82d543d93eaefae3f6348c
 SHA512:
-  metadata.gz: 0910bcf676f10ea3f0c7061a685400ee232350b39fe08866ed89558e6d3dc649cb6a6b0288041a59924c010825a23510fada8adc043466a37b56374b74ad1543
-  data.tar.gz: de9395b533f039629f2eef14bd583de33839afac67e225844616c5cdc22a6d3038b3852839e2f110b4f19e2e227ec946284d58bfc0f9dfd04792a645b966d1aa
+  metadata.gz: c871060bd3057105ddcd5266c29ad8947eb1e42c5b2c91048076da25c564c1bdeefb46270adadec74f0f8d3429974a4d90e89b62bbfe8752735e0bfcf40e6af8
+  data.tar.gz: 905361134b015f6c4d8ccd4cbbbdb7d767bd79866ed3a8bcc72cd68bb65e6321332616ce7d93383af1ebf5a84a5baf53a7abef56604e8eeb79acaffa3c7971bc

data/.ruby-version

@@ -1 +1 @@
-2.0.0-p247
+2.0.0-p353

data/.travis.yml

@@ -2,7 +2,6 @@ language: ruby
 rvm:
   - 1.9.3
   - 2.0.0
-  - jruby-19mode
 notifications:
   irc:
     channels:

data/CHANGELOG.md

@@ -1,3 +1,12 @@
+# 1.1.0
+
+* Enhancements
+  * Added Github cache builder and Github endpoint. You can now add Github organization's as endpoints and the cache builder will crawl each repository looking for repositories which contain SemVer tags and raw cookbook metadata.
+
+* Bug Fixes
+  * Fix cache builder crashes when builder encounters a cookbook with invalid or missing metadata.
+  * Invalid cookbooks won't be added to cache.
+
 # 1.0.0
 
 * Prioritize Cache Builders so their entries do not clobber one another

data/CONTRIBUTING.md

@@ -23,7 +23,7 @@ Install the latest version of [Bundler](http://gembundler.com)
 
 Clone the project
 
-    $ git clone git://github.com/RiotGames/berkshelf-api.git
+    $ git clone git://github.com/berkshelf/berkshelf-api.git
 
 and run:
 

data/README.md

@@ -1,8 +1,6 @@
 # Berkshelf::API
 [![Gem Version](https://badge.fury.io/rb/berkshelf-api.png)](http://badge.fury.io/rb/berkshelf-api)
-[![Build Status](https://secure.travis-ci.org/RiotGames/berkshelf-api.png?branch=master)](http://travis-ci.org/RiotGames/berkshelf-api)
-[![Dependency Status](https://gemnasium.com/RiotGames/berkshelf-api.png?travis)](https://gemnasium.com/RiotGames/berkshelf-api)
-[![Code Climate](https://codeclimate.com/github/RiotGames/berkshelf-api.png)](https://codeclimate.com/github/RiotGames/berkshelf-api)
+[![Build Status](https://secure.travis-ci.org/berkshelf/berkshelf-api.png?branch=master)](http://travis-ci.org/berkshelf/berkshelf-api)
 
 A server which indexes cookbooks from various sources and hosts it over a REST API
 
@@ -58,6 +56,25 @@ You may configure the endpoints to index by editing the JSON configuration file
 }
 ```
 
+### Github Organization
+
+Github limits the rate of requests to their API if not authenticated. For this reason the access_token option
+is required.
+
+```json
+{
+  "endpoints": [
+    {
+      "type": "github",
+      "options": {
+        "organization": "opscode-cookbooks",
+        "access_token": ""
+      }
+    }
+  ]
+}
+```
+
 ## Contributing
 
 1. Fork it
@@ -71,4 +88,4 @@ You may configure the endpoints to index by editing the JSON configuration file
 * Jamie Winsor (<jamie@vialstudios.com>)
 * Andrew Garson (<agarson@riotgames.com>)
 
-Thank you to all of our [Contributors](https://github.com/RiotGames/berkshelf-api/graphs/contributors), testers, and users.
+Thank you to all of our [Contributors](https://github.com/berkshelf/berkshelf-api/graphs/contributors), testers, and users.

data/berkshelf-api.gemspec

@@ -10,7 +10,7 @@ Gem::Specification.new do |spec|
   spec.email         = ["jamie@vialstudios.com", "agarson@riotgames"]
   spec.description   = %q{Berkshelf dependency API server}
   spec.summary       = %q{A server which indexes cookbooks from various sources and hosts it over a REST API}
-  spec.homepage      = "https://github.com/RiotGames/berkshelf-api"
+  spec.homepage      = "https://github.com/berkshelf/berkshelf-api"
   spec.license       = "Apache 2.0"
 
   spec.files         = `git ls-files`.split($/)
@@ -19,7 +19,7 @@ Gem::Specification.new do |spec|
   spec.require_paths = ["lib"]
   spec.required_ruby_version = ">= 1.9.3"
 
-  spec.add_dependency 'ridley', '~> 1.6'
+  spec.add_dependency 'ridley', '~> 2.0'
   spec.add_dependency 'celluloid', '~> 0.15'
   spec.add_dependency 'reel', '>= 0.4.0'
   spec.add_dependency 'grape', '~> 0.5.0'
@@ -28,6 +28,8 @@ Gem::Specification.new do |spec|
   spec.add_dependency 'retryable', '~> 1.3.3'
   spec.add_dependency 'archive', '= 0.0.2'
   spec.add_dependency 'buff-config', '~> 0.1'
+  spec.add_dependency 'octokit', '~> 2.6'
+  spec.add_dependency 'solve'
 
   spec.add_development_dependency 'bundler', '~> 1.3'
   spec.add_development_dependency 'thor', '~> 0.18.0'

data/lib/berkshelf/api/cache_builder/worker.rb

@@ -38,7 +38,6 @@ module Berkshelf::API
         def cookbooks
           raise RuntimeError, "must be implemented"
         end
-
       end
 
       class << self

data/lib/berkshelf/api/cache_builder/worker/chef_server.rb

@@ -6,6 +6,14 @@ module Berkshelf::API
 
         finalizer :finalize_callback
 
+        # @option options [String] :url
+        #   the URL of the target Chef Server
+        # @option options [String] :client_name
+        #   the name of the client for authenticating to the Chef Server
+        # @option options [String] :client_key
+        #   a client key for authenticating to the Chef Server
+        # @option options [Boolean] :ssl_verify
+        #   turn ssl verification off if you have an unsigned SSL certificate
         def initialize(options = {})
           @connection = Ridley::Client.new_link(server_url: options[:url], client_key: options[:client_key],
             client_name: options[:client_name], ssl: { verify: options[:ssl_verify] })

data/lib/berkshelf/api/cache_builder/worker/github.rb

@@ -0,0 +1,102 @@
+require 'octokit'
+require 'solve'
+require 'net/http'
+
+module Berkshelf::API
+  class CacheBuilder
+    module Worker
+      class Github < Worker::Base
+        worker_type "github"
+
+        # @return [String]
+        attr_reader :organization
+
+        # @option options [String] :organization
+        #   the organization to crawl for cookbooks
+        # @option options [String] :access_token
+        #   authentication token for accessing the Github organization. This is necessary
+        #   since Github throttles unauthenticated API requests
+        def initialize(options = {})
+          @connection   = Octokit::Client.new(access_token: options[:access_token])
+          @organization = options[:organization]
+          super(options)
+        end
+
+        # @return [Array<RemoteCookbook>]
+        #  The list of cookbooks this builder can find
+        def cookbooks
+          [].tap do |cookbook_versions|
+            connection.organization_repositories(organization).each do |repo|
+              connection.tags(repo.full_name).each do |tag|
+                if match = /^v(?<version>.*)$/.match(tag.name)
+                  begin
+                    next unless cookbook_metadata = load_metadata(repo.name, tag.name)
+
+                    if cookbook_metadata.version.to_s == match[:version].to_s
+                      cookbook_versions << RemoteCookbook.new(repo.name, cookbook_metadata.version,
+                        self.class.worker_type, repo.full_name, priority)
+                    else
+                      log.warn "Version found in metadata for #{repo.name} (#{tag.name}) does not " +
+                        "match the tag. Got #{cookbook_metadata.version}."
+                    end
+                  rescue Solve::Errors::InvalidVersionFormat
+                    log.debug "Ignoring tag #{tag.name} for: #{repo.name}. Does not conform to semver."
+                  rescue Octokit::NotFound
+                    log.debug "Ignoring tag #{tag.name} for: #{repo.name}. No raw metadata found."
+                  end
+                else
+                  log.debug "Version number cannot be parsed"
+                end
+              end
+            end
+          end
+        end
+
+        # Return the metadata of the given RemoteCookbook. If the metadata could not be found or parsed
+        # nil is returned.
+        #
+        # @param [RemoteCookbook] remote
+        #
+        # @return [Ridley::Chef::Cookbook::Metadata, nil]
+        def metadata(remote)
+          load_metadata(remote.name, "v#{remote.version}")
+        end
+
+        private
+
+          attr_reader :connection
+
+          # Helper function for loading metadata from a particular ref in a Github repository
+          #
+          # @param [String] repo
+          #   name of repository to load from
+          # @param [String] ref
+          #   reference, tag, or branch to load from
+          #
+          # @return [Ridley::Chef::Cookbook::Metadata, nil]
+          def load_metadata(repo, ref)
+            metadata_content  = read_content(repo, ref, Ridley::Chef::Cookbook::Metadata::RAW_FILE_NAME)
+            cookbook_metadata = Ridley::Chef::Cookbook::Metadata.new
+            cookbook_metadata.instance_eval(metadata_content)
+            cookbook_metadata
+          rescue => ex
+            nil
+          end
+
+          # Read content from a file from a particular ref in a Github repository
+          #
+          # @param [String] repo
+          #   name of the repository to load from
+          # @param [String] ref
+          #   reference, tag, or branch to load from
+          # @param [String] file
+          #   file to read content from
+          #
+          # @return [String]
+          def read_content(repo, ref, file)
+            Base64.decode64(connection.contents("#{organization}/#{repo}", path: file, ref: ref).content)
+          end
+      end
+    end
+  end
+end

data/lib/berkshelf/api/cache_builder/worker/opscode.rb

@@ -6,6 +6,8 @@ module Berkshelf::API
 
         finalizer :finalize_callback
 
+        # @param [Hash] options
+        #   see {API::SiteConnector::Opscode.new} for options
         def initialize(options = {})
           @connection = Berkshelf::API::SiteConnector::Opscode.pool_link(size: 25, args: [ options ])
           super
@@ -25,13 +27,17 @@ module Berkshelf::API
           end
         end
 
+        # Return the metadata of the given RemoteCookbook. If the metadata could not be found or parsed
+        # nil is returned.
+        #
         # @param [RemoteCookbook] remote
         #
-        # @return [Ridley::Chef::Cookbook::Metadata]
+        # @return [Ridley::Chef::Cookbook::Metadata, nil]
         def metadata(remote)
-          Dir.mktmpdir do |destination|
-            connection.download(remote.name, remote.version, destination)
-            load_metadata(destination, remote.name)
+          Dir.mktmpdir('metadata') do |destination|
+            if connection.download(remote.name, remote.version, destination)
+              load_metadata(destination, remote.name)
+            end
           end
         end
 
@@ -46,7 +52,9 @@ module Berkshelf::API
           def load_metadata(directory, cookbook)
             # The community site does not enforce the name of the cookbook contained in the archive
             # downloaded and extracted. This will just find the first metadata.json and load it.
-            file     = Dir["#{directory}/**/*/metadata.json"].first
+            file = Dir["#{directory}/**/*/metadata.json"].first
+            return nil if file.nil?
+
             metadata = File.read(file)
             Ridley::Chef::Cookbook::Metadata.from_json(metadata)
           rescue JSON::ParserError => ex

data/lib/berkshelf/api/cache_manager.rb

@@ -97,9 +97,10 @@ module Berkshelf::API
         work.map! do |remote|
           [ remote, worker.future(:metadata, remote) ]
         end.map! do |remote, metadata|
-          [remote, metadata.value]
+          metadata.value ? [remote, metadata.value] : nil
         end
-        created_cookbooks_with_metadata += work
+
+        created_cookbooks_with_metadata += work.compact
       end
 
       log.info "about to merge cookbooks"

data/lib/berkshelf/api/site_connector/opscode.rb

@@ -36,9 +36,12 @@ module Berkshelf::API
       #   time to wait between retries
       attr_reader :retry_interval
 
-      # @param [Faraday::Connection] connection
-      #   Optional parameter for setting the connection object
-      #   This should only be set manually for testing
+      # @option options [String] :url ({V1_API})
+      #   url of community site
+      # @option options [Integer] :retries (5)
+      #   how many download retry attempts to make
+      # @option options [Float] :retry_interval (0.5)
+      #   how long to wait before retrying again
       def initialize(options = {})
         options  = { url: V1_API, retries: 5, retry_interval: 0.5 }.merge(options)
         @api_uri = options[:url]
@@ -83,6 +86,10 @@ module Berkshelf::API
         end
       end
 
+      # Download the cookbook with the given name and version to the destination. The directory
+      # containing the extracted contents will be returned on success. On failure, nil will
+      # be returned.
+      #
       # @param [String] cookbook
       #   The name of the cookbook to download
       # @param [String] version
@@ -94,11 +101,17 @@ module Berkshelf::API
       def download(name, version, destination = Dir.mktmpdir)
         log.debug "downloading #{name}(#{version})"
         if uri = download_uri(name, version)
-          archive = stream(uri)
-          Archive.extract(archive.path, destination)
+          begin
+            archive = stream(uri)
+            Archive.extract(archive.path, destination)
+            destination
+          rescue => ex
+            log.warn "error downloading/extracting #{name} (#{version}): #{ex}"
+            nil
+          ensure
+            archive.unlink unless archive.nil?
+          end
         end
-      ensure
-        archive.unlink unless archive.nil?
       end
 
       # Return the location where a cookbook of the given name and version can be downloaded from

data/lib/berkshelf/api/version.rb

@@ -1,5 +1,5 @@
 module Berkshelf
   module API
-    VERSION = "1.0.0"
+    VERSION = "1.1.0"
   end
 end

data/spec/unit/berkshelf/api/cache_builder/worker/github_spec.rb

@@ -0,0 +1,71 @@
+require 'spec_helper'
+
+describe Berkshelf::API::CacheBuilder::Worker::Github do
+  describe "ClassMethods" do
+    subject { described_class }
+    its(:worker_type) { should eql("github") }
+  end
+
+  let(:connection) do
+    double('connection')
+  end
+
+  let(:good_tag) do
+    tag = double('good_tag')
+    tag.stub(:name) { 'v1.0.0' }
+    tag
+  end
+
+  let(:bad_tag) do
+    tag = double('good_tag')
+    tag.stub(:name) { 'beta2' }
+    tag
+  end
+
+  let :contents do
+    contents = double('contents')
+    contents.stub(:content) { 'dmVyc2lvbiAiMS4wLjAi' }
+    contents
+  end
+
+  let(:repo) do
+    repo = double('repo')
+    repo.stub(:full_name) { 'opscode-cookbooks/apt' }
+    repo.stub(:name) { 'apt' }
+    repo
+  end
+
+  let(:repos) do
+    [repo]
+  end
+
+  subject do
+    expect(Octokit::Client).to receive(:new) { connection }
+    described_class.new(organization: "opscode-cookbooks", access_token: "asdf")
+  end
+
+  describe "#cookbooks" do
+    before do
+      expect(connection).to receive(:organization_repositories) { repos }
+      expect(connection).to receive(:tags) { [good_tag, bad_tag] }
+      expect(connection).to receive(:contents).with("opscode-cookbooks/apt",
+        { path: "metadata.rb", ref: "v1.0.0"}) { contents }
+    end
+
+    it "returns an array containing an item for each valid cookbook on the server" do
+      expect(subject.cookbooks).to have(1).items
+    end
+
+    it "returns an array of RemoteCookbooks" do
+      subject.cookbooks.each do |cookbook|
+        expect(cookbook).to be_a(Berkshelf::API::RemoteCookbook)
+      end
+    end
+
+    it "each RemoteCookbook is tagged with a location_type matching the worker_type of the builder" do
+      subject.cookbooks.each do |cookbook|
+        expect(cookbook.location_type).to eql(described_class.worker_type)
+      end
+    end
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: berkshelf-api
 version: !ruby/object:Gem::Version
-  version: 1.0.0
+  version: 1.1.0
 platform: ruby
 authors:
 - Jamie Winsor
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-10-15 00:00:00.000000000 Z
+date: 2013-12-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ridley
@@ -17,14 +17,14 @@ dependencies:
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: '1.6'
+        version: '2.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: '1.6'
+        version: '2.0'
 - !ruby/object:Gem::Dependency
   name: celluloid
   requirement: !ruby/object:Gem::Requirement
@@ -137,6 +137,34 @@ dependencies:
     - - ~>
       - !ruby/object:Gem::Version
         version: '0.1'
+- !ruby/object:Gem::Dependency
+  name: octokit
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '2.6'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '2.6'
+- !ruby/object:Gem::Dependency
+  name: solve
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -206,6 +234,7 @@ files:
 - lib/berkshelf/api/cache_builder.rb
 - lib/berkshelf/api/cache_builder/worker.rb
 - lib/berkshelf/api/cache_builder/worker/chef_server.rb
+- lib/berkshelf/api/cache_builder/worker/github.rb
 - lib/berkshelf/api/cache_builder/worker/opscode.rb
 - lib/berkshelf/api/cache_manager.rb
 - lib/berkshelf/api/config.rb
@@ -233,6 +262,7 @@ files:
 - spec/support/chef_server.rb
 - spec/unit/berkshelf/api/application_spec.rb
 - spec/unit/berkshelf/api/cache_builder/worker/chef_server_spec.rb
+- spec/unit/berkshelf/api/cache_builder/worker/github_spec.rb
 - spec/unit/berkshelf/api/cache_builder/worker/opscode_spec.rb
 - spec/unit/berkshelf/api/cache_builder/worker_spec.rb
 - spec/unit/berkshelf/api/cache_builder_spec.rb
@@ -246,7 +276,7 @@ files:
 - spec/unit/berkshelf/api/rest_gateway_spec.rb
 - spec/unit/berkshelf/api/site_connector/opscode_spec.rb
 - spec/unit/berkshelf/api/srv_ctl_spec.rb
-homepage: https://github.com/RiotGames/berkshelf-api
+homepage: https://github.com/berkshelf/berkshelf-api
 licenses:
 - Apache 2.0
 metadata: {}
@@ -266,7 +296,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.0.3
+rubygems_version: 2.0.14
 signing_key: 
 specification_version: 4
 summary: A server which indexes cookbooks from various sources and hosts it over a
@@ -278,6 +308,7 @@ test_files:
 - spec/support/chef_server.rb
 - spec/unit/berkshelf/api/application_spec.rb
 - spec/unit/berkshelf/api/cache_builder/worker/chef_server_spec.rb
+- spec/unit/berkshelf/api/cache_builder/worker/github_spec.rb
 - spec/unit/berkshelf/api/cache_builder/worker/opscode_spec.rb
 - spec/unit/berkshelf/api/cache_builder/worker_spec.rb
 - spec/unit/berkshelf/api/cache_builder_spec.rb