puppet-forge-server 1.0.1

data/lib/puppet_forge_server/api/v3/releases.rb

@@ -0,0 +1,38 @@
+# -*- encoding: utf-8 -*-
+#
+# Copyright 2014 North Development AB
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+module PuppetForgeServer::Api::V3
+  module Releases
+    def get_releases(metadata)
+      metadata.map do |element|
+        name = element[:metadata].name.sub(/^[^-]+-/, '')
+
+        {
+            :uri => "/v3/releases/#{element[:metadata].name}-#{element[:metadata].version}",
+            :module => {
+                :uri => "/v3/modules/#{element[:metadata].name}",
+                :name => name
+            },
+            :metadata => element[:metadata].to_hash,
+            :version => element[:metadata].version,
+            :tags => element[:tags] ? element[:tags] : [element[:metadata].author, name],
+            :file_uri => "/v3/files#{element[:path]}",
+            :file_md5 => element[:checksum]
+        }
+      end.version_sort_by { |r| r[:version] }.reverse
+    end
+  end
+end

data/lib/puppet_forge_server/app/version1.rb

@@ -0,0 +1,67 @@
+# -*- encoding: utf-8 -*-
+#
+# Copyright 2014 North Development AB
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'sinatra/base'
+require 'json'
+
+module PuppetForgeServer::App
+  class Version1 < Sinatra::Base
+    include PuppetForgeServer::Api::V1::Releases
+    include PuppetForgeServer::Api::V1::Modules
+    include PuppetForgeServer::Utils::Buffer
+
+    def initialize(backends)
+      super(nil)
+      @backends = backends
+    end
+
+    get '/api/v1/releases.json' do
+      unless params[:module]
+        halt 400, {'error' => 'The number of version constraints in the query does not match the number of module names'}.to_json
+      end
+
+      author, name = params[:module].split '/'
+      version = params[:version] if params[:version]
+
+      metadata = @backends.map do |backend|
+        backend.get_metadata(author, name, {:version => version, :with_checksum => false})
+      end.flatten.uniq
+
+      halt 400, {'errors' => ["'#{params[:module]}' is not a valid module slug"]}.to_json if metadata.empty?
+
+      {"#{author}/#{name}" => get_releases(metadata)}.to_json
+    end
+
+    get '/api/v1/files/*' do
+      captures = params[:captures].first
+      buffer = get_buffer(@backends, captures)
+
+      halt 404, {'errors' => ['404 Not found']}.to_json unless buffer
+
+      content_type 'application/octet-stream'
+      attachment captures.split('/').last
+      download buffer
+    end
+
+    get '/modules.json' do
+      query = params[:q]
+      metadata = @backends.map do |backend|
+        backend.query_metadata(query, {:with_checksum => false})
+      end.flatten.uniq
+      get_modules(metadata).to_json
+    end
+  end
+end

data/lib/puppet_forge_server/app/version3.rb

@@ -0,0 +1,68 @@
+# -*- encoding: utf-8 -*-
+#
+# Copyright 2014 North Development AB
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'sinatra/base'
+
+module PuppetForgeServer::App
+  class Version3 < Sinatra::Base
+    include PuppetForgeServer::Api::V3::Releases
+    include PuppetForgeServer::Api::V3::Modules
+    include PuppetForgeServer::Utils::Buffer
+
+    def initialize(backends)
+      super(nil)
+      @backends = backends
+    end
+
+    get '/v3/releases' do
+      halt 400, {'error' => 'The number of version constraints in the query does not match the number of module names'}.to_json unless params[:module]
+
+      author, name, version = params[:module].split '-'
+      metadata = @backends.map do |backend|
+        backend.get_metadata(author, name, {:version => version})
+      end.flatten.uniq
+
+      halt 404, {'pagination' => {'next' => false}, 'results' => []}.to_json if metadata.empty?
+      {'pagination' => {'next' => false}, 'results' => get_releases(metadata)}.to_json
+    end
+
+    get '/v3/files/*' do
+      captures = params[:captures].first
+      buffer = get_buffer(@backends, captures)
+
+      halt 404, {'errors' => ['404 Not found']}.to_json unless buffer
+
+      content_type 'application/octet-stream'
+      attachment captures.split('/').last
+      download buffer
+    end
+
+    get '/v3/modules/:author-:name' do
+      author = params[:author]
+      name = params[:name]
+
+      halt 400, {'errors' => "'#{params[:module]}' is not a valid module slug"}.to_json unless author && name
+
+      metadata = @backends.map do |backend|
+        backend.get_metadata(author, name)
+      end.flatten.uniq
+
+      halt 404, {'errors' => ['404 Not found']}.to_json if metadata.empty?
+
+      get_modules(metadata).to_json
+    end
+  end
+end

data/lib/puppet_forge_server/backends/directory.rb

@@ -0,0 +1,67 @@
+# -*- encoding: utf-8 -*-
+#
+# Copyright 2014 North Development AB
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'digest/md5'
+require 'pathname'
+
+module PuppetForgeServer::Backends
+  class Directory
+    include PuppetForgeServer::Utils::Archiver
+
+    # Give highest priority to locally hosted modules
+    @@PRIORITY = 0
+    attr_reader :PRIORITY
+
+    def initialize(url)
+      @module_dir = url
+    end
+
+    def query_metadata(query, options = {})
+      get_file_metadata("*#{query}*.tar.gz", options)
+    end
+
+    def get_metadata(author, name, options = {})
+      version = options[:version] ? options[:version] : '*'
+      get_file_metadata("#{author}-#{name}-#{version}.tar.gz", options)
+    end
+
+    def get_file_buffer(relative_path)
+      path = File.join(File.expand_path(@module_dir), relative_path)
+      File.open(path, 'r') if File.exist?(path)
+    end
+
+    private
+    def read_metadata(archive_path)
+      metadata_file = read_entry(archive_path, %r[[^/]+/metadata\.json$])
+      JSON.parse(metadata_file)
+    rescue => error
+      warn "Error reading from module archive #{archive_path}: #{error}"
+      return nil
+    end
+
+    def get_file_metadata(file_name, options)
+      options = ({:with_checksum => true}).merge(options)
+      Dir["#{@module_dir}/**/#{file_name}"].map do |path|
+        checksum = Digest::MD5.hexdigest(File.read(path)) if options[:with_checksum] == true
+        {
+            :metadata => PuppetForgeServer::Models::Metadata.new(read_metadata(path)),
+            :checksum => checksum,
+            :path => "/#{Pathname.new(path).relative_path_from(Pathname.new(@module_dir))}"
+        }
+      end.compact
+    end
+  end
+end

data/lib/puppet_forge_server/backends/proxy.rb

@@ -0,0 +1,63 @@
+# -*- encoding: utf-8 -*-
+#
+# Copyright 2014 North Development AB
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'json'
+require 'digest/sha1'
+
+module PuppetForgeServer::Backends
+  class Proxy
+
+    def initialize(url, cache_dir, http_client)
+      @url = url
+      @cache_dir = File.join(cache_dir, Digest::SHA1.hexdigest(@url))
+      @http_client = http_client
+
+      # Create directory structure for all alphabetic letters
+      (10...36).each do |i|
+        FileUtils.mkdir_p(File.join(@cache_dir, i.to_s(36)))
+      end
+    end
+
+    def get_file_buffer(relative_path)
+      file_name = relative_path.split('/').last
+      File.join(@cache_dir, file_name[0], file_name)
+      path = Dir["#{@cache_dir}/**/#{file_name}"].first
+      unless File.exist?("#{path}")
+        buffer = download("/#{relative_path}")
+        File.open(File.join(@cache_dir, file_name[0], file_name), 'wb') do |file|
+          file.write(buffer.read)
+        end
+        path = File.join(@cache_dir, file_name[0], file_name)
+      end
+      File.open(path, 'rb')
+    rescue OpenURI::HTTPError
+      # ignore
+    end
+
+    protected
+    def get(relative_url)
+      @http_client.get(url(relative_url))
+    end
+
+    def download(relative_url)
+      @http_client.download(url(relative_url))
+    end
+
+    def url(relative_url)
+      @url + relative_url
+    end
+  end
+end

data/lib/puppet_forge_server/backends/proxy_v1.rb

@@ -0,0 +1,69 @@
+# -*- encoding: utf-8 -*-
+#
+# Copyright 2014 North Development AB
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'json'
+require 'digest/md5'
+
+module PuppetForgeServer::Backends
+  class ProxyV1 < PuppetForgeServer::Backends::Proxy
+
+    # Priority should be lower than v3 API proxies as v3 requires less API calls
+    @@PRIORITY = 15
+    attr_reader :PRIORITY
+
+    def initialize(url, cache_dir, http_client = PuppetForgeServer::Http::HttpClient.new)
+      super(url, cache_dir, http_client)
+    end
+
+    def get_metadata(author, name, options = {})
+      options = ({:with_checksum => true}).merge(options)
+      query ="#{author}/#{name}"
+      get_module_metadata(JSON.parse(get("/modules.json?q=#{query}")).select { |e| e['full_name'].match("#{query}") }, options)
+    end
+
+    def query_metadata(query, options = {})
+      options = ({:with_checksum => true}).merge(options)
+      get_module_metadata(JSON.parse(get("/modules.json?q=#{query}")).select { |e| e['full_name'].match("*#{query}*") }, options)
+    end
+
+    private
+    def read_metadata(element, release)
+      element['project_page'] = element['project_url']
+      element['name'] = element['full_name'] ? element['full_name'].gsub('/', '-') : element['name']
+      element['description'] = element['desc']
+      element['version'] = release['version'] ? release['version'] : element['version']
+      element['dependencies'] = release['dependencies'] ? release['dependencies'] : []
+      %w(project_url full_name releases tag_list desc).each { |key| element.delete(key) }
+      element
+    end
+
+    def get_module_metadata(modules, options)
+      modules.map do |element|
+        version = options['version'] ? "&version=#{options['version']}" : ''
+        JSON.parse(get("/api/v1/releases.json?module=#{element['author']}/#{element['name']}#{version}")).values.first.map do |release|
+          tags = element['tag_list'] ? element['tag_list'] : nil
+          raw_metadata = read_metadata(element, release)
+          {
+              :metadata => PuppetForgeServer::Models::Metadata.new(raw_metadata),
+              :checksum => options[:with_checksum] ? Digest::MD5.hexdigest(File.read(get_file_buffer(release['file']))) : nil,
+              :path => "#{release['file']}",
+              :tags => tags
+          }
+        end
+      end.compact
+    end
+  end
+end

data/lib/puppet_forge_server/backends/proxy_v3.rb

@@ -0,0 +1,63 @@
+# -*- encoding: utf-8 -*-
+#
+# Copyright 2014 North Development AB
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'json'
+
+module PuppetForgeServer::Backends
+  class ProxyV3 < PuppetForgeServer::Backends::Proxy
+
+    @@PRIORITY = 10
+    attr_reader :PRIORITY
+
+    def initialize(url, cache_dir, http_client = PuppetForgeServer::Http::HttpClient.new)
+      super(url, cache_dir, http_client)
+    end
+
+    def get_metadata(author, name, options = {})
+      options = ({:with_checksum => true}).merge(options)
+      query ="#{author}-#{name}"
+      releases = options[:version] ? [JSON.parse(get("/v3/releases/#{query}-#{options[:version]}"))] : get_all_release_pages("/v3/releases?module=#{query}")
+      get_release_metadata(releases)
+    end
+
+    def query_metadata(query, options = {})
+      author, name = query.split('-')
+      get_metadata(author, name, options) if author && name
+    end
+
+    private
+    def get_all_release_pages(next_page)
+      releases = []
+      begin
+        result = JSON.parse(get(next_page))
+        releases += result['results']
+        next_page = result['pagination']['next']
+      end while next_page
+      releases
+    end
+
+    def get_release_metadata(releases)
+      releases.map do |element|
+        {
+            :metadata => PuppetForgeServer::Models::Metadata.new(element['metadata']),
+            :checksum => element['file_md5'],
+            :path => element['file_uri'],
+            :tags => element['tags']
+        }
+      end.compact
+    end
+  end
+end

data/lib/puppet_forge_server/errors.rb

@@ -0,0 +1,20 @@
+# -*- encoding: utf-8 -*-
+#
+# Copyright 2014 North Development AB
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+module PuppetForgeServer::Errors
+  class Expected
+  end
+end