puppet-forge-server 1.8.0 → 1.9.0

data/lib/puppet_forge_server/app/views/modules.haml

@@ -20,7 +20,7 @@
     - modules.each do |element|
       %li{:class => element['private'] ? 'clearfix private' : 'clearfix'}
         .col
-          %h3= "#{element['owner']['username']}/#{element['name']}"
+          %a{:class => "h3", :href => "/module?name=#{element['current_release']['metadata']['name']}" }= "#{element['owner']['username']}/#{element['name']}"
           %p= element['current_release']['metadata']['summary']
           %span.release-info= "Version #{element['current_release']['metadata']['version']}"
           - if element['current_release']['metadata']['issues_url']

data/lib/puppet_forge_server/backends/directory.rb

@@ -56,9 +56,15 @@ module PuppetForgeServer::Backends
     end
 
     private
-    def read_metadata(archive_path)
-      metadata_file = read_from_archive(archive_path, %r[[^/]+/metadata\.json$])
-      JSON.parse(metadata_file)
+    def read_module_data(archive_path)
+      file_contents = read_from_archive(archive_path, %r[^([^/]+/)?(metadata\.json|README\.md)$])
+      metadata = JSON.parse(file_contents.find {|key, value| key =~ /metadata\.json/}[1])
+      begin
+        readme = file_contents.find {|key, value| key =~ /README\.md/}[1]
+      rescue
+        readme = nil
+      end
+      [metadata, readme]
     rescue => error
       warn "Error reading from module archive #{archive_path}: #{error}"
       return nil
@@ -80,14 +86,15 @@ module PuppetForgeServer::Backends
       options = ({:with_checksum => true}).merge(options)
       modules = []
       Dir["#{@module_dir}/**/#{file_name}"].each do |path|
-        metadata_raw = read_metadata(path)
+        metadata_raw, readme = read_module_data(path)
         if metadata_raw
           modules <<
             PuppetForgeServer::Models::Module.new({
             :metadata => parse_dependencies(PuppetForgeServer::Models::Metadata.new(normalize_metadata(metadata_raw))),
             :checksum => options[:with_checksum] == true ? Digest::MD5.file(path).hexdigest : nil,
             :path => "/#{File.basename(path)}",
-            :private => ! @readonly
+            :private => ! @readonly,
+            :readme => readme
           })
         else
           @log.error "Failed reading metadata from #{path}"

data/lib/puppet_forge_server/backends/proxy.rb

@@ -35,14 +35,18 @@ module PuppetForgeServer::Backends
 
     def get_file_buffer(relative_path)
       file_name = relative_path.split('/').last
-      File.join(@cache_dir, file_name[0].downcase, file_name)
+      target_file = File.join(@cache_dir, file_name[0].downcase, file_name)
       path = Dir["#{@cache_dir}/**/#{file_name}"].first
       unless File.exist?("#{path}")
         buffer = download("#{@file_path.chomp('/')}/#{relative_path}")
-        File.open(File.join(@cache_dir, file_name[0].downcase, file_name), 'wb') do |file|
-          file.write(buffer.read)
+        File.open(target_file, 'wb') do |file|
+          bytes = buffer.read
+          file.write(bytes)
+          @log.debug("Saved #{bytes.size} bytes in filesystem cache for path: #{relative_path}, target file: #{target_file}")
         end
-        path = File.join(@cache_dir, file_name[0].downcase, file_name)
+        path = target_file
+      else
+        @log.info("Filesystem cache HIT for path: #{relative_path}")
       end
       File.open(path, 'rb')
     rescue => e
@@ -59,6 +63,27 @@ module PuppetForgeServer::Backends
     protected
     attr_reader :log
 
+    def get_non_mutable(relative_url)
+      file_name = relative_url.split('/').last
+      target_file = File.join(@cache_dir, file_name[0].downcase, file_name)
+      path = Dir["#{@cache_dir}/**/#{file_name}"].first
+      unless File.exist?("#{path}")
+        buffer = get(relative_url)
+        File.open(target_file, 'wb') do |file|
+          file.write(buffer)
+          @log.debug("Saved #{buffer} bytes in filesystem cache for url: #{relative_url}, target file: #{target_file}")
+        end
+        path = target_file
+      else
+        @log.info("Filesystem cache HIT for url: #{relative_url}")
+      end
+      File.binread(path)
+    rescue => e
+      @log.error("#{self.class.name} failed getting non-mutable url '#{relative_url}'")
+      @log.error("Error: #{e}")
+      return nil
+    end
+
     def get(relative_url)
       @http_client.get(url(relative_url))
     end

data/lib/puppet_forge_server/backends/proxy_v3.rb

@@ -30,11 +30,11 @@ module PuppetForgeServer::Backends
     def get_metadata(author, name, options = {})
       query ="#{author}-#{name}"
       begin
-        releases = options[:version] ? [JSON.parse(get("/v3/releases/#{query}-#{options[:version]}"))] : get_all_result_pages("/v3/releases?module=#{query}")
+        releases = get_releases(query, options)
         get_modules(releases)
       rescue => e
-        @log.debug("#{self.class.name} failed querying metadata for '#{query}' with options #{options}")
-        @log.debug("Error: #{e}")
+        @log.error("#{self.class.name} failed querying metadata for '#{query}' with options #{options}")
+        @log.error("Error: #{e}")
         return nil
       end
     end
@@ -44,13 +44,26 @@ module PuppetForgeServer::Backends
         releases = get_all_result_pages("/v3/modules?query=#{query}").map {|element| element['current_release']}
         get_modules(releases)
       rescue => e
-        @log.debug("#{self.class.name} failed querying metadata for '#{query}' with options #{options}")
-        @log.debug("Error: #{e}")
+        @log.error("#{self.class.name} failed querying metadata for '#{query}' with options #{options}")
+        @log.error("Error: #{e}")
         return nil
       end
     end
 
     private
+
+    def get_releases(query, options = {})
+      version = options[:version]
+      unless version.nil?
+        url = "/v3/releases/#{query}-#{version}"
+        buffer = get_non_mutable(url)
+        release = JSON.parse(buffer)
+        [ release ]
+      else
+        get_all_result_pages("/v3/releases?module=#{query}")
+      end
+    end
+
     def get_all_result_pages(next_page)
       results = []
       begin
@@ -80,7 +93,8 @@ module PuppetForgeServer::Backends
           :checksum => element['file_md5'],
           :path => element['file_uri'].gsub(/^#{@@FILE_PATH}/, ''),
           :tags => (element['tags'] + (element['metadata']['tags'] ? element['metadata']['tags'] : [])).flatten.uniq,
-          :deleted_at => element['deleted_at']
+          :deleted_at => element['deleted_at'],
+          :readme => element['readme']
         })
       end
     end

data/lib/puppet_forge_server/http/http_client.rb

@@ -23,6 +23,15 @@ require 'net/http/post/multipart'
 
 module PuppetForgeServer::Http
   class HttpClient
+    include PuppetForgeServer::Utils::CacheProvider
+    include PuppetForgeServer::Utils::FilteringInspecter
+
+    def initialize(cache = nil)
+      cache = cache_instance if cache.nil?
+      cache.extend(PuppetForgeServer::Utils::FilteringInspecter)
+      @log = PuppetForgeServer::Logger.get
+      @cache = cache
+    end
 
     def post_file(url, file_hash, options = {})
       options = { :http => {}, :headers => {}}.merge(options)
@@ -45,11 +54,29 @@ module PuppetForgeServer::Http
       open_uri(url)
     end
 
+    def inspect
+      cache_inspected = @cache.inspect_without [ :@data ]
+      cache_inspected.gsub!(/>$/, ", @size=#{@cache.size}>")
+      inspected = inspect_without [ :@cache ]
+      inspected.gsub(/>$/, ", @cache=#{cache_inspected}>")
+    end
+
     private
+
     def open_uri(url)
-      ::Timeout.timeout(10) do
-        open(url, 'User-Agent' => "Puppet-Forge-Server/#{PuppetForgeServer::VERSION}", :allow_redirections => :safe)
+      hit_or_miss = @cache.include?(url) ? 'HIT' : 'MISS'
+      @log.info "Cache in RAM memory size: #{@cache.size}, #{hit_or_miss} for url: #{url}"
+      contents = @cache.fetch(url) do
+        tmpfile = ::Timeout.timeout(10) do
+          PuppetForgeServer::Logger.get.debug "Fetching data for url: #{url} from remote server"
+          open(url, 'User-Agent' => "Puppet-Forge-Server/#{PuppetForgeServer::VERSION}", :allow_redirections => :safe)
+        end
+        contents = tmpfile.read
+        tmpfile.close
+        contents
       end
+      @log.debug "Data for url: #{url} fetched, #{contents.size} bytes"
+      StringIO.new(contents)
     end
   end
 end

data/lib/puppet_forge_server/logger.rb

@@ -16,6 +16,7 @@
 
 
 require 'logger'
+require 'logger/colors'
 
 module PuppetForgeServer
   class Logger
@@ -45,7 +46,13 @@ module PuppetForgeServer
                       else
                         method_name
                     end
-      @loggers.each { |logger| logger.send(method_name, args.first) }
+      if args.size > 0
+        # setters
+        @loggers.each { |logger| logger.send(method_name, args.first) }
+      else
+        # getters
+        @loggers.collect { |logger| logger.send(method_name) }
+      end
     end
 
     def respond_to?(method_name, include_private = false)
@@ -70,4 +77,4 @@ module PuppetForgeServer
       end
     end
   end
-end
+end

data/lib/puppet_forge_server/models/builder.rb

@@ -27,6 +27,11 @@ module PuppetForgeServer::Models
     end
 
     def to_hash(obj = self)
+      # Quickly return if it's not one of ours
+      unless obj.kind_of? Builder
+        return obj
+      end
+      # Otherwise start building hash
       hash = {}
       obj.instance_variables.each do |var|
         var_value = obj.instance_variable_get(var)
@@ -41,4 +46,4 @@ module PuppetForgeServer::Models
       hash
     end
   end
-end
+end

data/lib/puppet_forge_server/models/metadata.rb

@@ -44,4 +44,4 @@ module PuppetForgeServer::Models
           @version.eql?(other.version)
     end
   end
-end
+end

data/lib/puppet_forge_server/models/module.rb

@@ -18,7 +18,7 @@
 module PuppetForgeServer::Models
   class Module < Builder
 
-    attr_accessor :metadata, :checksum, :path, :private, :deleted_at, :tags
+    attr_accessor :metadata, :checksum, :path, :private, :deleted_at, :tags, :readme
 
     def initialize(attributes)
       super(attributes)
@@ -26,4 +26,4 @@ module PuppetForgeServer::Models
     end
 
   end
-end
+end

data/lib/puppet_forge_server/server.rb

@@ -19,6 +19,7 @@ require 'rack/mount'
 module PuppetForgeServer
   class Server
     include PuppetForgeServer::Utils::OptionParser
+    include PuppetForgeServer::Utils::CacheProvider
     include PuppetForgeServer::Utils::Http
 
     def go(args)
@@ -27,6 +28,7 @@ module PuppetForgeServer
       begin
         options = parse_options(args)
         @log = logging(options)
+        configure_cache(options[:ram_cache_ttl], options[:ram_cache_size])
         backends = backends(options)
         server = build(backends, options[:webui_root])
         announce(options, backends)

data/lib/puppet_forge_server/utils/archiver.rb

@@ -22,9 +22,11 @@ module PuppetForgeServer::Utils
     def read_from_archive(archive, name_regex)
       tar = Gem::Package::TarReader.new(Zlib::GzipReader.open(archive))
       tar.rewind
+      files = {}
       tar.each do |obj|
-        return obj.read if obj.full_name =~ name_regex
+        files[obj.full_name] = obj.read if obj.file? && obj.full_name =~ name_regex
       end
+      return files unless files.empty?
       raise "Given name #{name_regex} not found in #{archive}"
     end
   end

data/lib/puppet_forge_server/utils/cache_provider.rb

@@ -0,0 +1,42 @@
+# -*- encoding: utf-8 -*-
+#
+# Copyright 2015 Centralny Osroder Informatyki (gov.pl)
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'lrucache'
+
+module PuppetForgeServer::Utils
+  module CacheProvider
+
+    opts = PuppetForgeServer::Utils::OptionParser.DEFAULT_OPTIONS
+    @@CACHE = LRUCache.new(:ttl => opts[:ram_cache_ttl], :max_size => opts[:ram_cache_size])
+
+    # Method for fetching application wide cache for fetching HTTP requests
+    #
+    # @return [LRUCache] a instance of cache for application
+    def cache_instance
+      @@CACHE
+    end
+
+    # Configure a application wide cache using LSUCache implementation
+    #
+    # @param [int] ttl a time to live for elements
+    # @param [int] size a maximum size for cache
+    def configure_cache(ttl, size)
+      @@CACHE = LRUCache.new(:ttl => ttl, :max_size => size)
+      PuppetForgeServer::Logger.get.info("Using RAM memory LRUCache with time to live of #{ttl}sec and max size of #{size} elements")
+      nil
+    end
+  end
+end

data/lib/puppet_forge_server/utils/encoding.rb

@@ -0,0 +1,46 @@
+# -*- encoding: utf-8 -*-
+#
+# Copyright 2015 Centralny Osroder Informatyki (gov.pl)
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'iconv'
+
+module PuppetForgeServer::Utils
+  module Encoding
+
+    @@ic = Iconv.new('UTF-8//IGNORE', 'UTF-8')
+
+    # Converts give text to valid UTF-8
+    # @param [string] text given string, can be null
+    # @return [string] output string in utf-8
+    def to_utf8(text)
+      replaced = text
+      unless replaced.nil?
+        replaced = replaced.force_encoding("UTF-8") if is_ascii_8bit?(replaced)
+        replaced = cleanup_utf8(replaced)
+      end
+      replaced
+    end
+
+    private
+
+    def is_ascii_8bit?(text)
+      text.encoding.name == 'ASCII-8BIT'
+    end
+
+    def cleanup_utf8(text)
+      @@ic.iconv(text)
+    end
+  end
+end

data/lib/puppet_forge_server/utils/filtering_inspecter.rb

@@ -0,0 +1,30 @@
+# -*- encoding: utf-8 -*-
+#
+# Copyright 2015 Centralny Osroder Informatyki (gov.pl)
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+module PuppetForgeServer::Utils
+  module FilteringInspecter
+    def self.inspect_without(object, variables)
+      filtered = object.instance_variables.reject { |n| variables.include? n }
+      vars = filtered.map { |n| "#{n}=#{object.instance_variable_get(n).inspect}" }
+      oid = object.object_id << 1
+      "#<%s:0x%x %s>" % [ object.class, oid, vars.join(', ') ]
+    end
+
+    def inspect_without(variables)
+      PuppetForgeServer::Utils::FilteringInspecter.inspect_without(self, variables)
+    end
+  end
+end

data/lib/puppet_forge_server/utils/md_renderer.rb

@@ -0,0 +1,54 @@
+# -*- encoding: utf-8 -*-
+#
+# Copyright 2015 North Development AB
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'tilt/redcarpet'
+
+module PuppetForgeServer::Utils
+  module MarkdownRenderer
+
+    class CustomRenderer < Redcarpet::Render::HTML
+      def block_code(code, lang)
+        output = "<pre>"
+        output << "<code class=\"prettyprint lang-puppet\">#{code}</code>"
+        output << "</pre>"
+      end
+    end
+
+    def markdown(text)
+      options = {
+        filter_html:     true,
+        with_toc_data:   true,
+        hard_wrap:       true,
+        prettify:        true
+      }
+
+      extensions = {
+        autolink:           true,
+        superscript:        true,
+        disable_indented_code_blocks: false,
+        fenced_code_blocks: true,
+        strikethrough: true,
+        quote: true,
+        tables: true
+      }
+
+      renderer = CustomRenderer.new(options)
+      markdown = Redcarpet::Markdown.new(renderer, extensions)
+      markdown.render(text)
+    end
+
+  end
+end