wonko_the_sane 0.1.2 → 0.1.4

data/lib/wonko_the_sane/util/configuration.rb

@@ -1,11 +1,63 @@
+require 'yaml'
+
 module WonkoTheSane
   module Util
     class Configuration
+      class Aws
+        attr_accessor :client_id
+        attr_accessor :client_secret
+        attr_accessor :bucket
+      end
+
+      class WonkoWeb
+        attr_accessor :host
+        attr_accessor :email
+        attr_accessor :token
+        attr_accessor :name
+      end
+
+      attr_reader :aws
+      attr_reader :wonkoweb
       attr_reader :lists
       attr_accessor :data_path
+      attr_accessor :out_dir
+
+      def initialize
+        @lists = []
+        @aws = Aws.new
+        @wonkoweb = WonkoWeb.new
+      end
+
+      def load_from_env
+        @aws.client_id = ENV['WTS_AWS_CLIENT_ID'] if ENV['WTS_AWS_CLIENT_ID']
+        @aws.client_secret = ENV['WTS_AWS_CLIENT_SECRET'] if ENV['WTS_AWS_CLIENT_SECRET']
+        @aws.bucket = ENV['WTS_AWS_BUCKET'] if ENV['WTS_AWS_BUCKET']
+
+        @wonkoweb.host = ENV['WTS_WONKOWEB_HOST'] if ENV['WTS_WONKOWEB_HOST']
+        @wonkoweb.email = ENV['WTS_WONKOWEB_EMAIL'] if ENV['WTS_WONKOWEB_EMAIL']
+        @wonkoweb.token = ENV['WTS_WONKOWEB_TOKEN'] if ENV['WTS_WONKOWEB_TOKEN']
+        @wonkoweb.token = ENV['WTS_WONKOWEB_NAME'] if ENV['WTS_WONKOWEB_NAME']
+
+        @data_path = ENV['WTS_DATA_PATH'] if ENV['WTS_DATA_PATH']
+        @out_dir = ENV['WTS_OUT_DIR'] if ENV['WTS_OUT_DIR']
+      end
+
+      def load_from_file(filename)
+        raw = YAML.load_file filename
+        @aws.client_id = raw['aws']['client_id']
+        @aws.client_secret = raw['aws']['client_secret']
+        @aws.bucket = raw['aws']['bucket']
+
+        @wonkoweb.host = raw['wonkoweb']['host']
+        @wonkoweb.email = raw['wonkoweb']['email']
+        @wonkoweb.token = raw['wonkoweb']['token']
+        @wonkoweb.name = raw['wonkoweb']['name']
+
+        @data_path = raw['data_path']
+        @out_dir = raw['out_dir']
+      end
 
       def register_list(list)
-        @lists ||= []
         case list
         when String
           register_list list.to_sym
@@ -24,7 +76,7 @@ module WonkoTheSane
           register_list ForgeFilesModsList.new(uid.to_s, urlId)
         end if sources[:forgefiles]
         sources[:jenkins].each do |obj|
-          register_list JenkinsVersionList.new(obj[:uid], obj[:url], obj[:artifact], obj[:fileRegex])
+          register_list JenkinsVersionList.new(obj[:uid], obj[:url], obj[:artifact], obj[:@file_regex])
         end if sources[:jenkins]
         sources[:curse].each do |obj|
           register_list CurseVersionList.new(obj[:uid], obj[:id], obj[:fileregex])

data/lib/wonko_the_sane/util/deep_storage_cache.rb

@@ -1,25 +1,27 @@
 require 'aws-sdk-resources'
 require 'wonko_the_sane/util/http_cache'
+require 'wonko_the_sane'
 
 module WonkoTheSane
   module Util
     class DeepStorageCache
       def initialize
         @resource = Aws::S3::Resource.new region: 'eu-west-1',
-                                          credentials: Aws::Credentials.new(Settings[:aws][:client_id],
-                                                                            Settings[:aws][:client_secret])
-        @bucket = @resource.bucket 'wonkoweb-02jandal-xyz'
+                                          credentials: Aws::Credentials.new(WonkoTheSane.configuration.aws.client_id,
+                                                                            WonkoTheSane.configuration.aws.client_secret)
+        @bucket = @resource.bucket WonkoTheSane.configuration.aws.bucket
 
         @manifest = @bucket.object 'manifest.json'
         @entries = @manifest.exists? ? JSON.parse(@manifest.get.body.read, symbolize_keys: true) : {}
       end
 
+      # fetch file (if not available), get info hash, upload metadata and file to S3, return info hash
       def get_info(url, options = {})
         return @entries[url] if @entries.key? url
 
         ctxt = options[:ctxt] || 'DeepStorageCache'
 
-        file = HTTPCache.file url, check_stale: false, ctxt: options[:ctxt]
+        file = HTTPCache.file url, check_stale: false, ctxt: ctxt
         info = self.class.info_for_file file, url
 
         @entries[url] = info
@@ -43,7 +45,7 @@ module WonkoTheSane
             object.put body: file,
                        content_md5: md5,
                        content_type: content_type,
-                       metadata: Hash[info.map { |k,v| [k.to_s, v.to_s]}]
+                       metadata: Hash[info.map { |k, v| [k.to_s, v.to_s] }]
             Logging.logger[ctxt].debug 'Backup successfully uploaded to S3'
           end
         end
@@ -52,9 +54,9 @@ module WonkoTheSane
       end
 
       def self.get_info(url, options = {})
-        if Settings[:aws][:client_id]
-          @@instance ||= DeepStorageCache.new
-          @@instance.get_info url, options
+        if WonkoTheSane.configuration.aws.client_id
+          @instance ||= DeepStorageCache.new
+          @instance.get_info url, options
         else
           info_for_file HTTPCache.file(url, check_stale: false, ctxt: options[:ctxt]), url
         end

data/lib/wonko_the_sane/util/extraction_cache.rb

@@ -7,7 +7,7 @@ class ExtractionCache
   def get(archive, type, file)
     out = path(archive, type, file)
     FileUtils.mkdir_p File.dirname(out) unless Dir.exist? File.dirname(out)
-    if not File.exist? out
+    unless File.exist? out
       if type == :zip
         Zip::File.open archive do |arch|
           File.write out, arch.glob(file).first.get_input_stream.read
@@ -15,15 +15,16 @@ class ExtractionCache
       end
     end
 
-    return File.read out
+    File.read out
   end
 
-  @@defaultCache = ExtractionCache.new 'cache/extraction'
   def self.get(archive, type, file)
-    @@defaultCache.get archive, type, file
+    @cache ||= ExtractionCache.new 'cache/extraction'
+    @cache.get archive, type, file
   end
 
   private
+
   def path(archive, type, file)
     @basedir + '/' + File.basename(archive) + '/' + file
   end

data/lib/wonko_the_sane/util/file_hash_cache.rb

@@ -10,7 +10,7 @@ class FileHashCache
     name = (file.is_a?(File) ? file.path : file).to_sym
     timestamp = (file.is_a?(File) ? file.mtime : File.mtime(file)).to_i
     size = file.is_a?(File) ? file.size : File.size(file)
-    if not @data[name] or not @data[name][:timestamp] == timestamp or not @data[name][:size] == size
+    if !@data[name] || !@data[name][:timestamp] == timestamp || !@data[name][:size] == size
       hash = digest(file.is_a?(File) ? file.read : File.read(file))
       @data[name] = {
           timestamp: timestamp,
@@ -19,7 +19,7 @@ class FileHashCache
       }
       File.write @file, JSON.pretty_generate(@data)
     end
-    return @data[name][:hash]
+    @data[name][:hash]
   end
 
   def digest(data)
@@ -30,13 +30,13 @@ class FileHashCache
     end
   end
 
-  @@defaultCache = FileHashCache.new 'cache/filehashes', :sha256
   def self.get(file)
-    @@defaultCache.get file
+    @sha256_cache ||= FileHashCache.new 'cache/filehashes.json', :sha256
+    @sha256_cache.get file
   end
 
-  @@md5Cache = FileHashCache.new 'cache/filehashes.md5', :md5
   def self.get_md5(file)
-    @@md5Cache.get file
+    @md5_cache ||= FileHashCache.new 'cache/filehashes.md5.json', :md5
+    @md5_cache.get file
   end
 end

data/lib/wonko_the_sane/util/http_cache.rb

@@ -1,17 +1,26 @@
-# http://www.ericson.net/content/2011/04/caching-http-requests-with-ruby/
-# TODO proper etags and other caching stuff
+# :cookie_jar
+# :timer, response.env[:duration]
+
+require 'faraday'
+require 'faraday_connection_pool'
+require 'faraday/http_cache'
+require 'faraday_middleware'
+require 'faraday-cookie_jar'
+require 'active_support/cache'
+require 'uri'
+
 class HTTPCache
   def initialize(basedir)
     @basedir = basedir
     FileUtils.mkdir_p @basedir unless Dir.exist? @basedir
-    @etags = {}
-    @etags = JSON.parse File.read(@basedir + '/etags.json') if File.exists? @basedir + '/etags.json'
+    @mutex = Mutex.new
+    @connections = {}
+    @store = ActiveSupport::Cache.lookup_store(:file_store, @basedir + '/faraday_cache/')
   end
 
   # HTTP GETs a url if it doesn't exist locally
   def get(ctxt, url, key, check_stale = true)
     fetch ctxt, url, key, check_stale
-    IO.read @basedir + '/' + key
   end
 
   def file(ctxt, url, key, check_stale = true)
@@ -20,124 +29,60 @@ class HTTPCache
   end
 
   private
+
   def fetch(ctxt, url, key, check_stale)
     cached_path = @basedir + '/' + key
     cached_dir = File.dirname cached_path
     FileUtils.mkdir_p cached_dir unless Dir.exist? cached_dir
 
-    TaskStack.in_background do
-      if should_check cached_path, check_stale
-        Logging.logger[ctxt.to_s].debug "DL: #{url}"
-        resp = http_get ctxt.to_s, url, cached_path
-        unless resp == nil
-          File.open(cached_path, 'w') do |f|
-            f.write resp.body
-          end
-        end
-      end
-    end
-  end
-
-  # get a file, using the local cached file modified timestamp to make sture we don't re-download stuff pointlessly
-  # this also *should* handle redirection properly
-  def http_get(ctxt, url, cached_path, limit = 10, http = nil)
-    # too many redirects...
-    raise ArgumentError, 'too many HTTP redirects' if limit == 0
-
-    uri = url.is_a?(URI) ? url : URI.parse(url)
+    return if File.exists?(cached_path) && !check_stale
 
-    local_date = Time.parse("1985-10-28")
-    local_date = File.mtime cached_path if File.exists? cached_path
-
-    if http.nil?
-      Net::HTTP.start uri.hostname, uri.port, :use_ssl => uri.scheme == 'https' do |http|
-        return http_get_internal ctxt, uri, cached_path, limit, http, local_date
-      end
-    else
-      return http_get_internal ctxt, uri, cached_path, limit, http, local_date
-    end
-  end
-  def http_get_internal(ctxt, uri, cached_path, limit = 10, http = nil, local_date = nil)
-    existing_etag = @etags[uri]
-
-    # start by doing a HEAD request
-    head_req = Net::HTTP::Head.new uri
-    head_req.add_field 'If-None-Match', existing_etag if existing_etag
-    head_req.add_field 'If-Modified-Since', local_date.httpdate
-    head_resp = http.request head_req
-
-    case head_resp
-    when Net::HTTPSuccess
-      # don't re-check this
-      checked cached_path
-
-      remote_date = head_resp['Last-Modified'] ? Time.httpdate(head_resp['Last-Modified']) : Time.now
-      new_etag = head_resp['ETag']
-
-      # if the remote resource has been modified later than the local file, grab it and return it
-      if remote_date > local_date || existing_etag != new_etag || !file_valid?(head_resp, cached_path)
-        req = Net::HTTP::Get.new(uri)
-        resp = http.request Net::HTTP::Get.new(uri)
-        Logging.logger[ctxt].debug 'GOT FULL FILE'
-
-        @etags[uri] = new_etag if new_etag
-        File.write @basedir + '/etags.json', JSON.pretty_generate(@etags)
-
-        return resp
-      else
-        Logging.logger[ctxt].debug 'CACHE HIT'
-        return nil
-      end
-    when Net::HTTPRedirection
-      if head_resp.code == "304"
-        Logging.logger[ctxt].debug 'CACHE HIT'
-        checked cached_path
-        return nil
+    TaskStack.in_background do
+      uri = URI.parse url
+      host = URI::HTTP.new(uri.scheme, uri.userinfo, uri.host, uri.port, nil, nil, nil, nil, nil).to_s
+
+      connection = nil
+      @mutex.synchronize do
+        connection_id = host + ctxt.to_s
+        @connections[connection_id] ||= create_faraday host, ctxt.to_s
+        connection = @connections[connection_id]
       end
+      Logging.logger[ctxt.to_s].debug "DL: #{url}"
 
-      location = head_resp['Location']
-      Logging.logger[ctxt].debug "Redirected to #{location} - code #{head_resp.code}"
-      newurl = URI.parse location
-      newurl = URI.join uri.to_s, location if newurl.relative?
-      return http_get ctxt, newurl, cached_path, limit - 1, http
-    else
-      Logging.logger[ctxt].warn "#{location} failed: #{head_resp.code}"
-      checked cached_path
-      return nil
+      response = connection.get uri.path
+      File.write cached_path, response.body
+      response.body
     end
   end
 
-  def file_valid?(response, path)
-    if response['Content-Length']
-      return false if response['Content-Length'].to_i != File.size(path)
-    end
-    if response['Content-MD5']
-      return false if response['Content-MD5'] != FileHashCache.get_md5(path)
+  def create_faraday(host, ctxt)
+    Faraday.new url: host do |faraday|
+      faraday.use :cookie_jar
+      faraday.response :raise_error
+      faraday.response :chunked
+      faraday.use :http_cache,
+                  logger: Logging.logger[ctxt],
+                  shared_cache: true,
+                  serializer: Marshal,
+                  store: @store
+      faraday.response :follow_redirects
+      faraday.request :retry, max: 2, interval: 0.05, interval_randomness: 0.5, backoff_factor: 2,
+                      exceptions: [Faraday::Error::ConnectionFailed]
+
+      faraday.adapter :net_http_pooled
     end
-    return true
   end
 
-  @@checked_paths = Set.new
-  def should_check(cached_path, check_stale)
-    # if the file doesn't exist locally, or we should check for stale cache
-    if !File.exist? cached_path or check_stale
-      # but only once per run
-      return !@@checked_paths.include?(cached_path)
-    end
-    # otherwise don't check
-    return false
-  end
+  public
 
-  def checked(cached_path)
-    @@checked_paths.add cached_path
-  end
+  class << self; attr_accessor :cache; end
+  self.cache = HTTPCache.new 'cache/network'
 
-  public
-  @@defaultCatcher = HTTPCache.new 'cache/network'
   def self.get(url, options = {})
-    @@defaultCatcher.get(options[:ctxt] || 'Download', url, (options.key?(:key) ? options[:key] : url), options[:check_stale] || false)
+    self.cache.get(options[:ctxt] || 'Download', url, (options.key?(:key) ? options[:key] : url), options[:check_stale] || false)
   end
+
   def self.file(url, options = {})
-    @@defaultCatcher.file(options[:ctxt] || 'Download', url, (options.key?(:key) ? options[:key] : url), options[:check_stale] || false)
+    self.cache.file(options[:ctxt] || 'Download', url, (options.key?(:key) ? options[:key] : url), options[:check_stale] || false)
   end
 end

data/lib/wonko_the_sane/util/maven_identifier.rb

@@ -13,26 +13,20 @@ module WonkoTheSane
         @artifact = parts[:artifact]
         @version = parts[:version]
         @classifier = parts[:classifier]
-        @extension = parts[:extension] ? parts[:extension] : 'jar'
+        @extension = parts[:extension] || 'jar'
       end
 
       def to_path
-        path = @group.gsub(/\./, '/') + '/' + @artifact + '/' + @version + '/' + @artifact + '-' + @version
-        if @classifier
-          path = path + '-' + @classifier
-        end
-        return path + '.' + @extension
+        path = "#{@group.gsub /\./, '/'}/#{@artifact}/#{@version}/#{@artifact}-#{@version}"
+        path = "#{path}-#{@classifier}" if @classifier
+        "#{path}.#{@extension}"
       end
 
       def to_name
-        name = @group + ':' + @artifact + ':' + @version
-        if @classifier
-          name = name + ':' + @classifier
-        end
-        if @extension != 'jar'
-          name = name + '@' + @extension
-        end
-        return name
+        name = "#{@group}:#{@artifact}:#{@version}"
+        name = "#{name}:#{@classifier}" if @classifier
+        name = "#{name}@#{@extension}" if @extension != 'jar'
+        name
       end
     end
   end

data/lib/wonko_the_sane/util/task_stack.rb

@@ -1,21 +1,22 @@
 class TaskStack
-  @@queue = []
+  class << self; attr_accessor :queue; end
+  self.queue = []
   def self.push(task)
-    @@queue.push task
+    self.queue.push task
   end
   def self.push_defered(task)
-    @@queue.unshift task
+    self.queue.unshift task
   end
   def self.pop
-    task = @@queue.pop
+    task = self.queue.pop
     task.call
   end
   def self.pop_all
-    self.pop until @@queue.empty?
+    self.pop until self.queue.empty?
   end
   def self.in_background(&block)
     thread = Thread.new &block
     TaskStack.pop_all
-    thread.join
+    thread.join.value
   end
 end