wonko_the_sane 0.1.1

data/lib/wonko_the_sane/util/deep_storage_cache.rb

@@ -0,0 +1,65 @@
+require 'aws-sdk-resources'
+require 'wonko_the_sane/util/http_cache'
+
+module WonkoTheSane
+  module Util
+    class DeepStorageCache
+      def initialize(manifest_file)
+        lines = File.read('.aws-credentials').split "\n"
+        @resource = Aws::S3::Resource.new region: 'eu-west-1', credentials: Aws::Credentials.new(lines[0], lines[1])
+        @bucket = @resource.bucket 'wonkoweb-02jandal-xyz'
+
+        @manifest = @bucket.object 'manifest.json'
+        @entries = @manifest.exists? ? JSON.parse(@manifest.get.body, symbolize_keys: true) : {}
+      end
+
+      def get_info(url, options = {})
+        return @entries[url] if @entries.key? url
+
+        ctxt = options[:ctxt] || 'DeepStorageCache'
+
+        file = HTTPCache.file url, check_stale: false, ctxt: options[:ctxt]
+        info = info_for_file file, url
+
+        @entries[url] = info
+        @manifest.put body: JSON.pretty_generate(@entries)
+
+        object = @bucket.object info[:file]
+        unless object.exists? && object.size == info[:size]
+          TaskStack.in_background do
+            # convert the hex-encoded md5 to a base64-encoded md5, which is what S3 expects
+            # http://anthonylewis.com/2011/02/09/to-hex-and-back-with-ruby/
+            md5 = [info[:md5].scan(/../).map { |x| x.hex.chr }.join].pack 'm0'
+
+            Logging.logger[ctxt].debug "Uploading backup of #{url} to S3..."
+            object.put body: file, content_md5: md5, metadata: Hash[info.map { |k,v| [k.to_s, v.to_s]}]
+            Logging.logger[ctxt].debug 'Backup successfully uploaded to S3'
+          end
+        end
+
+        info
+      end
+
+      def self.get_info(url, options = {})
+        if File.exists? '.aws-credentials'
+          @@instance ||= DeepStorageCache.new 'cache/deep_storage.json'
+          @@instance.get_info url, options
+        else
+          info_for_file HTTPCache.file(url, check_stale: false, ctxt: options[:ctxt]), url
+        end
+      end
+
+      private
+
+      def info_for_file(file, url)
+        {
+            url: url,
+            file: url.gsub(/[&:$@=+,?\\^`><\{\}\[\]#%'"~|]/, '_'),
+            size: file.size,
+            md5: FileHashCache.get_md5(file),
+            sha256: FileHashCache.get(file)
+        }
+      end
+    end
+  end
+end

data/lib/wonko_the_sane/util/extraction_cache.rb

@@ -0,0 +1,30 @@
+class ExtractionCache
+  def initialize(basedir)
+    @basedir = basedir
+    FileUtils.mkdir_p @basedir unless Dir.exist? @basedir
+  end
+
+  def get(archive, type, file)
+    out = path(archive, type, file)
+    FileUtils.mkdir_p File.dirname(out) unless Dir.exist? File.dirname(out)
+    if not File.exist? out
+      if type == :zip
+        Zip::File.open archive do |arch|
+          File.write out, arch.glob(file).first.get_input_stream.read
+        end
+      end
+    end
+
+    return File.read out
+  end
+
+  @@defaultCache = ExtractionCache.new 'cache/extraction'
+  def self.get(archive, type, file)
+    @@defaultCache.get archive, type, file
+  end
+
+  private
+  def path(archive, type, file)
+    @basedir + '/' + File.basename(archive) + '/' + file
+  end
+end

data/lib/wonko_the_sane/util/file_hash_cache.rb

@@ -0,0 +1,42 @@
+class FileHashCache
+  def initialize(file, algorithm)
+    @file = file
+    @algorithm = algorithm
+    @data = JSON.parse File.read(@file), symbolize_names: true if File.exists? @file
+    @data ||= {}
+  end
+
+  def get(file)
+    name = (file.is_a?(File) ? file.path : file).to_sym
+    timestamp = (file.is_a?(File) ? file.mtime : File.mtime(file)).to_i
+    size = file.is_a?(File) ? file.size : File.size(file)
+    if not @data[name] or not @data[name][:timestamp] == timestamp or not @data[name][:size] == size
+      hash = digest(file.is_a?(File) ? file.read : File.read(file))
+      @data[name] = {
+          timestamp: timestamp,
+          size: size,
+          hash: hash
+      }
+      File.write @file, JSON.pretty_generate(@data)
+    end
+    return @data[name][:hash]
+  end
+
+  def digest(data)
+    if @algorithm == :sha256
+      Digest::SHA256.hexdigest data
+    elsif @algorithm == :md5
+      Digest::MD5.hexdigest data
+    end
+  end
+
+  @@defaultCache = FileHashCache.new 'cache/filehashes', :sha256
+  def self.get(file)
+    @@defaultCache.get file
+  end
+
+  @@md5Cache = FileHashCache.new 'cache/filehashes.md5', :md5
+  def self.get_md5(file)
+    @@md5Cache.get file
+  end
+end

data/lib/wonko_the_sane/util/http_cache.rb

@@ -0,0 +1,143 @@
+# http://www.ericson.net/content/2011/04/caching-http-requests-with-ruby/
+# TODO proper etags and other caching stuff
+class HTTPCache
+  def initialize(basedir)
+    @basedir = basedir
+    FileUtils.mkdir_p @basedir unless Dir.exist? @basedir
+    @etags = {}
+    @etags = JSON.parse File.read(@basedir + '/etags.json') if File.exists? @basedir + '/etags.json'
+  end
+
+  # HTTP GETs a url if it doesn't exist locally
+  def get(ctxt, url, key, check_stale = true)
+    fetch ctxt, url, key, check_stale
+    IO.read @basedir + '/' + key
+  end
+
+  def file(ctxt, url, key, check_stale = true)
+    fetch ctxt, url, key, check_stale
+    File.new @basedir + '/' + key, 'r'
+  end
+
+  private
+  def fetch(ctxt, url, key, check_stale)
+    cached_path = @basedir + '/' + key
+    cached_dir = File.dirname cached_path
+    FileUtils.mkdir_p cached_dir unless Dir.exist? cached_dir
+
+    TaskStack.in_background do
+      if should_check cached_path, check_stale
+        Logging.logger[ctxt.to_s].debug "DL: #{url}"
+        resp = http_get ctxt.to_s, url, cached_path
+        unless resp == nil
+          File.open(cached_path, 'w') do |f|
+            f.write resp.body
+          end
+        end
+      end
+    end
+  end
+
+  # get a file, using the local cached file modified timestamp to make sture we don't re-download stuff pointlessly
+  # this also *should* handle redirection properly
+  def http_get(ctxt, url, cached_path, limit = 10, http = nil)
+    # too many redirects...
+    raise ArgumentError, 'too many HTTP redirects' if limit == 0
+
+    uri = url.is_a?(URI) ? url : URI.parse(url)
+
+    local_date = Time.parse("1985-10-28")
+    local_date = File.mtime cached_path if File.exists? cached_path
+
+    if http.nil?
+      Net::HTTP.start uri.hostname, uri.port, :use_ssl => uri.scheme == 'https' do |http|
+        return http_get_internal ctxt, uri, cached_path, limit, http, local_date
+      end
+    else
+      return http_get_internal ctxt, uri, cached_path, limit, http, local_date
+    end
+  end
+  def http_get_internal(ctxt, uri, cached_path, limit = 10, http = nil, local_date = nil)
+    existing_etag = @etags[uri]
+
+    # start by doing a HEAD request
+    head_req = Net::HTTP::Head.new uri
+    head_req.add_field 'If-None-Match', existing_etag if existing_etag
+    head_req.add_field 'If-Modified-Since', local_date.httpdate
+    head_resp = http.request head_req
+
+    case head_resp
+    when Net::HTTPSuccess
+      # don't re-check this
+      checked cached_path
+
+      remote_date = head_resp['Last-Modified'] ? Time.httpdate(head_resp['Last-Modified']) : Time.now
+      new_etag = head_resp['ETag']
+
+      # if the remote resource has been modified later than the local file, grab it and return it
+      if remote_date > local_date || existing_etag != new_etag || !file_valid?(head_resp, cached_path)
+        req = Net::HTTP::Get.new(uri)
+        resp = http.request Net::HTTP::Get.new(uri)
+        Logging.logger[ctxt].debug 'GOT FULL FILE'
+
+        @etags[uri] = new_etag if new_etag
+        File.write @basedir + '/etags.json', JSON.pretty_generate(@etags)
+
+        return resp
+      else
+        Logging.logger[ctxt].debug 'CACHE HIT'
+        return nil
+      end
+    when Net::HTTPRedirection
+      if head_resp.code == "304"
+        Logging.logger[ctxt].debug 'CACHE HIT'
+        checked cached_path
+        return nil
+      end
+
+      location = head_resp['Location']
+      Logging.logger[ctxt].debug "Redirected to #{location} - code #{head_resp.code}"
+      newurl = URI.parse location
+      newurl = URI.join uri.to_s, location if newurl.relative?
+      return http_get ctxt, newurl, cached_path, limit - 1, http
+    else
+      Logging.logger[ctxt].warn "#{location} failed: #{head_resp.code}"
+      checked cached_path
+      return nil
+    end
+  end
+
+  def file_valid?(response, path)
+    if response['Content-Length']
+      return false if response['Content-Length'].to_i != File.size(path)
+    end
+    if response['Content-MD5']
+      return false if response['Content-MD5'] != FileHashCache.get_md5(path)
+    end
+    return true
+  end
+
+  @@checked_paths = Set.new
+  def should_check(cached_path, check_stale)
+    # if the file doesn't exist locally, or we should check for stale cache
+    if !File.exist? cached_path or check_stale
+      # but only once per run
+      return !@@checked_paths.include?(cached_path)
+    end
+    # otherwise don't check
+    return false
+  end
+
+  def checked(cached_path)
+    @@checked_paths.add cached_path
+  end
+
+  public
+  @@defaultCatcher = HTTPCache.new 'cache/network'
+  def self.get(url, options = {})
+    @@defaultCatcher.get(options[:ctxt] || 'Download', url, (options.key?(:key) ? options[:key] : url), options[:check_stale] || false)
+  end
+  def self.file(url, options = {})
+    @@defaultCatcher.file(options[:ctxt] || 'Download', url, (options.key?(:key) ? options[:key] : url), options[:check_stale] || false)
+  end
+end

data/lib/wonko_the_sane/util/maven_identifier.rb

@@ -0,0 +1,39 @@
+module WonkoTheSane
+  module Util
+    class MavenIdentifier
+      attr_accessor :group
+      attr_accessor :artifact
+      attr_accessor :version
+      attr_accessor :classifier
+      attr_accessor :extension
+
+      def initialize(string)
+        parts = string.match /(?<group>[^:@]+):(?<artifact>[^:@]+):(?<version>[^:@]+)(:(?<classifier>[^:@]+))?(@(?<extension>[^:@]+))?/
+        @group = parts[:group]
+        @artifact = parts[:artifact]
+        @version = parts[:version]
+        @classifier = parts[:classifier]
+        @extension = parts[:extension] ? parts[:extension] : 'jar'
+      end
+
+      def to_path
+        path = @group.gsub(/\./, '/') + '/' + @artifact + '/' + @version + '/' + @artifact + '-' + @version
+        if @classifier
+          path = path + '-' + @classifier
+        end
+        return path + '.' + @extension
+      end
+
+      def to_name
+        name = @group + ':' + @artifact + ':' + @version
+        if @classifier
+          name = name + ':' + @classifier
+        end
+        if @extension != 'jar'
+          name = name + '@' + @extension
+        end
+        return name
+      end
+    end
+  end
+end

data/lib/wonko_the_sane/util/task_stack.rb

@@ -0,0 +1,21 @@
+class TaskStack
+  @@queue = []
+  def self.push(task)
+    @@queue.push task
+  end
+  def self.push_defered(task)
+    @@queue.unshift task
+  end
+  def self.pop
+    task = @@queue.pop
+    task.call
+  end
+  def self.pop_all
+    self.pop until @@queue.empty?
+  end
+  def self.in_background(&block)
+    thread = Thread.new &block
+    TaskStack.pop_all
+    thread.join
+  end
+end

data/lib/wonko_the_sane/version.rb

@@ -0,0 +1,3 @@
+module WonkoTheSane
+  VERSION = '0.1.1'
+end

data/lib/wonko_the_sane/version_index.rb

@@ -0,0 +1,33 @@
+class VersionIndex
+  attr_accessor :uid
+  attr_accessor :name
+  attr_reader :versions
+
+  def initialize(uid)
+    @uid = uid
+    @versions = []
+  end
+
+  def add_version(version)
+    if version.is_a? WonkoVersion
+      remove_version version # remove any previous versions
+      @versions << version
+    end
+  end
+  def remove_version(version)
+    @versions.select! do |ver|
+      version.version != ver.version
+    end
+  end
+  def self.get_full_version(version)
+    if File.exist? version.local_filename
+      Reader.read_version File.read(version.local_filename)
+    else
+      nil
+    end
+  end
+
+  def self.local_filename(uid)
+    'files/' + uid + '.json'
+  end
+end

data/lib/wonko_the_sane/version_parser.rb

@@ -0,0 +1,115 @@
+class VersionParser
+  private
+  @@cache = {}
+  def self.parse(string)
+    if @@cache.has_key? string
+      return @@cache[string]
+    end
+    appendix = string.scan(/\-.*$/).first
+    string = string.sub /\-.*$/, ''
+    sections = string.split '.'
+    sections.map! do |sec|
+      test = Integer sec rescue nil
+      if test.nil?
+        sec
+      else
+        test
+      end
+    end
+
+    result = {
+        appendix: appendix,
+        sections: sections
+    }
+    @@cache[string] = result
+    return result
+  end
+
+  def self.appendix_values(appendix)
+    str = appendix.scan /[a-zA-Z]*/
+    digits = appendix.scan(/\d*/).join.to_i
+    ret = case str
+      when 'a'
+        [0, digits]
+      when 'alpha'
+        [0, digits]
+      when 'b'
+        [1, digits]
+      when 'beta'
+        [1, digits]
+      when 'rc'
+        [2, digits]
+      when 'pre'
+        [2, digits]
+          end
+    return ret ? ret : [-1, digits]
+  end
+
+  def self.compare_values(first, second)
+    if first < second
+      -1
+    elsif first > second
+      1
+    else
+      0
+    end
+  end
+
+  public
+  def self.compare(string1, string2)
+    par1 = VersionParser.parse string1
+    par2 = VersionParser.parse string2
+    size = [par1[:sections].length, par2[:sections].length].max
+    ret = 0
+    size.times do |index|
+      val1 = par1[:sections].length > index ? par1[:sections][index] : 0
+      val2 = par2[:sections].length > index ? par2[:sections][index] : 0
+      if val1.is_a? Integer and val2.is_a? Integer
+        ret = VersionParser.compare_values val1, val2
+      elsif val1.is_a? Integer
+        ret = VersionParser.compare_values val1.to_s, val2
+      elsif val2.is_a? Integer
+        ret = VersionParser.compare_values val1, val2.to_s
+      else
+        ret = VersionParser.compare_values val1.to_s, val2.to_s
+      end
+      break unless ret == 0
+    end
+    if ret == 0
+      if par1[:appendix] and par2[:appendix]
+        appendix1 = VersionParser.appendix_values par1[:appendix]
+        appendix2 = VersionParser.appendix_values par2[:appendix]
+        ret = VersionParser.compare_values appendix1[0], appendix2[0]
+        if ret == 0
+          ret = VersionParser.compare_values appendix1[1], appendix2[1]
+        end
+      elsif par1[:appendix]
+        ret = -1
+      elsif par2[:appendix]
+        ret = 1
+      end
+    end
+    return ret
+  end
+
+  def self.less?(string1, string2)
+    VersionParser.compare(string1, string2) == -1
+  end
+  def self.greater?(string1, string2)
+    VersionParser.compare(string1, string2) == 1
+  end
+  def self.equal?(string1, string2)
+    VersionParser.compare(string1, string2) == 0
+  end
+  def self.less_or_equal?(string1, string2)
+    ret = VersionParser.compare string1, string2
+    ret == 0 or ret == -1
+  end
+  def self.greater_or_equal?(string1, string2)
+    ret = VersionParser.compare string1, string2
+    ret == 0 or ret == 1
+  end
+  def self.not_equal?(string1, string2)
+    VersionParser.compare(string1, string2) != 0
+  end
+end