wonko_the_sane 0.1.1 → 0.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: cc57f86a647b80182858a1980048fe31dde2dbc6
-  data.tar.gz: 7ddb8818395ea2e823cf61ac94aa26d362453f79
+  metadata.gz: 40c75f8cc35f2cedea382f8b22f0373f9199ab60
+  data.tar.gz: 419906fc73b134f8ddc6d994d2bab5ec8d277840
 SHA512:
-  metadata.gz: 48502a7dfd108e92d4cae999c99fa87ab236334ef771aec8f9cb6c942dc25a4ae9fc43ee0d81c79fed334415256d95e8cf2d47884767be965b23d25c96dfd678
-  data.tar.gz: 7addb60be6a1876d40cc473367916d4a406937cc9d492baa43b01a2c6d73e81ec9222329a245a0f6d110eb724e74d3efc920fe5babfc0c2a86284ebea90a1000
+  metadata.gz: bcba25e623bf105bc2aa37d9572e3a542ffc58c4eaa30f50d8dc576aa6af10a9472175736e4460687d24564e977070de45935b037001b39be6b7f2ea972f3a25
+  data.tar.gz: 52fd86ab7abd1262bf6b2096ef67e89a83641157b8fb9106787a2d71d4916d17bf85f1d0cecf369b64e0c8b6baa126c51302a84f281549851f32179744553e0c

data/bin/wonko_the_sane

@@ -1,3 +1,4 @@
+#!/usr/bin/env ruby
 require 'wonko_the_sane'
 require 'optparse'
 
@@ -21,7 +22,7 @@ Logging.logger['General'].info 'Welcome to WonkoTheSane!'.yellow
 
 WonkoTheSane.configure do |config|
     config.data_path = File.dirname(__FILE__) + '/../data'
-    config.register_lists_from 'data/sources.json'
+    config.register_lists_from_sources
     config.register_list VanillaVersionList
     config.register_list VanillaLegacyVersionList
     config.register_list LiteLoaderVersionList
@@ -82,6 +83,15 @@ OptionParser.new do |opts|
   opts.on '--update-nem', 'Updates sources.json with data from NEM' do
     update_nem
   end
+  opts.on '--set-config=VAL', 'Sets a configuration value' do |val|
+    parts = val.match /([^:=]*)([:=](.*))?/
+    if parts.length == 4 && !parts[3].nil?
+      Settings[parts[1]] = parts[3]
+    else
+      Settings.delete parts[1]
+    end
+    Settings.save! WonkoTheSane.settings_file
+  end
   opts.on '-h', '--help', 'Prints this help' do
     puts opts
     exit
@@ -89,3 +99,4 @@ OptionParser.new do |opts|
 end.parse!
 
 TaskStack.pop_all
+WonkoTheSane.wonkoweb_uploader.upload_changes!

data/data/minecraft.json

@@ -1,4 +1,4 @@
-{ 
+{
 "versions": [
 	{
 		"id": "1.5.2",

data/lib/wonko_the_sane.rb

@@ -4,9 +4,19 @@ require 'yajl/json_gem'
 require 'fileutils'
 require 'zip'
 require 'set'
+require 'uri'
+require 'net/http'
+require 'digest'
+require 'date'
+require 'time'
+require 'oga'
+require 'configliere'
+require 'httparty'
+require 'active_support/core_ext/hash/indifferent_access'
 
 require 'wonko_the_sane/version'
 require 'wonko_the_sane/tools/update_nem'
+require 'wonko_the_sane/util/benchmark'
 require 'wonko_the_sane/util/configuration'
 require 'wonko_the_sane/util/extraction_cache'
 require 'wonko_the_sane/util/deep_storage_cache'
@@ -30,6 +40,7 @@ require 'wonko_the_sane/versionlists/liteloader_version_list'
 require 'wonko_the_sane/versionlists/vanilla_legacy_version_list'
 require 'wonko_the_sane/versionlists/vanilla_version_list'
 
+require 'wonko_the_sane/wonkoweb_uploader'
 require 'wonko_the_sane/reader_writer'
 require 'wonko_the_sane/registry'
 require 'wonko_the_sane/rules'
@@ -39,16 +50,25 @@ require 'wonko_the_sane/version_parser'
 require 'wonko_the_sane/wonko_version'
 
 module WonkoTheSane
+  def self.wonkoweb_uploader
+    @uploader ||= WonkoWebUploader.new
+  end
+
   def self.lists
     configuration.lists
   end
 
+  def self.settings_file
+    File.dirname(__FILE__) + '/../wonko_the_sane.yml'
+  end
+
   def self.data(file)
     configuration.data_path + '/' + file
   end
 
   def self.data_json(file)
-    JSON.parse File.open(data(file), 'r'), symbolize_keys: true
+    res = JSON.parse File.read data file
+    res.is_a?(Hash) ? HashWithIndifferentAccess.new(res) : res
   end
 
   def self.set_data_json(file, obj)
@@ -64,3 +84,24 @@ module WonkoTheSane
   end
   private_class_method :configuration
 end
+
+Settings.use :config_block, :encrypted, :prompt, :define
+Settings({
+  aws: {
+      client_id: nil,
+      client_secret: nil
+  },
+  wonkoweb: {
+      host: nil,
+      email: nil,
+      token: nil
+  }
+})
+Settings.define 'aws.client_id', encrypted: true
+Settings.define 'aws.client_secret', encrypted: true
+Settings.define 'wonkoweb.host'
+Settings.define 'wonkoweb.name'
+Settings.define 'wonkoweb.token', encrypted: true
+Settings.read WonkoTheSane.settings_file
+Settings[:encrypt_pass] = ENV['ENCRYPT_PASS'] || (print 'Password: '; pwd = STDIN.noecho(&:gets).chomp; puts; pwd)
+Settings.resolve!

data/lib/wonko_the_sane/input/mojang_input.rb

@@ -114,16 +114,16 @@ class MojangInput
     file.client.downloads = object[:libraries].map do |obj|
       MojangInput.sanetize_mojang_library obj
     end.flatten 1
-    mainLib = VersionLibrary.new
-    mainLib.name = "net.minecraft:minecraft:#{file.version}"
-    mainLib.url = 'http://s3.amazonaws.com/Minecraft.Download/versions/' + file.version + '/' + file.version + '.jar'
-    file.client.downloads << mainLib
+    main_lib = VersionLibrary.new
+    main_lib.name = "net.minecraft:minecraft:#{file.version}"
+    main_lib.url = 'http://s3.amazonaws.com/Minecraft.Download/versions/' + file.version + '/' + file.version + '.jar'
+    file.client.downloads << main_lib
     file.client.launchMethod = 'minecraft'
 
-    serverLib = VersionLibrary.new
-    serverLib.name = "net.minecraft:minecraft_server:#{file.version}"
-    serverLib.url = 'http://s3.amazonaws.com/Minecraft.Download/versions/' + file.version + '/minecraft_server.' + file.version + '.jar'
-    file.server.downloads << serverLib
+    server_lib = VersionLibrary.new
+    server_lib.name = "net.minecraft:minecraft_server:#{file.version}"
+    server_lib.url = 'http://s3.amazonaws.com/Minecraft.Download/versions/' + file.version + '/minecraft_server.' + file.version + '.jar'
+    file.server.downloads << server_lib
     file.server.jarModTarget = "net.minecraft:minecraft_server:#{file.version}"
     file.server.launchMethod = 'java.jar'
 

data/lib/wonko_the_sane/reader_writer.rb

@@ -2,7 +2,7 @@ require 'hashie'
 
 module Reader
   def read_version_index(data)
-    json = Hashie::Mash.new JSON.parse(data, symbolize_names: true)
+    json = Hashie::Mash.new data
 
     index = VersionIndex.new json.uid
     index.name = json.name
@@ -21,11 +21,11 @@ module Reader
 
   def read_download(data, key)
     if data[key.to_sym]
-      return data[key.to_sym].map do |dl|
+      data[key.to_sym].map do |dl|
         Download.from_json key, dl
       end
     else
-      return []
+      []
     end
   end
 
@@ -49,7 +49,7 @@ module Reader
   end
 
   def read_version(data)
-    json = Hashie::Mash.new JSON.parse(data)
+    json = Hashie::Mash.new data
 
     file = WonkoVersion.new
     file.is_complete = true
@@ -73,11 +73,11 @@ module Reader
       end
     end if json[:data]
 
-    return file
+    file
   end
 
   def read_index(data)
-    JSON.parse data, symbolize_names: true
+    data.with_indifferent_access
   end
 end
 
@@ -96,7 +96,7 @@ module Writer
       json[:versions] << obj
     end
 
-    return JSON.pretty_generate json
+    json
   end
 
   def write_resource(side, resource, out)
@@ -134,25 +134,26 @@ module Writer
         formatVersion: 0,
         uid: version.uid,
         version: version.version,
-        time: version.time
+        time: version.time.to_s,
+        type: version.type,
+        data: [],
+        requires: []
     }
-    json[:type] = version.type                             if version.type and version.type != ''
     json[:requires] = version.requires.map do |req|
       obj = { uid: req.uid }
       obj[:version] = req.version if req.version
       obj
     end if version.requires and not version.requires.empty?
 
-    json[:data] = []
     write_resource(:client, version.client, json[:data]) if version.is_complete
     write_resource(:server, version.server, json[:data]) if version.is_complete
     write_resource(:common, version.common, json[:data]) if version.is_complete
 
-    JSON.pretty_generate json
+    json
   end
 
   def write_index(index)
-    JSON.pretty_generate index
+    index
   end
 end
 

data/lib/wonko_the_sane/registry.rb

@@ -1,20 +1,17 @@
 class Registry
   def version_index(uid)
-    if File.exist? VersionIndex.local_filename(uid)
-      $rw.read_version_index File.read(VersionIndex.local_filename uid)
-    else
-      VersionIndex.new uid
-    end
+    return VersionIndex.new uid unless File.exists? VersionIndex.local_filename uid
+    $rw.read_version_index JSON.parse File.read(VersionIndex.local_filename uid)
+  end
+
+  def store_version_index(index)
+    File.write index.local_filename, JSON.pretty_generate($rw.write_version_index index)
+    WonkoTheSane.wonkoweb_uploader.file_changed index.uid
   end
 
   def index
-    if File.exists? 'files/index.json'
-      $rw.read_index File.read('files/index.json')
-    else
-      {
-          index: []
-      }
-    end
+    return { index: [] } unless File.exists? 'files/index.json'
+    $rw.read_index JSON.parse File.read 'files/index.json'
   end
 
   def store(version)
@@ -23,34 +20,30 @@ class Registry
     else
       BaseSanitizer.sanitize(version, DownloadsFixer).each do |version|
         Dir.mkdir 'files/' + version.uid unless Dir.exist? 'files/' + version.uid
-        File.write version.local_filename, $rw.write_version(version)
+        File.write version.local_filename, JSON.pretty_generate($rw.write_version version)
+        WonkoTheSane.wonkoweb_uploader.version_changed version.uid, version.version
 
         vindex = version_index version.uid
         vindex.add_version version
-        File.write VersionIndex.local_filename(vindex.uid), $rw.write_version_index(vindex)
+        store_version_index vindex
 
         ind = index
+        next if ind[:index].find { |i| version.uid == i[:uid] } # early exit if the uid already exists in the index
         ind[:formatVersion] = 0
-        ind[:index].each do |i|
-          if version.uid == i[:uid]
-            return
-          end
-        end
-        ind[:index] << {
-            uid: version.uid
-        }
-        File.write 'files/index.json', $rw.write_index(ind)
+        ind[:index] << { uid: version.uid }
+        File.write 'files/index.json', JSON.pretty_generate($rw.write_index ind)
       end
     end
   rescue Exception => e
     Logging.logger[version.uid].error 'Unable to store: ' + version.version
     raise e
   end
+
   def retrieve(id, version)
     if File.exist? WonkoVersion.local_filename(id, version)
-      return $rw.read_version File.read(WonkoVersion.local_filename id, version)
+      $rw.read_version JSON.parse File.read(WonkoVersion.local_filename id, version)
     else
-      return nil
+      nil
     end
   end
 

data/lib/wonko_the_sane/util.rb

@@ -0,0 +1,17 @@
+module Util
+  def self.deep_map_keys(val, &block)
+    Util.deep_map_hash val do |k, v|
+      [yield(k), v]
+    end
+  end
+
+  def self.deep_map(val, &block)
+    if val.is_a? Array
+      val.map { |item| deep_map_hash item, &block }
+    elsif val.is_a? Hash
+      Hash[val.map { |k, v| yield k, deep_map_hash(v, &block) }]
+    else
+      yield nil, val
+    end
+  end
+end

data/lib/wonko_the_sane/util/benchmark.rb

@@ -0,0 +1,29 @@
+require 'benchmark'
+
+module WonkoTheSane
+  module Util
+    class Benchmark
+      def initialize
+        @timers = {}
+        @calls = {}
+      end
+
+      def benchmark(id, &block)
+        @timers[id.to_s] ||= ::Benchmark::Tms.new(0, 0, 0, 0, 0, id.to_s)
+        @calls[id.to_s] ||= 0
+        @calls[id.to_s] += 1
+        ret = nil
+        @timers[id.to_s].add! { ret = yield }
+        ret
+      end
+
+      def print_times(reset = false)
+        @timers.each do |id, timer|
+          puts "#{timer.label}: #{timer.real.round 2}s total, #{(timer.real / @calls[id]).round 2}s/call"
+        end
+        @timers = {} if reset
+        @calls = {} if reset
+      end
+    end
+  end
+end

data/lib/wonko_the_sane/util/configuration.rb

@@ -18,7 +18,7 @@ module WonkoTheSane
         end
       end
 
-      def register_lists_from(file)
+      def register_lists_from_sources
         sources = WonkoTheSane.data_json 'sources.json'
         sources[:forgefiles].each do |uid, urlId|
           register_list ForgeFilesModsList.new(uid.to_s, urlId)

data/lib/wonko_the_sane/util/deep_storage_cache.rb

@@ -4,13 +4,14 @@ require 'wonko_the_sane/util/http_cache'
 module WonkoTheSane
   module Util
     class DeepStorageCache
-      def initialize(manifest_file)
-        lines = File.read('.aws-credentials').split "\n"
-        @resource = Aws::S3::Resource.new region: 'eu-west-1', credentials: Aws::Credentials.new(lines[0], lines[1])
+      def initialize
+        @resource = Aws::S3::Resource.new region: 'eu-west-1',
+                                          credentials: Aws::Credentials.new(Settings[:aws][:client_id],
+                                                                            Settings[:aws][:client_secret])
         @bucket = @resource.bucket 'wonkoweb-02jandal-xyz'
 
         @manifest = @bucket.object 'manifest.json'
-        @entries = @manifest.exists? ? JSON.parse(@manifest.get.body, symbolize_keys: true) : {}
+        @entries = @manifest.exists? ? JSON.parse(@manifest.get.body.read, symbolize_keys: true) : {}
       end
 
       def get_info(url, options = {})
@@ -19,7 +20,7 @@ module WonkoTheSane
         ctxt = options[:ctxt] || 'DeepStorageCache'
 
         file = HTTPCache.file url, check_stale: false, ctxt: options[:ctxt]
-        info = info_for_file file, url
+        info = self.class.info_for_file file, url
 
         @entries[url] = info
         @manifest.put body: JSON.pretty_generate(@entries)
@@ -31,8 +32,18 @@ module WonkoTheSane
             # http://anthonylewis.com/2011/02/09/to-hex-and-back-with-ruby/
             md5 = [info[:md5].scan(/../).map { |x| x.hex.chr }.join].pack 'm0'
 
+            content_type = case url
+                           when /\.zip$/, /\.jar$/
+                             'application/zip'
+                           else
+                             ''
+                           end
+
             Logging.logger[ctxt].debug "Uploading backup of #{url} to S3..."
-            object.put body: file, content_md5: md5, metadata: Hash[info.map { |k,v| [k.to_s, v.to_s]}]
+            object.put body: file,
+                       content_md5: md5,
+                       content_type: content_type,
+                       metadata: Hash[info.map { |k,v| [k.to_s, v.to_s]}]
             Logging.logger[ctxt].debug 'Backup successfully uploaded to S3'
           end
         end
@@ -41,8 +52,8 @@ module WonkoTheSane
       end
 
       def self.get_info(url, options = {})
-        if File.exists? '.aws-credentials'
-          @@instance ||= DeepStorageCache.new 'cache/deep_storage.json'
+        if Settings[:aws][:client_id]
+          @@instance ||= DeepStorageCache.new
           @@instance.get_info url, options
         else
           info_for_file HTTPCache.file(url, check_stale: false, ctxt: options[:ctxt]), url
@@ -51,7 +62,7 @@ module WonkoTheSane
 
       private
 
-      def info_for_file(file, url)
+      def self.info_for_file(file, url)
         {
             url: url,
             file: url.gsub(/[&:$@=+,?\\^`><\{\}\[\]#%'"~|]/, '_'),

data/lib/wonko_the_sane/util/http_cache.rb

@@ -16,7 +16,7 @@ class HTTPCache
 
   def file(ctxt, url, key, check_stale = true)
     fetch ctxt, url, key, check_stale
-    File.new @basedir + '/' + key, 'r'
+    File.open @basedir + '/' + key, 'r'
   end
 
   private

data/lib/wonko_the_sane/version.rb

@@ -1,3 +1,3 @@
 module WonkoTheSane
-  VERSION = '0.1.1'
+  VERSION = '0.1.2'
 end

data/lib/wonko_the_sane/version_index.rb

@@ -14,19 +14,25 @@ class VersionIndex
       @versions << version
     end
   end
+
   def remove_version(version)
     @versions.select! do |ver|
       version.version != ver.version
     end
   end
+
   def self.get_full_version(version)
     if File.exist? version.local_filename
-      Reader.read_version File.read(version.local_filename)
+      Reader.read_version JSON.parse File.read(version.local_filename)
     else
       nil
     end
   end
 
+  def local_filename
+    VersionIndex.local_filename @uid
+  end
+
   def self.local_filename(uid)
     'files/' + uid + '.json'
   end

data/lib/wonko_the_sane/versionlists/base_version_list.rb

@@ -18,38 +18,36 @@ class BaseVersionList
 
   def refresh
     @lastError = nil
-    begin
-      versions = get_versions
+    versions = get_versions
 
-      # check if some versions aren't in @processed (likely new ones) and fetch and process them
-      versions.each do |version|
-        begin
-          next if not version
-          id = version.is_a?(Array) ? version.first : version
-          unless @processed.include? id
-            files = get_version version
-            next if files.nil? or (files.is_a? Array and files.empty?)
+    # check if some versions aren't in @processed (likely new ones) and fetch and process them
+    versions.each do |version|
+      begin
+        next if not version
+        id = version.is_a?(Array) ? version.first : version
+        unless @processed.include? id
+          files = get_version version
+          next if files.nil? or (files.is_a? Array and files.empty?)
 
-            files.flatten.each do |file|
-              file.is_complete = true
-              Registry.instance.store file
-            end if files and files.is_a? Array
-            files.is_complete = true if files and files.is_a? WonkoVersion
-            Registry.instance.store files if files and files.is_a? WonkoVersion
+          files.flatten.each do |file|
+            file.is_complete = true
+            Registry.instance.store file
+          end if files and files.is_a? Array
+          files.is_complete = true if files and files.is_a? WonkoVersion
+          Registry.instance.store files if files and files.is_a? WonkoVersion
 
-            @processed << id
-            write_cache_file
-          end
-        rescue => e
-          logger.error e.message
-          logger.warn e.backtrace.first
-          binding.pry if $stdout.isatty && ENV['DEBUG_ON_ERROR']
-          @lastError = e.message
+          @processed << id
+          write_cache_file
         end
       end
-
-      FileUtils.touch cache_file
     end
+
+    FileUtils.touch cache_file
+  rescue => e
+    logger.error e.message
+    logger.warn e.backtrace.first
+    binding.pry if $stdout.isatty && ENV['DEBUG_ON_ERROR']
+    @lastError = e.message
   end
 
   def logger
@@ -93,11 +91,11 @@ class BaseVersionList
   end
 
   def get_json(url)
-    JSON.parse HTTPCache.file(url, ctxt: @artifact, check_stale: true), symbolize_names: true
+    Yajl::Parser.parse HTTPCache.file(url, ctxt: @artifact, check_stale: true), symbolize_keys: true
   end
 
   def get_json_cached(url)
-    JSON.parse HTTPCache.file(url, ctxt: @artifact, check_stale: false), symbolize_names: true
+    Yajl::Parser.parse HTTPCache.file(url, ctxt: @artifact, check_stale: false), symbolize_keys: true
   end
 end
 Dir.mkdir 'cache' unless Dir.exist? 'cache'

data/lib/wonko_the_sane/versionlists/curse_version_list.rb

@@ -1,7 +1,3 @@
-require_relative 'base_version_list'
-require 'date'
-require 'time'
-
 class CurseVersionList < BaseVersionList
   def initialize(uid, curseId, fileRegex)
     super(uid)

data/lib/wonko_the_sane/wonko_version.rb

@@ -95,7 +95,7 @@ class VersionLibrary < Download
     @mavenBaseUrl = json[:mavenBaseUrl]
 
     # if the absolute url is equal to the expected maven url we clear the absolute url
-    if @url == url
+    if @mavenBaseUrl && @url == url
       @url = nil
     end
   end
@@ -134,7 +134,7 @@ class WonkoVersion
   attr_accessor :type
   attr_accessor :is_complete
 
-  attr_accessor :requires # list of Referenced
+  attr_accessor :requires # [Referenced]
 
   class Resources
     attr_accessor :traits

data/lib/wonko_the_sane/wonkoweb_uploader.rb

@@ -0,0 +1,162 @@
+module WonkoTheSane
+  class WonkoWeb
+    include HTTParty
+
+    class UploadError < ResponseError
+      attr_reader :errors
+
+      def initialize(errors)
+        @errors = errors
+      end
+    end
+
+    def initialize(benchmark)
+      self.class.base_uri Settings[:wonkoweb][:host]
+      self.class.headers 'X-Uploader-Name' => Settings[:wonkoweb][:name],
+                         'X-Uploader-Token' => Settings[:wonkoweb][:token],
+                         'X-WUR-Enabled' => 'true',
+                         'Content-Type' => 'application/json'
+
+      @benchmark = benchmark
+    end
+
+    def index
+      self.class.get(path 'index').with_indifferent_access
+    end
+
+    def file(uid)
+      self.class.get(path uid).with_indifferent_access
+    end
+
+    def version(uid, version)
+      self.class.get(path uid, version).with_indifferent_access
+    end
+
+    def upload_file(data)
+      return if data.nil?
+      uid = data[:uid]
+      theirs = @benchmark.benchmark('upload_file.theirs') { file uid rescue nil }
+
+      data.delete :versions # only stubs here
+      data[:name] ||= data[:uid]
+
+      upload_hash path, path(uid), data, theirs, false
+    end
+
+    def upload_version(data)
+      return if data.nil?
+      uid = data[:uid]
+      version = data[:version]
+      theirs = @benchmark.benchmark('upload_version.theirs') { version uid, version rescue nil }
+      upload_hash path(uid), path(uid, version), data, theirs, true
+    end
+
+    private
+
+    def upload_hash(create_path, update_path, ours, theirs, allow_overwrite)
+      caller = ours[:version] ? 'upload_version' : 'upload_file'
+      res = if theirs.nil? || (theirs[:errors] && theirs[:errors].size > 0)
+              # upload entire file
+              @benchmark.benchmark(caller + '.create') { self.class.post create_path, body: JSON.generate(ours) }
+            else
+              # upload changes
+
+              body = JSON.parse JSON.generate ours # round-trip to "normalize" things
+              # keep_if modifies in-place (why isn't it named keep_if! ????????)
+              body.keep_if { |k, v| !(theirs[k.to_sym] == v) }
+              body.keep_if { |k, v| theirs[k.to_sym].nil? || theirs[k.to_sym].to_s.empty? } unless allow_overwrite
+              @benchmark.benchmark(caller + '.update') { self.class.patch update_path, body: JSON.generate(body) } unless body.empty?
+            end
+
+      if !res.nil? && (res.code >= 300 || JSON.parse(res.body).key?('errors'))
+        binding.pry
+        fail UploadError.new res.body[:errors]
+      end
+    end
+
+    def path(*args)
+      return '/api/v1.json' if args.nil? || args.empty?
+      '/api/v1/' + args.map(&:to_s).join('/') + '.json'
+    end
+  end
+
+  class WonkoWebUploader
+    def initialize
+      data = JSON.parse File.read 'uploader_queue.json' if File.exists? 'uploader_queue.json'
+      data ||= {}
+      data = data.with_indifferent_access
+      @changed_files = data[:files] || []
+      @changed_versions = data[:versions] || {}
+
+      @benchmark = WonkoTheSane::Util::Benchmark.new
+      @client = WonkoWeb.new @benchmark
+    end
+
+    def file_changed(uid)
+      @changed_files << uid
+      save_changes!
+    end
+
+    def version_changed(uid, version)
+      @changed_versions[uid] ||= []
+      @changed_versions[uid] << version
+      save_changes!
+    end
+
+    def upload_changes!
+      @changed_files.uniq!
+      return if Settings[:wonkoweb][:host].nil?
+
+      existing_uids = @benchmark.benchmark('initial index') { @client.index }[:index].map { |obj| obj[:uid] }
+      logger.info "Uploading #{@changed_files.size} changed files to WonkoWeb at #{Settings[:wonkoweb][:host]}..."
+      @changed_files.dup.each do |uid|
+        begin
+          @client.upload_file $rw.write_version_index Registry.instance.version_index uid
+          existing_uids << uid
+          @changed_files.delete uid
+        rescue => e
+          logger.error "Unable to upload file for #{uid} to WonkoWeb: #{e.message}"
+          logger.warn e.backtrace.first
+          binding.pry if $stdout.isatty && ENV['DEBUG_ON_ERROR']
+        end
+      end
+      logger.info 'Done.'
+      @benchmark.print_times true
+
+      num_versions = @changed_versions.collect { |k, v| v.size }.inject :+
+      logger.info "Uploading #{num_versions} changed versions to WonkoWeb at #{Settings[:wonkoweb][:host]}..."
+      @changed_versions.dup.each do |uid, versions|
+        versions.dup.each do |version|
+          begin
+            @client.upload_file $rw.write_version_index Registry.instance.version_index uid unless existing_uids.include? uid
+            @client.upload_version $rw.write_version Registry.instance.retrieve uid, version
+            versions.delete version
+          rescue => e
+            logger.error "Unable to upload version #{version} of #{uid} to WonkoWeb: #{e.message}"
+            logger.warn e.backtrace.first
+            binding.pry if $stdout.isatty && ENV['DEBUG_ON_ERROR']
+          end
+          @changed_versions.delete uid if versions.empty?
+        end
+      end
+      logger.info 'Done.'
+      @benchmark.print_times true
+
+      save_changes!
+    end
+
+    private
+
+    def logger
+      Logging.logger[self]
+    end
+
+    def save_changes!
+      data = {
+          files: @changed_files,
+          versions: @changed_versions
+      }
+      File.write 'uploader_queue.json', JSON.pretty_generate(data)
+    end
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: wonko_the_sane
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.1.2
 platform: ruby
 authors:
 - 02JanDal
@@ -10,7 +10,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-05-17 00:00:00.000000000 Z
+date: 2015-05-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -250,6 +250,48 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: configliere
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: httparty
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: 
 email:
 - jan@dalheimer.de
@@ -276,6 +318,8 @@ files:
 - lib/wonko_the_sane/rules.rb
 - lib/wonko_the_sane/timestamps.rb
 - lib/wonko_the_sane/tools/update_nem.rb
+- lib/wonko_the_sane/util.rb
+- lib/wonko_the_sane/util/benchmark.rb
 - lib/wonko_the_sane/util/configuration.rb
 - lib/wonko_the_sane/util/deep_storage_cache.rb
 - lib/wonko_the_sane/util/extraction_cache.rb
@@ -295,6 +339,7 @@ files:
 - lib/wonko_the_sane/versionlists/vanilla_legacy_version_list.rb
 - lib/wonko_the_sane/versionlists/vanilla_version_list.rb
 - lib/wonko_the_sane/wonko_version.rb
+- lib/wonko_the_sane/wonkoweb_uploader.rb
 homepage: https://github.com/MultiMC/WonkoTheSane
 licenses:
 - MIT
@@ -321,4 +366,3 @@ specification_version: 4
 summary: Pre-processing and format-unification of various resources mainly related
   to Minecraft
 test_files: []
-has_rdoc: