flox 0.3.0 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b43c6043976e6f113bd8f1882c82ca6f106385b7279dbff99ea87e7eec3059ef
-  data.tar.gz: 9166d78906c48085337b996fbf73f554a7aee8912d20d5e18e5b247ad13f463d
+  metadata.gz: f366e3d238defdbc4a7d2c94613eaed5a0877eaaad1aba8afa56392ff75caf25
+  data.tar.gz: ef61dd08c7dc87f040557fcef8d3b00e603311516b6c6cd58f461763c692629a
 SHA512:
-  metadata.gz: 3edae31aac538f8ff71ac335cbc75b2393a2620c1c98aa63736634c62a5e815aff98be75f6b6df20898ab7356a119aab80e02b0b4c16708fb8bf985f4bb112e8
-  data.tar.gz: 0a34f6e3de1631f64ba71a3f2c9100ffe9c51bce75611378f1f8fa66f6a51658e93c40325e04542fb4502d63964b7db5e824b87fc1ac07360b783d66b6c4ade6
+  metadata.gz: 24da9b6e59adf749c515f71bfec553bcbf42975a159e8fab4fbedb95ee82950941325d6376dc6f6ae0d5a07252684413df428bd667668a67c1cbc0777f5ee166
+  data.tar.gz: cbaa7a929062f4d35c05e99215a0cb783c674aef3b94b8d133a87e9edef527987be27b7a9ae11138261f2c350c567f3f78d0f1fcc5021cd1542868e7dab6ffa9

data/README.md

@@ -33,6 +33,12 @@ As an example, here's how to download all the log files of a certain day that ha
 
 Note that you need to authorize with a "Hero" key. A hero is a special Flox player that has super-user rights; create it in the Flox online interface.
 
+You can use that same key to export all the entities of a specific type. Like this:
+
+    flox export_entities --game_id="id" --game_key="key" --hero_key="key" --type="Player"
+
+This will download JSON files with all player data into a subfolder of the working directory. Beware: you will need to add an index for the "updatedAt" property to this entity type first (which is done in the online interface, as well). This makes it possible that repeating the same export will only download new and changed files.
+
 ## How to use the Flox SDK
 
 The Ruby client is designed to be used not in games, but in scripts that help you operate your games. You can use it e.g. to automatically download your leaderboards or certain entities for backup. It's also easy to utilize Flox via 'irb', allowing quick introspection into your server data.

data/bin/flox

@@ -4,18 +4,16 @@
 ## Copyright: Copyright 2014 Gamua
 ## License:   Simplified BSD
 
-require 'flox'
-require 'fileutils'
-require 'slop'
-
-# Helper class that does the heavy lifting.
-class Worker
+require "flox"
+require "fileutils"
+require "slop"
 
+class LogExporter
   attr_reader :flox
 
-  def initialize(args={})
+  def initialize(args = {})
     base_url = args[:url] || Flox::DEFAULT_URL
-    game_id  = args[:game_id]
+    game_id = args[:game_id]
     game_key = args[:game_key]
     hero_key = args[:hero_key]
 
@@ -25,7 +23,7 @@ class Worker
     puts "done."
   end
 
-  def load_logs(args={})
+  def run(args = {})
     query = args[:query]
     limit = args[:limit]
     destination = args[:destination] || Dir.pwd
@@ -52,7 +50,82 @@ class Worker
       puts("Saved #{local_path}")
     end
   end
+end
+
+class EntityExporter
+  def run(args = {})
+    base_url = args[:url] || Flox::DEFAULT_URL
+    game_id = args[:game_id]
+    game_key = args[:game_key]
+    hero_key = args[:hero_key]
+    entity_type = args[:type]
+
+    puts("Exporting entities of type '#{entity_type}'. Beware, this will take a while!")
+    puts("  -> Precondition: the entity MUST have an index on the property 'updatedAt'!")
+    puts("  -> If you stop and restart, the exporter will continue where it left off.")
+    puts("  -> Running the exporter again will download only the changed / new entities.")
+    print("Progress: ")
+
+    batch_size = 10
+    num_threads = 4
+    out_folder = "export/#{entity_type}"
+    FileUtils.mkdir_p out_folder
+
+    status_file = "#{out_folder}/_export_status.json"
+    start_index = File.exist?(status_file) ?
+      JSON.parse(IO.read(status_file))["index"].to_i : 0
+
+    thread_state = [start_index] * num_threads
+    signal_queue = Queue.new
+
+    threads = (0...num_threads).map do |thread_index|
+      Thread.new do
+        flox = Flox.new(game_id, game_key, base_url)
+        flox.login_with_key(hero_key)
+
+        query = Flox::Query.new(entity_type)
+        query.order_by = "updatedAt ASC"
+        query.limit = batch_size
+        query.offset = start_index + thread_index * batch_size
+        sleep thread_index
+
+        loop do
+          begin
+            signal_queue << { :thread_index => thread_index,
+                              :result_index => query.offset }
+            results = flox.find_entities(query)
+            break if results.count == 0
+            results.each do |player|
+              IO.write("#{out_folder}/#{player.id}.json", player.to_json)
+            end
+          rescue StandardError => e
+            print "\nError: #{e.message}"
+            if e.message.include? "index is missing"
+              abort("\n")
+            else
+              puts " - retrying."
+            end
+          else
+            query.offset += batch_size * num_threads
+            print "."
+          end
+        end
+      end
+    end
+
+    signal_thread = Thread.new do
+      while thread_signal = signal_queue.pop
+        thread_state[thread_signal[:thread_index]] = thread_signal[:result_index]
+        smallest_index = thread_state.min
+        IO.write(status_file, { :index => smallest_index }.to_json)
+      end
+    end
 
+    threads.each { |thr| thr.join }
+    signal_queue.close
+
+    puts "\nExport completed!"
+  end
 end
 
 #
@@ -67,11 +140,9 @@ def shared_options
 end
 
 begin
-
   ARGV << "-h" if ARGV.empty?
   Slop.parse(ARGV, :help => true) do
-
-    on :v, "version", 'Print the current version of the Flox Gem.' do
+    on :v, "version", "Print the current version of the Flox Gem." do
       puts "Gamua Flox v#{Flox::VERSION}"
     end
 
@@ -84,12 +155,21 @@ begin
       on :l, "limit=", "Maximum number of logs to download", as: Integer
 
       run do |opts, args|
-        Worker.new(opts).load_logs(opts)
+        LogExporter.new(opts).run(opts)
       end
     end
 
-  end
+    command "export_entities" do
+      description "Fetch all entitites of a specific type from the server."
 
+      shared_options
+      on :d, "type=", "The entity type", :required => true
+
+      run do |opts, args|
+        EntityExporter.new.run(opts)
+      end
+    end
+  end
 rescue Slop::Error, Flox::Error => error
   puts error
 end

data/lib/flox/version.rb

@@ -9,6 +9,6 @@
 class Flox
 
   # The current version of the Flox SDK.
-  VERSION = '0.3.0'
+  VERSION = '0.4.0'
 
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: flox
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.4.0
 platform: ruby
 authors:
 - Daniel Sperl