spout 0.10.0.beta7 → 0.10.0.beta8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: d4418b6291f10a58bcf649b785114423a462aca8
-  data.tar.gz: 16a62a4c3357173af4808bd5a5ab9d767f692296
+  metadata.gz: 97f132b532821e57396f65c1752060dac1a33c14
+  data.tar.gz: f87fc93a7f9a60c34e81f602c47a770bbd29eba9
 SHA512:
-  metadata.gz: fdaeae598fccf2e5d29be1d491efc4951d02e56eaeb2917fb91a944b8eaea24d5fd4780bebb281116d42e8271fc63798694a53a1bdc3a69d0a926e7535a684de
-  data.tar.gz: 38b272dfe518b4e230b6e4eec679b00838259c0fc3b3d0e1b3f5af39a56325ce702c1af3542a885e7c9e71c066ed1a8c38a72fc30a7ae1be6fc593bb07c9f576
+  metadata.gz: 7c9ed942121864d02d02bdcdac9115e8b04093e368f31dfd9a4e37541726f5ff803654a93b6d62c872c314efb40f4757bf759f6d1098281dc91f090bbd201229
+  data.tar.gz: 741c7cfd6b2f365aff937f8d2508a04aaaef81fad297cca5773f3c2a82e1759cda0aba69a68d6e72a6d6547a39151f8915d64ac7e0d688ee5f605dd5a39fe216

data/lib/spout/commands/deploy.rb

@@ -1,5 +1,6 @@
 require 'colorize'
 require 'net/http'
+require 'io/console'
 
 require 'spout/helpers/config_reader'
 require 'spout/helpers/quietly'
@@ -48,13 +49,20 @@ module Spout
       def initialize(argv, version)
         @environment = argv[1].to_s
         @version = version
-        @skip_checks = (argv.delete('--skip-checks') != nil)
+        @skip_checks = (argv.delete('--skip-checks') != nil or argv.delete('--no-checks') != nil)
 
-        @skip_graphs = (argv.delete('--skip-graphs') != nil)
-        @skip_images = (argv.delete('--skip-images') != nil)
-        @skip_server_updates = (argv.delete('--skip-server-updates') != nil)
+        @skip_graphs = (argv.delete('--skip-graphs') != nil or argv.delete('--no-graphs') != nil)
+        @skip_images = (argv.delete('--skip-images') != nil or argv.delete('--no-images') != nil)
+        @clean = (argv.delete('--clean') != nil or argv.delete('--no-resume'))
+        @skip_server_updates = (argv.delete('--skip-server-updates') != nil or argv.delete('--no-server-updates') != nil)
 
-        run_all
+        @token = argv.select{|a| /^--token=/ =~ a}.collect{|a| a.gsub(/^--token=/, '')}.first
+
+        begin
+          run_all
+        rescue Interrupt, IRB::Abort
+          puts "\nINTERRUPTED".colorize(:red)
+        end
       end
 
       def run_all
@@ -66,6 +74,7 @@ module Spout
           graph_generation unless @skip_graphs
           image_generation unless @skip_images
           dataset_uploads
+          data_dictionary_uploads
           trigger_server_updates unless @skip_server_updates
         rescue DeployError
         end
@@ -170,7 +179,18 @@ module Spout
       def user_authorization
         puts  "  Get your token here: " + "#{@url}/token".colorize(:blue).on_white.underline
         print "     Enter your token: "
-        @token =  STDIN.gets.chomp
+        @token = STDIN.noecho(&:gets).chomp if @token.to_s.strip == ''
+
+        response = Spout::Helpers::JsonRequest.get("#{@url}/datasets/#{@slug}/a/#{@token}/editor.json")
+
+        if response.kind_of?(Hash) and response['editor']
+          puts "AUTHORIZED".colorize(:green)
+        else
+          puts "UNAUTHORIZED".colorize(:red)
+          puts "#{INDENT}You are not set as an editor on the #{@slug} dataset or you mistyped your token."
+          raise DeployError
+        end
+
         # failure ''
         # puts "PASS".colorize(:green)
       end
@@ -178,24 +198,31 @@ module Spout
       def graph_generation
         # failure ''
         require 'spout/commands/graphs'
-        Spout::Commands::Graphs.new([], @version, true, @url, @slug, @token)
+        argv = []
+        argv << "--clean" if @clean
+        Spout::Commands::Graphs.new(argv, @version, true, @url, @slug, @token)
         puts "\r     Graph Generation: " + "DONE          ".colorize(:green)
       end
 
       def image_generation
         # failure ''
         require 'spout/commands/images'
-        Spout::Commands::Images.new([], [], [], @version, [], true, @url, @slug, @token)
+        argv = []
+        argv << "--clean" if @clean
+        Spout::Commands::Images.new(argv, [], [], @version, [], true, @url, @slug, @token)
         puts "\r     Image Generation: " + "DONE          ".colorize(:green)
       end
 
       def dataset_uploads
         available_folders = (Dir.exist?('csvs') ? Dir.entries('csvs').select{|e| File.directory? File.join('csvs', e) }.reject{|e| [".",".."].include?(e)}.sort : [])
-
         semantic = Spout::Helpers::Semantic.new(@version, available_folders)
-
         csv_directory = semantic.selected_folder
 
+        if @version != csv_directory
+          puts "\r      Dataset Uploads: " + "SKIPPED - #{csv_directory} CSV dataset already on server".colorize(:blue)
+          return
+        end
+
         csv_files = Dir.glob("csvs/#{csv_directory}/*.csv")
 
         csv_files.each_with_index do |csv_file, index|
@@ -205,13 +232,36 @@ module Spout
         puts "\r      Dataset Uploads: " + "DONE          ".colorize(:green)
       end
 
+      def data_dictionary_uploads
+        print   "   Dictionary Uploads:"
+
+        require 'spout/commands/exporter'
+        Spout::Commands::Exporter.new(@version, ['--quiet'])
+
+        csv_files = Dir.glob("dd/#{@version}/*.csv")
+        csv_files.each_with_index do |csv_file, index|
+          print "\r   Dictionary Uploads: " + "#{index + 1} of #{csv_files.count}".colorize(:green)
+          response = Spout::Helpers::SendFile.post("#{@url}/datasets/#{@slug}/upload_dataset_csv.json", csv_file, @version, @token)
+        end
+        puts "\r   Dictionary Uploads: " + "DONE          ".colorize(:green)
+      end
+
       def trigger_server_updates
         print "Launch Server Scripts: "
         response = Spout::Helpers::JsonRequest.get("#{@url}/datasets/#{@slug}/a/#{@token}/refresh_dictionary.json?version=#{@version}")
-        if response and response['refresh'] == 'success'
+        if response.kind_of?(Hash) and response['refresh'] == 'success'
           puts "DONE".colorize(:green)
+        elsif response.kind_of?(Hash) and response['refresh'] == 'notagfound'
+          puts "FAIL".colorize(:red)
+          puts "#{INDENT}Tag not found in repository, resolve using: " + "git push --tags".colorize(:white)
+          raise DeployError
+        elsif response.kind_of?(Hash) and response['refresh'] == 'gitrepodoesnotexist'
+          puts "FAIL".colorize(:red)
+          puts "#{INDENT}Dataset data dictionary git repository has not been cloned on the server. Contact server admin.".colorize(:white)
+          raise DeployError
         else
           puts "FAIL".colorize(:red)
+          raise DeployError
         end
       end
 

data/lib/spout/commands/exporter.rb

@@ -9,7 +9,7 @@ module Spout
   module Commands
     class Exporter
       def initialize(standard_version, argv)
-        @csv_file = argv[1].to_s
+        @quiet = (argv.delete('--quiet') != nil)
         @standard_version = standard_version
         @config = Spout::Helpers::ConfigReader.new
         expanded_export!
@@ -19,7 +19,7 @@ module Spout
 
       def expanded_export!
         folder = "dd/#{@standard_version}"
-        puts "      create".colorize( :green ) + "  #{folder}"
+        puts "      create".colorize( :green ) + "  #{folder}" unless @quiet
         FileUtils.mkpath folder
 
         generic_export(folder, 'variables', %w(id display_name description type units domain labels calculation))
@@ -29,7 +29,7 @@ module Spout
 
       def generic_export(folder, type, keys, include_domain_name = false)
         export_file = export_file_name(type)
-        puts "      export".colorize( :blue ) + "  #{folder}/#{export_file}"
+        puts "      export".colorize( :blue ) + "  #{folder}/#{export_file}" unless @quiet
         CSV.open("#{folder}/#{export_file}", "wb") do |csv|
           if include_domain_name
             csv << ['folder', 'domain_id'] + keys

data/lib/spout/commands/graphs.rb

@@ -20,6 +20,10 @@ module Spout
         @slug = slug
         @token = token
 
+        argv = variables
+
+        @clean = (argv.delete('--no-resume') != nil or argv.delete('--clean'))
+
         @config = Spout::Helpers::ConfigReader.new
 
         if Spout::Helpers::ChartTypes::get_json(@config.visit, 'variable') == nil
@@ -52,17 +56,33 @@ module Spout
         @variable_files = Dir.glob('variables/**/*.json')
 
         t = Time.now
-        FileUtils.mkpath "graphs/#{@standard_version}"
+        @graphs_folder = File.join("graphs", @standard_version)
+        FileUtils.mkpath @graphs_folder
 
         @subject_loader = Spout::Helpers::SubjectLoader.new(@variable_files, @valid_ids, @standard_version, @number_of_rows, @config.visit)
 
         @subject_loader.load_subjects_from_csvs!
         @subjects = @subject_loader.subjects
+
+        load_current_progress
+
         compute_tables_and_charts
 
         puts "Took #{Time.now - t} seconds." if @subjects.size > 0 and not @deploy_mode
       end
 
+      def load_current_progress
+        @progress_file = File.join(@graphs_folder, ".progress.json")
+        @progress = JSON.parse(File.read(@progress_file)) rescue @progress = {}
+        @progress = {} if @clean
+      end
+
+      def save_current_progress
+        File.open(@progress_file,"w") do |f|
+          f.write(@progress.to_json)
+        end
+      end
+
       def compute_tables_and_charts
         variable_files_count = @variable_files.count
         @variable_files.each_with_index do |variable_file, file_index|
@@ -79,7 +99,8 @@ module Spout
             puts "#{file_index+1} of #{variable_files_count}: #{variable_file.gsub(/(^variables\/|\.json$)/, '').gsub('/', ' / ')}"
           end
 
-
+          @progress[variable_name] ||= {}
+          next if (not @deploy_mode and @progress[variable_name]['generated'] == true) or (@deploy_mode and @progress[variable_name]['uploaded'] == true)
 
           stats = {
             charts: {},
@@ -109,13 +130,18 @@ module Spout
             end
           end
 
-          chart_json_file = File.join('graphs', @standard_version, "#{json['id']}.json")
+          chart_json_file = File.join(@graphs_folder, "#{json['id']}.json")
           File.open(chart_json_file, 'w') { |file| file.write( JSON.pretty_generate(stats) + "\n" ) }
 
-          if @deploy_mode
-            send_to_server(chart_json_file)
+          @progress[variable_name]['generated'] = true
+
+          if @deploy_mode and not @progress[variable_name]['uploaded'] == true
+            response = send_to_server(chart_json_file)
+            @progress[variable_name]['uploaded'] = (response.kind_of?(Hash) and response['upload'] == 'success')
           end
 
+          save_current_progress
+
         end
       end
 

data/lib/spout/commands/images.rb

@@ -24,6 +24,7 @@ module Spout
         @variable_files = Dir.glob('variables/**/*.json')
         @standard_version = standard_version
         @pretend = (argv.delete('--pretend') != nil)
+        @clean = (argv.delete('--no-resume') != nil or argv.delete('--clean'))
         @sizes = sizes
         @types = types
 
@@ -34,17 +35,32 @@ module Spout
         @config = Spout::Helpers::ConfigReader.new
 
         t = Time.now
-        FileUtils.mkpath "graphs/#{@standard_version}"
+        @images_folder = File.join("images", @standard_version)
+        FileUtils.mkpath @images_folder
 
         @subject_loader = Spout::Helpers::SubjectLoader.new(@variable_files, @valid_ids, @standard_version, @number_of_rows, @config.visit)
 
         @subject_loader.load_subjects_from_csvs!
         @subjects = @subject_loader.subjects
 
+        load_current_progress
+
         compute_images
         puts "Took #{Time.now - t} seconds." if @subjects.size > 0 and not @deploy_mode
       end
 
+      def load_current_progress
+        @progress_file = File.join(@images_folder, ".progress.json")
+        @progress = JSON.parse(File.read(@progress_file)) rescue @progress = {}
+        @progress = {} if @clean
+      end
+
+      def save_current_progress
+        File.open(@progress_file,"w") do |f|
+          f.write(@progress.to_json)
+        end
+      end
+
       def compute_images
 
         options_folder = "images/#{@standard_version}"
@@ -67,6 +83,10 @@ module Spout
             puts "#{file_index+1} of #{variable_files_count}: #{variable_file.gsub(/(^variables\/|\.json$)/, '').gsub('/', ' / ')}"
           end
 
+          @progress[variable_name] ||= {}
+
+          next if (not @deploy_mode and @progress[variable_name]['generated'] == true) or (@deploy_mode and @progress[variable_name]['uploaded'] == true)
+
           filtered_subjects = @subjects.select{ |s| s.send(@config.visit) != nil }
 
           chart_json = Spout::Helpers::ChartTypes::chart_histogram(@config.visit, filtered_subjects, json, variable_name)
@@ -103,16 +123,21 @@ module Spout
                 }
               eos
             end
-            run_phantom_js("#{json['id']}-lg.png", 600, tmp_options_file) if @sizes.size == 0 or @sizes.include?('lg')
-            run_phantom_js("#{json['id']}.png",     75, tmp_options_file) if @sizes.size == 0 or @sizes.include?('sm')
+            run_phantom_js(variable_name, "#{variable_name}-lg.png", 600, tmp_options_file) if @sizes.size == 0 or @sizes.include?('lg')
+            run_phantom_js(variable_name, "#{variable_name}.png",     75, tmp_options_file) if @sizes.size == 0 or @sizes.include?('sm')
+            @progress[variable_name]['uploaded'] = (@deploy_mode and @progress[variable_name]['failed'] != true)
+            save_current_progress
           end
 
         end
         File.delete(tmp_options_file) if File.exist?(tmp_options_file)
       end
 
-      def run_phantom_js(png_name, width, tmp_options_file)
-        graph_path = File.join(Dir.pwd, 'images', @standard_version, png_name)
+      def run_phantom_js(variable_name, png_name, width, tmp_options_file)
+        @progress[variable_name]['generated'] ||= []
+        @progress[variable_name]['uploaded_files'] ||= []
+
+        image_path = File.join(Dir.pwd, 'images', @standard_version, png_name)
         directory = File.join( File.dirname(__FILE__), '..', 'support', 'javascripts' )
 
         open_command = if RUBY_PLATFORM.match(/mingw/) != nil
@@ -121,19 +146,29 @@ module Spout
           'phantomjs'
         end
 
-        phantomjs_command = "#{open_command} #{directory}/highcharts-convert.js -infile #{tmp_options_file} -outfile #{graph_path} -scale 2.5 -width #{width} -constr Chart"
+        phantomjs_command = "#{open_command} #{directory}/highcharts-convert.js -infile #{tmp_options_file} -outfile #{image_path} -scale 2.5 -width #{width} -constr Chart"
 
         if @pretend
           puts phantomjs_command
         else
-          `#{phantomjs_command}`
+          if not @progress[variable_name]['generated'].include?(png_name) or not File.exist?(png_name) or (File.exist?(png_name) and File.size(png_name) == 0)
+            `#{phantomjs_command}`
+            @progress[variable_name]['generated'] << png_name
+          end
 
-          send_to_server(graph_path) if @deploy_mode
+          if @deploy_mode and not @progress[variable_name]['uploaded_files'].include?(png_name)
+            response = send_to_server(image_path)
+            if response.kind_of?(Hash) and response['upload'] == 'success'
+              @progress[variable_name]['uploaded_files'] << png_name
+            else
+              @progress[variable_name]['upload_failed'] = true
+            end
+          end
         end
       end
 
       def send_to_server(file)
-        response = Spout::Helpers::SendFile.post("#{@url}/datasets/#{@slug}/upload_graph.json", file, @standard_version, @token, 'images')
+        Spout::Helpers::SendFile.post("#{@url}/datasets/#{@slug}/upload_graph.json", file, @standard_version, @token, 'images')
       end
 
     end

data/lib/spout/version.rb

@@ -3,7 +3,7 @@ module Spout
     MAJOR = 0
     MINOR = 10
     TINY = 0
-    BUILD = "beta7" # nil, "pre", "rc", "rc2"
+    BUILD = "beta8" # nil, "pre", "rc", "rc2"
 
     STRING = [MAJOR, MINOR, TINY, BUILD].compact.join('.')
   end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: spout
 version: !ruby/object:Gem::Version
-  version: 0.10.0.beta7
+  version: 0.10.0.beta8
 platform: ruby
 authors:
 - Remo Mueller
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-10-17 00:00:00.000000000 Z
+date: 2014-10-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake