spout 0.11.0.beta1 → 0.11.0.beta2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 35179a9011f963b875618eeb8fd07854c58f9cce
-  data.tar.gz: 321e836a1a80d07092e0bbe5d921f712db8dcb18
+  metadata.gz: a7a873eabc85e3035b8e784628e510d8e976f15b
+  data.tar.gz: 90c06d0bac0ef4c772c82da29760d4007591407b
 SHA512:
-  metadata.gz: d811aca192cc49aa05a8fc5dd5c4f1523f977b0fc2eb0e56c6f6ed5cd7f8295d73b69ccd93382183d3199c84986dff828c1faa611a39038b2b2bcb32ef14405f
-  data.tar.gz: 9ba01d7cb5f51f977b9433a8186f69241585bbba0eb2beb238c7794fe3c1eaa548da3450d527692747a950786ca02f609dad7e90058bc539a28ec371fbe4a521
+  metadata.gz: a7cee73fe2aa1bec7762300e8fe24c29462a6db61fa7149bd81274950e5da9ef3071602718f061db7d603db52a5ad56ed97f2377620a08ac96144f4dfc75f2cb
+  data.tar.gz: 9c9038e3ea7ff18eae3485622f11d12f8523e656d331ad9274d969c7c70863cf63ad0b729e76ec5b1de7cbf2c6f59fcd946a2312d9a5602ca2c52ddf1c9ef809

data/LICENSE

@@ -1,4 +1,4 @@
-Copyright (c) 2014 Division of Medicine Program of Sleep Medicine Epidemiology
+Copyright (c) 2016 Division of Medicine Program of Sleep Medicine Epidemiology
 https://github.com/sleepepi
 
 

data/README.md

@@ -144,8 +144,8 @@ class DictionaryTest < Minitest::Test
 
   VALID_UNITS = ['minutes', 'hours']
 
-  @variables.select{|v| v.type == 'numeric' or v.type == 'integer'}.each do |variable|
-    define_method("test_valid_units: "+variable.path) do
+  @variables.select { |v| %w(numeric integer).include?(v.type) }.each do |variable|
+    define_method("test_units: #{variable.path}") do
       puts variable.class
       # => Spout::Models::Variable
       assert VALID_UNITS.include?(variable.units)
@@ -165,8 +165,8 @@ class DictionaryTest < Minitest::Test
   # iterators that can be used to write custom tests
   include Spout::Helpers::Iterators
 
-  @variables.select{|v| ['numeric','integer'].include?(v.type)}.each do |variable|
-    define_method("test_at_least_two_labels: "+variable.path) do
+  @variables.select { |v| %w(numeric integer).include?(v.type) }.each do |variable|
+    define_method("test_at_least_two_labels: #{variable.path}") do
       assert_operator 2, :<=, variable.labels.size
     end
   end

data/lib/spout/commands/deploy.rb

@@ -8,6 +8,7 @@ require 'spout/helpers/quietly'
 require 'spout/helpers/send_file'
 require 'spout/helpers/semantic'
 require 'spout/helpers/json_request'
+require 'spout/helpers/json_request_generic'
 
 # - **User Authorization**
 #   - User authenticates via token, the user must be a dataset editor
@@ -48,10 +49,13 @@ module Spout
         @version = version
         @skip_checks = !(argv.delete('--skip-checks').nil? && argv.delete('--no-checks').nil?)
 
-        @skip_graphs = !(argv.delete('--skip-graphs').nil? && argv.delete('--no-graphs').nil?)
-        @skip_csvs = !(argv.delete('--skip-csvs').nil? && argv.delete('--no-csvs').nil?)
+        @skip_variables = !(argv.delete('--skip-variables').nil? && argv.delete('--no-variables').nil?)
+        @skip_dataset = !(argv.delete('--skip-dataset').nil? && argv.delete('--no-dataset').nil?)
+        @skip_dictionary = !(argv.delete('--skip-dictionary').nil? && argv.delete('--no-dictionary').nil?)
+        @skip_documentation = !(argv.delete('--skip-documentation').nil? && argv.delete('--no-documentation').nil?)
         @clean = !(argv.delete('--no-resume').nil? && argv.delete('--clean').nil?)
-        @skip_server_updates = !(argv.delete('--skip-server-updates').nil? && argv.delete('--no-server-updates').nil?)
+        @skip_server_scripts = !(argv.delete('--skip-server-scripts').nil? && argv.delete('--no-server-scripts').nil?)
+        @archive_only = !(argv.delete('--archive-only').nil?)
 
         token_arg = argv.find { |arg| /^--token=/ =~ arg }
         argv.delete(token_arg)
@@ -61,13 +65,10 @@ module Spout
         argv.delete(rows_arg)
         @number_of_rows = rows_arg.gsub(/--rows=/, '').to_i if rows_arg
 
-        # puts "token_arg: #{token_arg}".colorize(:red)
-        # puts "@token: #{@token}".colorize(:red)
-        # puts "rows_arg: #{rows_arg}".colorize(:red)
-        # puts "@number_of_rows: #{@number_of_rows}".colorize(:red)
-
         @argv = argv
 
+        @created_folders = []
+
         begin
           run_all
         rescue Interrupt
@@ -77,28 +78,19 @@ module Spout
 
       def run_all
         config_file_load
-        version_check unless @skip_checks
-        test_check unless @skip_checks
+        version_check
+        test_check
         user_authorization
-        load_subjects_from_csvs unless @skip_graphs
-        graph_generation unless @skip_graphs
-        dataset_uploads unless @skip_csvs
-        data_dictionary_uploads unless @skip_csvs
-        markdown_uploads unless @skip_csvs
-        trigger_server_updates unless @skip_server_updates
+        upload_variables
+        dataset_uploads
+        data_dictionary_uploads
+        markdown_uploads
+        trigger_server_updates
+        set_default_dataset_version
       rescue DeployError
         # Nothing on Deploy Error
       end
 
-      def load_subjects_from_csvs
-        @dictionary_root = Dir.pwd
-        @variable_files = Dir.glob(File.join(@dictionary_root, 'variables', '**', '*.json'))
-
-        @subject_loader = Spout::Helpers::SubjectLoader.new(@variable_files, [], @version, @number_of_rows, @config.visit)
-        @subject_loader.load_subjects_from_csvs!
-        @subjects = @subject_loader.subjects
-      end
-
       def config_file_load
         print '   `.spout.yml` Check: '
         @config = Spout::Helpers::ConfigReader.new
@@ -142,6 +134,11 @@ module Spout
       #   - `CHANGELOG.md` top line should include version, ex: `## 0.1.0`
       #   - "v#{VERSION}" matches HEAD git tag annotation
       def version_check
+        if @skip_checks
+          puts '        Version Check: ' + 'SKIP'.colorize(:blue)
+          return
+        end
+
         stdout = quietly do
           `git status --porcelain`
         end
@@ -179,6 +176,11 @@ module Spout
       end
 
       def test_check
+        if @skip_checks
+          puts '          Spout Tests: ' + 'SKIP'.colorize(:blue)
+          return
+        end
+
         print "          Spout Tests: "
 
         stdout = quietly do
@@ -192,7 +194,7 @@ module Spout
           failure message
         end
 
-        puts "       Spout Coverage: " + "SKIP".colorize(:blue)
+        puts '       Spout Coverage: ' + 'SKIP'.colorize(:blue)
       end
 
       def user_authorization
@@ -207,43 +209,69 @@ module Spout
         else
           puts 'UNAUTHORIZED'.colorize(:red)
           puts "#{INDENT}You are not set as an editor on the #{@slug} dataset or you mistyped your token."
-          raise DeployError
+          fail DeployError
         end
 
         # failure ''
         # puts 'PASS'.colorize(:green)
       end
 
+      def upload_variables
+        if @skip_variables
+          puts '     Upload Variables: ' + 'SKIP'.colorize(:blue)
+          return
+        end
+        load_subjects_from_csvs
+        graph_generation
+      end
+
+      def load_subjects_from_csvs
+        @dictionary_root = Dir.pwd
+        @variable_files = Dir.glob(File.join(@dictionary_root, 'variables', '**', '*.json'))
+        @subject_loader = Spout::Helpers::SubjectLoader.new(@variable_files, [], @version, @number_of_rows, @config.visit)
+        @subject_loader.load_subjects_from_csvs!
+        @subjects = @subject_loader.subjects
+      end
+
       def graph_generation
         # failure ''
         require 'spout/commands/graphs'
         @argv << '--clean' if @clean
         Spout::Commands::Graphs.new(@argv, @version, true, @url, @slug, @token, @webserver_name, @subjects)
-        puts "\r     Graph Generation: " + 'DONE          '.colorize(:green)
+        puts "\r     Upload Variables: " + 'DONE          '.colorize(:green)
       end
 
       def dataset_uploads
-        available_folders = (Dir.exist?('csvs') ? Dir.entries('csvs').select { |e| File.directory? File.join('csvs', e) }.reject { |e| ['.', '..'].include?(e) }.sort : [])
-        semantic = Spout::Helpers::Semantic.new(@version, available_folders)
-        csv_directory = semantic.selected_folder
-
-        if @version != csv_directory
-          puts "\r      Dataset Uploads: " + "SKIPPED - #{csv_directory} CSV dataset already on server".colorize(:blue)
+        if @skip_dataset
+          puts '      Dataset Uploads: ' + 'SKIP'.colorize(:blue)
           return
         end
 
+        available_folders = (Dir.exist?('csvs') ? Dir.entries('csvs').select { |e| File.directory? File.join('csvs', e) }.reject { |e| ['.', '..'].include?(e) }.sort : [])
+        semantic = Spout::Helpers::Semantic.new(@version, available_folders)
+        csv_directory = semantic.selected_folder
         csv_files = Dir.glob("csvs/#{csv_directory}/**/*.csv")
 
         csv_files.each_with_index do |csv_file, index|
           print "\r      Dataset Uploads: " + "#{index + 1} of #{csv_files.count}".colorize(:green)
-          folder = csv_file.gsub(/^csvs/, '').gsub(/#{File.basename(csv_file)}$/, '')
-          response = Spout::Helpers::SendFile.post("#{@url}/api/v1/dictionary/upload_dataset_csv.json", csv_file, @version, @token, @slug, folder)
+          folder = csv_file.gsub(%r{^csvs/#{csv_directory}}, '').gsub(/#{File.basename(csv_file)}$/, '')
+          folder = folder.gsub(%r{/$}, '')
+          @created_folders << "datasets#{folder}"
+          @created_folders << 'datasets/archive'
+          @created_folders << "datasets/archive/#{@version}#{folder}"
+          upload_file(csv_file, "datasets#{folder}") unless @archive_only
+          upload_file(csv_file, "datasets/archive/#{@version}#{folder}")
         end
-        puts "\r      Dataset Uploads: " + "DONE          ".colorize(:green)
+        puts "\r      Dataset Uploads: " + 'DONE          '.colorize(:green)
       end
 
       def data_dictionary_uploads
-        print   '   Dictionary Uploads:'
+        if @skip_dictionary
+          puts '   Dictionary Uploads: ' + 'SKIP'.colorize(:blue)
+          return
+        end
+
+        print '   Dictionary Uploads:'
 
         require 'spout/commands/exporter'
         Spout::Commands::Exporter.new(@version, ['--quiet'])
@@ -251,44 +279,75 @@ module Spout
         csv_files = Dir.glob("dd/#{@version}/*.csv")
         csv_files.each_with_index do |csv_file, index|
           print "\r   Dictionary Uploads: " + "#{index + 1} of #{csv_files.count}".colorize(:green)
-          response = Spout::Helpers::SendFile.post("#{@url}/api/v1/dictionary/upload_dataset_csv.json", csv_file, @version, @token, @slug, nil)
+          @created_folders << 'datasets'
+          @created_folders << 'datasets/archive'
+          @created_folders << "datasets/archive/#{@version}"
+          upload_file(csv_file, 'datasets') unless @archive_only
+          upload_file(csv_file, "datasets/archive/#{@version}")
         end
         puts "\r   Dictionary Uploads: " + 'DONE          '.colorize(:green)
       end
 
       def markdown_uploads
-        print   'Documentation Uploads:'
+        if @skip_documentation
+          puts 'Documentation Uploads: ' + 'SKIP'.colorize(:blue)
+          return
+        end
+
+        print 'Documentation Uploads:'
         markdown_files = Dir.glob(%w(CHANGELOG.md KNOWNISSUES.md))
         markdown_files.each_with_index do |markdown_file, index|
           print "\rDocumentation Uploads: " + "#{index + 1} of #{markdown_files.count}".colorize(:green)
-          Spout::Helpers::SendFile.post("#{@url}/api/v1/dictionary/upload_dataset_csv.json", markdown_file, @version, @token, @slug, nil)
+          @created_folders << 'datasets'
+          @created_folders << 'datasets/archive'
+          @created_folders << "datasets/archive/#{@version}"
+          upload_file(markdown_file, 'datasets') unless @archive_only
+          upload_file(markdown_file, "datasets/archive/#{@version}")
         end
         puts "\rDocumentation Uploads: " + 'DONE          '.colorize(:green)
       end
 
       def trigger_server_updates
+        if @skip_server_scripts
+          puts 'Launch Server Scripts: ' + 'SKIP'.colorize(:blue)
+          return
+        end
+
         print 'Launch Server Scripts: '
-        response = Spout::Helpers::JsonRequest.get("#{@url}/datasets/#{@slug}/a/#{@token}/refresh_dictionary.json?version=#{@version}")
+        params = { auth_token: @token, dataset: @slug, version: @version, folders: @created_folders.compact.uniq }
+        (response, status) = Spout::Helpers::JsonRequestGeneric.post("#{@url}/api/v1/dictionary/refresh.json", params)
         if response.is_a?(Hash) && response['refresh'] == 'success'
           puts 'DONE'.colorize(:green)
-        elsif response.is_a?(Hash) && response['refresh'] == 'notagfound'
-          puts 'FAIL'.colorize(:red)
-          puts "#{INDENT}Tag not found in repository, resolve using: " + 'git push --tags'.colorize(:white)
-          raise DeployError
-        elsif response.is_a?(Hash) && response['refresh'] == 'gitrepodoesnotexist'
-          puts 'FAIL'.colorize(:red)
-          puts "#{INDENT}Dataset data dictionary git repository has not been cloned on the server. Contact server admin.".colorize(:white)
-          raise DeployError
         else
           puts 'FAIL'.colorize(:red)
-          raise DeployError
+          fail DeployError
+        end
+      end
+
+      def set_default_dataset_version
+        if @archive_only
+          puts '  Set Default Version: ' + 'SKIP'.colorize(:blue)
+          return
+        end
+
+        print '  Set Default Version: '
+        params = { auth_token: @token, dataset: @slug, version: @version }
+        (response, status) = Spout::Helpers::JsonRequestGeneric.post("#{@url}/api/v1/dictionary/update_default_version.json", params)
+        if response.is_a?(Hash) && response['version_update'] == 'success'
+          puts @version.to_s.colorize(:green)
+        else
+          failure("#{INDENT}Unable to set default version\n#{INDENT}to " + @version.to_s.colorize(:white) + ' for ' + @slug.to_s.colorize(:white) + ' dataset.')
         end
       end
 
       def failure(message)
         puts 'FAIL'.colorize(:red)
         puts message
-        raise DeployError
+        fail DeployError
+      end
+
+      def upload_file(file, folder)
+        Spout::Helpers::SendFile.post("#{@url}/api/v1/dictionary/upload_file.json", file, @version, @token, @slug, folder)
       end
     end
   end

data/lib/spout/commands/graphs.rb

@@ -108,7 +108,7 @@ module Spout
           next unless Spout::Models::Subject.method_defined?(variable.id)
 
           if @deploy_mode
-            print "\r     Graph Generation: " + "#{"% 3d" % ((file_index+1)*100/variable_files_count)}% Uploaded".colorize(:white)
+            print "\r     Upload Variables: " + "#{"% 3d" % ((file_index+1)*100/variable_files_count)}% Uploaded".colorize(:white)
           else
             puts "#{file_index + 1} of #{variable_files_count}: #{variable.folder}#{variable.id}"
           end

data/lib/spout/commands/help.rb

@@ -132,17 +132,24 @@ Usage: spout deploy NAME
 
 NAME is the name of the webserver listed in `.spout.yml` file
 Optional Flags:
-  --clean         Regenerate all variables (default is to resume
-                  where command last left off)
-  --rows=N        Limit the number of rows read from CSVs to a
-                  maximum of N rows
-  <variable>      Only deploy specified variable(s)
-                  Ex: spout deploy production age gender
-  --skip-checks   Skips Spout checks
-  --skip-graphs   Skip generation of variable graphs
-  --skip-csvs     Skip upload of Data Dictionary and Dataset CSVs
-  --token=TOKEN   Provide token via command-line for automated
-                  processes
+  --clean               Regenerate all variables (default is to
+                        resume where command last left off)
+  --rows=N              Limit the number of rows read from CSVs
+                        to a maximum of N rows
+  <variable>            Only deploy specified variable(s)
+                        Ex: spout deploy production age gender
+  --skip-checks         Skip Spout checks
+  --skip-variables      Skip upload of dataset variables
+  --skip-dataset        Skip upload of dataset CSVs
+  --skip-dictionary     Skip upload of data dictionary
+  --skip-documentation  Skip upload of CHANGELOG.md and
+                        KNOWNISSUES.md
+  --skip-server-scripts Skips server refreshing datasets folder
+                        to list newly uploaded files
+  --archive-only        Only upload files to the datasets/archive
+                        folder and not in the root datasets folder
+  --token=TOKEN         Provide token via command-line for
+                        automated processes
 
 More information here:
 

data/lib/spout/models/coverage_result.rb

@@ -17,7 +17,7 @@ module Spout
 
       def load_json(column)
         file = Dir.glob("variables/**/#{column.to_s.downcase}.json", File::FNM_CASEFOLD).first
-        @file_name_test = (file != nil)
+        @file_name_test = !file.nil?
         @json = JSON.parse(File.read(file)) rescue @json = {}
         @json_id_test = (@json['id'].to_s.downcase == column)
       end

data/lib/spout/templates/test/dictionary_test.rb

@@ -1,5 +1,6 @@
 require 'test_helper'
 
+# Launches default Spout tests and custom tests for specific to this dictionary
 class DictionaryTest < Minitest::Test
   # This line includes all default Spout Dictionary tests
   include Spout::Tests
@@ -10,18 +11,17 @@ class DictionaryTest < Minitest::Test
 
   # Example 1: Create custom tests to show that `integer` and `numeric` variables have a valid unit type
   # VALID_UNITS = ['minutes', 'hours'] # Add your own valid units to this array
-  # @variables.select{|v| v.type == 'numeric' or v.type == 'integer'}.each do |variable|
-  #   define_method("test_units: "+variable.path) do
-  #     message = "\"#{variable.units}\"".colorize( :red ) + " invalid units.\n" +
-  #               "             Valid types: " +
-  #               VALID_UNITS.sort.collect{|u| u.inspect.colorize( :white )}.join(', ')
+  # @variables.select { |v| %w(numeric integer).include?(v.type) }.each do |variable|
+  #   define_method("test_units: #{variable.path}") do
+  #     message = "\"#{variable.units}\"".colorize(:red) + " invalid units.\n             Valid types: " +
+  #               VALID_UNITS.sort.collect { |u| u.inspect.colorize(:white) }.join(', ')
   #     assert VALID_UNITS.include?(variable.units), message
   #   end
   # end
 
   # Example 2: Create custom tests to show that variables have 2 or more labels.
-  # @variables.select{|v| ['numeric','integer'].include?(v.type)}.each do |variable|
-  #   define_method("test_at_least_two_labels: "+variable.path) do
+  # @variables.select { |v| %w(numeric integer).include?(v.type) }.each do |variable|
+  #   define_method("test_at_least_two_labels: #{variable.path}") do
   #     assert_operator 2, :<=, variable.labels.size
   #   end
   # end

data/lib/spout/templates/travis.yml

@@ -1,3 +1,4 @@
+sudo: false
 language: ruby
 rvm:
   - 2.3.0

data/lib/spout/version.rb

@@ -3,7 +3,7 @@ module Spout
     MAJOR = 0
     MINOR = 11
     TINY = 0
-    BUILD = 'beta1' # 'pre', 'rc', 'rc2', nil
+    BUILD = 'beta2' # 'pre', 'rc', 'rc2', nil
 
     STRING = [MAJOR, MINOR, TINY, BUILD].compact.join('.')
   end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: spout
 version: !ruby/object:Gem::Version
-  version: 0.11.0.beta1
+  version: 0.11.0.beta2
 platform: ruby
 authors:
 - Remo Mueller
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-01-11 00:00:00.000000000 Z
+date: 2016-01-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler