spout 0.1.0 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 25fe4f0b1ed9242f0e8d1689e52bc372e285161f
-  data.tar.gz: e69ffdc296da47898fddf7f1a44d25ab3eb49b9c
+  metadata.gz: b17c5cb1d0059611bb4ebb9807ffb83f5015a765
+  data.tar.gz: 3c3e5d7493590ecfffcb6a7534d73b4ce0f56bee
 SHA512:
-  metadata.gz: 00b32e110a7dd21da41d5a7738f9802d33b7bf607fb581821e6fc375b992e9bc80abc10be3da2f140294363742f4ef89c2a8b8df97dfd07aca248f212a087586
-  data.tar.gz: c62938092d52b3a67318139fe2a33cacf836c67a2d9b6d46674ec17d02e546139b4acb456dfb7ebec1aa5a933f9c33e8ba8147184b332aa24b19231e729f6b30
+  metadata.gz: 50846a11d41e8654a283a5836bd9d5496745bdf4c0707e2a7494ceca86633ac743ac2f9fda054185a5ebe6f0c72bf218414c4b7d71fa05deda1541ab368f2bad
+  data.tar.gz: 08da58a7d5533b2210144e66d30158e825131a24f968f956cd7d64aae584084b3c97aed6acdb00d47351132158ca44dbab0aa9862874681d010ba4d08438d8b3

data/CHANGELOG.md

@@ -1,4 +1,11 @@
-## 0.1.0
+## 0.2.0 (June 26, 2013)
+
+### Enhancements
+- Domains can now be imported using `spout import_domains CSVFILE`
+- Data Dictionary can now be exported to the Hybrid data dictionary CSV format along with an optional version number:
+  - `spout hybrid [1.0.0]`
+
+## 0.1.0 (May 21, 2013)
 
 ### Enhancements
 - Existing Data Dictionaries can be converted to JSON format from a CSV file

data/README.md

@@ -34,7 +34,7 @@ spout import data_dictionary.csv
 
 The CSV should contain at minimal the two column headers:
 
-`id`: This column will give the variable it's name, and also be used to name the file, i.e. `<id>.json`
+`id`: This column will give the variable its name, and also be used to name the file, i.e. `<id>.json`
 `folder`: This can be blank, however it is used to place variables into a folder hiearchy. The folder column can contain forward slashes `/` to place a variable into a subfolder. An example may be, `id`: `myvarid`, `folder`: `Demographics/Subfolder` would create a file `variables/Demographics/Subfolder/myvarid.json`
 
 Other columns that will be interpreted include:
@@ -64,6 +64,24 @@ Other columns that will be interpreted include:
 
 All other columns get grouped into a hash labeled `other`.
 
+#### Importing domains from an existing CSV file
+
+```
+spout import_domains data_dictionary_domains.csv
+```
+
+The CSV should contain at minimal three column headers:
+
+`domain_id`: The name of the associated domain for the choice/option.
+`value`: The value of the choice/option.
+`display_name`: The display name of the choice/option.
+
+Other columns that are imported include:
+
+`description`: A longer description of the choice/option.
+`folder`: The name of the folder path where the domain resides.
+
+
 ### Test your repository
 
 If you created your data dictionary repository using `spout new`, you can go ahead and test using:
@@ -99,7 +117,7 @@ Then run either `spout test` or `bundle exec rake` to run your tests.
 
 ### Create a CSV Data Dictionary from your JSON repository
 
-Provide an optional version parameter to name the folder the CSVs will be generated in, defaults to 1.0.0 currently.
+Provide an optional version parameter to name the folder the CSVs will be generated in, defaults to 1.0.0.
 
 ```
 spout export
@@ -116,3 +134,18 @@ or
 ```
 bundle exec rake dd:create [VERSION=1.0.0]
 ```
+
+
+### Export to the Hybrid Data Dictionary format from your JSON repository
+
+Exporting to a format compatible with [Hybrid](https://github.com/sleepepi/hybrid) is also available.
+
+```
+spout hybrid
+```
+
+You can optionally provide a version string
+
+```
+spout hybrid [1.0.0]
+```

data/lib/spout/actions.rb

@@ -11,8 +11,12 @@ module Spout
         system "bundle exec rake"
       when 'import', 'i', 'im', 'imp', '--import', '-i', '-im', '-imp'
         import_from_csv(argv)
+      when 'import_domain', '--import_domain', 'import_domains', '--import_domains'
+        import_from_csv(argv, 'domains')
       when 'export', 'e', 'ex', 'exp', '--export', '-e', '-ex', '-exp'
         new_data_dictionary_export(argv)
+      when 'hybrid', '-hybrid', '--hybrid', 'y', 'hy', '-y', '-hy'
+        new_data_dictionary_export(argv, 'hybrid')
       else
         help
       end
@@ -20,7 +24,7 @@ module Spout
 
     protected
 
-      def import_from_csv(argv)
+      def csv_usage
         usage = <<-EOT
 
 Usage: spout import CSVFILE
@@ -28,12 +32,15 @@ Usage: spout import CSVFILE
 The CSVFILE must be the location of a valid CSV file.
 
 EOT
+        usage
+      end
 
+      def import_from_csv(argv, type = "")
         csv_file = File.join(argv[1].to_s.strip)
         if File.exists?(csv_file)
-          system "bundle exec rake dd:import CSV=#{csv_file}"
+          system "bundle exec rake dd:import CSV=#{csv_file} #{'TYPE='+type if type.to_s != ''}"
         else
-          puts usage
+          puts csv_usage
         end
       end
 
@@ -43,23 +50,28 @@ EOT
 Usage: spout COMMAND [ARGS]
 
 The most common spout commands are:
-  [n]ew       Create a new Spout dictionary. "spout new my_dd" creates a
-              new data dictionary called MyDD in "./my_dd"
-  [t]est      Running the test file
-  [i]mport    Import a CSV file into the JSON repository
-  [e]xport    Export the JSON respository to a CSV
-  [v]ersion   Returns the version of Spout
-
-Each command can be referenced by the first letter: Ex: `spout t`, for test
+  [n]ew             Create a new Spout dictionary.
+                    "spout new my_dd" creates a new data
+                    dictionary called MyDD in "./my_dd"
+  [t]est            Running the test file
+  [i]mport          Import a CSV file into the JSON repository
+  [e]xport [1.0.0]  Export the JSON respository to a CSV
+ h[y]brid  [1.0.0]  Export the JSON repository in the Hybrid
+                    Dictionary format
+  [v]ersion         Returns the version of Spout
+
+Commands can be referenced by the first letter:
+  Ex: `spout t`, for test
 
 EOT
         puts help_message
       end
 
-      def new_data_dictionary_export(argv)
+      def new_data_dictionary_export(argv, type = '')
         version = argv[1].to_s.gsub(/[^a-zA-Z0-9\.-]/, '_').strip
         version_string = (version == '' ? "" : "VERSION=#{version}")
-        system "bundle exec rake dd:create #{version_string}"
+        type_string =  type.to_s == '' ? "" : "TYPE=#{type}"
+        system "bundle exec rake dd:create #{version_string} #{type_string}"
       end
 
       def new_template_dictionary(argv)

data/lib/spout/tasks/engine.rake

@@ -19,75 +19,231 @@ namespace :dd do
   desc 'Create Data Dictionary from repository'
   task :create do
 
-    folder = "dd/#{ENV['VERSION'] || Spout::Application.new.version}"
+    folder = "dd/#{ENV['VERSION'] || '1.0.0'}"
     FileUtils.mkpath folder
 
-    CSV.open("#{folder}/variables.csv", "wb") do |csv|
-      keys = %w(id display_name description type units domain labels calculation)
-      csv << ['folder'] + keys
-      Dir.glob("variables/**/*.json").each do |file|
-        if json = JSON.parse(File.read(file)) rescue false
-          variable_folder = file.gsub(/variables\//, '').split('/')[0..-2].join('/')
-          csv << [variable_folder] + keys.collect{|key| json[key].kind_of?(Array) ? json[key].join(';') : json[key].to_s}
-        end
-      end
-    end
-    CSV.open("#{folder}/domains.csv", "wb") do |csv|
-      keys = %w(value display_name description)
-      csv << ['folder', 'id'] + keys
-      Dir.glob("domains/**/*.json").each do |file|
-        if json = JSON.parse(File.read(file)) rescue false
-          domain_folder = file.gsub(/domains\//, '').split('/')[0..-2].join('/')
-          domain_name = file.gsub(/domains\//, '').split('/').last.to_s.gsub(/.json/, '')
-          json.each do |hash|
-            csv << [domain_folder, domain_name] + keys.collect{|key| hash[key]}
-          end
-        end
-      end
+    case ENV['TYPE']
+    when 'hybrid'
+      hybrid_export(folder)
+    else
+      standard_export(folder)
     end
 
+
     puts "Data Dictionary Created in #{folder}"
   end
 
   desc 'Initialize JSON repository from a CSV file: CSV=datadictionary.csv'
   task :import do
-    additional_csv_info =  "\n\nFor additional information on specifying CSV column headers before import see:\n\n    " + "https://github.com/sleepepi/spout#generate-a-new-repository-from-an-existing-csv-file".colorize( :light_cyan ) + "\n\n"
-
     puts ENV['CSV'].inspect
     if File.exists?(ENV['CSV'].to_s)
-      CSV.parse( File.open(ENV['CSV'].to_s, 'r:iso-8859-1:utf-8'){|f| f.read}, headers: true ) do |line|
-        row = line.to_hash
-        if not row.keys.include?('id')
-          puts "\nMissing column header `".colorize( :red ) + "id".colorize( :light_cyan ) + "` in data dictionary.".colorize( :red ) + additional_csv_info
-          exit(1)
+      ENV['TYPE'] == 'domains' ? import_domains : import_variables
+    else
+      puts "\nPlease specify a valid CSV file.".colorize( :red ) + additional_csv_info
+    end
+  end
+end
+
+def standard_export(folder)
+  CSV.open("#{folder}/variables.csv", "wb") do |csv|
+    keys = %w(id display_name description type units domain labels calculation)
+    csv << ['folder'] + keys
+    Dir.glob("variables/**/*.json").each do |file|
+      if json = JSON.parse(File.read(file)) rescue false
+        variable_folder = variable_folder_path(file)
+        csv << [variable_folder] + keys.collect{|key| json[key].kind_of?(Array) ? json[key].join(';') : json[key].to_s}
+      end
+    end
+  end
+  CSV.open("#{folder}/domains.csv", "wb") do |csv|
+    keys = %w(value display_name description)
+    csv << ['folder', 'domain_id'] + keys
+    Dir.glob("domains/**/*.json").each do |file|
+      if json = JSON.parse(File.read(file)) rescue false
+        domain_folder = domain_folder_path(file)
+        domain_name = extract_domain_name(file)
+        json.each do |hash|
+          csv << [domain_folder, domain_name] + keys.collect{|key| hash[key]}
+        end
+      end
+    end
+  end
+end
+
+def extract_domain_name(file)
+  file.gsub(/domains\//, '').split('/').last.to_s.gsub(/.json/, '')
+end
+
+def domain_folder_path(file)
+  file.gsub(/domains\//, '').split('/')[0..-2].join('/')
+end
+
+def variable_folder_path(file)
+  file.gsub(/variables\//, '').split('/')[0..-2].join('/')
+end
+
+def hybrid_concept_type(json)
+  if json['hybrid'] and json['hybrid']['type'].to_s != ''
+    json['hybrid']['type']
+  else
+    hybrid_concept_type_map(json['type'])
+  end
+end
+
+def hybrid_concept_type_map(variable_type)
+  hybrid_types = { "choices" => "categorical",
+                   "numeric" => "continuous",
+                   "integer" => "continuous",
+                   "string" => "free text",
+                   "text" => "free text",
+                   "date" => "datetime",
+                   "time" => "datetime",
+                   "file" => "free text" }
+  hybrid_types[variable_type] || variable_type
+end
+
+def hybrid_property(json, property)
+  json['hybrid'] ? json['hybrid'][property] : ''
+end
+
+def hybrid_export(folder)
+  domain_parents = {}
+  CSV.open("#{folder}/hybrid.csv", "wb") do |csv|
+    csv << ["Folder", "Short Name", "Description", "Concept Type", "Units", "Terms", "Internal Terms", "Parents", "Children", "Field Values", "Sensitivity", "Display Name", "Commonly Used", "Calculation", "Source Name", "Source File"]
+    Dir.glob("variables/**/*.json").each do |file|
+      if json = JSON.parse(File.read(file)) rescue false
+        if json['domain'].to_s != ''
+          domain_parents[json['domain'].to_s.downcase] ||= []
+          domain_parents[json['domain'].to_s.downcase] << json['id'].to_s
         end
-        next if row['id'] == ''
-        folder = File.join('variables', row.delete('folder').to_s)
-        FileUtils.mkpath folder
-        hash = {}
-        id = row.delete('id')
-        hash['id'] = id
-        hash['display_name'] = row.delete('display_name')
-        hash['description'] = row.delete('description').to_s
-        hash['type'] = row.delete('type')
-        domain = row.delete('domain').to_s
-        hash['domain'] = domain if domain != ''
-        units = row.delete('units').to_s
-        hash['units'] = units if units != ''
-        calculation = row.delete('calculation').to_s
-        hash['calculation'] = calculation if calculation != ''
-        labels = row.delete('labels').to_s.split(';')
-        hash['labels'] = labels if labels.size > 0
-        hash['other'] = row unless row.empty?
-
-        file_name = File.join(folder, id.downcase + '.json')
-        File.open(file_name, 'w') do |file|
-          file.write(JSON.pretty_generate(hash))
+        row = [
+          variable_folder_path(file),                             # Folder
+          json['id'],                                             # Short Name
+          json['description'],                                    # Description
+          hybrid_concept_type(json),                              # Concept Type
+          json['units'],                                          # Units
+          (json['labels'] || []).join(';'),                       # Terms
+          '',                                                     # Internal Terms
+          '',                                                     # Parents
+          '',                                                     # Children
+          '',                                                     # Field Values
+          hybrid_property(json, 'access level'),                  # Sensitivity
+          json['display_name'],                                   # Display Name
+          hybrid_property(json, 'most commonly used'),            # Commonly Used
+          json['calculation'],                                    # Calculation
+          hybrid_property(json, 'SOURCE'),                        # Source Name
+          hybrid_property(json, 'filename')                       # Source File
+        ]
+        csv << row
+      end
+    end
+    Dir.glob("domains/**/*.json").each do |file|
+      if json = JSON.parse(File.read(file)) rescue false
+        json.each do |option|
+          row = [
+            domain_folder_path(file),                             # Folder
+            extract_domain_name(file)+'_'+option['value'].to_s,   # Short Name
+            option['description'],                                # Description
+            'boolean',                                            # Concept Type
+            '',                                                   # Units
+            '',                                                   # Terms
+            option['value'],                                      # Internal Terms
+            (domain_parents[extract_domain_name(file).downcase] || []).join(';'), # Parents
+            '',                                                   # Children
+            '',                                                   # Field Values
+            '0',                                                  # Sensitivity
+            option['display_name'],                               # Display Name
+            '',                                                   # Commonly Used
+            '',                                                   # Calculation
+          ]
+          csv << row
         end
-        puts "      create".colorize( :green ) + "  #{file_name}"
       end
-    else
-      puts "\nPlease specify a valid CSV file.".colorize( :red ) + additional_csv_info
     end
   end
 end
+
+def import_variables
+  CSV.parse( File.open(ENV['CSV'].to_s, 'r:iso-8859-1:utf-8'){|f| f.read}, headers: true ) do |line|
+    row = line.to_hash
+    if not row.keys.include?('id')
+      puts "\nMissing column header `".colorize( :red ) + "id".colorize( :light_cyan ) + "` in data dictionary.".colorize( :red ) + additional_csv_info
+      exit(1)
+    end
+    next if row['id'] == ''
+    folder = File.join('variables', row.delete('folder').to_s)
+    FileUtils.mkpath folder
+    hash = {}
+    id = row.delete('id')
+    hash['id'] = id
+    hash['display_name'] = row.delete('display_name')
+    hash['description'] = row.delete('description').to_s
+    hash['type'] = row.delete('type')
+    domain = row.delete('domain').to_s
+    hash['domain'] = domain if domain != ''
+    units = row.delete('units').to_s
+    hash['units'] = units if units != ''
+    calculation = row.delete('calculation').to_s
+    hash['calculation'] = calculation if calculation != ''
+    labels = row.delete('labels').to_s.split(';')
+    hash['labels'] = labels if labels.size > 0
+    hash['other'] = row unless row.empty?
+
+    file_name = File.join(folder, id.downcase + '.json')
+    File.open(file_name, 'w') do |file|
+      file.write(JSON.pretty_generate(hash) + "\n")
+    end
+    puts "      create".colorize( :green ) + "  #{file_name}"
+  end
+end
+
+def import_domains
+  domains = {}
+
+  CSV.parse( File.open(ENV['CSV'].to_s, 'r:iso-8859-1:utf-8'){|f| f.read}, headers: true ) do |line|
+    row = line.to_hash
+    if not row.keys.include?('domain_id')
+      puts "\nMissing column header `".colorize( :red ) + "domain_id".colorize( :light_cyan ) + "` in data dictionary.".colorize( :red ) + additional_csv_info
+      exit(1)
+    end
+    if not row.keys.include?('value')
+      puts "\nMissing column header `".colorize( :red ) + "value".colorize( :light_cyan ) + "` in data dictionary.".colorize( :red ) + additional_csv_info
+      exit(1)
+    end
+    if not row.keys.include?('display_name')
+      puts "\nMissing column header `".colorize( :red ) + "display_name".colorize( :light_cyan ) + "` in data dictionary.".colorize( :red ) + additional_csv_info
+      exit(1)
+    end
+
+    next if row['domain_id'].to_s == '' or row['value'].to_s == '' or row['display_name'].to_s == ''
+    folder = File.join('domains', row['folder'].to_s).gsub(/[^a-zA-Z0-9_\/\.-]/, '_')
+    domain_name = row['domain_id'].to_s.gsub(/[^a-zA-Z0-9_\/\.-]/, '_')
+    domains[domain_name] ||= {}
+    domains[domain_name]["folder"] = folder
+    domains[domain_name]["options"] ||= []
+
+    hash = {}
+    hash['value'] = row.delete('value').to_s
+    hash['display_name'] = row.delete('display_name').to_s
+    hash['description'] = row.delete('description').to_s
+
+    domains[domain_name]["options"] << hash
+  end
+
+  domains.each do |domain_name, domain_hash|
+    folder = domain_hash["folder"]
+    FileUtils.mkpath folder
+
+    file_name = File.join(folder, domain_name.downcase + '.json')
+
+    File.open(file_name, 'w') do |file|
+      file.write(JSON.pretty_generate(domain_hash["options"]) + "\n")
+    end
+    puts "      create".colorize( :green ) + "  #{file_name}"
+  end
+
+end
+
+def additional_csv_info
+  "\n\nFor additional information on specifying CSV column headers before import see:\n\n    " + "https://github.com/sleepepi/spout#generate-a-new-repository-from-an-existing-csv-file".colorize( :light_cyan ) + "\n\n"
+end

data/lib/spout/version.rb

@@ -1,7 +1,7 @@
 module Spout
   module VERSION #:nodoc:
     MAJOR = 0
-    MINOR = 1
+    MINOR = 2
     TINY = 0
     BUILD = nil # nil, "pre", "rc", "rc2"
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: spout
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.0
 platform: ruby
 authors:
 - Remo Mueller
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-05-21 00:00:00.000000000 Z
+date: 2013-06-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake