spout 0.6.0.beta4 → 0.6.0.beta5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 68fc0a3c4fab7e033261d72e655de1bdfbc2754e
-  data.tar.gz: eb62cad1ad96fc30c3ce3b3dc2a331d59ef3cb26
+  metadata.gz: 334f5a8e2330e851b8334cf6109d73dfef8486f6
+  data.tar.gz: 95347c28437cbe8866dbf11a903c0e877ce7bb48
 SHA512:
-  metadata.gz: 75387420b459a3d3b57aae192ebf9f0a0a7607bac20c41f72a094a5cb1ebd40726d71d8844efbb5631832796b6d642c6777cd78c8685a237c2544f176508c33f
-  data.tar.gz: f10ca5e16cb21f4d2ab94e55aa56ef92df57cecce2affd6c7a878ce8b8afca9b995eb0b68c1f626c3febd1dac2de5410f9fdecaa437f1e636b5f03958b1c7320
+  metadata.gz: 3797558300ae6080c8371ab11d9cbbf92c7174dd1c2bd59341d3c1b84fb88fed299b0409cb6b12018032694ecd06b2e265621e5a77948fdfaecc3f25d795d9e2
+  data.tar.gz: 0f139f427db50c0ec1019ecc62bdc881c87d71e16d637d7788340dac4df0a42ecea2c15fa5020d1b8c72a8412b0bdaf24153b52a8a5f856b3780275f46cbac75

data/lib/spout/tasks/engine.rake

@@ -46,66 +46,51 @@ namespace :dd do
 
   desc 'Match CSV dataset with JSON repository'
   task :coverage do
-    puts csvs = Dir.glob("dd/csvs/*.csv")
+    require 'spout/tests/variable_type_validation'
 
-    all_column_headers = []
-
-    variable_json_ids = []
     choice_variables = []
-    variable_file_names = []
 
     Dir.glob("variables/**/*.json").each do |file|
       if json = JSON.parse(File.read(file)) rescue false
-        variable_json_ids << json['id']
         choice_variables << json['id'] if json['type'] == 'choices'
       end
-      variable_file_names << file.split('/').last.to_s.split('.json').first.to_s
     end
 
-    csvs.each do |csv_file|
-      csv_name = csv_file.split('/').last.to_s
-      column_headers = []
-
-      CSV.parse( File.open(csv_file, 'r:iso-8859-1:utf-8'){|f| f.read} ) do |line|
-        column_headers = line.collect{|l| [csv_name, l.to_s.downcase]}
-        break # Only read first line
-      end
+    all_column_headers = []
+    value_hash = {}
+    csv_names = []
 
-      all_column_headers += column_headers
-    end
+    Dir.glob("dd/csvs/*.csv").each do |csv_file|
+      csv_name = csv_file.split('/').last.to_s
+      csv_names << csv_name
+      puts "\nParsing: #{csv_name}"
 
-    value_hash = {}
-    row_count = 0
+      column_headers = []
+      row_count = 0
 
-    csvs.each do |csv_file|
-      total_row_count = CSV.readlines(csv_file, 'r:iso-8859-1:utf-8').size - 1
       CSV.parse( File.open(csv_file, 'r:iso-8859-1:utf-8'){|f| f.read}, headers: true ) do |line|
-        puts "ROW: #{row_count+1} of #{total_row_count}"
         row = line.to_hash
+        column_headers = row.collect{|key, val| [csv_name, key.to_s.downcase]} if row_count == 0
+
+        print "." if row_count % 100 == 0
+
         choice_variables.each do |column_name|
           value_hash[column_name] ||= []
           value_hash[column_name] = value_hash[column_name] | [row[column_name]] if row[column_name]
         end
 
         row_count += 1
-        # break if row_count > 10
       end
+
+      print "done\n"
+
+      all_column_headers += column_headers
     end
 
     @matching_results = []
 
     all_column_headers.each do |csv, column|
-      file = Dir.glob("variables/**/#{column}.json").first
-      valid_values = []
-      variable_type = ''
-      if json = JSON.parse(File.read(file)) rescue false
-        variable_type = json['type']
-        if variable_type == 'choices'
-          valid_values = load_valid_domain_values(json['domain'])
-        end
-      end
-
-      scr = SpoutCoverageResult.new(csv, column, variable_file_names, variable_json_ids, variable_type, valid_values, value_hash[column])
+      scr = SpoutCoverageResult.new(csv, column, value_hash[column])
       @matching_results << [ csv, column, scr ]
     end
 
@@ -113,16 +98,14 @@ namespace :dd do
 
     @coverage_results = []
 
-    csvs.each do |csv_file|
-      csv_name = csv_file.split('/').last.to_s
-
+    csv_names.each do |csv_name|
       total_column_count = @matching_results.select{|mr| mr[0] == csv_name}.count
       mapped_column_count = @matching_results.select{|mr| mr[0] == csv_name and mr[2].number_of_errors == 0}.count
-
       @coverage_results << [ csv_name, total_column_count, mapped_column_count ]
     end
 
     coverage_file = File.join(Dir.pwd, 'dd', 'index.html')
+    puts "\nGenerating: index.html\n\n"
 
     File.open(coverage_file, 'w+') do |file|
       name = 'index.html'
@@ -134,39 +117,64 @@ namespace :dd do
     open_command = 'start' if RUBY_PLATFORM.match(/mingw/) != nil
 
     system "#{open_command} #{coverage_file}" if ['start', 'open'].include?(open_command)
-    puts coverage_file
+    puts "#{coverage_file}\n\n"
   end
 
 end
 
-def load_valid_domain_values(domain_name)
-  values = []
-  file = Dir.glob("domains/**/#{domain_name}.json").first
-  if json = JSON.parse(File.read(file)) rescue false
-    values = json.collect{|hash| hash['value']}
-  end
-  values
-end
-
 class SpoutCoverageResult
-  attr_accessor :error, :error_message, :file_name_test, :json_id_test, :values_test, :valid_values, :csv_values, :variable_type
+  attr_accessor :error, :error_message, :file_name_test, :json_id_test, :values_test, :valid_values, :csv_values, :variable_type_test, :json, :domain_test
+
+  def initialize(csv, column, csv_values)
+    load_json(column)
+    load_valid_values
 
-  def initialize(csv, column, variable_file_names, variable_json_ids, variable_type, valid_values, csv_values)
-    # puts "Initialize"
-    @file_name_test = variable_file_names.include?(column)
-    @json_id_test = variable_json_ids.include?(column)
-    @variable_type = variable_type
-    @valid_values = valid_values
     @csv_values = csv_values
     @values_test = check_values
+    @variable_type_test = check_variable_type
+    @domain_test = check_domain_specified
+  end
+
+  def load_json(column)
+    file = Dir.glob("variables/**/#{column}.json").first
+    @file_name_test = (file != nil)
+    @json = JSON.parse(File.read(file)) rescue @json = {}
+    @json_id_test = (@json['id'].downcase == column)
+  end
+
+  def load_valid_values
+    valid_values = []
+    if @json['type'] == 'choices'
+      file = Dir.glob("domains/**/#{@json['domain']}.json").first
+      if json = JSON.parse(File.read(file)) rescue false
+        valid_values = json.collect{|hash| hash['value']}
+      end
+    end
+    @valid_values = valid_values
   end
 
   def number_of_errors
-    @file_name_test && @json_id_test && @values_test ? 0 : 1
+    @file_name_test && @json_id_test && @values_test && @variable_type_test && @domain_test ? 0 : 1
   end
 
   def check_values
-    variable_type != 'choices' || (valid_values | csv_values.compact).size == valid_values.size
+    @json['type'] != 'choices' || (@valid_values | @csv_values.compact).size == @valid_values.size
+  end
+
+  def check_variable_type
+    Spout::Tests::VariableTypeValidation::VALID_VARIABLE_TYPES.include?(@json['type'])
+  end
+
+  def check_domain_specified
+    if @json['type'] != 'choices'
+      true
+    else
+      domain_file = Dir.glob("domains/**/#{@json['domain']}.json").first
+      if domain_json = JSON.parse(File.read(domain_file)) rescue false
+        return domain_json.kind_of?(Array)
+      end
+      false
+    end
   end
 
   def errored?

data/lib/spout/version.rb

@@ -3,7 +3,7 @@ module Spout
     MAJOR = 0
     MINOR = 6
     TINY = 0
-    BUILD = "beta4" # nil, "pre", "rc", "rc2"
+    BUILD = "beta5" # nil, "pre", "rc", "rc2"
 
     STRING = [MAJOR, MINOR, TINY, BUILD].compact.join('.')
   end

data/lib/spout/views/index.html.erb

@@ -6,7 +6,7 @@
     <meta name="author" content="">
     <link rel="shortcut icon" href="data:image/vnd.microsoft.icon;base64,AAABAAEAEBAAAAEAIABoBAAAFgAAACgAAAAQAAAAIAAAAAEAIAAAAAAAAAQAABMLAAATCwAAAAAAAAAAAAAAAAAAAAAAIAAAAMIAAAAiAAAAAAAAAAAAAAAAAAAARAAAAEEAAAAcAAAAwQAAACEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAAAADCAAAAIgAAAAAAAAAAAAAAAAAAAEQAAABBAAAAHAAAAMEAAAAhAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAgAAAAwwAAACIAAAAAAAAAAAAAAAAAAABEAAAAQQAAABwAAADBAAAAIQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAHAAAAKoAAAAeAAAAAAAAAAAAAAAAAAAARAAAAEEAAAAYAAAAqwAAAB4AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAWAAAACAAAAB8AAAAfAAAAAAAAAEQAAABDAAAAAAAAABgAAAAIAAAAHQAAAB0AAAAEAAAAAAAAAAAAAAAAAAAAAAAAABkAAACuAAAArQAAABQAAABCAAAAQwAAAAAAAAAAAAAAGAAAAK0AAACrAAAAFgAAAAAAAAAAAAAAAAAAAAAAAAADAAAAGQAAABkAAAAAAAAARAAAAEMAAAAAAAAAAAAAAAMAAAAZAAAAGQAAAAMAAAAAAAAATAAAAEwAAABOAAAATAAAAEoAAABKAAAASQAAAHwAAAB7AAAASgAAAEwAAABMAAAASgAAAEoAAABMAAAATAAAAFAAAABnAAAA0AAAAGcAAABRAAAAUgAAAE8AAACAAAAAfgAAAE0AAABQAAAAUAAAAFAAAABSAAAAUgAAAFIAAAAAAAAAIAAAAL8AAAAfAAAAAAAAAAAAAAAAAAAARQAAAFkAAAAbAAAAHgAAAB8AAAAbAAAABAAAAAAAAAAAAAAAAAAAACAAAAC/AAAAHwAAAAIAAAAXAAAAAAAAAEgAAADPAAAAwwAAAMMAAADFAAAArAAAABgAAAAAAAAAAAAAAAAAAAAgAAAAvwAAAB4AAAAcAAAAqwAAABkAAABDAAAAWgAAAB0AAAAgAAAAHwAAABsAAAAEAAAAAAAAAAAAAAAAAAAAHAAAAKgAAAAaAAAAIAAAAMMAAAAdAAAAQgAAAEMAAAAAAAAABAAAABsAAAAfAAAAIAAAACEAAAAhAAAAAAAAAAQAAAAXAAAAAgAAACIAAADDAAAAHQAAAEIAAABDAAAAAAAAABkAAACrAAAAwwAAAMIAAADCAAAAwQAAAAAAAAAAAAAAAAAAAAAAAAAiAAAAwwAAAB0AAABCAAAAQwAAAAAAAAAEAAAAHQAAACEAAAAgAAAAIAAAACAAAAAAAAAAAAAAAAAAAAAAAAAAIgAAAMMAAAAdAAAAQgAAAEMAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAjg8AAI4PAACODwAAjg8AAIABAADgYQAA4mEAAAAAAAAAAAAAjgMAAIIDAACAAwAAgEAAAIBAAADwQAAA8H8AAA==">
 
-    <title>Spout Dataset and Data Dictionary Coverage</title>
+    <title>Spout Coverage</title>
 
     <!-- Bootstrap core CSS -->
     <link href="http://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
@@ -36,6 +36,10 @@ body {
   /* Set the fixed height of the footer here */
   height: 60px;
   background-color: #f5f5f5;
+}
+code.success {
+  color: #468847;
+  background-color: #dff0d8;
 }
     </style>
   </head>
@@ -45,7 +49,7 @@ body {
     <div class="navbar navbar-inverse navbar-fixed-top" role="navigation">
       <div class="container">
         <div class="navbar-header">
-          <a class="navbar-brand" href="#">Spout Dataset and Data Dictionary Coverage</a>
+          <a class="navbar-brand" href="#">Spout Coverage</a>
         </div>
       </div>
     </div>
@@ -83,44 +87,57 @@ body {
         <thead>
           <tr>
             <th>CSV</th>
-            <th>Dataset Column Header</th>
+            <th>Column</th>
             <th>JSON File</th>
-            <th>Found in Data Dictionary</th>
+            <th>Variable ID</th>
             <th>Variable Type</th>
+            <th>Variable Domain</th>
             <th>Values</th>
           </tr>
         </thead>
         <tbody>
           <% @matching_results.each do |csv, column, scr| %>
             <tr>
-              <td><code><%= csv %></code></td>
+              <td><code class="<%= 'success' if scr.number_of_errors == 0 %>"><%= csv %></code></td>
               <td><%= column %></td>
               <td>
                 <% if scr.file_name_test %>
-                  <span class="text-success">File Found</span>
+                  <div class="text-success" style="text-align:center"><span class="glyphicon glyphicon-ok"></span></div>
                 <% else %>
-                  <span class="text-danger">No JSON File Found, expecting: <code><%= column %>.json</code></span>
+                  <span class="text-danger"><code><%= column %>.json</code> missing</span>
                 <% end %>
               </td>
               <td>
-                <% if scr.json_id_test %>
-                  <span class="text-success">JSON ID Found</span>
-                <% else %>
-                  <span class="text-danger">No Matching JSON ID</span>
-                  <pre>{
-  "id": "---", ...
-}</pre>
+                <% if scr.file_name_test %>
+                  <% if scr.json_id_test %>
+                    <code class="success">"id": "<%= column %>"</code>
+                  <% else %>
+                    <code>"id": <%= scr.json['id'].inspect %></code>
+                  <% end %>
                 <% end %>
               </td>
-              <td><%= scr.variable_type %></td>
               <td>
-                <% if scr.values_test %>
-                  <span class="text-success">Valid Values</span>
-                <% else %>
-                  VALID: <%= scr.valid_values %><br /><br />
-                  Bad Values:
-                  <% (scr.csv_values.compact - scr.valid_values).each do |value| %>
-                    <code><%= value %></code>
+                <% if scr.file_name_test %>
+                  <code class="<%= 'success' if scr.variable_type_test %>">"type": <%= scr.json['type'].inspect %></code>
+                <% end %>
+              </td>
+              <td>
+                <% if scr.json['type'] == 'choices' and scr.file_name_test %>
+                  <% if scr.domain_test or scr.json['domain'].to_s.strip == '' %>
+                    <code class="<%= 'success' if scr.domain_test %>">"domain": <%= scr.json['domain'].inspect %></code>
+                  <% else %>
+                    <span class="text-danger"><code><%= scr.json['domain'] %>.json</code> missing</span>
+                  <% end %>
+                <% end %>
+              </td>
+              <td style="white-space:nowrap">
+                <% if scr.json['type'] == 'choices' %>
+                  <% if scr.values_test %>
+                    <div class="text-success" style="text-align:center"><span class="glyphicon glyphicon-ok"></span></div>
+                  <% else %>
+                    <% (scr.valid_values + scr.csv_values.compact.sort).uniq.each do |value| %>
+                      <code class="<%= 'success' if scr.valid_values.include?(value) %>"><%= value %></code>
+                    <% end %>
                   <% end %>
                 <% end %>
               </td>
@@ -133,7 +150,7 @@ body {
 
     <div id="footer">
       <div class="container">
-        <p class="text-muted" style="margin: 20px 0;">Generated by <a href="https://github.com/sleepepi/spout">Spout v<%= Spout::VERSION::STRING %></a></p>
+        <p class="text-muted" style="margin: 20px 0;">Generated by <a href="https://github.com/sleepepi/spout">Spout</a> v<%= Spout::VERSION::STRING %></p>
       </div>
     </div>
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: spout
 version: !ruby/object:Gem::Version
-  version: 0.6.0.beta4
+  version: 0.6.0.beta5
 platform: ruby
 authors:
 - Remo Mueller