spout 0.6.0.beta2 → 0.6.0.beta3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: f4ae8ef2362ef67fc7cb4cb86560fabee408ef7b
-  data.tar.gz: aa825b0f03d9def1e26e784c09eb205424896258
+  metadata.gz: e85fa1a3ea963f131b61ce1952d10f1744c510a1
+  data.tar.gz: 2569c7aa7cbf7ce5bab644c128707e70a6ac659c
 SHA512:
-  metadata.gz: 815ae970ecfdf3ba9db2aa6f2095eff92608df08312e7903fdaf23e15243aca4a842147ada5031faa7fee594646b6f9632b0bfa1be87de408f7e41fe32d45562
-  data.tar.gz: 1b55348b328af02eee3e2ee189f88778584f09cc1a82eff5160d4258c8bdf78646facb23c3dd2baa515897260ed223cb2a6d4ec9dce49c1ba23af178051210c3
+  metadata.gz: c7766b6aadc863f2159fa7e8ba33d10be37d97feb2f4a71317049941cd29d3a7bcab1128c3613ed945a4da62ea11aea073870b8461b1dd86330b2d95c83e6f95
+  data.tar.gz: 35005ff394184b10aaad88955b7129401c81ee8ee883b91cf952eb0e4c68a88709c583eacb48e64a1e50785ea0f036e694172871038825533a6ce347297a8fbd

data/CHANGELOG.md

@@ -3,6 +3,7 @@
 ### Enhancement
 - Added `spout match` command that generates a coverage report of how well a dataset matches the data dictionary
   - Spout Match generates a viewable report in `dd\index.html` that shows which columns are covered in CSVs located in `dd\csvs`
+  - Spout Match checks that all collected values for a variable with a domain exist in the associated domain
 - **Gem Changes**
   - Updated to colorize 0.6.0
 

data/lib/spout/tasks/engine.rake

@@ -72,19 +72,40 @@ namespace :dd do
       all_column_headers += column_headers
     end
 
+    value_hash = {}
+    row_count = 0
 
-    all_column_headers
+    csvs.each do |csv_file|
+      CSV.parse( File.open(csv_file, 'r:iso-8859-1:utf-8'){|f| f.read}, headers: true ) do |line|
+        row = line.to_hash
+        row.each do |column_name, value|
+          value_hash[column_name] ||= []
+          value_hash[column_name] = value_hash[column_name] | [value]
+        end
+
+        row_count += 1
+        # break if row_count > 10
+      end
+    end
 
     @matching_results = []
 
     all_column_headers.each do |csv, column|
-      file_name_test = variable_file_names.include?(column)
-      json_id_test = variable_json_ids.include?(column)
-      # SpoutCoverageResult
-      @matching_results << [ csv, column, file_name_test, json_id_test ]
+      file = Dir.glob("variables/**/#{column}.json").first
+      valid_values = []
+      variable_type = ''
+      if json = JSON.parse(File.read(file)) rescue false
+        variable_type = json['type']
+        if variable_type == 'choices'
+          valid_values = load_valid_domain_values(json['domain'])
+        end
+      end
+
+      scr = SpoutCoverageResult.new(csv, column, variable_file_names, variable_json_ids, variable_type, valid_values, value_hash[column])
+      @matching_results << [ csv, column, scr ]
     end
 
-    @matching_results.sort!{|a,b| [(a[2] && a[3] ? 1 : 0), a[0].to_s, a[1].to_s] <=> [(b[2] && b[3] ? 1 : 0), b[0].to_s, b[1].to_s]}
+    @matching_results.sort!{|a,b| [b[2].number_of_errors, a[0].to_s, a[1].to_s] <=> [a[2].number_of_errors, b[0].to_s, b[1].to_s]}
 
     @coverage_results = []
 
@@ -92,7 +113,7 @@ namespace :dd do
       csv_name = csv_file.split('/').last.to_s
 
       total_column_count = @matching_results.select{|mr| mr[0] == csv_name}.count
-      mapped_column_count = @matching_results.select{|mr| mr[0] == csv_name and mr[2] and mr[3]}.count
+      mapped_column_count = @matching_results.select{|mr| mr[0] == csv_name and mr[2].number_of_errors == 0}.count
 
       @coverage_results << [ csv_name, total_column_count, mapped_column_count ]
     end
@@ -114,6 +135,45 @@ namespace :dd do
 
 end
 
+def load_valid_domain_values(domain_name)
+  values = []
+  file = Dir.glob("domains/**/#{domain_name}.json").first
+  if json = JSON.parse(File.read(file)) rescue false
+    values = json.collect{|hash| hash['value']}
+  end
+  values
+end
+
+class SpoutCoverageResult
+  attr_accessor :error, :error_message, :file_name_test, :json_id_test, :values_test, :valid_values, :csv_values, :variable_type
+
+  def initialize(csv, column, variable_file_names, variable_json_ids, variable_type, valid_values, csv_values)
+    # puts "Initialize"
+    @file_name_test = variable_file_names.include?(column)
+    @json_id_test = variable_json_ids.include?(column)
+    @variable_type = variable_type
+    @valid_values = valid_values
+    @csv_values = csv_values
+    @values_test = check_values
+  end
+
+  def number_of_errors
+    @file_name_test && @json_id_test && @values_test ? 0 : 1
+  end
+
+  def check_values
+    variable_type != 'choices' || (valid_values | csv_values.compact).size == valid_values.size
+  end
+
+  def errored?
+    error == true
+  end
+end
+
+def number_with_delimiter(number, delimiter = ",")
+  number.to_s.gsub(/(\d)(?=(\d\d\d)+(?!\d))/, "\\1#{delimiter}")
+end
+
 def standard_version
   version = File.open('VERSION', &:readline).strip rescue ''
   version == '' ? '1.0.0' : version
@@ -249,12 +309,3 @@ end
 def additional_csv_info
   "\n\nFor additional information on specifying CSV column headers before import see:\n\n    " + "https://github.com/sleepepi/spout#generate-a-new-repository-from-an-existing-csv-file".colorize( :light_cyan ) + "\n\n"
 end
-
-
-# class SpoutCoverageResult
-#   attr_accessor :error, :error_message
-
-#   def errored?
-#     error == true
-#   end
-# end

data/lib/spout/version.rb

@@ -3,7 +3,7 @@ module Spout
     MAJOR = 0
     MINOR = 6
     TINY = 0
-    BUILD = "beta2" # nil, "pre", "rc", "rc2"
+    BUILD = "beta3" # nil, "pre", "rc", "rc2"
 
     STRING = [MAJOR, MINOR, TINY, BUILD].compact.join('.')
   end

data/lib/spout/views/index.html.erb

@@ -6,7 +6,7 @@
     <meta name="author" content="">
     <link rel="shortcut icon" href="data:image/vnd.microsoft.icon;base64,AAABAAEAEBAAAAEAIABoBAAAFgAAACgAAAAQAAAAIAAAAAEAIAAAAAAAAAQAABMLAAATCwAAAAAAAAAAAAAAAAAAAAAAIAAAAMIAAAAiAAAAAAAAAAAAAAAAAAAARAAAAEEAAAAcAAAAwQAAACEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAAAADCAAAAIgAAAAAAAAAAAAAAAAAAAEQAAABBAAAAHAAAAMEAAAAhAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAgAAAAwwAAACIAAAAAAAAAAAAAAAAAAABEAAAAQQAAABwAAADBAAAAIQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAHAAAAKoAAAAeAAAAAAAAAAAAAAAAAAAARAAAAEEAAAAYAAAAqwAAAB4AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAWAAAACAAAAB8AAAAfAAAAAAAAAEQAAABDAAAAAAAAABgAAAAIAAAAHQAAAB0AAAAEAAAAAAAAAAAAAAAAAAAAAAAAABkAAACuAAAArQAAABQAAABCAAAAQwAAAAAAAAAAAAAAGAAAAK0AAACrAAAAFgAAAAAAAAAAAAAAAAAAAAAAAAADAAAAGQAAABkAAAAAAAAARAAAAEMAAAAAAAAAAAAAAAMAAAAZAAAAGQAAAAMAAAAAAAAATAAAAEwAAABOAAAATAAAAEoAAABKAAAASQAAAHwAAAB7AAAASgAAAEwAAABMAAAASgAAAEoAAABMAAAATAAAAFAAAABnAAAA0AAAAGcAAABRAAAAUgAAAE8AAACAAAAAfgAAAE0AAABQAAAAUAAAAFAAAABSAAAAUgAAAFIAAAAAAAAAIAAAAL8AAAAfAAAAAAAAAAAAAAAAAAAARQAAAFkAAAAbAAAAHgAAAB8AAAAbAAAABAAAAAAAAAAAAAAAAAAAACAAAAC/AAAAHwAAAAIAAAAXAAAAAAAAAEgAAADPAAAAwwAAAMMAAADFAAAArAAAABgAAAAAAAAAAAAAAAAAAAAgAAAAvwAAAB4AAAAcAAAAqwAAABkAAABDAAAAWgAAAB0AAAAgAAAAHwAAABsAAAAEAAAAAAAAAAAAAAAAAAAAHAAAAKgAAAAaAAAAIAAAAMMAAAAdAAAAQgAAAEMAAAAAAAAABAAAABsAAAAfAAAAIAAAACEAAAAhAAAAAAAAAAQAAAAXAAAAAgAAACIAAADDAAAAHQAAAEIAAABDAAAAAAAAABkAAACrAAAAwwAAAMIAAADCAAAAwQAAAAAAAAAAAAAAAAAAAAAAAAAiAAAAwwAAAB0AAABCAAAAQwAAAAAAAAAEAAAAHQAAACEAAAAgAAAAIAAAACAAAAAAAAAAAAAAAAAAAAAAAAAAIgAAAMMAAAAdAAAAQgAAAEMAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAjg8AAI4PAACODwAAjg8AAIABAADgYQAA4mEAAAAAAAAAAAAAjgMAAIIDAACAAwAAgEAAAIBAAADwQAAA8H8AAA==">
 
-    <title>Starter Template for Bootstrap</title>
+    <title>Spout Dataset and Data Dictionary Coverage</title>
 
     <!-- Bootstrap core CSS -->
     <link href="http://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
@@ -20,8 +20,22 @@
     <script src="https://ajax.googleapis.com/ajax/libs/jquery/1.11.0/jquery.min.js"></script>
     <script src="http://netdna.bootstrapcdn.com/bootstrap/3.1.1/js/bootstrap.min.js"></script>
     <style type="text/css">
+html {
+  position: relative;
+  min-height: 100%;
+}
 body {
   padding-top: 50px;
+  /* Margin bottom by footer height */
+  margin-bottom: 60px;
+}
+#footer {
+  position: absolute;
+  bottom: 0;
+  width: 100%;
+  /* Set the fixed height of the footer here */
+  height: 60px;
+  background-color: #f5f5f5;
 }
     </style>
   </head>
@@ -58,8 +72,8 @@ body {
                   <span class="text-muted">---</span>
                 <% end %>
               </td>
-              <td><%= total_column_count %></td>
-              <td><%= total_column_count - mapped_column_count %> </td>
+              <td><%= number_with_delimiter( total_column_count ) %></td>
+              <td><%= number_with_delimiter( total_column_count - mapped_column_count ) %></td>
             </tr>
           <% end %>
         </tbody>
@@ -72,22 +86,24 @@ body {
             <th>Dataset Column Header</th>
             <th>JSON File</th>
             <th>Found in Data Dictionary</th>
+            <th>Variable Type</th>
+            <th>Values</th>
           </tr>
         </thead>
         <tbody>
-          <% @matching_results.each do |csv, column, file_name_test, json_id_test| %>
+          <% @matching_results.each do |csv, column, scr| %>
             <tr>
               <td><code><%= csv %></code></td>
               <td><%= column %></td>
               <td>
-                <% if file_name_test %>
+                <% if scr.file_name_test %>
                   <span class="text-success">File Found</span>
                 <% else %>
                   <span class="text-danger">No JSON File Found, expecting: <code><%= column %>.json</code></span>
                 <% end %>
               </td>
               <td>
-                <% if json_id_test %>
+                <% if scr.json_id_test %>
                   <span class="text-success">JSON ID Found</span>
                 <% else %>
                   <span class="text-danger">No Matching JSON ID</span>
@@ -96,6 +112,18 @@ body {
 }</pre>
                 <% end %>
               </td>
+              <td><%= scr.variable_type %></td>
+              <td>
+                <% if scr.values_test %>
+                  <span class="text-success">Valid Values</span>
+                <% else %>
+                  VALID: <%= scr.valid_values %><br /><br />
+                  Bad Values:
+                  <% (scr.csv_values.compact - scr.valid_values).each do |value| %>
+                    <code><%= value %></code>
+                  <% end %>
+                <% end %>
+              </td>
             </tr>
           <% end %>
         </tbody>
@@ -103,5 +131,10 @@ body {
 
     </div><!-- /.container -->
 
+    <div id="footer">
+      <div class="container">
+        <p class="text-muted" style="margin: 20px 0;">Generated by <a href="https://github.com/sleepepi/spout">Spout v<%= Spout::VERSION::STRING %></a></p>
+      </div>
+    </div>
 
 </body></html>

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: spout
 version: !ruby/object:Gem::Version
-  version: 0.6.0.beta2
+  version: 0.6.0.beta3
 platform: ruby
 authors:
 - Remo Mueller
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-02-28 00:00:00.000000000 Z
+date: 2014-03-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake