spout 0.6.0 → 0.7.0.beta1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 2ced3e644fac755624b158cc68ca0fc305c2ee58
-  data.tar.gz: 91b6e4f0c6974794664d2a61b32b6c3c0faeff48
+  metadata.gz: a0751d0dabac934f14dba3f6a34b7ab0ad181a64
+  data.tar.gz: 5b341a7e4f0c9558c7235a4ed9237ef4815ebe8f
 SHA512:
-  metadata.gz: b59ef048213f41475dbf1472882b89f6642f98d92b49d9d3fb492bcc68615d4dcaa84fd7487d8d3c97a2d4de397bc41ab8fdeb0825d542b6ce2d247e3ff147e2
-  data.tar.gz: 8e3e4d94be396306ba4aa40371e204af0ca57825badc3b3f526ef865a66fbcbefe3f337801daa63222d1fc1f9fd23f5748b593f61c03adc345ab3960cf3c0526
+  metadata.gz: 5f58f2ad27f7feebeb7815ff2fc612b0c7f8a4eb2c66033282e42a05296f0c68c9a2e6fa00b2f0f4ae5feae3e1ba54f30c9a8c40bc74843f0c2f314868dabffd
+  data.tar.gz: fc3c91243f3a0fc3518db19e2b0fc543702d1da9826774a7ab521d35595bdcb7f2dcb2b4d3b94aa3d5ef65872a571fda73deba79773fc4862e8c9482ae87179d

data/CHANGELOG.md

@@ -1,3 +1,8 @@
+## 0.7.0
+
+### Enhancements
+- Added `spout graphs` command that generates pie charts and histograms of each variable in a dataset
+
 ## 0.6.0 (March 7, 2014)
 
 ### Enhancement

data/README.md

@@ -146,6 +146,17 @@ You can optionally provide a version string
 spout export [1.0.0]
 ```
 
+### Generate graphs for data in your dataset
+
+Spout lets you generate graphs for each variable defined in your dataset. Make sure to run `spout coverage` first to validate that your data dictionary and dataset match.
+
+This command will take some time, and requires [PhantomJS](http://phantomjs.org/) to be installed on your system.
+
+```
+spout graphs
+```
+
+Generated graphs will be placed in: `./graphs/`
 
 ### Export to the Hybrid Data Dictionary format from your JSON repository
 

data/lib/spout/actions.rb

@@ -21,6 +21,8 @@ module Spout
         new_data_dictionary_export(argv, 'hybrid')
       when 'coverage', '-coverage', '--coverage', 'c', '-c'
         coverage_report(argv)
+      when 'graphs', '-graphs', '--graphs', 'g', '-g'
+        generate_graphs(argv)
       else
         help
       end
@@ -66,6 +68,9 @@ The most common spout commands are:
                     dictionary format
   [c]overage        Coverage report, requires dataset CSVs
                     in `<project_name>/csvs/`
+  [g]raphs          Generates graphs for each variable in a
+                    dataset and places them
+                    in `<project_name>/graphs/`
   [v]ersion         Returns the version of Spout
 
 Commands can be referenced by the first letter:
@@ -120,6 +125,10 @@ EOT
         system "bundle exec rake spout:coverage"
       end
 
+      def generate_graphs(argv)
+        system "bundle exec rake spout:graphs"
+      end
+
     private
 
       def copy_file(template_file, file_name = '')

data/lib/spout/commands/graphs.rb

@@ -0,0 +1,178 @@
+require 'csv'
+require 'fileutils'
+require 'rubygems'
+require 'json'
+
+module Spout
+  module Commands
+    class Graphs
+
+      def initialize
+
+        total_index_count = Dir.glob("variables/**/*.json").count
+
+        last_completed = 0
+
+        tmp_options_file = File.join( 'graphs', 'options.json' )
+
+        Dir.glob("csvs/*.csv").each do |csv_file|
+          puts "Working on: #{csv_file}"
+          t = Time.now
+          csv_table = CSV.table(csv_file,  encoding: 'iso-8859-1').by_col!
+          puts "Loaded #{csv_file} in #{Time.now - t} seconds."
+
+          total_header_count = csv_table.headers.count
+          csv_table.headers.each_with_index do |header, index|
+            puts "Column #{ index + 1 } of #{ total_header_count } for #{header} in #{csv_file}"
+            if variable_file = Dir.glob("variables/**/#{header.downcase}.json", File::FNM_CASEFOLD).first
+              json = JSON.parse(File.read(variable_file)) rescue json = nil
+              next unless json
+              next unless ["choices", "numeric", "integer"].include?(json["type"])
+
+              basename = File.basename(variable_file).gsub(/\.json$/, '').downcase
+              col_data = csv_table[header]
+
+              case json["type"] when "choices"
+                domain_file = Dir.glob("domains/**/#{json['domain']}.json").first
+                domain_json = JSON.parse(File.read(domain_file)) rescue domain_json = nil
+                next unless domain_json
+
+                create_pie_chart_options_file(col_data, tmp_options_file, domain_json)
+              when "numeric", "integer"
+                create_line_chart_options_file(col_data, tmp_options_file, json["units"])
+              else
+                next
+              end
+
+              run_phantom_js("#{basename}-lg.png", 600, tmp_options_file)
+              run_phantom_js("#{basename}.png",     75, tmp_options_file)
+            end
+          end
+        end
+        File.delete(tmp_options_file) if File.exists?(tmp_options_file)
+      end
+
+      def graph_values(col_data)
+        categories = []
+
+        col_data = col_data.select{|v| !['', 'null'].include?(v.to_s.strip.downcase)}.collect(&:to_f)
+
+        all_integers = false
+        all_integers = (col_data.count{|i| i.denominator != 1} == 0)
+
+        minimum = col_data.min || 0
+        maximum = col_data.max || 100
+
+        default_max_buckets = 30
+        max_buckets = all_integers ? [maximum - minimum + 1, default_max_buckets].min : default_max_buckets
+        bucket_size = (maximum - minimum + 1).to_f / max_buckets
+
+        (0..(max_buckets-1)).each do |bucket|
+          val_min = (bucket_size * bucket) + minimum
+          val_max = bucket_size * (bucket + 1) + minimum
+          # Greater or equal to val_min, less than val_max
+          # categories << "'#{val_min} to #{val_max}'"
+          categories << "#{all_integers || (maximum - minimum) > (default_max_buckets / 2)  ? val_min.round : "%0.02f" % val_min}"
+        end
+
+        new_values = []
+        (0..max_buckets-1).each do |bucket|
+          val_min = (bucket_size * bucket) + minimum
+          val_max = bucket_size * (bucket + 1) + minimum
+          # Greater or equal to val_min, less than val_max
+          new_values << col_data.count{|i| i >= val_min and i < val_max}
+        end
+
+        values = []
+
+        values << { name: '', data: new_values, showInLegend: false }
+
+        [ values, categories ]
+      end
+
+
+      def create_pie_chart_options_file(values, options_file, domain_json)
+
+        values.select!{|v| !['', 'null'].include?(v.to_s.strip.downcase) }
+        counts = values.group_by{|a| a}.collect{|k,v| [(domain_json.select{|h| h['value'] == k.to_s}.first['display_name'] rescue (k.to_s == '' ? 'NULL' : k)), v.count]}
+
+        total_count = counts.collect(&:last).inject(&:+)
+
+        data = counts.collect{|value, count| [value, (count * 100.0 / total_count)]}
+
+        File.open(options_file, "w") do |outfile|
+          outfile.puts <<-eos
+            {
+              "title": {
+                "text": ""
+              },
+
+              "credits": {
+                  "enabled": false,
+              },
+              "series": [{
+                        "type": "pie",
+                        "name": "",
+                        "data": #{data.to_json}
+              }]
+            }
+          eos
+        end
+      end
+
+
+      def create_line_chart_options_file(values, options_file, units)
+        ( series, categories ) = graph_values(values)
+
+        File.open(options_file, "w") do |outfile|
+          outfile.puts <<-eos
+            {
+              "chart": {
+                  "type": "areaspline"
+              },
+              "title": {
+                "text": ""
+              },
+              "credits": {
+                  "enabled": false,
+              },
+              "xAxis": {
+                  "categories": #{categories.to_json},
+                  "labels": {
+                      "step": #{(categories.size.to_f / 12).ceil}
+                  },
+                  "title": {
+                      "text": #{units.to_json}
+                  }
+              },
+              "yAxis": {
+                  "maxPadding": 0,
+                  "minPadding": 0,
+                  "title": {
+                    "text": "Count"
+                  }
+              },
+              "series": #{series.to_json}
+            }
+          eos
+        end
+      end
+
+      def run_phantom_js(png_name, width, tmp_options_file)
+        graph_path = File.join(Dir.pwd, 'graphs', png_name)
+        directory = File.join( File.dirname(__FILE__), '..', 'support', 'javascripts' )
+
+        open_command = if RUBY_PLATFORM.match(/mingw/) != nil
+          'phantomjs.exe'
+        else
+          'phantomjs'
+        end
+
+        phantomjs_command = "#{open_command} #{directory}/highcharts-convert.js -infile #{tmp_options_file} -outfile #{graph_path} -scale 2.5 -width #{width} -constr Chart"
+        # puts phantomjs_command
+        `#{phantomjs_command}`
+      end
+
+    end
+  end
+end

data/lib/spout/support/javascripts/data.js

@@ -0,0 +1,17 @@
+/*
+ Data plugin for Highcharts
+
+ (c) 2012-2013 Torstein Hønsi
+ Last revision 2013-06-07
+
+ License: www.highcharts.com/license
+*/
+(function(h){var k=h.each,m=function(b,a){this.init(b,a)};h.extend(m.prototype,{init:function(b,a){this.options=b;this.chartOptions=a;this.columns=b.columns||this.rowsToColumns(b.rows)||[];this.columns.length?this.dataFound():(this.parseCSV(),this.parseTable(),this.parseGoogleSpreadsheet())},getColumnDistribution:function(){var b=this.chartOptions,a=b&&b.chart&&b.chart.type,c=[];k(b&&b.series||[],function(b){c.push((h.seriesTypes[b.type||a||"line"].prototype.pointArrayMap||[0]).length)});this.valueCount=
+{global:(h.seriesTypes[a||"line"].prototype.pointArrayMap||[0]).length,individual:c}},dataFound:function(){this.parseTypes();this.findHeaderRow();this.parsed();this.complete()},parseCSV:function(){var b=this,a=this.options,c=a.csv,d=this.columns,f=a.startRow||0,i=a.endRow||Number.MAX_VALUE,j=a.startColumn||0,e=a.endColumn||Number.MAX_VALUE,g=0;c&&(c=c.replace(/\r\n/g,"\n").replace(/\r/g,"\n").split(a.lineDelimiter||"\n"),k(c,function(c,h){var n=b.trim(c),p=n.indexOf("#")===0;h>=f&&h<=i&&!p&&n!==""&&
+(n=c.split(a.itemDelimiter||","),k(n,function(b,a){a>=j&&a<=e&&(d[a-j]||(d[a-j]=[]),d[a-j][g]=b)}),g+=1)}),this.dataFound())},parseTable:function(){var b=this.options,a=b.table,c=this.columns,d=b.startRow||0,f=b.endRow||Number.MAX_VALUE,i=b.startColumn||0,j=b.endColumn||Number.MAX_VALUE,e;a&&(typeof a==="string"&&(a=document.getElementById(a)),k(a.getElementsByTagName("tr"),function(a,b){e=0;b>=d&&b<=f&&k(a.childNodes,function(a){if((a.tagName==="TD"||a.tagName==="TH")&&e>=i&&e<=j)c[e]||(c[e]=[]),
+c[e][b-d]=a.innerHTML,e+=1})}),this.dataFound())},parseGoogleSpreadsheet:function(){var b=this,a=this.options,c=a.googleSpreadsheetKey,d=this.columns,f=a.startRow||0,i=a.endRow||Number.MAX_VALUE,j=a.startColumn||0,e=a.endColumn||Number.MAX_VALUE,g,h;c&&jQuery.getJSON("https://spreadsheets.google.com/feeds/cells/"+c+"/"+(a.googleSpreadsheetWorksheet||"od6")+"/public/values?alt=json-in-script&callback=?",function(a){var a=a.feed.entry,c,k=a.length,m=0,o=0,l;for(l=0;l<k;l++)c=a[l],m=Math.max(m,c.gs$cell.col),
+o=Math.max(o,c.gs$cell.row);for(l=0;l<m;l++)if(l>=j&&l<=e)d[l-j]=[],d[l-j].length=Math.min(o,i-f);for(l=0;l<k;l++)if(c=a[l],g=c.gs$cell.row-1,h=c.gs$cell.col-1,h>=j&&h<=e&&g>=f&&g<=i)d[h-j][g-f]=c.content.$t;b.dataFound()})},findHeaderRow:function(){k(this.columns,function(){});this.headerRow=0},trim:function(b){return typeof b==="string"?b.replace(/^\s+|\s+$/g,""):b},parseTypes:function(){for(var b=this.columns,a=b.length,c,d,f,i;a--;)for(c=b[a].length;c--;)d=b[a][c],f=parseFloat(d),i=this.trim(d),
+i==f?(b[a][c]=f,f>31536E6?b[a].isDatetime=!0:b[a].isNumeric=!0):(d=this.parseDate(d),a===0&&typeof d==="number"&&!isNaN(d)?(b[a][c]=d,b[a].isDatetime=!0):b[a][c]=i===""?null:i)},dateFormats:{"YYYY-mm-dd":{regex:"^([0-9]{4})-([0-9]{2})-([0-9]{2})$",parser:function(b){return Date.UTC(+b[1],b[2]-1,+b[3])}}},parseDate:function(b){var a=this.options.parseDate,c,d,f;a&&(c=a(b));if(typeof b==="string")for(d in this.dateFormats)a=this.dateFormats[d],(f=b.match(a.regex))&&(c=a.parser(f));return c},rowsToColumns:function(b){var a,
+c,d,f,i;if(b){i=[];c=b.length;for(a=0;a<c;a++){f=b[a].length;for(d=0;d<f;d++)i[d]||(i[d]=[]),i[d][a]=b[a][d]}}return i},parsed:function(){this.options.parsed&&this.options.parsed.call(this,this.columns)},complete:function(){var b=this.columns,a,c,d=this.options,f,i,j,e,g,k;if(d.complete){this.getColumnDistribution();b.length>1&&(a=b.shift(),this.headerRow===0&&a.shift(),a.isDatetime?c="datetime":a.isNumeric||(c="category"));for(e=0;e<b.length;e++)if(this.headerRow===0)b[e].name=b[e].shift();i=[];
+for(e=0,k=0;e<b.length;k++){f=h.pick(this.valueCount.individual[k],this.valueCount.global);j=[];for(g=0;g<b[e].length;g++)j[g]=[a[g],b[e][g]!==void 0?b[e][g]:null],f>1&&j[g].push(b[e+1][g]!==void 0?b[e+1][g]:null),f>2&&j[g].push(b[e+2][g]!==void 0?b[e+2][g]:null),f>3&&j[g].push(b[e+3][g]!==void 0?b[e+3][g]:null),f>4&&j[g].push(b[e+4][g]!==void 0?b[e+4][g]:null);i[k]={name:b[e].name,data:j};e+=f}d.complete({xAxis:{type:c},series:i})}}});h.Data=m;h.data=function(b,a){return new m(b,a)};h.wrap(h.Chart.prototype,
+"init",function(b,a,c){var d=this;a&&a.data?h.data(h.extend(a.data,{complete:function(f){a.series&&k(a.series,function(b,c){a.series[c]=h.merge(b,f.series[c])});a=h.merge(f,a);b.call(d,a,c)}}),a):b.call(d,a,c)})})(Highcharts);