idata 0.1.28 → 0.1.29

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: d40de88a668b52fa14ae83259cf63b59697a1deb
-  data.tar.gz: 759fd49acead65db9206210d9db3054b4fe4e4a1
+  metadata.gz: d37af89a0b0962c026e20d63637f3a750030429f
+  data.tar.gz: a67cdfd07638f57788c837667c01e4f259867fa6
 SHA512:
-  metadata.gz: bc2af89be0e37eb04be0cc237e063b5ade86c22e3028726df179ac68bd0d3fc58aab9cde81782215a89cc1d0fc8a9445ad3425da0e3c6e8e8ce690c9153001b3
-  data.tar.gz: 349f28149ca318f7bd42fcb38685cba845992c6b0e380351392fe1b2d6b34cb982409721d44de91ce783ab4df1d103a3bd59d678fb6498f723db47bdb9d666c8
+  metadata.gz: 58eec49a072286d87b643f58be7c456e0d9140c7341b0816fc46b790241a9ef07fff0c2e86f72b2eda17f3c76defebb3259bd0fc50f912a9e90ff0fda2cd0511
+  data.tar.gz: bd83cd1f605863ed36bdf6c71236233bed41a0830013683b6906c9834fe23b12d482807eef38d318f6dec364ab14d55d8017734555ec01a63b6faa0270d67f10

data/README.md

@@ -1,14 +1,26 @@
-# OVERVIEW
+# Overview
 We provide some useful utilities for validating data in a PostgreSQL data table
 These utilities can be used as simple terminal commands and can be installed by:
 
     gem install idata
 
+idata comes along with the following commands:
+* iload
+* ivalidate
+* ipatch
+* ieval
+* iexport
+* imerge
+* isanitize
+
+Run a command with --help switch for the details
+
 Prequisites:
 * PostgreSQL 9.0 or above
 * Ruby 2.0 or above
+* An auto ID field is required for data table to be validated using ivalidate 
 
-# USAGE
+# Usage
 Suppose we have an `items` table, and we want to validate its records against certain criteria like:
 
 * `item_id` must not be null
@@ -65,7 +77,7 @@ For example, the following two checks are equivalent:
 Note: run `ivalidate --help` to see the full list of supported switches
 
 
-# PUT IT ALL TOGETHER
+# Put it all together
 You can put several `ivalidate` commands (for several data tables) in one single bash/sh file.
 Besides `ivalidate`, we also support some other utilities to:
 + Load data from text files to SQL tables

data/bin/ivalidate

@@ -73,6 +73,10 @@ parser = OptionParser.new("", 24) do |opts|
   opts.on("--pretty", "Use more human-readable error message") do |v|
     $options[:pretty] = v
   end
+  
+  opts.on("--case-insensitive", "Use more human-readable error message") do |v|
+    $options[:case_insensitive] = v
+  end
 
   opts.on("-h", "--host HOST", "PostgreSQL host") do |v|
     $options[:host] = v
@@ -185,6 +189,15 @@ class String
     
     "(#{sql})"
   end
+  
+  def lower
+    a = self.split(/\s*,\s*/)
+    sql = a.map{|s|
+      "lower(#{s})"
+    }.join(",")
+    
+    sql
+  end
 end
 
 
@@ -215,11 +228,15 @@ ActiveRecord::Base.connection.execute(pre_sql)
 $options[:unique].each do |field|
   begin
     puts "Checking uniqueness: #{field}"
-
+    if $options[:case_insensitive]
+      f_lower = field.lower
+    else
+      f_lower = field
+    end
     uniq_sql = <<-eos
       UPDATE #{$options[:table]} SET #{$options[:log_to]} = array_to_string(string_to_array(#{$options[:log_to]}, ' || ') || string_to_array('[#{field}] is not unique', ' || '), ' || ')
       WHERE id IN (
-        SELECT unnest(array_agg(id)) FROM #{$options[:table]} GROUP BY #{field}
+        SELECT unnest(array_agg(id)) FROM #{$options[:table]} GROUP BY #{f_lower}
         HAVING count(*) > 1
       ) AND #{field.not_null_sql};
     eos
@@ -239,19 +256,24 @@ $options[:consistent_by].each do |fields|
     
     raise "input must be in field1|field2 format" if fields.count != 2
     
-    f1 = fields.first
-    f2 = fields.last
+    f1_case = f1 = fields.first
+    f2_case = f2 = fields.last
+    
+    if $options[:case_insensitive]
+      f1_case = f1_case.lower
+      f2_case = f2_case.lower
+    end
     
-    puts "Checking uniqueness: #{f1} | #{f2}"
+    puts "Checking consistency: #{f1} | #{f2}"
 
     uniq_sql = <<-eos
-      UPDATE #{$options[:table]} SET #{$options[:log_to]} = array_to_string(string_to_array(#{$options[:log_to]}, ' || ') || string_to_array('same [#{f2}] but with different #{f1}', ' || '), ' || ')
+      UPDATE #{$options[:table]} SET #{$options[:log_to]} = array_to_string(string_to_array(#{$options[:log_to]}, ' || ') || string_to_array('same [#{f2}] but with different [#{f1}]', ' || '), ' || ')
       WHERE id IN
       (
         SELECT unnest(array_agg(id)) FROM #{$options[:table]}
         WHERE #{f1.not_null_sql} AND #{f2.not_null_sql}
-        GROUP BY #{f2}
-        HAVING COUNT(distinct #{f1}) > 1
+        GROUP BY #{f2_case}
+        HAVING COUNT(distinct #{f1_case}) > 1
       );
     eos
     
@@ -340,19 +362,25 @@ $options[:cross_reference].each do |value|
       raise "Error: Wrong argument for --cross-reference switch"
       exit(0)
     end
-
+    
     field = values[0]
     ref_table = values[1]
     ref_field = values[2]
 
     puts "Checking data integrity: #{value}"
+    
+    if $options[:case_insensitive]
+      join_condition = "on lower(origin.#{field}) = lower(target.#{ref_field})"
+    else
+      join_condition = "on origin.#{field} = target.#{ref_field}"
+    end
 
     # @todo: poor performance here, think of a better SQL!!!
     ref_sql = <<-eos
       UPDATE #{$options[:table]} SET #{$options[:log_to]} = array_to_string(string_to_array(#{$options[:log_to]}, ' || ') || string_to_array('#{field} does not reference #{values[1]}.#{values[2]}', ' || '), ' || ')
       WHERE #{field} IN (
         SELECT origin.#{field} from #{$options[:table]} origin LEFT JOIN #{ref_table} target
-        on origin.#{field} = target.#{ref_field}
+        #{join_condition}
         where target.#{ref_field} is null
       ) AND #{field} IS NOT NULL AND length(trim(#{field})) <> 0;
     eos

data/lib/idata/version.rb

@@ -1,3 +1,3 @@
 module Idata
-  VERSION = "0.1.28"
+  VERSION = "0.1.29"
 end

data/sample.sh

@@ -94,7 +94,7 @@ ivalidate --table=$ITEM \
 # Step 3 - Generate summary report
 ###################################################################################
 # After the validation step above, an additional field named validation_errors
-# is added to every table. In case the record does not pass a validation creterion, a corresponding error shall be logged to this field
+# is added to every table. In case the record does not pass a validation criterion, a corresponding error shall be logged to this field
 # One record may have more than one error logged
 # 
 # You can simply look at the validation_errors field to see errors associated to a record
@@ -104,7 +104,6 @@ ivalidate --table=$ITEM \
 #    2 Extract the first 1000 sample records for every error
 #    3 Put all together into one single Excel report
 
-
 # 1) Create error summary report table and write to /tmp/summary.csv
 # This can be done using the iexport utility which can generate a CSV file from a data table or from a custom query
 # Run iexport --help for more information

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: idata
 version: !ruby/object:Gem::Version
-  version: 0.1.28
+  version: 0.1.29
 platform: ruby
 authors:
 - Nghi Pham
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-07-21 00:00:00.000000000 Z
+date: 2014-07-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler