idata 0.1.6 → 0.1.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 673cf283d4058de28221f3876c91f125b485c23f
-  data.tar.gz: d19d74ba799782ff5369da19151886eafbc758ee
+  metadata.gz: 63a5dd0a3ae8f3eddca79de0caf67073852e0e13
+  data.tar.gz: dcd2ffe755ea9e61e07a0e419489b983e52addc8
 SHA512:
-  metadata.gz: c72c6773422e856ee2e041f492814fa483ab3abff6a86b92e3df9cf2f28d908f002ee1373fc243bd83a9fb4eff65a52b99f2b16d96cab2c2e2427e6850b522d5
-  data.tar.gz: a4cd5113b26d8cbda3329b0aed56fa176e1f1cf7cf8e6294dd46938573863f9a8b8604d498c42248fb899de08c152f04dbcf3594996d90da22a1a650c54bfc0e
+  metadata.gz: 6c5b90be452b015cb5e4f422667555fe80d0fffc0b77007c1ac8bc1589f77557ca3aa3bd86b402ce21a381a7df65a5572251fd7a57585f8a42556ead6d42e658
+  data.tar.gz: d7aa54e1ad652cf60889069b5c73c53676176da2f89b586abeae6029c4c79d0c9d87ae61b865a653df8c6e203bafb337cf78bb8fce121a32baca2ff9a2bf3f21

data/bin/iexport

@@ -47,6 +47,14 @@ parser = OptionParser.new("", 24) do |opts|
   opts.on("--delim DELIMITER", "CSV delimiter") do |v|
     $options[:delim] = v
   end
+  
+  opts.on("--include FIELDS", "") do |v|
+    $options[:include] = v
+  end
+  
+  opts.on("--exclude FIELDS", "") do |v|
+    $options[:exclude] = v
+  end
 
   opts.on("--linebreak CHAR", "") do |v|
     $options[:linebreak] = v
@@ -163,6 +171,19 @@ $options[:quote_empty] ||= false
 $options[:select] ||= '*'
 $options[:where] ||= 'true'
 
+$options[:include] ||= ""
+
+
+if $options[:exclude]
+  $options[:exclude] = $options[:exclude].split(/\s*,\s*/)
+else
+  $options[:exclude] = []
+end
+
+$options[:select] = [$options[:select].split(/\s*,\s*/) + $options[:include].split(/\s*,\s*/)].join(", ")
+
+p $options[:select]
+
 #$options = {host: 'localhost', database: 'db', username: 'postgres', password: 'postgres', table: 'products', listen: 5432}
 ActiveRecord::Base.establish_connection(
   'adapter' => 'postgresql',
@@ -183,14 +204,14 @@ CSV.open($options[:output], "wb", :quote_char => $options[:quote_char], :col_sep
   first = scope.first
   
   if first
-    csv << first.attributes.keys if $options[:headers]
+    csv << first.attributes.delete_if{|k,v| k.nil? || $options[:exclude].include?(k) }.keys if $options[:headers]
   else
     # only use .column_names when table is empty, to ensure fields match values
     csv << Product.column_names if $options[:headers]
   end
   
   scope.each do |item|
-    attrs = item.attributes.values
+    attrs = item.attributes.delete_if{|k,v| k.nil? || $options[:exclude].include?(k) }.values
     if !$options[:quote_empty]
       attrs.map!{|e| e == "" ? nil : e }
     end

data/bin/iload

@@ -14,7 +14,7 @@ require 'rubygems'
 require 'digest/sha1'
 require 'fileutils'
 
-SUPPORTED_INPUT_FORMATS = ['CSV', 'FX']
+SUPPORTED_INPUT_FORMATS = ['CSV', 'FX', 'RPT']
 POSTGRESQL_PORT = 5432
 CSV_DEFAULT_DELIMITER = ','
 CSV_DEFAULT_QUOTE = '"'
@@ -34,6 +34,10 @@ parser = OptionParser.new("", 24) do |opts|
   opts.on("--delim DELIMITER", "Field DELIMITER (for CSV format only - default to COMMA ',')") do |v|
     $options[:delim] = v
   end
+  
+  opts.on("--null NULL", "") do |v|
+    $options[:null] = v
+  end
 
   opts.on("--quote QUOTE", "Default to '\"'") do |v|
     $options[:quote] = v
@@ -161,6 +165,14 @@ ActiveRecord::Base.establish_connection(
   'timeout' => 15000
 )
 
+$csv_converters = []
+$csv_converters << :null_converter if $options[:null]
+
+CSV::Converters[:null_converter] = lambda{ |s|
+  return nil if s == $options[:null]
+  return s
+}
+
 class String
   def underscore
     return self if self.nil?
@@ -177,14 +189,14 @@ class MyParser
   end
 
   def run  
-    load_fx if $options[:format] == 'FX'
+    load_fx if $options[:format] == 'FX' || $options[:format] == 'RPT'
     load_csv if $options[:format] == 'CSV'
   end
 
   def load_csv
     # Load CSV data from input file to a temp array    
     csv_data = []
-    CSV.foreach($options[:input], :col_sep => $options[:delim], :quote_char => $options[:quote]) do |csv|
+    CSV.foreach($options[:input], :col_sep => $options[:delim], :quote_char => $options[:quote], :converters => $csv_converters) do |csv|
       csv_data << csv
     end
     
@@ -214,6 +226,13 @@ class MyParser
       csv << headers
       data.each_with_index{|s, index|
         record = s.unpack(ranges).map{|e| e.strip}
+        
+        # take advantage of CSV converters
+        $csv_converters.each {|converter|
+          converter_lambda = CSV::Converters[converter]
+          record.map!(&converter_lambda)
+        }
+        
         csv << record
       }
     end

data/bin/ivalidate

@@ -135,7 +135,7 @@ if $options[:database].nil?
 end
 
 if $options[:username].nil?
-  puts "\nPlease specify PostgreSQL username: -d\n\n"
+  puts "\nPlease specify PostgreSQL username: -u\n\n"
   exit
 end
 
@@ -179,7 +179,7 @@ $options[:unique].each do |field|
   puts "Checking unique fields: #{field}"
 
   uniq_sql = <<-eos
-    UPDATE #{$options[:table]} SET #{$options[:log_to]} = array_to_string(string_to_array(#{$options[:log_to]}, ' || ') || string_to_array('#{field}-not-unique', ' || '), ' || ')
+    UPDATE #{$options[:table]} SET #{$options[:log_to]} = array_to_string(string_to_array(#{$options[:log_to]}, ' || ') || string_to_array('#{field} not unique', ' || '), ' || ')
     WHERE #{field} IN (
       SELECT #{field} FROM #{$options[:table]} GROUP BY #{field}
       HAVING count(*) > 1
@@ -196,7 +196,7 @@ $options[:not_null].each do |field|
   puts "Checking not-null fields: #{field}"
 
   not_null_sql = <<-eos
-    UPDATE #{$options[:table]} SET #{$options[:log_to]} = array_to_string(string_to_array(#{$options[:log_to]}, ' || ') || string_to_array('#{field}-null', ' || '), ' || ')
+    UPDATE #{$options[:table]} SET #{$options[:log_to]} = array_to_string(string_to_array(#{$options[:log_to]}, ' || ') || string_to_array('#{field} null', ' || '), ' || ')
     WHERE #{field} IS NULL OR length(trim(#{field})) = 0;
   eos
   
@@ -212,7 +212,7 @@ $options[:match].each do |value|
   puts "Checking REGEXP matching: #{regexp}"
 
   match_sql = <<-eos
-    UPDATE #{$options[:table]} SET #{$options[:log_to]} = array_to_string(string_to_array(#{$options[:log_to]}, ' || ') || string_to_array('#{regexp.gsub("'", "''")}-not-matched', ' || '), ' || ')
+    UPDATE #{$options[:table]} SET #{$options[:log_to]} = array_to_string(string_to_array(#{$options[:log_to]}, ' || ') || string_to_array('#{field} not match [#{regexp.gsub("'", "''")}]', ' || '), ' || ')
     WHERE #{field} IS NOT NULL AND length(trim(#{field})) <> 0 AND #{field} !~ '#{regexp}';
   eos
   
@@ -228,7 +228,7 @@ $options[:not_match].each do |value|
   puts "Checking REGEXP not matching: #{regexp}"
 
   not_match_sql = <<-eos
-    UPDATE #{$options[:table]} SET #{$options[:log_to]} = array_to_string(string_to_array(#{$options[:log_to]}, ' || ') || string_to_array('#{regexp.gsub("'", "''")}-unexpectedly-matched', ' || '), ' || ')
+    UPDATE #{$options[:table]} SET #{$options[:log_to]} = array_to_string(string_to_array(#{$options[:log_to]}, ' || ') || string_to_array('#{field} match [#{regexp.gsub("'", "''")}]', ' || '), ' || ')
     WHERE #{field} IS NOT NULL AND length(trim(#{field})) <> 0 AND #{field} ~ '#{regexp}';
   eos
   
@@ -254,7 +254,7 @@ $options[:cross_reference].each do |value|
 
   # @todo: poor performance here, think of a better SQL!!!
   ref_sql = <<-eos
-    UPDATE #{$options[:table]} SET #{$options[:log_to]} = array_to_string(string_to_array(#{$options[:log_to]}, ' || ') || string_to_array('#{field}-not-referenced-to-#{values[1]}.#{values[2]}', ' || '), ' || ')
+    UPDATE #{$options[:table]} SET #{$options[:log_to]} = array_to_string(string_to_array(#{$options[:log_to]}, ' || ') || string_to_array('#{field} not referenced to #{values[1]}.#{values[2]}', ' || '), ' || ')
     WHERE #{field} NOT IN (
       SELECT #{ref_field} FROM #{ref_table}
     ) AND #{field} IS NOT NULL AND length(trim(#{field})) <> 0;

data/lib/idata/version.rb

@@ -1,3 +1,3 @@
 module Idata
-  VERSION = "0.1.6"
+  VERSION = "0.1.9"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: idata
 version: !ruby/object:Gem::Version
-  version: 0.1.6
+  version: 0.1.9
 platform: ruby
 authors:
 - Nghi Pham
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-04-24 00:00:00.000000000 Z
+date: 2014-04-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler