idata 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 27e594b49b95b536a5d01ed6f90155751104ec1a
+  data.tar.gz: a4e52832bccdc2ab55ae33e7e6f659857b800991
+SHA512:
+  metadata.gz: f55f15be3fbc959bcfacdb68f2b45e5b97fa560ee953890a66eb5a4715366019240e61c1b9ef36df9ed77d0ba396d6e38b32177a555d75aeb4f0deb04ae99757
+  data.tar.gz: 0e8061590a8d5da84ee9f74d137c2f6d022a58009d9d56ca965241c10e064a9cde59fb1f0d784cab643d88d19c1cdbe3a2f688ce2bbfcc42f4cd50b14e2bcfda

data/.gitignore

@@ -0,0 +1,17 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in idata.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2014 TODO: Write your name
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,29 @@
+# Idata
+
+TODO: Write a gem description
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'idata'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install idata
+
+## Usage
+
+TODO: Write usage instructions here
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/Rakefile

@@ -0,0 +1 @@
+require "bundler/gem_tasks"

data/bin/ieval

@@ -0,0 +1,222 @@
+# DATA LOADER
+#
+# @author Nghi Pham
+# @date April 2014
+#
+# The script loads data from a fixed-width text file or a CSV file and fills in
+# a corresponding table in the specified database
+# Issue ruby load.rb --help for guideline/examples
+#
+# @todo Make the script OOP
+# @todo Constantize default values
+require 'optparse'
+require 'csv'
+require 'active_record'
+require 'rubygems'
+require 'digest/sha1'
+require 'fileutils'
+
+# Default configuration settings
+POSTGRESQL_PORT = 5432
+
+# Parse script inputs
+$options = {}
+parser = OptionParser.new("", 24) do |opts|
+  opts.banner = "\nProgram: Data Loader\nAuthor: MCKI\n\n"
+
+  opts.on("-e", "--eval SCRIPT", "Ruby SCRIPT to execute") do |v|
+    $options[:eval] = v
+  end
+
+  opts.on("-t", "--table TABLE", "Table name to be created") do |v|
+    $options[:table] = v
+  end
+
+  opts.on("-h", "--host HOST", "PostgreSQL host") do |v|
+    $options[:host] = v
+  end
+
+  opts.on("-d", "--database DATABASE", "PostgreSQL database") do |v|
+    $options[:database] = v
+  end
+
+  opts.on("-u", "--username USER", "PostgreSQL username") do |v|
+    $options[:username] = v
+  end
+
+  opts.on("-p", "--password PASSWORD", "PostgreSQL password") do |v|
+    $options[:password] = v
+  end
+
+  opts.on("-l", "--listen PORT", "PostgreSQL listen port (default to 5432)") do |v|
+    $options[:listen] = v
+  end
+
+  opts.on_tail('--help', 'Displays this help') do
+		puts opts, "", help
+    exit
+	end
+end
+
+# Help message
+def help
+  return <<-eos
+
+EXAMPLES
+-------------------------------------------------------
+Run a custom Ruby command on the every record represented by the `item` variable
+
+    ruby eval.rb --host=localhost --username=postgres --password=postgres \\
+                 --table=vendors --database=db \\
+                 --eval="item.last_name = item.full_name.split(/\\s+/).last"
+
+eos
+end
+
+begin
+  parser.parse!
+rescue SystemExit => ex
+  exit
+end
+
+# Certain arguments are required
+if $options[:eval].nil?
+  puts "\nPlease specify Ruby script to execute: -e\n\n"
+  exit
+end
+
+if $options[:table].nil?
+  puts "\nPlease specify SQL table name: -t\n\n"
+  exit
+end
+
+if $options[:host].nil?
+  puts "\nPlease specify host name: -h\n\n"
+  exit
+end
+
+if $options[:database].nil?
+  puts "\nPlease specify PostgreSQL database name: -d\n\n"
+  exit
+end
+
+if $options[:username].nil?
+  puts "\nPlease specify PostgreSQL username: -d\n\n"
+  exit
+end
+
+# Default in case arguments are not provided
+$options[:listen] ||= POSTGRESQL_PORT
+
+# Connect to PostgreSQL
+# $options = {host: 'localhost', database: 'db', username: 'postgres', password: 'postgres', table: 'products', listen: 5432}
+ActiveRecord::Base.establish_connection(
+  'adapter' => 'postgresql',
+  'host' => $options[:host],
+  'database' => $options[:database],
+  'username' => $options[:username],
+  'password' => $options[:password],
+  'port' => $options[:listen],
+  'timeout' => 15000
+)
+
+class Product < ActiveRecord::Base
+  self.primary_key = :id
+  self.table_name = $options[:table]
+end
+
+# Temp file & temp table name
+$tmpfile = "/tmp/#{Digest::SHA1.hexdigest(rand(100000).to_s)}.csv"
+$tmpname = "_tmp_#{Digest::SHA1.hexdigest(rand(100000).to_s)}"
+
+# Check if PostgreSQL host is local
+def local?
+  return ['localhost', '127.0.0.1'].include?($options[:host])
+end
+
+# Execute query
+def query(*query_str)
+  ActiveRecord::Base.connection.execute(query_str.join("; "))
+end
+
+# Create table from CSV
+def create_table_from_csv(name, csv_path)
+  # Get headers
+  csv = CSV.open(csv_path, :headers => true)
+  first = csv.first
+  unless first
+    raise "File Empty!!!"
+  end
+  
+  # sanitize
+  headers = first.headers
+  headers.each_with_index {|e, index|
+    if e.nil? or e.empty?
+      headers[index] = "column_#{index + 1}"
+    end
+  }
+  headers.map!{|e| e.downcase.underscore }
+  
+  # check if every field name is unique
+  if headers.count != headers.uniq.count
+    raise "Field name must be UNIQUE: \nPlease check your input headers: [#{headers.join(', ')}]"
+  end
+
+  # Create table
+  create_table_sql = headers.map{|e| e == 'id' ? "\"#{e}\" integer" : "\"#{e}\" text"}.join(",")
+  create_table_sql = "drop table if exists #{name}; create table #{name}( #{create_table_sql} );"
+  query(create_table_sql)
+
+  # Dump data
+  insert_data_sql = headers.map{|e| "\"#{e}\""}.join(",")
+  insert_data_sql = "COPY #{name}( #{insert_data_sql} ) FROM '#{csv_path}' DELIMITER ',' CSV HEADER;"
+
+  # Change output file permission so that postgres user can read it
+  begin
+    FileUtils.chmod 0755, csv_path
+  rescue Exception => ex
+    puts "Error while changing file permission"
+  end
+
+  if local?
+    query(insert_data_sql)
+  else
+    puts "\nWARNING: pushing data to remote server [#{$options[:host]}].\nBe sure you have the correct version of `psql` command installed\n\n"
+    insert_data_sql = "PGPASSWORD=#{$options[:username]} psql -U #{$options[:username]} -h #{$options[:host]} -p #{$options[:listen]} #{$options[:database]} -c \"\\#{insert_data_sql}\""
+
+    `#{insert_data_sql}`
+    `PGPASSWORD=""`
+  end
+end
+
+if !local? and $options[:password].nil?
+  puts "\n\nERROR: You are connecting to a remote server\nPlease make sure you specify SQL password: --password \n\n"
+  exit
+end
+
+fields_changed = []
+
+CSV.open($tmpfile, "wb", :col_sep => ',') do |csv|
+  csv << Product.first.attributes.keys
+  
+  Product.all.each do |item|
+    # Execute the script provided on every record
+    eval($options[:eval])
+    fields_changed += item.changes.keys
+    fields_changed.uniq!
+    csv << item.attributes.values
+  end
+end
+
+unless fields_changed.empty?
+  create_table_from_csv($tmpname, $tmpfile)
+  update_sql = fields_changed.map{|f| "\"#{f}\" = tmp.\"#{f}\""}.join(", ")
+  update_sql = "UPDATE #{$options[:table]} origin SET #{update_sql} FROM #{$tmpname} tmp WHERE origin.id = tmp.id"
+  query(update_sql)
+
+  cleanup_sql = "DROP TABLE #{$tmpname};"
+  query(cleanup_sql)
+
+  puts "\nDone\n\n"
+end
+

data/bin/iexport

@@ -0,0 +1,163 @@
+# DATA LOADER
+#
+# @author Nghi Pham
+# @date April 2014
+#
+# The script loads data from a fixed-width text file or a CSV file and fills in
+# a corresponding table in the specified database
+# Issue ruby load.rb --help for guideline/examples
+#
+require 'optparse'
+require 'csv'
+require 'active_record'
+require 'rubygems'
+require 'digest/sha1'
+require 'fileutils'
+
+SUPPORTED_INPUT_FORMATS = ['CSV']
+POSTGRESQL_PORT = 5432
+CSV_DEFAULT_DELIMITER = ','
+
+$options = {}
+parser = OptionParser.new("", 24) do |opts|
+  opts.banner = "\nProgram: Data Exporter\nAuthor: MCKI\n\n"
+
+  opts.on("-f", "--format FORMAT", "Output file format") do |v|
+    $options[:format] = v
+  end
+
+  opts.on("--delim DELIMITER", "CSV delimiter") do |v|
+    $options[:delim] = v
+  end
+
+  opts.on("-t", "--table TABLE", "Table name to be created") do |v|
+    $options[:table] = v
+  end
+
+  opts.on("-s", "--select FIELDS", "Fields to export") do |v|
+    $options[:select] = v
+  end
+
+  opts.on("-w", "--where CONDITION", "CONDITION") do |v|
+    $options[:where] = v
+  end
+
+  opts.on("-o", "--output FILE", "Output file") do |v|
+    $options[:output] = v
+  end
+
+  opts.on("-h", "--host HOST", "PostgreSQL host") do |v|
+    $options[:host] = v
+  end
+
+  opts.on("-d", "--database DATABASE", "PostgreSQL database") do |v|
+    $options[:database] = v
+  end
+
+  opts.on("-u", "--username USER", "PostgreSQL username") do |v|
+    $options[:username] = v
+  end
+
+  opts.on("-p", "--password PASSWORD", "PostgreSQL password") do |v|
+    $options[:password] = v
+  end
+
+  opts.on("-l", "--listen PORT", "PostgreSQL listen port (default to 5432)") do |v|
+    $options[:listen] = v
+  end
+
+  opts.on_tail('--help', 'Displays this help') do
+		puts opts, "", help
+    exit
+	end
+end
+
+def help
+  return <<-eos
+
+EXAMPLES
+-------------------------------------------------------
+    
+  ruby export.rb --host=localhost --username=postgres --database=db --table=items \\
+                 --output=/tmp/data.csv --format=csv --delim="\\t" \\
+                 --select="id, name AS vendor_name, age" --where="active = 1"
+
+eos
+end
+
+begin
+  parser.parse!
+rescue SystemExit => ex
+  exit
+end
+
+if $options[:format].nil?
+  puts "\nPlease specify input file format: -f\n\n"
+  exit
+end
+
+# downcase for consistency
+$options[:format].upcase!
+
+if !SUPPORTED_INPUT_FORMATS.include?($options[:format])
+  puts "\nInvalid input file format, supported formats are: #{SUPPORTED_INPUT_FORMATS.join(', ')}\n\n"
+  exit
+end
+
+if $options[:table].nil?
+  puts "\nPlease specify SQL table name: -t\n\n"
+  exit
+end
+
+if $options[:output].nil?
+  puts "\nPlease specify output file: -o\n\n"
+  exit
+end
+
+if $options[:host].nil?
+  puts "\nPlease specify host name: -h\n\n"
+  exit
+end
+
+if $options[:database].nil?
+  puts "\nPlease specify PostgreSQL database name: -d\n\n"
+  exit
+end
+
+if $options[:username].nil?
+  puts "\nPlease specify PostgreSQL username: -d\n\n"
+  exit
+end
+
+# Default value
+$options[:listen] ||= POSTGRESQL_PORT
+$options[:delim] ||= CSV_DEFAULT_DELIMITER
+$options[:delim] = "\t" if $options[:delim] == '\t'
+$options[:select] ||= '*'
+$options[:where] ||= 'true'
+
+#$options = {host: 'localhost', database: 'db', username: 'postgres', password: 'postgres', table: 'products', listen: 5432}
+ActiveRecord::Base.establish_connection(
+  'adapter' => 'postgresql',
+  'host' => $options[:host],
+  'database' => $options[:database],
+  'username' => $options[:username],
+  'password' => $options[:password],
+  'port' => $options[:listen],
+  'timeout' => 15000
+)
+
+class Product < ActiveRecord::Base
+  self.table_name = $options[:table]
+end
+
+CSV.open($options[:output], "wb", :col_sep => $options[:delim]) do |csv|
+  scope = Product.select($options[:select]).where($options[:where])
+  csv << scope.first.attributes.keys
+  
+  scope.each do |item|
+    csv << item.attributes.values
+  end
+end
+
+puts "\nFile #{$options[:output]} created!\n\n"

data/bin/iload

@@ -0,0 +1,276 @@
+# DATA LOADER
+#
+# @author Nghi Pham
+# @date April 2014
+#
+# The script loads data from a fixed-width text file or a CSV file and fills in
+# a corresponding table in the specified database
+# Issue ruby load.rb --help for guideline/examples
+#
+require 'optparse'
+require 'csv'
+require 'active_record'
+require 'rubygems'
+require 'digest/sha1'
+require 'fileutils'
+
+SUPPORTED_INPUT_FORMATS = ['CSV', 'FX']
+POSTGRESQL_PORT = 5432
+CSV_DEFAULT_DELIMITER = ','
+
+$options = {}
+parser = OptionParser.new("", 24) do |opts|
+  opts.banner = "\nProgram: Data Loader\nAuthor: MCKI\n\n"
+
+  opts.on("-i", "--input INPUT", "INPUT text file (fixed-width)") do |v|
+    $options[:input] = v
+  end
+
+  opts.on("-f", "--format FORMAT", "Input file format, available values include (CSV|FX) ") do |v|
+    $options[:format] = v
+  end
+
+  opts.on("--delim DELIMITER", "Field DELIMITER (for CSV format only - default to COMMA ',')") do |v|
+    $options[:delim] = v
+  end
+
+  # opts.on("-o", "--output CSV", "Temporary CSV output file") do |v|
+  #   $options[:output] = v
+  # end
+
+  opts.on("-t", "--table TABLE", "Table name to be created") do |v|
+    $options[:table] = v
+  end
+
+  opts.on("-h", "--host HOST", "PostgreSQL host") do |v|
+    $options[:host] = v
+  end
+
+  opts.on("-d", "--database DATABASE", "PostgreSQL database") do |v|
+    $options[:database] = v
+  end
+
+  opts.on("-u", "--username USER", "PostgreSQL username") do |v|
+    $options[:username] = v
+  end
+
+  opts.on("-p", "--password PASSWORD", "PostgreSQL password") do |v|
+    $options[:password] = v
+  end
+
+  opts.on("-l", "--listen PORT", "PostgreSQL listen port (default to 5432)") do |v|
+    $options[:listen] = v
+  end
+
+  opts.on_tail('--help', 'Displays this help') do
+		puts opts, "", help
+    exit
+	end
+end
+
+def help
+  return <<-eos
+
+EXAMPLES
+-------------------------------------------------------
+Load data from text file and store to a table name "vendors"
+
+    ruby load.rb --host=localhost --username=postgres --password=postgres \\
+                 --input=/home/administrator/VendorMaster.txt --format=fx \\
+                 --table=vendors --database=db
+
+
+The same usage in short form:
+  
+    ruby load.rb -h localhost -u postgres -p postgres \\
+                 -i /home/administrator/VendorMaster.txt -f fx \\
+                 -t vendors -d db
+
+eos
+end
+
+begin
+  parser.parse!
+rescue SystemExit => ex
+  exit
+end
+
+if $options[:input].nil?
+  puts "\nPlease specify input file: -i\n\n"
+  exit
+end
+
+if $options[:format].nil?
+  puts "\nPlease specify input file format: -f\n\n"
+  exit
+end
+
+# downcase for consistency
+$options[:format].upcase!
+
+if !SUPPORTED_INPUT_FORMATS.include?($options[:format])
+  puts "\nInvalid input file format, supported formats are: #{SUPPORTED_INPUT_FORMATS.join(', ')}\n\n"
+  exit
+end
+
+if $options[:table].nil?
+  puts "\nPlease specify table name: -t\n\n"
+  exit
+end
+
+unless File.exists?($options[:input])
+  puts "\nFile does not exist"
+  exit
+end
+
+if $options[:host].nil?
+  puts "\nPlease specify host name: -h\n\n"
+  exit
+end
+
+if $options[:database].nil?
+  puts "\nPlease specify PostgreSQL database name: -d\n\n"
+  exit
+end
+
+if $options[:username].nil?
+  puts "\nPlease specify PostgreSQL username: -d\n\n"
+  exit
+end
+
+# Default value
+$options[:listen] ||= POSTGRESQL_PORT
+$options[:delim] ||= CSV_DEFAULT_DELIMITER
+$options[:delim] = "\t" if $options[:delim] == '\t'
+$tmpfile = "/tmp/#{Digest::SHA1.hexdigest(rand(100000).to_s)}.csv"
+
+# Database dump
+ActiveRecord::Base.establish_connection(
+  'adapter' => 'postgresql',
+  'host' => $options[:host],
+  'database' => $options[:database],
+  'username' => $options[:username],
+  'password' => $options[:password],
+  'port' => $options[:listen],
+  'timeout' => 15000
+)
+
+class String
+  def underscore
+    return self if self.nil?
+    return self.strip.gsub(/[^a-z0-9]+/, "_")
+  end
+end
+
+class MyParser
+  def initialize
+    # remote server always requires password
+    if !local? and $options[:password].nil?
+      raise "You are connecting to a remote server\nPlease make sure you specify SQL password: --password "
+    end
+  end
+
+  def run  
+    load_fx if $options[:format] == 'FX'
+    load_csv if $options[:format] == 'CSV'
+  end
+
+  def load_csv
+    # Copy file to /tmp/ folder to prevent Permission Error 
+    FileUtils.cp $options[:input], $tmpfile
+    create_table_from_csv($tmpfile)
+  end
+
+  def load_fx
+    # Load data
+    data = IO.read($options[:input]).split("\n")
+    header = data.shift
+    headers = header.scan(/[^\s]+\s+/)
+
+    # Parse
+    ranges = headers.map{|s| "a#{s.size}"}.join("")
+    headers.map!{|s| s.downcase.strip }
+
+    # Write
+    CSV.open($tmpfile, "wb", :col_sep => CSV_DEFAULT_DELIMITER, quote_char: "\b") do |csv|
+      csv << headers
+
+      data.each_with_index{|s, index|
+        record = s.unpack(ranges).map{|e| e.strip}
+        csv << record
+      }
+    end
+
+    create_table_from_csv($tmpfile)
+  end
+
+  def create_table_from_csv(csv_path)
+    # Get headers
+    csv = CSV.open(csv_path, :headers => true, :col_sep => $options[:delim], quote_char: "\b")
+    
+    first = csv.first
+    unless first
+      raise "File Empty!!!"
+    end
+    
+    # sanitize
+    headers = first.headers
+    headers.each_with_index {|e, index|
+      if e.nil? or e.empty?
+        headers[index] = "column_#{index + 1}"
+      end
+    }
+    headers.map!{|e| e.downcase.underscore }
+    
+    # check if every field name is unique
+    if headers.count != headers.uniq.count
+      raise "Field name must be UNIQUE: \nPlease check your input headers: [#{headers.sort.join(', ')}]"
+    end
+
+    # Create table
+    create_table_sql = headers.map{|e| "\"#{e}\" text"}.join(",")
+    create_table_sql = "drop table if exists #{$options[:table]}; create table #{$options[:table]}( id serial not null, #{create_table_sql} );"
+    query(create_table_sql)
+
+    # Dump data
+    pg_tab = "E'\t'" if $options[:delim] == "\t"
+    insert_data_sql = headers.map{|e| "\"#{e}\""}.join(",")
+    insert_data_sql = "COPY #{$options[:table]}( #{insert_data_sql} ) FROM '#{csv_path}' DELIMITER #{pg_tab} CSV HEADER QUOTE E'\b';"
+    
+    # Change output file permission so that postgres user can read it
+    begin
+      FileUtils.chmod 0755, csv_path
+    rescue Exception => ex
+      puts "Error while changing file permission"
+    end
+
+    if local?
+      query(insert_data_sql)
+    else
+      puts "\nWARNING: pushing data to remote server [#{$options[:host]}].\nBe sure you have the correct version of `psql` command installed\n\n"
+      insert_data_sql = "PGPASSWORD=#{$options[:username]} psql -U #{$options[:username]} -h #{$options[:host]} -p #{$options[:listen]} #{$options[:database]} -c \"\\#{insert_data_sql}\""
+
+      `#{insert_data_sql}`
+      `PGPASSWORD=""`
+    end
+    
+    puts "\nTable `#{$options[:table]}` loaded \n\n"
+  end
+
+  private
+  def query(*query_str)
+    ActiveRecord::Base.connection.execute(query_str.join("; "))
+  end
+
+  def local?
+    return ['localhost', '127.0.0.1'].include?($options[:host])
+  end
+end
+
+begin
+  e = MyParser.new
+  e.run
+rescue Exception => ex
+  puts "\n\n*********** ERROR ***********\n#{ex.message}\n\n"
+  exit
+end

data/bin/ipatch

@@ -0,0 +1,107 @@
+# DATA PATCHER
+#
+# @author Nghi Pham
+# @date April 2014
+#
+# Any custom patches to the database goes here
+#
+require 'optparse'
+require 'csv'
+require 'active_record'
+require 'rubygems'
+
+# Default values
+POSTGRESQL_PORT = 5432
+
+# User input
+$options = {}
+parser = OptionParser.new("", 24) do |opts|
+  opts.banner = "\nProgram: Data Patcher\nAuthor: MCKI\n\n"
+
+  opts.on("-q", "--query QUERY", "Custom query") do |v|
+    $options[:query] = v
+  end
+
+  opts.on("-h", "--host HOST", "PostgreSQL host") do |v|
+    $options[:host] = v
+  end
+
+  opts.on("-d", "--database DATABASE", "PostgreSQL database") do |v|
+    $options[:database] = v
+  end
+
+  opts.on("-u", "--username USER", "PostgreSQL username") do |v|
+    $options[:username] = v
+  end
+
+  opts.on("-p", "--password PASSWORD", "PostgreSQL password") do |v|
+    $options[:password] = v
+  end
+
+  opts.on("-l", "--listen PORT", "PostgreSQL listen port (default to 5432)") do |v|
+    $options[:listen] = v
+  end
+
+  opts.on_tail('--help', 'Displays this help') do
+		puts opts, "", help
+    exit
+	end
+end
+
+def help
+  return <<-eos
+
+EXAMPLES
+-------------------------------------------------------
+Send custom SQL to a data table
+
+    ruby patch.rb --host=localhost --username=postgres --password=postgres --database=db \\
+                  --query="ALTER TABLE vendors ADD COLUMN tmp text; UPDATE vendors SET tmp = vendor_number"
+
+eos
+end
+
+begin
+  parser.parse!
+rescue SystemExit => ex
+  exit
+end
+
+if $options[:query].nil?
+  puts "\nPlease specify SQL query: -q\n\n"
+  exit
+end
+
+if $options[:host].nil?
+  puts "\nPlease specify host name: -h\n\n"
+  exit
+end
+
+if $options[:database].nil?
+  puts "\nPlease specify PostgreSQL database name: -d\n\n"
+  exit
+end
+
+if $options[:username].nil?
+  puts "\nPlease specify PostgreSQL username: -d\n\n"
+  exit
+end
+
+# Default value
+$options[:listen] ||= POSTGRESQL_PORT
+
+# Database
+ActiveRecord::Base.establish_connection(
+  'adapter' => 'postgresql',
+  'host' => $options[:host],
+  'database' => $options[:database],
+  'username' => $options[:username],
+  'password' => $options[:password],
+  'port' => $options[:listen],
+  'timeout' => 15000
+)
+
+# Execute the query, use it at your own risk!!!
+ActiveRecord::Base.connection.execute($options[:query])
+
+puts "\nDone!\n\n"

data/bin/isanitize

@@ -0,0 +1,4 @@
+file = ARGV[0]
+s = IO.read(file)
+s = s.encode('UTF-8', 'binary', invalid: :replace, undef: :replace, replace: '')
+File.open(file, 'wb') {|f| f.write(s)}

data/bin/ivalidate

@@ -0,0 +1,258 @@
+# DATA VALIDATOR
+#
+# @author Nghi Pham
+# @date April 2014
+#
+# Data validation includes:
+# * Uniqueness
+# * Integrity (cross reference)
+# * Data type: numeric, text, enum, etc.
+# * Data format: text size, text values, enum, inclusion, exclusion, etc.
+#
+# Issue ruby load.rb --help for guideline/examples
+#
+require 'optparse'
+require 'csv'
+require 'active_record'
+require 'rubygems'
+require 'digest/sha1'
+
+$options = {
+  :unique => [],
+  :not_null => [],
+  :match => [],
+  :not_match => [],
+  :cross_reference => []
+}
+parser = OptionParser.new("", 24) do |opts|
+  opts.banner = "\nProgram: Data Validator\nAuthor: MCKI\n\n"
+
+  opts.on("--unique FIELD", "Check if FIELD is unique") do |v|
+    $options[:unique] << v
+  end
+
+  opts.on("--not-null FIELD", "Check if FIELD is not null or empty") do |v|
+    $options[:not_null] << v
+  end
+
+  opts.on("--match REGEXP", "Check if FIELD matches REGEXP") do |v|
+    $options[:match] << v
+  end
+
+  opts.on("--not-match REGEXP", "Check if FIELD does not match REGEXP") do |v|
+    $options[:not_match] << v
+  end
+
+  opts.on("--cross-reference EXPR", "Reference") do |v|
+    $options[:cross_reference] << v
+  end
+
+  opts.on("-t", "--table TABLE", "Table to verify") do |v|
+    $options[:table] = v
+  end
+
+  opts.on("--log-to FIELD", "Field to log error to") do |v|
+    $options[:log_to] = v
+  end
+
+  opts.on("-h", "--host HOST", "PostgreSQL host") do |v|
+    $options[:host] = v
+  end
+
+  opts.on("-d", "--database DATABASE", "PostgreSQL database") do |v|
+    $options[:database] = v
+  end
+
+  opts.on("-u", "--username USER", "PostgreSQL username") do |v|
+    $options[:username] = v
+  end
+
+  opts.on("-p", "--password PASSWORD", "PostgreSQL password") do |v|
+    $options[:password] = v
+  end
+
+  opts.on("-l", "--listen PORT", "PostgreSQL listen port (default to 5432)") do |v|
+    $options[:listen] = v
+  end
+
+  opts.on_tail('--help', 'Displays this help') do
+		puts opts, "", help
+    exit
+	end
+end
+
+def help
+  return <<-eos
+
+EXAMPLES
+-------------------------------------------------------
+Validate `mfrs` table against several rules and write error logs to errors field:
+
+    ruby validate.rb -h localhost -u postgres -p postgres -d db \\
+                --table=mfrs \\
+                --log-to=errors \\
+                --unique="mfr_lic" \\
+                --unique="mfr_number" \\
+                --not-null="mfr_number" \\
+                --not-null="mfr_lic" \\
+                --match="mfr_number/^2.*/" \\
+                --match="mfr_name/^[a-zA-Z]+/" \\
+                --not-match="status/(failed|inactive|wrong)/"
+
+eos
+end
+
+begin
+  parser.parse!
+rescue SystemExit => ex
+  exit
+end
+
+if $options[:table].nil?
+  puts "\nPlease specify table name: -t\n\n"
+  exit
+end
+
+if $options[:log_to].nil?
+  puts "\nPlease specify field to log errors: --log-to\n\n"
+  exit
+end
+
+if $options[:host].nil?
+  puts "\nPlease specify host name: -h\n\n"
+  exit
+end
+
+if $options[:database].nil?
+  puts "\nPlease specify PostgreSQL database name: -d\n\n"
+  exit
+end
+
+if $options[:username].nil?
+  puts "\nPlease specify PostgreSQL username: -d\n\n"
+  exit
+end
+
+# Default value
+$options[:listen] ||= 5432
+$options[:output] ||= "/tmp/#{Digest::SHA1.hexdigest(rand(100000).to_s)}.csv"
+
+# Database dump
+ActiveRecord::Base.establish_connection(
+  'adapter' => 'postgresql',
+  'host' => $options[:host],
+  'database' => $options[:database],
+  'username' => $options[:username],
+  'password' => $options[:password],
+  'port' => $options[:listen],
+  'timeout' => 15000
+)
+
+# --------------------------------------------------------------------
+# Preliminary check
+# Add column errors if not yet exists
+# --------------------------------------------------------------------
+pre_sql = <<-eos
+  DO $$ 
+    BEGIN
+      BEGIN
+        ALTER TABLE #{$options[:table]} ADD COLUMN #{$options[:log_to]} text default '';
+      EXCEPTION
+        WHEN duplicate_column THEN RAISE NOTICE 'column #{$options[:log_to]} already exists';
+      END;
+    END;
+  $$
+eos
+
+ActiveRecord::Base.connection.execute(pre_sql)
+
+# --------------------------------------------------------------------
+# Check unique field
+# --------------------------------------------------------------------
+$options[:unique].each do |field|
+  puts "Checking unique fields: #{field}"
+
+  uniq_sql = <<-eos
+    UPDATE #{$options[:table]} SET #{$options[:log_to]} = array_to_string(string_to_array(#{$options[:log_to]}, ' || ') || string_to_array('#{field}-not-unique', ' || '), ' || ')
+    WHERE #{field} IN (
+      SELECT #{field} FROM #{$options[:table]} GROUP BY #{field}
+      HAVING count(*) > 1
+    ) AND #{field} IS NOT NULL AND length(trim(#{field})) <> 0;
+  eos
+  
+  ActiveRecord::Base.connection.execute(uniq_sql)
+end
+
+# --------------------------------------------------------------------
+# Check not-null field
+# --------------------------------------------------------------------
+$options[:not_null].each do |field|
+  puts "Checking not-null fields: #{field}"
+
+  not_null_sql = <<-eos
+    UPDATE #{$options[:table]} SET #{$options[:log_to]} = array_to_string(string_to_array(#{$options[:log_to]}, ' || ') || string_to_array('#{field}-null', ' || '), ' || ')
+    WHERE #{field} IS NULL OR length(trim(#{field})) = 0;
+  eos
+  
+  ActiveRecord::Base.connection.execute(not_null_sql)
+end
+
+# --------------------------------------------------------------------
+# Check field satisfying --match=FIELD/REGEXP/
+# --------------------------------------------------------------------
+$options[:match].each do |value|
+  field = value[/^[^\/]+/]
+  regexp = value[/(?<=\/).*(?=\/)/]
+  puts "Checking REGEXP matching: #{regexp}"
+
+  match_sql = <<-eos
+    UPDATE #{$options[:table]} SET #{$options[:log_to]} = array_to_string(string_to_array(#{$options[:log_to]}, ' || ') || string_to_array('#{regexp.gsub("'", "''")}-not-matched', ' || '), ' || ')
+    WHERE #{field} IS NOT NULL AND length(trim(#{field})) <> 0 AND #{field} !~ '#{regexp}';
+  eos
+  
+  ActiveRecord::Base.connection.execute(match_sql)
+end
+
+# --------------------------------------------------------------------
+# Check field satisfying --not-match=FIELD/REGEXP/
+# --------------------------------------------------------------------
+$options[:not_match].each do |value|
+  field = value[/^[^\/]+/]
+  regexp = value[/(?<=\/).*(?=\/)/]
+  puts "Checking REGEXP not matching: #{regexp}"
+
+  not_match_sql = <<-eos
+    UPDATE #{$options[:table]} SET #{$options[:log_to]} = array_to_string(string_to_array(#{$options[:log_to]}, ' || ') || string_to_array('#{regexp.gsub("'", "''")}-unexpectedly-matched', ' || '), ' || ')
+    WHERE #{field} IS NOT NULL AND length(trim(#{field})) <> 0 AND #{field} ~ '#{regexp}';
+  eos
+  
+  ActiveRecord::Base.connection.execute(not_match_sql)
+end
+
+# --------------------------------------------------------------------
+# Check field satisfying --cross-reference
+# --------------------------------------------------------------------
+$options[:cross_reference].each do |value|
+  values = value.split(/[|\.]/)
+
+  if values.size != 3
+    raise "Error: Wrong argument for --cross-reference switch"
+    exit(0)
+  end
+
+  field = values[0]
+  ref_table = values[1]
+  ref_field = values[2]
+
+  puts "Checking data integrity: #{value}"
+
+  # @todo: poor performance here, think of a better SQL!!!
+  ref_sql = <<-eos
+    UPDATE #{$options[:table]} SET #{$options[:log_to]} = array_to_string(string_to_array(#{$options[:log_to]}, ' || ') || string_to_array('#{field}-not-referenced-to-#{values[1]}.#{values[2]}', ' || '), ' || ')
+    WHERE #{field} NOT IN (
+      SELECT #{ref_field} FROM #{ref_table}
+    ) AND #{field} IS NOT NULL AND length(trim(#{field})) <> 0;
+  eos
+  
+  ActiveRecord::Base.connection.execute(ref_sql)
+end

data/idata.gemspec

@@ -0,0 +1,27 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'idata/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "idata"
+  spec.version       = Idata::VERSION
+  spec.authors       = ["Nghi Pham"]
+  spec.email         = ["minhnghivn@gmail.com"]
+  spec.description   = %q{Tools for importing data from raw files}
+  spec.summary       = %q{Tools for importing data from raw files}
+  spec.homepage      = ""
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files`.split($/)
+  #spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.executables  = ["iload", "ieval", "ipatch", "ivalidate", "iexport", "isanitize"]
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+  
+  spec.add_development_dependency "bundler", ">= 1.3"
+  spec.add_development_dependency "rake", ">= 0.9"
+
+  spec.add_dependency "rails", ">= 4.0"
+  spec.add_dependency "pg"
+end

data/lib/idata/version.rb

@@ -0,0 +1,3 @@
+module Idata
+  VERSION = "0.0.1"
+end

data/lib/idata.rb

@@ -0,0 +1,5 @@
+require "idata/version"
+
+module Idata
+  # Your code goes here...
+end

metadata

@@ -0,0 +1,120 @@
+--- !ruby/object:Gem::Specification
+name: idata
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Nghi Pham
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-04-21 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '1.3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '1.3'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0.9'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0.9'
+- !ruby/object:Gem::Dependency
+  name: rails
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '4.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '4.0'
+- !ruby/object:Gem::Dependency
+  name: pg
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Tools for importing data from raw files
+email:
+- minhnghivn@gmail.com
+executables:
+- iload
+- ieval
+- ipatch
+- ivalidate
+- iexport
+- isanitize
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/ieval
+- bin/iexport
+- bin/iload
+- bin/ipatch
+- bin/isanitize
+- bin/ivalidate
+- idata.gemspec
+- lib/idata.rb
+- lib/idata/version.rb
+homepage: ''
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.2.1
+signing_key: 
+specification_version: 4
+summary: Tools for importing data from raw files
+test_files: []