idata 1.0.0 → 1.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 61fc4d12442729bbdf41101831b69d10227bf8dd
-  data.tar.gz: 6614f00c45bf15d9e7daaa7f661180b1a5215794
+  metadata.gz: da7b55510deb9dc064fdbb66ae899b7c01d47379
+  data.tar.gz: 9af104c24dd81b1e4d4347f717d360ddd4ba639c
 SHA512:
-  metadata.gz: 7e98710d6ca73f91e52a31e7c9f17d8d3e235155ab194a51943f245e305de71c8021cf8dbf2ccc0848f2814e714d3381178ee9eed66d1a9e11f67ac9a4f8b2e6
-  data.tar.gz: bb32b89fa342436dcff667a460a4e2aab1b5e39db3e9d98e1dc0aa42410e6bf9233c4e3eb89a4764893841f35bc6cb241e1eda37c4cbfbcc3df91ee51739e25c
+  metadata.gz: 231e0051b69e0a402b3f606f08ae69b351ca63116db4d0bbe835b22ac9aaf268ea563135931c13b9f795c9d78090d446c4188e2175e2192840792e6c4ef0067e
+  data.tar.gz: 7592dbb8b9c6dda8e7756c7c36f17543b39e094f44f06d587a11e0141ccf06029ffc23966a2361475ffbe51082fff2afff01a61325b0936b33d836d77f713c07

data/bin/iload

@@ -8,9 +8,21 @@
 # a corresponding table in the specified database
 # Issue ruby load.rb --help for guideline/examples
 #
+
+require 'idata'
 require 'optparse'
 require 'csv'
-require 'active_record'
+
+begin
+  require 'active_record'
+rescue Exception => ex
+end
+
+begin
+  require 'activerecord'
+rescue Exception => ex
+end
+
 require 'rubygems'
 require 'digest/sha1'
 require 'fileutils'
@@ -29,9 +41,13 @@ end
 
 $options = {}
 parser = OptionParser.new("", 24) do |opts|
-  opts.banner = "\nProgram: Data Loader\nAuthor: MCKi Team\nDescription: the program takes an input text file and creates a corresponding data table\n\n"
+  opts.banner = "Program: iload #{Idata::VERSION}\nAuthor: Gaugau\n\nUsage: iload <client name> [option]\n       iload [options]\n"
+  opts.version = Idata::VERSION
 
-  opts.on("-i", "--input INPUT", "Input text file") do |v|
+  opts.separator ""
+  opts.separator "Command options:"
+
+  opts.on("-i", "--input INPUT", "Input file") do |v|
     $options[:input] = v
   end
 
@@ -51,10 +67,6 @@ parser = OptionParser.new("", 24) do |opts|
     $options[:quote] = v
   end
 
-  # opts.on("-o", "--output CSV", "Temporary CSV output file") do |v|
-  #   $options[:output] = v
-  # end
-
   opts.on("-t", "--table TABLE", "Table name to be created") do |v|
     $options[:table] = v
   end
@@ -62,6 +74,9 @@ parser = OptionParser.new("", 24) do |opts|
   opts.on("--drop", "") do |v|
     $options[:drop] = v
   end
+  
+  opts.separator ""
+  opts.separator "Connection options, can be ommited if <client name> is already specified:"
 
   opts.on("-h", "--host HOST", "PostgreSQL host") do |v|
     $options[:host] = v
@@ -82,25 +97,41 @@ parser = OptionParser.new("", 24) do |opts|
   opts.on("-l", "--listen PORT", "PostgreSQL listen port (default to 5432)") do |v|
     $options[:listen] = v
   end
+  
+  opts.separator ""
+  opts.separator "Common options:"
 
   opts.on_tail('--help', 'Displays this help') do
-		puts opts, "", help
+    puts opts, "", help
     exit
-	end
+  end
+	
 end
 
 def help
   return <<-eos
-
-EXAMPLES
+Example:
 -------------------------------------------------------
-Load data from text file and store to a table name "vendors"
+Load data from text file and store to a table name "items"
+
+    iload --host=localhost --username=postgres --password=postgres --database=db 
+          --table=items --input=/user/items.csv
+
+In short form, with client-name specified:
+
+    iload maricopa --table=items --input=/user/items.csv
 
-    iload --host=localhost --username=postgres --password=postgres \\
-          --database=db --table=vendors \\
-          --input=/home/administrator/VendorMaster.txt \\
-          --format=CSV --delim=$'\\t'
+To use the short form, the following environment variables must be present:
 
+    MAINDBHOST
+    MAINDBSER
+    MAINDBNAME
+    MAINDBPORT
+
+Column delimiter will be auto-detected, in case you want to overwrite the default, use --format/-f.
+For example, tell the program to use TAB as delimiter:
+    
+    iload maricopa --table=items --input=/user/items.csv --format=$'\\t'
 
 eos
 end
@@ -124,7 +155,6 @@ $options[:format].upcase! if $options[:format]
 $options[:format] ||= CSV_DEFAULT_FORMAT
 $options[:listen] ||= POSTGRESQL_PORT unless $options[:client]
 $options[:username] ||= POSTGRESQL_USERNAME unless $options[:client]
-$options[:delim] ||= CSV_DEFAULT_DELIMITER
 $options[:quote] ||= CSV_DEFAULT_QUOTE
 $options[:drop] ||= false
 
@@ -141,6 +171,9 @@ unless File.exists?($options[:input])
   error "file `#{$options[:input]}` not found!"
 end
 
+# auto detect delimiter
+$options[:delim] ||= Idata::Detector::new($options[:input]).find
+
 if !SUPPORTED_INPUT_FORMATS.include?($options[:format])
   error "invalid input file format, supported formats are: #{SUPPORTED_INPUT_FORMATS.join(', ')}"
 end
@@ -347,12 +380,15 @@ class MyParser
     end
     
     # Execute    
-    `#{insert_data_sql}`
+    `#{insert_data_sql} > /dev/null`
     
-    # Clean up
-    File.delete(csv_path) if File.exists?(csv_path)
-    
-    puts "Table `#{$options[:table]}` loaded\n"
+    if $?.exitstatus == 0 
+      # Clean up
+      File.delete(csv_path) if File.exists?(csv_path)
+      puts "Table `#{$options[:table]}` loaded\n"
+    else
+      puts "Something went wrong!"
+    end
   end
 
   private

data/lib/idata.rb

@@ -1,5 +1,8 @@
 require "idata/version"
+require "idata/detector"
 
 module Idata
   # Your code goes here...
 end
+
+

data/lib/idata/detector.rb

@@ -0,0 +1,60 @@
+require 'csv'
+module Idata
+  class Detector
+    DEFAULT_DELIMITER = ","
+    COMMON_DELIMITERS = [DEFAULT_DELIMITER, "|", "\t", ";"]
+    SAMPLE_SIZE = 100
+
+    def initialize(file)
+      @file = file
+      @sample = `head -n #{SAMPLE_SIZE} #{@file}`
+      @sample_lines = @sample.split(/[\r\n]+/)
+      @candidates = COMMON_DELIMITERS.map { |delim|
+        [delim, @sample.scan(delim).count]
+      }.to_h.select{|k,v| v > 0}
+    end
+
+    def find
+      return DEFAULT_DELIMITER if @candidates.empty? # for example, file with only one header
+      return find_same_occurence || find_valid || find_max_occurence || DEFAULT_DELIMITER
+    end
+    
+    # just work
+    def find_valid
+      selected = @candidates.select { |delim, count|
+        begin
+          CSV.parse(@sample, col_sep: delim)
+          true
+        rescue Exception => ex
+          false
+        end
+      }.keys
+
+      return selected.first if selected.count == 1
+      return DEFAULT_DELIMITER if selected.include?(DEFAULT_DELIMITER)
+    end
+
+    # high confident level
+    def find_same_occurence
+      selected = @candidates.select { |delim, count|
+        begin
+          CSV.parse(@sample, col_sep: delim).select{|e| !e.empty? }.map{|e| e.count}.uniq.count == 1
+        rescue Exception => ex
+          false
+        end
+      }.keys
+
+      return selected.first if selected.count == 1
+      return DEFAULT_DELIMITER if selected.include?(DEFAULT_DELIMITER)
+    end
+    
+    # most occurence
+    def find_max_occurence
+      selected = @candidates.select{|k,v| v == @candidates.sort_by(&:last).last }.keys
+
+      return selected.first if selected.count == 1
+      return DEFAULT_DELIMITER if selected.include?(DEFAULT_DELIMITER)
+    end
+  end
+end
+

data/lib/idata/version.rb

@@ -1,3 +1,3 @@
 module Idata
-  VERSION = "1.0.0"
+  VERSION = "1.1.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: idata
 version: !ruby/object:Gem::Version
-  version: 1.0.0
+  version: 1.1.0
 platform: ruby
 authors:
 - Nghi Pham
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-03-17 00:00:00.000000000 Z
+date: 2016-03-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -86,6 +86,7 @@ files:
 - full.sh
 - idata.gemspec
 - lib/idata.rb
+- lib/idata/detector.rb
 - lib/idata/version.rb
 - sample.sh
 homepage: https://github.com/minhnghivn/idata