insertica 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: f9cfab140cfaee0294ccda564b13de12adc6841a
+  data.tar.gz: ae3cab639e68a0967ca46eb27a74283d9d343e92
+SHA512:
+  metadata.gz: 10b4ecf013350727c8de2c800725dd3542b54ca23848ab078f17c01c8020b4e5918f023407cb05d994837d8b24ece745093215dc5b99b575fde2635a2569d29f
+  data.tar.gz: 8c77d2545e14e2e649aa7ac527e2bae2b8fd5ec25effc426be1e35ac1e2abea1f269a75852cb304706510d14dd77e5b6ac9c84e05b53620e33ab6f9d4bada9ed

data/bin/insertica

@@ -0,0 +1,20 @@
+#!/usr/bin/env ruby
+
+require './lib/insertica.rb'
+require 'thor'
+
+class InserticaExecutable < Thor
+  package_name "insertica"
+
+  desc "load", "Load a delimited file into Vertica."
+  method_option :username, aliases: "-u", required: true, type: :string, desc: "Specifies the username to use with Vertica."
+  method_option :password, aliases: "-p", required: true, type: :string, desc: "Specifies the password to use with Vertica."
+  method_option :host, aliases: "-h", default: 'localhost', desc: "Specifies the host of the Vertica database."
+  method_option :port, aliases: "-p", default: 5433, desc: "Specifies the port of the Vertica database."
+  def load(filename)
+    table = Insertica::Table.new(filename)
+    table.insert(options)
+  end
+end
+
+InserticaExecutable.start

data/lib/insertica/column.rb

@@ -0,0 +1,42 @@
+require 'insertica/column_type'
+
+module Insertica
+  class Column
+    ESCAPE_CHARACTERS = ["\n", "\t", "\""]
+
+    attr_accessor :name
+    attr_accessor :values
+    attr_accessor :type
+
+    def initialize(name, values = [])
+      @name   = name
+      @type   = nil
+      @values = values
+    end
+
+    def finalize
+      @type = ColumnType.new(@values)
+      escape_strings if @type.needs_escaping?
+
+      self
+    end
+
+    def definition
+      "#{@name} #{@type}"
+    end
+
+    def filler_definition
+      "#{@name}_filler FILLER VARCHAR"
+    end
+
+    def fix_nulls_definition
+      "#{@name} AS CASE WHEN #{@name}_filler = '' THEN NULL ELSE #{@name}_filler::#{@type} END"
+    end
+
+    private
+
+    def escape_strings
+      ESCAPE_CHARACTERS.each { |character| @values.map! { |value| value.gsub("#{character}","\\#{character}") unless value.nil? } }
+    end
+  end
+end

data/lib/insertica/column_type.rb

@@ -0,0 +1,77 @@
+require 'date'
+
+module Insertica
+  class ColumnType
+    attr_accessor :vertica_type
+
+    DATETIME_TYPE  = 'TIMESTAMP'
+    TRUECLASS_TYPE = 'BOOLEAN'
+    INTEGER_TYPE   = 'INTEGER'
+    FLOAT_TYPE     = 'FLOAT'
+    STRING_TYPE    = 'VARCHAR'
+
+    def initialize(values = [])
+      @vertica_type = guess_type(values)
+    end
+
+    def to_s
+      @vertica_type
+    end
+
+    def needs_escaping?
+      STRING_TYPE == @vertica_type
+    end
+
+    private
+
+    def guess_type(values)
+      if all_nil?(values)
+        STRING_TYPE
+      elsif datetime?(values)
+        DATETIME_TYPE
+      elsif trueclass?(values)
+        TRUECLASS_TYPE
+      elsif integer?(values)
+        INTEGER_TYPE
+      elsif float?(values)
+        FLOAT_TYPE
+      else 
+        STRING_TYPE
+      end
+    end
+
+    def all_nil?(values)
+      values.all? { |value| value.nil? }
+    end
+
+    def datetime?(values)
+      values.all? do |value|
+        value.nil? || !!DateTime.iso8601(value)
+      end
+    rescue
+      false
+    end
+
+    def trueclass?(values)
+      values.all? do |value|
+        value.nil? || value.class == FalseClass || value.class == TrueClass 
+      end
+    end
+
+    def integer?(values)
+      values.all? do |value|
+        value.nil? || Integer(value) == Float(value)
+      end
+    rescue
+      false
+    end
+
+    def float?(values)
+      values.all? do |value|
+        value.nil? || !!Float(value)
+      end
+    rescue
+      false
+    end
+  end
+end

data/lib/insertica/table.rb

@@ -0,0 +1,98 @@
+require 'insertica/column'
+
+require 'json/ext'
+require 'vertica'
+
+module Insertica
+  class Table
+    attr_accessor :schema_name
+    attr_accessor :table_name
+
+    def initialize(schema_name, filename)
+      @filename    = filename
+      @schema_name = schema_name
+      @table_name  = File.basename(filename, '.*')
+      @file        = File.read(@filename)
+
+      columns and rows
+    end
+
+    def full_name
+      "#{schema_name}.#{table_name}"
+    end
+
+    def definition
+      @definition ||= @columns.values.map { |column| "#{column.definition}" }.join(",\n")
+    end
+
+    def filler_definition
+      @filler_definition ||= [
+        @columns.values.map { |column| "#{column.filler_definition}" }.join(",\n"),
+        @columns.values.map { |column| "#{column.fix_nulls_definition}" }.join(",\n")
+      ].join(",\n")
+    end
+
+    def insert(**options)
+      vertica_connection = Vertica.connect({
+        host:     options[:host],
+        user:     options[:user],
+        password: options[:password],
+        port:     options[:port]
+      })
+
+      drop_statement   = "DROP TABLE IF EXISTS #{full_name} CASCADE"
+      create_statement = "CREATE TABLE #{full_name} (#{definition})"
+      copy_statement   = "COPY #{full_name} (#{filler_definition}) FROM STDIN DELIMITER '\t' ENCLOSED BY '\"' NULL AS 'NULL' ABORT ON ERROR"
+
+      vertica_connection.query(drop_statement)
+      vertica_connection.query(create_statement)
+      vertica_connection.copy(copy_statement) do |stdin|
+        stdin << to_tsv
+      end
+      vertica_connection.query("COMMIT")
+    end
+
+    def rows
+      @rows ||= generate_rows
+    end
+
+    def columns
+      @columns ||= generate_columns
+    end
+
+    def to_tsv
+      @rows.map do |row|
+        "\"#{row.join("\"\t\"")}\"\n"
+      end.join("")
+    end
+
+    private
+
+    def generate_columns
+      columns = {}
+      @numer_of_rows = @file.lines.length
+
+      @file.lines.each_with_index do |line, index|
+        json_line = JSON.parse(line)
+        json_line.keys.each do |key|
+          columns[key] = Column.new(key) if columns[key].nil?
+          columns[key].values[index] = json_line[key]
+        end
+      end
+      columns.each { |key, column| column.finalize }
+      columns
+    end
+
+    def generate_rows
+      rows = []
+      @numer_of_rows.times do |i|
+        row = []
+        @columns.values.each do |column|
+          row << column.values[i]
+        end
+        rows << row
+      end
+      rows
+    end
+  end
+end

data/lib/insertica.rb

@@ -0,0 +1,3 @@
+$: << './lib'
+
+require 'insertica/table'

metadata

@@ -0,0 +1,121 @@
+--- !ruby/object:Gem::Specification
+name: insertica
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Nick Evans
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-02-25 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: vertica
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.11.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.11.1
+- !ruby/object:Gem::Dependency
+  name: thor
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.18.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.18.1
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: A simple tool to insert data into Vertica. Currently only supports JSON
+  data.
+email: 
+executables:
+- insertica
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/insertica/column.rb
+- lib/insertica/column_type.rb
+- lib/insertica/table.rb
+- lib/insertica.rb
+- bin/insertica
+homepage: 
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.0.14
+signing_key: 
+specification_version: 4
+summary: A simple tool to insert data into Vertica.
+test_files: []
+has_rdoc: