infobright-loader 0.0.1

data/Rakefile

@@ -0,0 +1,2 @@
+#!/usr/bin/env rake
+require "bundler/gem_tasks"

data/bin/infobright-loader

@@ -0,0 +1,45 @@
+#!/usr/bin/env ruby
+
+# Copyright (c) 2012 SnowPlow Analytics Ltd. All rights reserved.
+#
+# This program is licensed to you under the Apache License Version 2.0,
+# and you may not use this file except in compliance with the Apache License Version 2.0.
+# You may obtain a copy of the Apache License Version 2.0 at http://www.apache.org/licenses/LICENSE-2.0.
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the Apache License Version 2.0 is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the Apache License Version 2.0 for the specific language governing permissions and limitations there under.
+
+# Author::    Alex Dean (mailto:support@snowplowanalytics.com)
+# Copyright:: Copyright (c) 2012 SnowPlow Analytics Ltd
+# License::   Apache License Version 2.0
+
+$LOAD_PATH.unshift File.dirname(__FILE__)
+
+require 'infobright-loader/cli/config'
+require 'infobright-loader/cli/loader'
+require 'infobright-loader/loader'
+
+# This Ruby script is the command-line interface to the
+# Infobright Ruby Loader.
+#
+begin
+  config = InfobrightLoader::Cli::Config.get_config()
+  InfobrightLoader::Cli::Loader.load(config)
+
+rescue InfobrightLoader::Loader::LoadError => le
+  $stderr.puts(le.message)
+  exit 1
+rescue InfobrightLoader::Cli::Config::ConfigError => e
+  $stderr.puts(e.message)
+  exit 1
+rescue SystemExit => e
+  exit 1
+rescue Exception => e
+  $stderr.puts("Unexpected error: " + e.message)
+  $stderr.puts(e.backtrace.join("\n"))
+  exit 1
+end
+
+exit 0

data/control-file/template.yml

@@ -0,0 +1,23 @@
+# Example control file for Infobright Ruby Loader
+
+# Can be overridden at the command line...
+:load:
+  :processes: ADD HERE
+:database:
+  :name: ADD HERE
+  :username: ADD HERE # Or leave blank to default to the user running the script
+  :password: ADD HERE # Or leave blank if no password
+:data_format:
+  :separator: ADD HERE
+  :encloser: ADD HERE # Or leave blank if no encloser
+# ... end of variables overridable at command line.
+
+# Map of tables to populate, along with files to load for each table
+:data_loads:
+  # For each table, list the data files to load
+  TABLE_NAME_1:
+    - PATH/TO/FILE-1
+    - PATH/TO/FILE-2
+  TABLE_NAME_2:
+    - PATH/TO/FILE-3
+    - PATH/TO/FILE-4

data/infobright-loader.gemspec

@@ -0,0 +1,35 @@
+# Copyright (c) 2012 SnowPlow Analytics Ltd. All rights reserved.
+#
+# This program is licensed to you under the Apache License Version 2.0,
+# and you may not use this file except in compliance with the Apache License Version 2.0.
+# You may obtain a copy of the Apache License Version 2.0 at http://www.apache.org/licenses/LICENSE-2.0.
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the Apache License Version 2.0 is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the Apache License Version 2.0 for the specific language governing permissions and limitations there under.
+
+# Author::    Alex Dean (mailto:support@snowplowanalytics.com)
+# Copyright:: Copyright (c) 2012 SnowPlow Analytics Ltd
+# License::   Apache License Version 2.0
+
+# -*- encoding: utf-8 -*-
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'infobright-loader/version'
+
+Gem::Specification.new do |gem|
+  gem.authors       = ["Alex Dean <support@snowplowanalytics.com>"]
+  gem.email         = ["support@snowplowanalytics.com"]
+  gem.description   = %q{Loads data files into Infobright}
+  gem.summary       = %{Infobright Ruby Loader (IRL) is a data loader for Infobright Community Edition (ICE) and Enterprise Edition (IEE), built as a Ruby gem. Inspired by ParaFlex (the bash equivalent)}
+  gem.homepage      = "http://snowplowanalytics.com"
+
+  gem.files         = `git ls-files`.split($\)
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.name          = InfobrightLoader::NAME
+  gem.version       = InfobrightLoader::VERSION
+  gem.platform      = Gem::Platform::RUBY
+  gem.require_paths = ["lib"]
+end

data/lib/infobright-loader.rb

@@ -0,0 +1,5 @@
+require "infobright-loader/loader"
+
+module InfobrightLoader
+  # Your code goes here...
+end

data/lib/infobright-loader/cli/config.rb

@@ -0,0 +1,204 @@
+# Copyright (c) 2012 SnowPlow Analytics Ltd. All rights reserved.
+#
+# This program is licensed to you under the Apache License Version 2.0,
+# and you may not use this file except in compliance with the Apache License Version 2.0.
+# You may obtain a copy of the Apache License Version 2.0 at http://www.apache.org/licenses/LICENSE-2.0.
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the Apache License Version 2.0 is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the Apache License Version 2.0 for the specific language governing permissions and limitations there under.
+
+# Author::    Alex Dean (mailto:support@snowplowanalytics.com)
+# Copyright:: Copyright (c) 2012 SnowPlow Analytics Ltd
+# License::   Apache License Version 2.0
+
+require 'optparse'
+require 'date'
+require 'yaml'
+
+require 'infobright-loader/db'
+
+# Config module to hold functions related to CLI argument parsing
+# and config file reading to support the daily ETL job.
+module InfobrightLoader
+  module Cli
+    module Config
+
+      # What are we called?
+      SCRIPT_NAME = InfobrightLoader::NAME
+
+      # For errors
+      class ConfigError < ArgumentError; end
+
+      # Configuration for loading all the files from a specific directory into
+      # a specific table
+      LoadHashConfig = Struct.new(:load_hash, :db, :processes, :separator, :encloser)
+      
+      # Configuration for loading a set of tables from a set of files (where
+      # each table can have multiple files loaded into it)
+      LoadFolderConfig = Struct.new(:folder, :table, :db, :separator, :encloser)
+
+      # Validates and returns the configuration.
+      #
+      # The configuration returned will either be
+      # a LoadMapConfig or a LoadFolderConfig.
+      def get_config()
+
+        options = Config.parse_args()
+
+        if options[:control].nil?
+
+          config = LoadFolderConfig.new
+          config.db = InfobrightLoader::Db::DbConfig.new(options[:db], options[:username], options[:password]) 
+          config.separator = options[:separator]
+          config.encloser = options[:encloser] 
+          config.folder = options[:folder] 
+          config.table = options[:table] 
+
+        else
+
+          yaml = YAML.load_file(options[:control])
+
+          # Set the overridable fields if they haven't been overridden at the command-line
+          config = LoadHashConfig.new
+          get_or_else = lambda {|x, y| x.nil? ? y : x }
+          config.processes = get_or_else.call(options[:processes], yaml[:load][:processes])
+          config.separator = get_or_else.call(options[:separator], yaml[:data_format][:separator])
+          config.encloser = get_or_else.call(options[:encloser], yaml[:data_format][:encloser])
+
+          db_name = get_or_else.call(options[:db], yaml[:database][:name])
+          db_username = get_or_else.call(options[:username], yaml[:database][:username])
+          db_password = get_or_else.call(options[:password], yaml[:database][:password])
+          config.db = InfobrightLoader::Db::DbConfig.new(db_name, db_username, db_password)
+
+          # Finally grab the load map
+          config.load_hash = yaml[:data_loads]
+
+          # Check we have everything now
+          if db_name.nil?
+            raise ConfigError, "Database name not specified"
+          end
+          if config.processes.nil?
+            raise ConfigError, "Number of processes not specified"
+          end
+          if config.separator.nil? or config.separator.empty?
+            raise ConfigError, "Separator not specified - have you escaped ('\\') it in your control file?"
+          end      
+
+          # Check that number of processes is a positive integer
+          unless config.processes.to_i > 0
+            raise ConfigError, "Parallel load processes '#{config.processes}' is not a positive integer"
+          end
+          config.processes = config.processes.to_i # A kitten dies, mutably.
+
+          # Check that we have some tables to load
+          if config.load_hash.nil? or config.load_hash.empty?
+            raise ConfigError, "Must specify at least one table to load"
+          end
+
+        end
+
+        config # Return either our LoadFolderConfig or our LoadHashConfig
+      end
+      module_function :get_config
+
+      # Parse the command-line arguments
+      # Returns: the hash of parsed options
+      def parse_args()
+
+        # Handle command-line arguments
+        options = {}
+        optparse = OptionParser.new do |opts|
+
+          opts.banner = "Usage: %s [options]" % SCRIPT_NAME
+          opts.separator ""
+          opts.separator "Specify a control file:"
+
+          opts.on('-c', '--control FILE', 'control file') { |config| options[:control] = config }
+          opts.on('-x', '--processes INT', 'optional number of parallel processes to run *') { |config| options[:processes] = config }
+
+          opts.separator ""
+          opts.separator "Or load a table from a folder of data files:"
+
+          opts.on('-d', '--db NAME', 'database name *') { |config| options[:db] = config }
+          opts.on('-u', '--username NAME', 'database username *') { |config| options[:username] = config }
+          opts.on('-p', '--password NAME', 'database password *') { |config| options[:password] = config }
+
+          opts.on('-t', '--table NAME', 'table to load data files into') { |config| options[:table] = config }
+          opts.on('-f', '--folder DIR', 'directory containing data files to load') { |config| options[:folder] = config }
+          opts.on('-s', '--separator CHAR', 'optional field separator, defaults to pipe bar (|) *') { |config| options[:separator] = config }
+          opts.on('-e', '--encloser CHAR', 'optional field encloser, defaults to none *') { |config| options[:encloser] = config }
+
+          opts.separator ""
+          opts.separator "* overrides the same setting in the control file if control file also specified"
+
+          opts.separator ""
+          opts.separator "Common options:"
+
+          opts.on_tail('-h', '--help', 'Show this message') { puts opts; exit }
+          opts.on_tail('-v', "--version", "Show version") do
+            puts "%s %s" % [SCRIPT_NAME, InfobrightLoader::VERSION]
+            exit
+          end
+        end
+
+        # Run OptionParser's structural validation
+        begin
+          optparse.parse!
+        rescue OptionParser::InvalidOption, OptionParser::MissingArgument
+          raise ConfigError, "#{$!.to_s}\n#{optparse}"
+        end
+
+        # If no control file given, most of the options are required
+        if options[:control].nil?
+
+          # Set defaults if necessary
+          options[:separator] ||= '|'
+          options[:encloser]  ||= ''
+
+          # First check we have all the options we need
+          mandatory = [:db, :table, :folder]
+          missing = mandatory.select{ |param| options[param].nil? }
+          if not missing.empty?
+            raise ConfigError, "No control file specified, so missing options: #{missing.join(', ')}\n#{optparse}"
+          end
+
+          # Check our folder exists and is not empty
+          unless File.directory?(options[:folder])
+            raise ConfigError, "Specified folder '#{options[:folder]}' not found"
+          end
+          if (Dir.entries(options[:folder]) - %w{ . .. }).empty?
+            raise ConfigError, "Specified folder '#{options[:folder]}' is empty"
+          end
+
+          # Check user didn't try to override processes
+          unless options[:processes].nil?
+            raise ConfigError, "Limited to one process when loading only one table"
+          end
+
+          # Add trailing slash if needed to the folder
+          trail = lambda {|str| return str[-1].chr != '/' ? str << '/' : str}
+          options[:folder] = trail.call(options[:folder])
+
+        # We are working with the control file
+        else
+
+          # Check we don't have a conflict of purpose
+          unless options[:folder].nil? and options[:table].nil?
+            raise ConfigError, "Specifying a control file as well as a folder and table does not make sense"
+          end
+
+          # Check the control file exists
+          unless File.file?(options[:control])
+            raise ConfigError, "Control file '#{options[:control]}' does not exist, or is not a file."
+          end
+        end
+
+        options
+      end
+      module_function :parse_args
+
+    end
+  end
+end

data/lib/infobright-loader/cli/loader.rb

@@ -0,0 +1,66 @@
+# Copyright (c) 2012 SnowPlow Analytics Ltd. All rights reserved.
+#
+# This program is licensed to you under the Apache License Version 2.0,
+# and you may not use this file except in compliance with the Apache License Version 2.0.
+# You may obtain a copy of the Apache License Version 2.0 at http://www.apache.org/licenses/LICENSE-2.0.
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the Apache License Version 2.0 is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the Apache License Version 2.0 for the specific language governing permissions and limitations there under.
+
+# Author::    Alex Dean (mailto:support@snowplowanalytics.com)
+# Copyright:: Copyright (c) 2012 SnowPlow Analytics Ltd
+# License::   Apache License Version 2.0
+
+require 'infobright-loader/cli/config'
+require 'infobright-loader/loader'
+
+# Loader determines which load action to run.
+#
+# It's a selective wrapper around
+# load_from_folder() and load_from_map()
+module InfobrightLoader
+  module Cli
+    module Loader
+
+      # Determine what type of config we have,
+      # and then call the appropriate load
+      def load(config)
+
+        failures = []
+
+        case config
+        when InfobrightLoader::Cli::Config::LoadFolderConfig
+          failures = InfobrightLoader::Loader::load_from_folder(
+            config.folder,
+            config.table,
+            config.db,
+            config.separator,
+            config.encloser
+          )
+
+        when InfobrightLoader::Cli::Config::LoadHashConfig
+          failures = InfobrightLoader::Loader::load_from_hash(
+            config.load_hash,
+            config.db,
+            config.processes,
+            config.separator,
+            config.encloser
+          )
+
+        else
+          raise ConfigError, "config argument passed to Cli::Loader::load() must be a LoadFolderConfig or a LoadHashConfig"
+        end
+
+        unless failures.empty?
+          error = "Load of following files failed (reason in brackets):\n" + \
+                  failures.map{|f| " - " + f}.join("\n")
+          raise InfobrightLoader::Loader::LoadError, error
+        end
+      end
+      module_function :load
+
+    end
+  end
+end

data/lib/infobright-loader/db.rb

@@ -0,0 +1,90 @@
+# Copyright (c) 2012 SnowPlow Analytics Ltd. All rights reserved.
+#
+# This program is licensed to you under the Apache License Version 2.0,
+# and you may not use this file except in compliance with the Apache License Version 2.0.
+# You may obtain a copy of the Apache License Version 2.0 at http://www.apache.org/licenses/LICENSE-2.0.
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the Apache License Version 2.0 is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the Apache License Version 2.0 for the specific language governing permissions and limitations there under.
+
+# Author::    Alex Dean (mailto:support@snowplowanalytics.com)
+# Copyright:: Copyright (c) 2012 SnowPlow Analytics Ltd
+# License::   Apache License Version 2.0
+
+require 'infobright-loader/loader'
+
+module InfobrightLoader
+  module Db
+
+    # Configuration for accessing an Infobright database.
+    # :username and :password can be nil
+    DbConfig = Struct.new(:name, :username, :password)
+
+    # Is mysql-ib running and the
+    # Infobright server accessible?
+    def running?(db)
+      ib = ib_command_from(db)
+      `#{ib} -e \\\\q > /dev/null 2>&1`
+      ($?.to_i == 0)
+    end
+    module_function :running?
+  
+    # Does the database exist and can
+    # we access it?
+    def db_exists?(db)
+      ib = ib_command_from(db)
+      `#{ib} -D #{db.name} -e \\\\q > /dev/null 2>&1`
+      ($?.to_i == 0)
+    end
+    module_function :db_exists?
+
+    # Does the table exist?
+    def table_exists?(table, db)
+      ib = ib_command_from(db)
+      `echo "desc "#{table}";" | #{ib} -D #{db.name} > /dev/null 2>&1`
+      ($?.to_i == 0)
+    end
+    module_function :table_exists?    
+
+    # Load data
+    def load_file(file, table, db, separator='|', encloser='')
+
+      # Make sure seperator and encloser are escaped if either is " or '
+      escaper = lambda { |c| (c == '"' || c == "'") ? "\\" + c : c }
+      separator = escaper.call(separator)
+      encloser = escaper.call(encloser)
+
+      # Check the file exists
+      unless File.file?(file)
+        raise InfobrightLoader::Loader::LoadError, "file does not exist, or is not a file"
+      end
+
+      ib = ib_command_from(db)
+      load = "LOAD DATA INFILE '#{file}' " + \
+             "INTO TABLE #{table} " + \
+             "FIELDS TERMINATED BY '#{separator}' ENCLOSED BY '#{encloser}'; "
+
+      stdout_err = `echo "#{load}" | #{ib} -D #{db.name} 2>&1`
+      ret_val = $?.to_i
+      unless ret_val == 0
+        raise InfobrightLoader::Loader::LoadError, "mysql-ib error code #{ret_val}: #{stdout_err}"
+      end
+    end
+    module_function :load_file
+
+    private
+    
+    # Get path to Infobright's mysql-ib client,
+    # then add in username and password as
+    # necessary
+    def ib_command_from(db)
+      `locate mysql-ib`[0...-1] + \
+        (db.username.nil? ? '' : " -u #{db.username}") + \
+        (db.password.nil? ? '' : " --password=#{db.password}")
+    end
+    module_function :ib_command_from
+
+  end
+end