infobright-loader 0.0.1

data/lib/infobright-loader/loader.rb

@@ -0,0 +1,175 @@
+# Copyright (c) 2012 SnowPlow Analytics Ltd. All rights reserved.
+#
+# This program is licensed to you under the Apache License Version 2.0,
+# and you may not use this file except in compliance with the Apache License Version 2.0.
+# You may obtain a copy of the Apache License Version 2.0 at http://www.apache.org/licenses/LICENSE-2.0.
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the Apache License Version 2.0 is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the Apache License Version 2.0 for the specific language governing permissions and limitations there under.
+
+# Author::    Alex Dean (mailto:support@snowplowanalytics.com)
+# Copyright:: Copyright (c) 2012 SnowPlow Analytics Ltd
+# License::   Apache License Version 2.0
+
+require 'thread'
+
+require 'infobright-loader/db'
+
+module InfobrightLoader
+  module Loader
+
+    # For errors
+    class LoadError < ArgumentError; end
+
+    # Load a single table in Infobright with
+    # the contents of a single folder
+    def load_from_folder(folder, table, db, separator='|', encloser='')
+
+      # Let's loop through and grab all absolute paths to all the files in this folder, recursively
+      load_hash = {}
+      load_hash[table] = Dir["#{folder}**/*"].find_all{|f| File.file?(f)}.map{|f| File.expand_path(f)}
+
+      # Check we have some files to load
+      unless load_hash[table].any?
+        raise LoadError, "No files to load in folder #{folder}"
+      end
+
+      # Now we have converted the folder and table
+      # into a map, we can use load_from_map()
+      load_from_hash(load_hash, db, 1, separator, encloser)
+    end
+    module_function :load_from_folder
+
+    # Load Infobright using a hash of
+    # tables to filenames.
+    def load_from_hash(load_hash, db, processes=10, separator='|', encloser='')
+
+      # Check we have some tables
+      t_count = load_hash.length
+
+      # Some validation about the load we're going to do
+      case 
+      when t_count == 0
+        raise LoadError, "We have no tables to populate"
+      when t_count < processes
+        puts "We have only #{t_count} table(s) to populate, reducing processes from #{processes} to #{t_count}" # TODO: move to Ruby logger?
+        processes = t_count
+      end
+
+      # Now let's check MySQL server is accessible
+      unless InfobrightLoader::Db.running?(db)
+        raise LoadError, "Default MySQL server cannot be found or is not running"
+      end
+
+      # Now let's check that we can access the database
+      unless InfobrightLoader::Db.db_exists?(db)
+        raise LoadError, "Database #{db.name} cannot be found or user lacks sufficient privileges"
+      end      
+
+      # Now we're ready to start with the load - either parallel or serial
+      if t_count == 1
+        table, files = load_hash.first
+        failures = load_table(files, table, db, separator, encloser)
+      else
+        failures = load_parallel(load_hash, db, processes, separator, encloser)
+        # failures = load_serial(load_hash, db, separator, encloser) # For debugging without worrying about threads.
+      end
+
+      failures # Return failures
+    end
+    module_function :load_from_hash
+
+    private
+
+    # Load a single table
+    def load_table(files, table, db, separator, encloser)
+
+      failures = [] # Tables we didn't manage to load
+
+      files.each { |f|
+        puts "Loading file #{f} into table #{db.name}.#{table}" # TODO: move to Ruby logger?
+        begin
+          InfobrightLoader::Db.load_file(f, table, db, separator, encloser)
+        rescue LoadError => le
+          puts "LOAD ERROR: %s" % le # TODO: move to Ruby logger?
+          failures << "%s (%s)" % [f, le]
+        end
+      }
+      failures
+    end
+    module_function :load_table
+
+    # Perform a serial load
+    # Only used for debugging
+    def load_serial(load_hash, db, separator, encloser)
+
+      files_not_loaded = []
+
+      load_hash.keys.each { |k|
+        failures = load_table(load_hash[k], k, db, separator, encloser)
+        unless failures.empty?
+          files_not_loaded.concat failures
+        end
+      }
+      files_not_loaded
+    end
+    module_function :load_serial
+
+    # Perform a parallel load
+    def load_parallel(load_hash, db, processes, separator, encloser)
+
+      tables_to_load = load_hash.keys
+      table = nil
+      threads = []
+      files_not_loaded = []
+      complete = false
+      mutex = Mutex.new
+
+      # If an exception is thrown in a thread that isn't handled, die quickly
+      Thread.abort_on_exception = true
+
+      # Create Ruby threads to concurrently execute Infobright loads
+      for i in (0...processes)
+        
+        # Each thread pops a table off the tables_to_load array, and loads files into it.
+        # We loop until there are no more tables to populate.
+        threads << Thread.new do
+          loop do
+
+            # Critical section
+            # Only allow one thread to modify the array at any time
+            mutex.synchronize do
+              if tables_to_load.length == 0
+                complete = true
+              end
+              table = tables_to_load.pop
+            end
+
+            # Let's quit if we have no table to load
+            break if complete # Exit the thread
+
+            # Otherwise let's run through and do all the loads for this table
+            failures = load_table(load_hash[table], table, db, separator, encloser)
+
+            # Also critical: only one thread should update the failures
+            # list at a time
+            mutex.synchronize do
+              unless failures.empty?
+                files_not_loaded.concat failures
+              end
+            end
+
+          end
+        end
+      end
+
+      # Wait for threads to finish
+      threads.each { |aThread|  aThread.join }
+      files_not_loaded
+    end
+    module_function :load_parallel
+
+  end
+end

data/lib/infobright-loader/version.rb

@@ -0,0 +1,4 @@
+module InfobrightLoader
+  NAME = "infobright-loader"
+  VERSION = "0.0.1"
+end

data/tests/manual/bad-files/.gitignore

@@ -0,0 +1 @@
+control-file-out.yml

data/tests/manual/bad-files/control-file-raw.yml

@@ -0,0 +1,50 @@
+# Control file for Infobright Ruby Loader control file test
+#
+# Is processed with sed to finalise <<x>> variables before
+# the test is run
+
+:load:
+  :processes: 3 # Test with 3 processes populating 5 tables
+:database:
+  :name: wrong # Will override at the command line to irl_tests
+  :username: <<USERNAME>>
+  :password: <<PASSWORD>>
+:data_format:
+  :separator: \|
+  :encloser: \' # Will override at the command line to "
+:data_loads:
+  b:
+    - <<PATH>>/data/b/b_1.txt # Bad data
+    - <<PATH>>/data/b/b_2.txt
+    - <<PATH>>/data/b/b_3.txt
+    - <<PATH>>/data/b/b_4.txt
+    - <<PATH>>/data/b/b_5.txt
+    - <<PATH>>/data/b/b_6.txt
+  c:
+    - <<PATH>>/data/c/c_1.txt
+    - <<PATH>>/data/c/c_2.txt
+    - <<PATH>>/data/c/BAD_3.txt
+    - <<PATH>>/data/c/c_4.txt
+    - <<PATH>>/data/c/c_5.txt
+    - <<PATH>>/data/c/c_6.txt
+  d:
+    - <<PATH>>/data/d/BAD_1.txt
+    - <<PATH>>/data/d/d_2.txt
+    - <<PATH>>/data/d/d_3.txt
+    - <<PATH>>/data/d/d_4.txt
+    - <<PATH>>/data/d/d_5.txt # Bad data
+    - <<PATH>>/data/d/d_6.txt
+  e:
+    - <<PATH>>/data/e/e_1.txt
+    - <<PATH>>/data/e/e_2.txt
+    - <<PATH>>/data/e/e_3.txt
+    - <<PATH>>/data/e/BAD_4.txt
+    - <<PATH>>/data/e/e_5.txt
+    - <<PATH>>/data/e/BAD_6.txt
+  f:
+    - <<PATH>>/data/f/f_1.txt
+    - <<PATH>>/data/f/f_2.txt
+    - <<PATH>>/data/f/f_3.txt
+    - <<PATH>>/data/f/f_4.txt
+    - <<PATH>>/data/f/f_5.txt
+    - <<PATH>>/data/f/f_6.txt

data/tests/manual/bad-files/data/b/b_1.txt

@@ -0,0 +1,6 @@
+"1"|"B.1"
+"2"|"B.2"
+"3"|"B.3"
+"4"|"B.4"
+bad
+"6"|"B.6"

data/tests/manual/bad-files/data/b/b_2.txt

@@ -0,0 +1,6 @@
+"7"|"B.7"
+"8"|"B.8"
+"9"|"B.9"
+"10"|"B.10"
+"11"|"B.11"
+"12"|"B.12"

data/tests/manual/bad-files/data/b/b_3.txt

@@ -0,0 +1,6 @@
+"13"|"B.13"
+"14"|"B.14"
+"15"|"B.15"
+"16"|"B.16"
+"17"|"B.17"
+"18"|"B.18"

data/tests/manual/bad-files/data/b/b_4.txt

@@ -0,0 +1,6 @@
+"19"|"B.19"
+"20"|"B.20"
+"21"|"B.21"
+"22"|"B.22"
+"23"|"B.23"
+"24"|"B.24"

data/tests/manual/bad-files/data/b/b_5.txt

@@ -0,0 +1,6 @@
+"25"|"B.25"
+"26"|"B.26"
+"27"|"B.27"
+"28"|"B.28"
+"29"|"B.29"
+"30"|"B.30"

data/tests/manual/bad-files/data/b/b_6.txt

@@ -0,0 +1,6 @@
+"31"|"B.31"
+"32"|"B.32"
+"33"|"B.33"
+"34"|"B.34"
+"35"|"B.35"
+"36"|"B.36"

data/tests/manual/bad-files/data/c/c_1.txt

@@ -0,0 +1,6 @@
+"1"|"C.1"
+"2"|"C.2"
+"3"|"C.3"
+"4"|"C.4"
+"5"|"C.5"
+"6"|"C.6"

data/tests/manual/bad-files/data/c/c_2.txt

@@ -0,0 +1,6 @@
+"7"|"C.7"
+"8"|"C.8"
+"9"|"C.9"
+"10"|"C.10"
+"11"|"C.11"
+"12"|"C.12"

data/tests/manual/bad-files/data/c/c_3.txt

@@ -0,0 +1,6 @@
+"13"|"C.13"
+"14"|"C.14"
+"15"|"C.15"
+"16"|"C.16"
+"17"|"C.17"
+"18"|"C.18"

data/tests/manual/bad-files/data/c/c_4.txt

@@ -0,0 +1,6 @@
+"19"|"C.19"
+"20"|"C.20"
+"21"|"C.21"
+"22"|"C.22"
+"23"|"C.23"
+"24"|"C.24"

data/tests/manual/bad-files/data/c/c_5.txt

@@ -0,0 +1,6 @@
+"25"|"C.25"
+"26"|"C.26"
+"27"|"C.27"
+"28"|"C.28"
+"29"|"C.29"
+"30"|"C.30"

data/tests/manual/bad-files/data/c/c_6.txt

@@ -0,0 +1,6 @@
+"31"|"C.31"
+"32"|"C.32"
+"33"|"C.33"
+"34"|"C.34"
+"35"|"C.35"
+"36"|"C.36"

data/tests/manual/bad-files/data/d/d_1.txt

@@ -0,0 +1,6 @@
+"1"|"D.1"
+"2"|"D.2"
+"3"|"D.3"
+"4"|"D.4"
+"5"|"D.5"
+"6"|"D.6"

data/tests/manual/bad-files/data/d/d_2.txt

@@ -0,0 +1,6 @@
+"7"|"D.7"
+"8"|"D.8"
+"9"|"D.9"
+"10"|"D.10"
+"11"|"D.11"
+"12"|"D.12"

data/tests/manual/bad-files/data/d/d_3.txt

@@ -0,0 +1,6 @@
+"13"|"D.13"
+"14"|"D.14"
+"15"|"D.15"
+"16"|"D.16"
+"17"|"D.17"
+"18"|"D.18"

data/tests/manual/bad-files/data/d/d_4.txt

@@ -0,0 +1,6 @@
+"19"|"D.19"
+"20"|"D.20"
+"21"|"D.21"
+"22"|"D.22"
+"23"|"D.23"
+"24"|"D.24"

data/tests/manual/bad-files/data/d/d_5.txt

@@ -0,0 +1,6 @@
+"25"|"D.25"
+"26"|"D.26"
+bad
+"28"|"D.28"
+"29"|"D.29"
+"30"|"D.30"

data/tests/manual/bad-files/data/d/d_6.txt

@@ -0,0 +1,6 @@
+"31"|"D.31"
+"32"|"D.32"
+"33"|"D.33"
+"34"|"D.34"
+"35"|"D.35"
+"36"|"D.36"

data/tests/manual/bad-files/data/e/e_1.txt

@@ -0,0 +1,6 @@
+"1"|"E.1"
+"2"|"E.2"
+"3"|"E.3"
+"4"|"E.4"
+"5"|"E.5"
+"6"|"E.6"

data/tests/manual/bad-files/data/e/e_2.txt

@@ -0,0 +1,6 @@
+"7"|"E.7"
+"8"|"E.8"
+"9"|"E.9"
+"10"|"E.10"
+"11"|"E.11"
+"12"|"E.12"

data/tests/manual/bad-files/data/e/e_3.txt

@@ -0,0 +1,6 @@
+"13"|"E.13"
+"14"|"E.14"
+"15"|"E.15"
+"16"|"E.16"
+"17"|"E.17"
+"18"|"E.18"

data/tests/manual/bad-files/data/e/e_4.txt

@@ -0,0 +1,6 @@
+"19"|"E.19"
+"20"|"E.20"
+"21"|"E.21"
+"22"|"E.22"
+"23"|"E.23"
+"24"|"E.24"

data/tests/manual/bad-files/data/e/e_5.txt

@@ -0,0 +1,6 @@
+"25"|"E.25"
+"26"|"E.26"
+"27"|"E.27"
+"28"|"E.28"
+"29"|"E.29"
+"30"|"E.30"

data/tests/manual/bad-files/data/e/e_6.txt

@@ -0,0 +1,6 @@
+"31"|"E.31"
+"32"|"E.32"
+"33"|"E.33"
+"34"|"E.34"
+"35"|"E.35"
+"36"|"E.36"

data/tests/manual/bad-files/data/f/f_1.txt

@@ -0,0 +1,6 @@
+"1"|"F.1"
+"2"|"F.2"
+"3"|"F.3"
+"4"|"F.4"
+"5"|"F.5"
+"6"|"F.6"

data/tests/manual/bad-files/data/f/f_2.txt

@@ -0,0 +1,6 @@
+"7"|"F.7"
+"8"|"F.8"
+"9"|"F.9"
+"10"|"F.10"
+"11"|"F.11"
+"12"|"F.12"

data/tests/manual/bad-files/data/f/f_3.txt

@@ -0,0 +1,6 @@
+"13"|"F.13"
+"14"|"F.14"
+"15"|"F.15"
+"16"|"F.16"
+"17"|"F.17"
+"18"|"F.18"