infobright-loader 0.0.1

data/tests/manual/bad-files/data/f/f_4.txt

@@ -0,0 +1,6 @@
+"19"|"F.19"
+"20"|"F.20"
+"21"|"F.21"
+"22"|"F.22"
+"23"|"F.23"
+"24"|"F.24"

data/tests/manual/bad-files/data/f/f_5.txt

@@ -0,0 +1,6 @@
+"25"|"F.25"
+"26"|"F.26"
+"27"|"F.27"
+"28"|"F.28"
+"29"|"F.29"
+"30"|"F.30"

data/tests/manual/bad-files/data/f/f_6.txt

@@ -0,0 +1,6 @@
+"31"|"F.31"
+"32"|"F.32"
+"33"|"F.33"
+"34"|"F.34"
+"35"|"F.35"
+"36"|"F.36"

data/tests/manual/bad-files/run_test.sh

@@ -0,0 +1,40 @@
+#!/bin/bash
+# Copyright (c) 2012 SnowPlow Analytics Ltd. All rights reserved.
+#
+# This program is licensed to you under the Apache License Version 2.0,
+# and you may not use this file except in compliance with the Apache License Version 2.0.
+# You may obtain a copy of the Apache License Version 2.0 at http://www.apache.org/licenses/LICENSE-2.0.
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the Apache License Version 2.0 is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the Apache License Version 2.0 for the specific language governing permissions and limitations there under.
+
+# Author::    Alex Dean (mailto:support@snowplowanalytics.com)
+# Copyright:: Copyright (c) 2012 SnowPlow Analytics Ltd
+# License::   Apache License Version 2.0
+
+# input parameters
+if [ $# != 2 ]; then
+    echo; echo 'syntax: '$0' <username> <password>'; echo
+    exit 1
+else
+    # assign parameters
+    USERNAME=$1
+    PASSWORD=$2
+fi
+
+echo "Running folder-based test..."
+SQL=`locate mysql-ib`
+PWD=`pwd`
+
+echo "Setting up Infobright"
+cat setup.sql | ${SQL} -u ${USERNAME} --password=${PASSWORD}
+
+echo "Running Infobright Ruby Loader in control file mode"
+sed "s|<<USERNAME>>|$USERNAME|;s|<<PASSWORD>>|$PASSWORD|;s|<<PATH>>|$PWD|" ./control-file-raw.yml > ./control-file-out.yml
+bundle exec infobright-loader -c control-file-out.yml -d irl_tests -e \" # Override a couple of args to test that too
+rm ./control-file-out.yml
+
+echo "Verifying the load into Infobright - please visually inspect:"
+cat verify.sql | ${SQL} -u ${USERNAME} --password=${PASSWORD}

data/tests/manual/bad-files/setup.sql

@@ -0,0 +1,8 @@
+drop database if exists irl_tests;
+create database irl_tests;
+use irl_tests;
+create table b (id int, description varchar(255)) engine=brighthouse;
+create table c (id int, description varchar(255)) engine=brighthouse;
+create table d (id int, description varchar(255)) engine=brighthouse;
+create table e (id int, description varchar(255)) engine=brighthouse;
+create table f (id int, description varchar(255)) engine=brighthouse;

data/tests/manual/bad-files/verify.sql

@@ -0,0 +1,10 @@
+use irl_tests;
+
+-- We should be able to make a nice table joining up all the rows in each table
+select b.id as row, b.description as b_desc, c.description as c_desc, d.description as d_desc, e.description as e_desc, f.description as f_desc
+  from b
+  left join c on b.id = c.id
+  left join d on b.id = d.id
+  left join e on b.id = e.id
+  left join f on b.id = f.id
+ order by b.id;

data/tests/manual/control-file/.gitignore

@@ -0,0 +1 @@
+control-file-out.yml

data/tests/manual/control-file/control-file-raw.yml

@@ -0,0 +1,50 @@
+# Control file for Infobright Ruby Loader control file test
+#
+# Is processed with sed to finalise <<x>> variables before
+# the test is run
+
+:load:
+  :processes: 3 # Test with 3 processes populating 5 tables
+:database:
+  :name: wrong # Will override at the command line to irl_tests
+  :username: <<USERNAME>>
+  :password: <<PASSWORD>>
+:data_format:
+  :separator: \|
+  :encloser: \' # Will override at the command line to "
+:data_loads:
+  b:
+    - <<PATH>>/data/b/b_1.txt
+    - <<PATH>>/data/b/b_2.txt
+    - <<PATH>>/data/b/b_3.txt
+    - <<PATH>>/data/b/b_4.txt
+    - <<PATH>>/data/b/b_5.txt
+    - <<PATH>>/data/b/b_6.txt
+  c:
+    - <<PATH>>/data/c/c_1.txt
+    - <<PATH>>/data/c/c_2.txt
+    - <<PATH>>/data/c/c_3.txt
+    - <<PATH>>/data/c/c_4.txt
+    - <<PATH>>/data/c/c_5.txt
+    - <<PATH>>/data/c/c_6.txt
+  d:
+    - <<PATH>>/data/d/d_1.txt
+    - <<PATH>>/data/d/d_2.txt
+    - <<PATH>>/data/d/d_3.txt
+    - <<PATH>>/data/d/d_4.txt
+    - <<PATH>>/data/d/d_5.txt
+    - <<PATH>>/data/d/d_6.txt
+  e:
+    - <<PATH>>/data/e/e_1.txt
+    - <<PATH>>/data/e/e_2.txt
+    - <<PATH>>/data/e/e_3.txt
+    - <<PATH>>/data/e/e_4.txt
+    - <<PATH>>/data/e/e_5.txt
+    - <<PATH>>/data/e/e_6.txt
+  f:
+    - <<PATH>>/data/f/f_1.txt
+    - <<PATH>>/data/f/f_2.txt
+    - <<PATH>>/data/f/f_3.txt
+    - <<PATH>>/data/f/f_4.txt
+    - <<PATH>>/data/f/f_5.txt
+    - <<PATH>>/data/f/f_6.txt

data/tests/manual/control-file/data/b/b_1.txt

@@ -0,0 +1,6 @@
+"1"|"B.1"
+"2"|"B.2"
+"3"|"B.3"
+"4"|"B.4"
+"5"|"B.5"
+"6"|"B.6"

data/tests/manual/control-file/data/b/b_2.txt

@@ -0,0 +1,6 @@
+"7"|"B.7"
+"8"|"B.8"
+"9"|"B.9"
+"10"|"B.10"
+"11"|"B.11"
+"12"|"B.12"

data/tests/manual/control-file/data/b/b_3.txt

@@ -0,0 +1,6 @@
+"13"|"B.13"
+"14"|"B.14"
+"15"|"B.15"
+"16"|"B.16"
+"17"|"B.17"
+"18"|"B.18"

data/tests/manual/control-file/data/b/b_4.txt

@@ -0,0 +1,6 @@
+"19"|"B.19"
+"20"|"B.20"
+"21"|"B.21"
+"22"|"B.22"
+"23"|"B.23"
+"24"|"B.24"

data/tests/manual/control-file/data/b/b_5.txt

@@ -0,0 +1,6 @@
+"25"|"B.25"
+"26"|"B.26"
+"27"|"B.27"
+"28"|"B.28"
+"29"|"B.29"
+"30"|"B.30"

data/tests/manual/control-file/data/b/b_6.txt

@@ -0,0 +1,6 @@
+"31"|"B.31"
+"32"|"B.32"
+"33"|"B.33"
+"34"|"B.34"
+"35"|"B.35"
+"36"|"B.36"

data/tests/manual/control-file/data/c/c_1.txt

@@ -0,0 +1,6 @@
+"1"|"C.1"
+"2"|"C.2"
+"3"|"C.3"
+"4"|"C.4"
+"5"|"C.5"
+"6"|"C.6"

data/tests/manual/control-file/data/c/c_2.txt

@@ -0,0 +1,6 @@
+"7"|"C.7"
+"8"|"C.8"
+"9"|"C.9"
+"10"|"C.10"
+"11"|"C.11"
+"12"|"C.12"

data/tests/manual/control-file/data/c/c_3.txt

@@ -0,0 +1,6 @@
+"13"|"C.13"
+"14"|"C.14"
+"15"|"C.15"
+"16"|"C.16"
+"17"|"C.17"
+"18"|"C.18"

data/tests/manual/control-file/data/c/c_4.txt

@@ -0,0 +1,6 @@
+"19"|"C.19"
+"20"|"C.20"
+"21"|"C.21"
+"22"|"C.22"
+"23"|"C.23"
+"24"|"C.24"

data/tests/manual/control-file/data/c/c_5.txt

@@ -0,0 +1,6 @@
+"25"|"C.25"
+"26"|"C.26"
+"27"|"C.27"
+"28"|"C.28"
+"29"|"C.29"
+"30"|"C.30"

data/tests/manual/control-file/data/c/c_6.txt

@@ -0,0 +1,6 @@
+"31"|"C.31"
+"32"|"C.32"
+"33"|"C.33"
+"34"|"C.34"
+"35"|"C.35"
+"36"|"C.36"

data/tests/manual/control-file/data/d/d_1.txt

@@ -0,0 +1,6 @@
+"1"|"D.1"
+"2"|"D.2"
+"3"|"D.3"
+"4"|"D.4"
+"5"|"D.5"
+"6"|"D.6"

data/tests/manual/control-file/data/d/d_2.txt

@@ -0,0 +1,6 @@
+"7"|"D.7"
+"8"|"D.8"
+"9"|"D.9"
+"10"|"D.10"
+"11"|"D.11"
+"12"|"D.12"

data/tests/manual/control-file/data/d/d_3.txt

@@ -0,0 +1,6 @@
+"13"|"D.13"
+"14"|"D.14"
+"15"|"D.15"
+"16"|"D.16"
+"17"|"D.17"
+"18"|"D.18"

data/tests/manual/control-file/data/d/d_4.txt

@@ -0,0 +1,6 @@
+"19"|"D.19"
+"20"|"D.20"
+"21"|"D.21"
+"22"|"D.22"
+"23"|"D.23"
+"24"|"D.24"

data/tests/manual/control-file/data/d/d_5.txt

@@ -0,0 +1,6 @@
+"25"|"D.25"
+"26"|"D.26"
+"27"|"D.27"
+"28"|"D.28"
+"29"|"D.29"
+"30"|"D.30"

data/tests/manual/control-file/data/d/d_6.txt

@@ -0,0 +1,6 @@
+"31"|"D.31"
+"32"|"D.32"
+"33"|"D.33"
+"34"|"D.34"
+"35"|"D.35"
+"36"|"D.36"

data/tests/manual/control-file/data/e/e_1.txt

@@ -0,0 +1,6 @@
+"1"|"E.1"
+"2"|"E.2"
+"3"|"E.3"
+"4"|"E.4"
+"5"|"E.5"
+"6"|"E.6"

data/tests/manual/control-file/data/e/e_2.txt

@@ -0,0 +1,6 @@
+"7"|"E.7"
+"8"|"E.8"
+"9"|"E.9"
+"10"|"E.10"
+"11"|"E.11"
+"12"|"E.12"

data/tests/manual/control-file/data/e/e_3.txt

@@ -0,0 +1,6 @@
+"13"|"E.13"
+"14"|"E.14"
+"15"|"E.15"
+"16"|"E.16"
+"17"|"E.17"
+"18"|"E.18"

data/tests/manual/control-file/data/e/e_4.txt

@@ -0,0 +1,6 @@
+"19"|"E.19"
+"20"|"E.20"
+"21"|"E.21"
+"22"|"E.22"
+"23"|"E.23"
+"24"|"E.24"

data/tests/manual/control-file/data/e/e_5.txt

@@ -0,0 +1,6 @@
+"25"|"E.25"
+"26"|"E.26"
+"27"|"E.27"
+"28"|"E.28"
+"29"|"E.29"
+"30"|"E.30"

data/tests/manual/control-file/data/e/e_6.txt

@@ -0,0 +1,6 @@
+"31"|"E.31"
+"32"|"E.32"
+"33"|"E.33"
+"34"|"E.34"
+"35"|"E.35"
+"36"|"E.36"

data/tests/manual/control-file/data/f/f_1.txt

@@ -0,0 +1,6 @@
+"1"|"F.1"
+"2"|"F.2"
+"3"|"F.3"
+"4"|"F.4"
+"5"|"F.5"
+"6"|"F.6"

data/tests/manual/control-file/data/f/f_2.txt

@@ -0,0 +1,6 @@
+"7"|"F.7"
+"8"|"F.8"
+"9"|"F.9"
+"10"|"F.10"
+"11"|"F.11"
+"12"|"F.12"

data/tests/manual/control-file/data/f/f_3.txt

@@ -0,0 +1,6 @@
+"13"|"F.13"
+"14"|"F.14"
+"15"|"F.15"
+"16"|"F.16"
+"17"|"F.17"
+"18"|"F.18"

data/tests/manual/control-file/data/f/f_4.txt

@@ -0,0 +1,6 @@
+"19"|"F.19"
+"20"|"F.20"
+"21"|"F.21"
+"22"|"F.22"
+"23"|"F.23"
+"24"|"F.24"

data/tests/manual/control-file/data/f/f_5.txt

@@ -0,0 +1,6 @@
+"25"|"F.25"
+"26"|"F.26"
+"27"|"F.27"
+"28"|"F.28"
+"29"|"F.29"
+"30"|"F.30"

data/tests/manual/control-file/data/f/f_6.txt

@@ -0,0 +1,6 @@
+"31"|"F.31"
+"32"|"F.32"
+"33"|"F.33"
+"34"|"F.34"
+"35"|"F.35"
+"36"|"F.36"

data/tests/manual/control-file/run_test.sh

@@ -0,0 +1,40 @@
+#!/bin/bash
+# Copyright (c) 2012 SnowPlow Analytics Ltd. All rights reserved.
+#
+# This program is licensed to you under the Apache License Version 2.0,
+# and you may not use this file except in compliance with the Apache License Version 2.0.
+# You may obtain a copy of the Apache License Version 2.0 at http://www.apache.org/licenses/LICENSE-2.0.
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the Apache License Version 2.0 is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the Apache License Version 2.0 for the specific language governing permissions and limitations there under.
+
+# Author::    Alex Dean (mailto:support@snowplowanalytics.com)
+# Copyright:: Copyright (c) 2012 SnowPlow Analytics Ltd
+# License::   Apache License Version 2.0
+
+# input parameters
+if [ $# != 2 ]; then
+    echo; echo 'syntax: '$0' <username> <password>'; echo
+    exit 1
+else
+    # assign parameters
+    USERNAME=$1
+    PASSWORD=$2
+fi
+
+echo "Running folder-based test..."
+SQL=`locate mysql-ib`
+PWD=`pwd`
+
+echo "Setting up Infobright"
+cat setup.sql | ${SQL} -u ${USERNAME} --password=${PASSWORD}
+
+echo "Running Infobright Ruby Loader in control file mode"
+sed "s|<<USERNAME>>|$USERNAME|;s|<<PASSWORD>>|$PASSWORD|;s|<<PATH>>|$PWD|" ./control-file-raw.yml > ./control-file-out.yml
+bundle exec infobright-loader -c control-file-out.yml -d irl_tests -e \" # Override a couple of args to test that too
+rm ./control-file-out.yml
+
+echo "Verifying the load into Infobright - please visually inspect:"
+cat verify.sql | ${SQL} -u ${USERNAME} --password=${PASSWORD}

data/tests/manual/control-file/setup.sql

@@ -0,0 +1,8 @@
+drop database if exists irl_tests;
+create database irl_tests;
+use irl_tests;
+create table b (id int, description varchar(255)) engine=brighthouse;
+create table c (id int, description varchar(255)) engine=brighthouse;
+create table d (id int, description varchar(255)) engine=brighthouse;
+create table e (id int, description varchar(255)) engine=brighthouse;
+create table f (id int, description varchar(255)) engine=brighthouse;

data/tests/manual/control-file/verify.sql

@@ -0,0 +1,10 @@
+use irl_tests;
+
+-- We should be able to make a nice table joining up all the rows in each table
+select b.id as row, b.description as b_desc, c.description as c_desc, d.description as d_desc, e.description as e_desc, f.description as f_desc
+  from b
+  left join c on b.id = c.id
+  left join d on b.id = d.id
+  left join e on b.id = e.id
+  left join f on b.id = f.id
+ order by b.id;

data/tests/manual/folder/data/a/a_1.txt

@@ -0,0 +1,6 @@
+"1"|"row 1"
+"2"|"row 2"
+"3"|"row 3"
+"4"|"row 4"
+"5"|"row 5"
+"6"|"row 6"