dbexpect 0.11.0

data/Gemfile

@@ -0,0 +1,20 @@
+source "http://rubygems.org"
+# Add dependencies required to use your gem here.
+# Example:
+#   gem "activesupport", ">= 2.3.5"
+
+# Add dependencies to develop your gem here.
+# Include everything needed to run rake, tests, features, etc.
+
+
+group :development do
+  gem 'ruby-odbc'
+  gem "rdoc", "~> 3.12"
+  gem "bundler"
+  gem "jeweler", "~> 1.8.3"
+  gem "rspec", "~> 2.8.0"
+  gem 'pg'
+  gem "pry"
+  gem "ZenTest"
+end
+

data/Gemfile.lock

@@ -0,0 +1,45 @@
+GEM
+  remote: http://rubygems.org/
+  specs:
+    ZenTest (4.8.2)
+    coderay (1.0.7)
+    diff-lcs (1.1.3)
+    git (1.2.5)
+    jeweler (1.8.4)
+      bundler (~> 1.0)
+      git (>= 1.2.5)
+      rake
+      rdoc
+    json (1.7.5)
+    method_source (0.8)
+    pg (0.14.0)
+    pry (0.9.10)
+      coderay (~> 1.0.5)
+      method_source (~> 0.8)
+      slop (~> 3.3.1)
+    rake (0.9.2.2)
+    rdoc (3.12)
+      json (~> 1.4)
+    rspec (2.8.0)
+      rspec-core (~> 2.8.0)
+      rspec-expectations (~> 2.8.0)
+      rspec-mocks (~> 2.8.0)
+    rspec-core (2.8.0)
+    rspec-expectations (2.8.0)
+      diff-lcs (~> 1.1.2)
+    rspec-mocks (2.8.0)
+    ruby-odbc (0.99994)
+    slop (3.3.3)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  ZenTest
+  bundler
+  jeweler (~> 1.8.3)
+  pg
+  pry
+  rdoc (~> 3.12)
+  rspec (~> 2.8.0)
+  ruby-odbc

data/LICENSE.txt

@@ -0,0 +1,2 @@
+Copyright (c) 2012 Beau Fabry
+

data/README.md

@@ -0,0 +1,106 @@
+dbexpect
+=======
+
+dbexpect is a domain specific language written in ruby for testing ETL solutions.
+
+Taking cues from Rspec in structure and usage, the point is to enable
+the specification of unit test data, job running, and expected outcomes
+for an entity/test cases in a ruby file using an internal DSL that is
+targeted at this kind of testing.
+
+dbexpect is ideally suited to creating automated unit tests for
+individual ETL jobs in a data warehousing or data migration project.
+Helping to ensure correctness of the job initially developed, and
+catching problems later on when someone makes a change that could affect
+existing functionality.
+
+Sample test
+---------
+
+    describe "Moving customers from source to target" do
+      @src = table(:dbexpect_src,:dbexpect_src,:customers_src)
+      @tgt = table(:dbexpect_tgt,:dbexpect_tgt,:customers_tgt)
+
+      etl_run_command "ruby etl2.rb"
+
+      expect_total_rows @tgt, 1
+
+      describe "it should upcase customer names" do
+        insert_into @src,
+          [:id,:name],
+          [[1,"Fred"]]
+
+        expect_rows @tgt,
+          [:id,:name],
+          [[1,"FRED"]]
+      end
+
+      describe "it should not migrate smith (because screw that guy)" do
+        insert_into @src,
+          [:id,:name],
+          [[1,"Smith"]]
+
+        # expect no rows
+      end
+    end
+
+Installation
+------------
+    gem install dbexpect
+    
+Create a database.yml file in a folder where you want to store your
+tests, and set up connections for each of the databases you want dbexpect
+to talk to. Each of the connections will need to have an ODBC connection
+defined as well.
+
+database.yml:
+
+    database1:
+      database: odbc_dsn
+      username: barry
+      password: secret
+
+    database2:
+      database: odbc_dsn2
+      username: shaz
+      password: secret
+
+Usage
+-----
+Assuming a folder structure for your tests that looks like this:
+
+    /
+    |-database.yml
+    |
+    |- defaults/
+    |          |- defaults_for_tablex.rb
+    |
+    |- tests/
+            |- test1.rb
+            |- test2.rb
+
+To run the tests in test1.rb:
+
+    prompt:/$ dbexpect tests/test1.rb
+
+There is a sample dbexpect project at
+http://github.com/C3/dbexpect_example for more information.
+
+
+License
+-------
+
+Copyright 2012 C3 Business Solutions. See COPYING for further details.
+
+dbexpect is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+dbexpect is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with dbexpect.  If not, see <http://www.gnu.org/licenses/>.

data/Rakefile

@@ -0,0 +1,65 @@
+# Copyright 2012 C3 Business Solutions
+#
+#    This file is part of dbexpect.
+#
+#    dbexpect is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License as published by
+#    the Free Software Foundation, either version 3 of the License, or
+#    (at your option) any later version.
+#
+#    dbexpect is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with dbexpect.  If not, see <http://www.gnu.org/licenses/>.
+# encoding: utf-8
+
+require 'rubygems'
+require 'bundler'
+begin
+  Bundler.setup(:default, :development)
+rescue Bundler::BundlerError => e
+  $stderr.puts e.message
+  $stderr.puts "Run `bundle install` to install missing gems"
+  exit e.status_code
+end
+require 'rake'
+
+require 'jeweler'
+Jeweler::Tasks.new do |gem|
+  # gem is a Gem::Specification... see http://docs.rubygems.org/read/chapter/20 for more options
+  gem.name = "dbexpect"
+  gem.homepage = "http://github.com/bfabry/dbexpect"
+  gem.license = "PRIVATE"
+  gem.summary = %Q{Generates inserts for test data from DSL}
+  gem.description = %Q{As above}
+  gem.email = "beau.fabry@c3businesssolutions.com"
+  gem.authors = ["Beau Fabry"]
+  # dependencies defined in Gemfile
+end
+Jeweler::RubygemsDotOrgTasks.new
+
+require 'rspec/core'
+require 'rspec/core/rake_task'
+RSpec::Core::RakeTask.new(:spec) do |spec|
+  spec.pattern = FileList['spec/**/*_spec.rb']
+end
+
+RSpec::Core::RakeTask.new(:rcov) do |spec|
+  spec.pattern = 'spec/**/*_spec.rb'
+  spec.rcov = true
+end
+
+task :default => :spec
+
+require 'rdoc/task'
+Rake::RDocTask.new do |rdoc|
+  version = File.exist?('VERSION') ? File.read('VERSION') : ""
+
+  rdoc.rdoc_dir = 'rdoc'
+  rdoc.title = "dbexpect #{version}"
+  rdoc.rdoc_files.include('README*')
+  rdoc.rdoc_files.include('lib/**/*.rb')
+end

data/VERSION

@@ -0,0 +1 @@
+0.11.0

data/bin/dbexpect

@@ -0,0 +1,33 @@
+# Copyright 2012 C3 Business Solutions
+#
+#    This file is part of dbexpect.
+#
+#    dbexpect is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License as published by
+#    the Free Software Foundation, either version 3 of the License, or
+#    (at your option) any later version.
+#
+#    dbexpect is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with dbexpect.  If not, see <http://www.gnu.org/licenses/>.
+
+#!/usr/bin/env ruby
+
+puts "dbexpect Copyright 2012 C3 Business Solutions"
+puts "This program comes with ABSOLUTELY NO WARRANTY"
+
+begin
+  require 'dbexpect'
+rescue LoadError
+  $LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+  require 'dbexpect'
+end
+
+db = Database.hash_from_config
+ret = Dbexpect.new().run_test(ARGV.shift,db,CommandRunner.new)
+
+exit(ret)

data/bin/great-expectations

@@ -0,0 +1,28 @@
+# Copyright 2012 C3 Business Solutions
+#
+#    This file is part of dbexpect.
+#
+#    dbexpect is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License as published by
+#    the Free Software Foundation, either version 3 of the License, or
+#    (at your option) any later version.
+#
+#    dbexpect is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with dbexpect.  If not, see <http://www.gnu.org/licenses/>.
+
+#!/usr/bin/env ruby
+
+begin
+  require 'dbexpect'
+rescue LoadError
+  $LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+  require 'dbexpect'
+end
+
+target_db = Database.from_dsn(ARGV.shift)
+exit(Dbexpect.new().great_expectations(ARGV.shift,target_db))

data/bin/ready-rig

@@ -0,0 +1,27 @@
+# Copyright 2012 C3 Business Solutions
+#
+#    This file is part of dbexpect.
+#
+#    dbexpect is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License as published by
+#    the Free Software Foundation, either version 3 of the License, or
+#    (at your option) any later version.
+#
+#    dbexpect is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with dbexpect.  If not, see <http://www.gnu.org/licenses/>.
+
+#!/usr/bin/env ruby
+
+begin
+  require 'dbexpect'
+rescue LoadError
+  $LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+  require 'dbexpect'
+end
+db = Database.from_dsn(ARGV.shift)
+exit(Dbexpect.new().setup_test(ARGV.shift,db))

data/database.yml

@@ -0,0 +1,9 @@
+dbexpect_src:
+  database: testgen_src
+  username: testgen
+  password: testgen
+
+dbexpect_tgt:
+  database: testgen_tgt
+  username: testgen
+  password: testgen

data/lib/dbexpect.rb

@@ -0,0 +1,19 @@
+# Copyright 2012 C3 Business Solutions
+#
+#    This file is part of dbexpect.
+#
+#    dbexpect is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License as published by
+#    the Free Software Foundation, either version 3 of the License, or
+#    (at your option) any later version.
+#
+#    dbexpect is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with dbexpect.  If not, see <http://www.gnu.org/licenses/>.
+require_relative 'dbexpect/dbexpect'
+require_relative 'dbexpect/database'
+require_relative 'dbexpect/command_runner'

data/lib/dbexpect/command_runner.rb

@@ -0,0 +1,21 @@
+# Copyright 2012 C3 Business Solutions
+#
+#    This file is part of dbexpect.
+#
+#    dbexpect is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License as published by
+#    the Free Software Foundation, either version 3 of the License, or
+#    (at your option) any later version.
+#
+#    dbexpect is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with dbexpect.  If not, see <http://www.gnu.org/licenses/>.
+class CommandRunner
+  def run(command)
+    system(command)
+  end
+end

data/lib/dbexpect/console_formatter.rb

@@ -0,0 +1,40 @@
+# Copyright 2012 C3 Business Solutions
+#
+#    This file is part of dbexpect.
+#
+#    dbexpect is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License as published by
+#    the Free Software Foundation, either version 3 of the License, or
+#    (at your option) any later version.
+#
+#    dbexpect is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with dbexpect.  If not, see <http://www.gnu.org/licenses/>.
+class ConsoleFormatter
+  def initialize(output)
+    @output = output
+  end
+
+  def notify_passed
+    @output.puts "Passed all expectations\n"
+  end
+
+  def notify_failed(failed_expectations)
+    failed_expectations.traverse do |depth, description, expectations|
+      @output.puts(('  ' * depth) + description + ":\n")
+
+      expectations.collect(&:failure_message).each do |msg|
+        @output.puts(('  ' * depth) + ' ' + msg + "\n")
+      end
+    end
+    @output.puts "Failed to meet expectations\n"
+  end
+
+  def format_sql(sql)
+    @output.puts sql
+  end
+end

data/lib/dbexpect/d_s_l_parser.rb

@@ -0,0 +1,124 @@
+# Copyright 2012 C3 Business Solutions
+#
+#    This file is part of dbexpect.
+#
+#    dbexpect is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License as published by
+#    the Free Software Foundation, either version 3 of the License, or
+#    (at your option) any later version.
+#
+#    dbexpect is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with dbexpect.  If not, see <http://www.gnu.org/licenses/>.
+require_relative 'expectation_tree_node'
+require 'set'
+
+class DSLParser
+
+  attr_accessor :commands
+
+  def initialize
+    @tables = {}
+    @tree_nodes = [ExpectationTreeNode.new('->')]
+    @files_loaded = Set.new
+    @commands = []
+  end
+
+  def expectation_tree
+    @tree_nodes.first
+  end
+
+  def parse(script)
+    instance_eval(script)
+  end
+
+  def tables
+    @tables.values
+  end
+
+protected
+  def requires(file)
+    unless @files_loaded.include?(file)
+      instance_eval(File.read(file))
+    end
+    @files_loaded << file
+  end
+
+  def etl_run_command(command)
+    @commands << command
+  end
+
+  def dirty(table)
+    table.dirty = true
+  end
+
+  def expect_total_rows(table, count)
+    @tree_nodes.last.add([table.row_count_check(count)])
+  end
+
+  def describe(description,&block)
+    new_node = @tree_nodes.last.create_child(description)
+    @tree_nodes << new_node
+    instance_eval(&block)
+    @tree_nodes.pop
+  end
+
+  def defaults_for(table, columns)
+    __set_defaults(table,:set_default,columns)
+  end
+
+  def expected_defaults(table, columns)
+    __set_defaults(table,:set_expected_default,columns)
+  end
+
+  def __set_defaults(table, method, columns)
+    columns.each do |col, value|
+      table.send(method, col, wrap(value))
+    end
+  end
+
+  def insert_into(table,row_columns,rows)
+    __add_rows(table, :add_fixture_row, row_columns, rows)
+  end
+
+  def expect_rows(table, row_columns, rows)
+    @tree_nodes.last.add __add_rows(table, :add_expected_row, row_columns, rows)
+  end
+
+  def __add_rows(table, row_method, row_columns, rows)
+    rows.collect do |row_values|
+      wrapped = row_values.map {|v| wrap(v) }
+      table.send(row_method,Hash[ row_columns.zip(wrapped)])
+    end
+  end
+
+  def all_tables(col_values)
+    col_values.each do |col,value|
+      @tables.each {|x,t| t.set_default(col, wrap(value)) }
+    end
+  end
+
+  def table(db_name,schema,tablename)
+    @tables[db_name.to_s + schema.to_s + tablename.to_s] ||= Table.new(db_name,schema,tablename)
+  end
+
+  NULL = DbNull.new
+  def null; DbNull.new; end
+
+  def wrap(val)
+    case val
+    when DbSequence
+      val
+    when DbNull
+      val
+    when nil
+      NULL
+    else
+      DbString.new(val)
+    end
+  end
+end