converge 0.0.1.pre.alpha.2
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/.gitignore +14 -0
- data/.travis.yml +17 -0
- data/CONTRIBUTING.md +33 -0
- data/Gemfile +6 -0
- data/LICENSE +22 -0
- data/README.md +40 -0
- data/Rakefile +32 -0
- data/converge.gemspec +29 -0
- data/lib/converge.rb +32 -0
- data/lib/converge/db.rb +6 -0
- data/lib/converge/db/connection.rb +16 -0
- data/lib/converge/db/connection/table.rb +59 -0
- data/lib/converge/db/table.rb +4 -0
- data/lib/converge/db/table/base.rb +24 -0
- data/lib/converge/db/table/loader.rb +26 -0
- data/lib/converge/db/table/merger.rb +19 -0
- data/lib/converge/db/table/partitioner/bookend.rb +23 -0
- data/lib/converge/db/table/staging.rb +22 -0
- data/lib/converge/db/table/target.rb +15 -0
- data/lib/converge/model.rb +21 -0
- data/lib/converge/version.rb +3 -0
- data/test/integration/converge_test.rb +89 -0
- data/test/integration/db_test_helper.rb +12 -0
- data/test/integration/mock/connection.rb +20 -0
- data/test/integration/mock/table.rb +22 -0
- data/test/integration/mock_adapter.rb +4 -0
- data/test/integration/test_helper.rb +23 -0
- data/test/unit/converge/db/table/base_test.rb +24 -0
- data/test/unit/converge/db/table/loader_test.rb +75 -0
- data/test/unit/converge/db/table/merger_test.rb +28 -0
- data/test/unit/converge/db/table/staging_test.rb +7 -0
- data/test/unit/converge/db/table/target_test.rb +10 -0
- data/test/unit/test_helper.rb +21 -0
- metadata +187 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: 4844beacb6a4e9fd06ab828198fb6be4eb63b4f6
|
4
|
+
data.tar.gz: 83806209ddeb54f71971a1260a7516afedcc7dc1
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: f9a22f9924a3bad3a90f3ed2a855b9273b76c39f3fab185111b8b3002441b89286d06d00b89ca35d8c31dec32b12f898a97e681ff6f8086fccf76e0f6f362935
|
7
|
+
data.tar.gz: 724cf8af3aa27b277713cf3e6985293a3aabf684f436db536dec5d0849fe965b089fe07570634157dfc9926cac92c647d54c2fe3d364deb6704b0121cdae5122
|
data/.gitignore
ADDED
data/.travis.yml
ADDED
data/CONTRIBUTING.md
ADDED
@@ -0,0 +1,33 @@
|
|
1
|
+
Contributing to Converge
|
2
|
+
========================
|
3
|
+
|
4
|
+
First of all, thanks for your interest in contributing to Converge!
|
5
|
+
|
6
|
+
Below are a list of guidelines to help keep this project sane in the face of open source entropy.
|
7
|
+
|
8
|
+
How To Submit A Patch
|
9
|
+
---------------------
|
10
|
+
|
11
|
+
Any patch you feel would make this a better project is welcome, including but not limited to: bug fixes, new features, refactoring, and better documentation.
|
12
|
+
|
13
|
+
Make a fork of this project, and hack to your heart's content. Once you have something you think would be great to include into master, feel free to open a pull request. Please keep in mind the following guidelines, which will make it more likely for your pull request to be accepted:
|
14
|
+
|
15
|
+
1. Please have test cases, both in the unit suite and the integration suite, that fail when your code is not present.
|
16
|
+
1. Please open a topic branch in your fork, and create your pull request against that.
|
17
|
+
* Use a separate topic for each feature and/or bug fix you want to submit.
|
18
|
+
1. Be logically consistent in your commits. Make them as small as possible, so they can tell a story the maintainer can understand.
|
19
|
+
1. Keeping a consistent style in the codebase is important for maintainability. When possible, please stick to the guidelines within [this guide](https://github.com/bbatsov/ruby-style-guide).
|
20
|
+
1. Pay attention to things like code climate and test coverage. A drop in these metrics will not necessarily preclude a patch from being accepted, but we'd much prefer to keep these numbers high.
|
21
|
+
|
22
|
+
Please keep in mind your contributions will be licensed the same as the rest of Converge, under the MIT license.
|
23
|
+
|
24
|
+
It may take the developer a day or two to get to your pull request. If your request has not been responded to in a few days, though, feel free to ping the developer team, preferably through a comment on the pull.
|
25
|
+
|
26
|
+
How To Report An Issue
|
27
|
+
----------------------
|
28
|
+
|
29
|
+
Issue reports are welcome, even if not accompanied by a pull request. Keep in mind that submitting a pull request will mean your issue likely gets resolved sooner, because less work will be left to do.
|
30
|
+
|
31
|
+
The most important thing an issue can do is get a developer as close as possible to seeing what you are seeing, in the case of a bug report. In the case of a feature request, you want to get them to the point where they don't see what you want to see, so they can make a decision about whether and how to include the proposed feature.
|
32
|
+
|
33
|
+
More about good bug reports [here](http://www.chiark.greenend.org.uk/~sgtatham/bugs.html).
|
data/Gemfile
ADDED
data/LICENSE
ADDED
@@ -0,0 +1,22 @@
|
|
1
|
+
Copyright (c) 2014 Ed Carrel
|
2
|
+
|
3
|
+
MIT License
|
4
|
+
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining
|
6
|
+
a copy of this software and associated documentation files (the
|
7
|
+
"Software"), to deal in the Software without restriction, including
|
8
|
+
without limitation the rights to use, copy, modify, merge, publish,
|
9
|
+
distribute, sublicense, and/or sell copies of the Software, and to
|
10
|
+
permit persons to whom the Software is furnished to do so, subject to
|
11
|
+
the following conditions:
|
12
|
+
|
13
|
+
The above copyright notice and this permission notice shall be
|
14
|
+
included in all copies or substantial portions of the Software.
|
15
|
+
|
16
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
17
|
+
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
18
|
+
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
19
|
+
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
20
|
+
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
21
|
+
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
22
|
+
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
data/README.md
ADDED
@@ -0,0 +1,40 @@
|
|
1
|
+
# Converge
|
2
|
+
|
3
|
+
Data merger for Redshift
|
4
|
+
|
5
|
+
## Installation
|
6
|
+
|
7
|
+
Add this line to your application's Gemfile:
|
8
|
+
|
9
|
+
```ruby
|
10
|
+
gem 'converge'
|
11
|
+
```
|
12
|
+
|
13
|
+
And then execute:
|
14
|
+
|
15
|
+
$ bundle
|
16
|
+
|
17
|
+
Or install it yourself as:
|
18
|
+
|
19
|
+
$ gem install converge
|
20
|
+
|
21
|
+
## Usage
|
22
|
+
|
23
|
+
TODO: Write usage instructions here
|
24
|
+
|
25
|
+
API Documentation
|
26
|
+
-------------
|
27
|
+
|
28
|
+
See [RubyDoc](http://rubydoc.info/github/azanar/converge/index)
|
29
|
+
|
30
|
+
Contributors
|
31
|
+
------------
|
32
|
+
|
33
|
+
See [Contributing](CONTRIBUTING.md) for details.
|
34
|
+
|
35
|
+
License
|
36
|
+
-------
|
37
|
+
|
38
|
+
©2014 Ed Carrel. Released under the MIT License.
|
39
|
+
|
40
|
+
See [License](LICENSE) for details.
|
data/Rakefile
ADDED
@@ -0,0 +1,32 @@
|
|
1
|
+
require "bundler/gem_tasks"
|
2
|
+
|
3
|
+
require 'coveralls/rake/task'
|
4
|
+
|
5
|
+
require 'rake/testtask'
|
6
|
+
|
7
|
+
task :test do
|
8
|
+
Rake::Task['test:unit'].invoke
|
9
|
+
Rake::Task['test:integration'].invoke
|
10
|
+
end
|
11
|
+
|
12
|
+
namespace :test do
|
13
|
+
Rake::TestTask.new("integration") do |t|
|
14
|
+
t.libs << "test"
|
15
|
+
t.libs << "config"
|
16
|
+
t.test_files = FileList['test/integration/**/*_test.rb']
|
17
|
+
t.verbose = true
|
18
|
+
end
|
19
|
+
|
20
|
+
Rake::TestTask.new("unit") do |t|
|
21
|
+
t.libs << "test"
|
22
|
+
t.libs << "config"
|
23
|
+
t.test_files = FileList['test/unit/**/*_test.rb']
|
24
|
+
t.verbose = true
|
25
|
+
end
|
26
|
+
end
|
27
|
+
|
28
|
+
Coveralls::RakeTask.new
|
29
|
+
|
30
|
+
task :test_with_coveralls => ['test:unit', 'test:integration', 'coveralls:push']
|
31
|
+
|
32
|
+
task :default => :test_with_coveralls
|
data/converge.gemspec
ADDED
@@ -0,0 +1,29 @@
|
|
1
|
+
# coding: utf-8
|
2
|
+
lib = File.expand_path('../lib', __FILE__)
|
3
|
+
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
4
|
+
require 'converge/version'
|
5
|
+
|
6
|
+
Gem::Specification.new do |spec|
|
7
|
+
spec.name = "converge"
|
8
|
+
spec.version = Converge::VERSION
|
9
|
+
spec.authors = ["Ed Carrel"]
|
10
|
+
spec.email = ["edward@carrel.org"]
|
11
|
+
spec.summary = %q{Data merger for Redshift}
|
12
|
+
spec.description = %q{Data merger}
|
13
|
+
spec.homepage = "https://github.com/azanar/converge"
|
14
|
+
spec.license = "MIT"
|
15
|
+
|
16
|
+
spec.files = `git ls-files -z`.split("\x0")
|
17
|
+
spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
|
18
|
+
spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
|
19
|
+
spec.require_paths = ["lib"]
|
20
|
+
|
21
|
+
spec.add_runtime_dependency 'hydrogen', '~> 0'
|
22
|
+
spec.add_runtime_dependency 'activesupport', '~> 4'
|
23
|
+
|
24
|
+
spec.add_development_dependency 'test-unit', '~> 3'
|
25
|
+
spec.add_development_dependency 'mocha', '~> 1'
|
26
|
+
spec.add_development_dependency 'simplecov', '~> 0'
|
27
|
+
spec.add_development_dependency "bundler", "~> 1.6"
|
28
|
+
spec.add_development_dependency "rake", "~> 10.0"
|
29
|
+
end
|
data/lib/converge.rb
ADDED
@@ -0,0 +1,32 @@
|
|
1
|
+
require 'logger'
|
2
|
+
|
3
|
+
require "hydrogen"
|
4
|
+
|
5
|
+
require "converge/db"
|
6
|
+
require "converge/model"
|
7
|
+
require "converge/version"
|
8
|
+
|
9
|
+
module Converge
|
10
|
+
module_function
|
11
|
+
def logger
|
12
|
+
return @logger if @logger
|
13
|
+
|
14
|
+
@logger = Logger.new(STDOUT)
|
15
|
+
@logger.formatter = proc { |severity, datetime, progname, msg|
|
16
|
+
"[#{datetime}, #{severity}] #{msg}\n"
|
17
|
+
}
|
18
|
+
@logger
|
19
|
+
end
|
20
|
+
|
21
|
+
def logger=(logger)
|
22
|
+
@logger = logger
|
23
|
+
end
|
24
|
+
|
25
|
+
def env
|
26
|
+
@_env ||= ActiveSupport::StringInquirer.new(ENV["QUASAR_ENV"] || ENV["RAILS_ENV"] || "development")
|
27
|
+
end
|
28
|
+
|
29
|
+
def env=(environment)
|
30
|
+
@_env = ActiveSupport::StringInquirer.new(environment)
|
31
|
+
end
|
32
|
+
end
|
data/lib/converge/db.rb
ADDED
@@ -0,0 +1,59 @@
|
|
1
|
+
module Converge
|
2
|
+
module DB
|
3
|
+
class Connection
|
4
|
+
class Table
|
5
|
+
def initialize(conn, table, opts = {})
|
6
|
+
@conn = conn
|
7
|
+
@table = table
|
8
|
+
@stale = false
|
9
|
+
end
|
10
|
+
|
11
|
+
def name
|
12
|
+
@table.name
|
13
|
+
end
|
14
|
+
|
15
|
+
def columns
|
16
|
+
@table.columns
|
17
|
+
end
|
18
|
+
|
19
|
+
def key
|
20
|
+
@table.key
|
21
|
+
end
|
22
|
+
|
23
|
+
def update(table)
|
24
|
+
conn_table.update(table)
|
25
|
+
@stale = true
|
26
|
+
end
|
27
|
+
|
28
|
+
def insert(table)
|
29
|
+
conn_table.insert(table)
|
30
|
+
@stale = true
|
31
|
+
end
|
32
|
+
|
33
|
+
def copy(object_collection)
|
34
|
+
conn_table.copy(object_collection)
|
35
|
+
@stale = true
|
36
|
+
end
|
37
|
+
|
38
|
+
def truncate
|
39
|
+
conn_table.truncate
|
40
|
+
end
|
41
|
+
|
42
|
+
def finalize
|
43
|
+
if @stale
|
44
|
+
conn_table.finalize
|
45
|
+
@stale = false
|
46
|
+
else
|
47
|
+
Converge.logger.warn "Called finalize on a finalized table"
|
48
|
+
end
|
49
|
+
end
|
50
|
+
|
51
|
+
private
|
52
|
+
|
53
|
+
def conn_table
|
54
|
+
@conn.table(self)
|
55
|
+
end
|
56
|
+
end
|
57
|
+
end
|
58
|
+
end
|
59
|
+
end
|
@@ -0,0 +1,24 @@
|
|
1
|
+
module Converge
|
2
|
+
module DB
|
3
|
+
class Table
|
4
|
+
module Base
|
5
|
+
def initialize(model)
|
6
|
+
@model = model
|
7
|
+
@stale = false
|
8
|
+
end
|
9
|
+
|
10
|
+
def columns
|
11
|
+
@model.columns
|
12
|
+
end
|
13
|
+
|
14
|
+
def key
|
15
|
+
@model.key
|
16
|
+
end
|
17
|
+
|
18
|
+
def loader(connection)
|
19
|
+
Loader.new(self, connection)
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
@@ -0,0 +1,26 @@
|
|
1
|
+
module Converge
|
2
|
+
module DB
|
3
|
+
class Table
|
4
|
+
class Loader
|
5
|
+
def initialize(table, conn)
|
6
|
+
@table = table
|
7
|
+
@conn = conn
|
8
|
+
@truncated = false #assume we aren't truncated.
|
9
|
+
end
|
10
|
+
|
11
|
+
def load(object_collection)
|
12
|
+
@table.truncate
|
13
|
+
@table.copy(object_collection)
|
14
|
+
@truncated = false
|
15
|
+
end
|
16
|
+
|
17
|
+
def finalize
|
18
|
+
unless @truncated
|
19
|
+
@table.truncate
|
20
|
+
@truncated = true
|
21
|
+
end
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
@@ -0,0 +1,19 @@
|
|
1
|
+
module Converge
|
2
|
+
module DB
|
3
|
+
class Table
|
4
|
+
class Merger
|
5
|
+
def initialize(source, conn)
|
6
|
+
@conn = conn
|
7
|
+
@conn_source = @conn.table(source)
|
8
|
+
end
|
9
|
+
|
10
|
+
def merge(destination)
|
11
|
+
conn_destination = @conn.table(destination)
|
12
|
+
conn_destination.update(@conn_source)
|
13
|
+
conn_destination.insert(@conn_source)
|
14
|
+
conn_destination.finalize
|
15
|
+
end
|
16
|
+
end
|
17
|
+
end
|
18
|
+
end
|
19
|
+
end
|
@@ -0,0 +1,23 @@
|
|
1
|
+
require 'converge/db/table/partitioner'
|
2
|
+
module Converge
|
3
|
+
module DB
|
4
|
+
class Table
|
5
|
+
module Partitioner
|
6
|
+
class Bookend
|
7
|
+
include Partitioner
|
8
|
+
def initialize(source)
|
9
|
+
@source = source
|
10
|
+
end
|
11
|
+
|
12
|
+
def matches?
|
13
|
+
true
|
14
|
+
end
|
15
|
+
|
16
|
+
def partition_for(dest)
|
17
|
+
@source
|
18
|
+
end
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
23
|
+
end
|
@@ -0,0 +1,22 @@
|
|
1
|
+
require 'converge/db/table/base'
|
2
|
+
|
3
|
+
require 'converge/db/table/loader'
|
4
|
+
require 'converge/db/table/merger'
|
5
|
+
|
6
|
+
module Converge
|
7
|
+
module DB
|
8
|
+
class Table
|
9
|
+
class Staging
|
10
|
+
include Converge::DB::Table::Base
|
11
|
+
|
12
|
+
def name
|
13
|
+
"#{@model.name}_staging"
|
14
|
+
end
|
15
|
+
|
16
|
+
def merger(connection)
|
17
|
+
Merger.new(self, connection)
|
18
|
+
end
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
@@ -0,0 +1,21 @@
|
|
1
|
+
module Converge
|
2
|
+
class Model
|
3
|
+
def initialize(config, model)
|
4
|
+
@config = config
|
5
|
+
@model = model
|
6
|
+
end
|
7
|
+
|
8
|
+
def name
|
9
|
+
@model.table_name
|
10
|
+
end
|
11
|
+
|
12
|
+
def columns
|
13
|
+
@config.columns || @model.columns
|
14
|
+
end
|
15
|
+
|
16
|
+
|
17
|
+
def remove_quotes?
|
18
|
+
@config.remove_quotes?
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
@@ -0,0 +1,89 @@
|
|
1
|
+
require File.expand_path('../test_helper', __FILE__)
|
2
|
+
|
3
|
+
require File.expand_path('../db_test_helper', __FILE__)
|
4
|
+
|
5
|
+
require File.expand_path('../mock_adapter', __FILE__)
|
6
|
+
|
7
|
+
require 'converge'
|
8
|
+
|
9
|
+
require 'active_support'
|
10
|
+
require 'active_support/core_ext/integer/time'
|
11
|
+
require 'active_support/core_ext/date/calculations'
|
12
|
+
require 'active_support/core_ext/time/calculations'
|
13
|
+
|
14
|
+
class Converge::IntegrationTest < Test::Unit::TestCase
|
15
|
+
include Converge::DBTestHelper
|
16
|
+
|
17
|
+
setup do
|
18
|
+
@columns= %w{id mock_col_1 mock_col_2}
|
19
|
+
@target_name = "mock_models"
|
20
|
+
@staging_name = "mock_models_staging"
|
21
|
+
|
22
|
+
@mock_socket = mock
|
23
|
+
@mock_db = Converge::Mock::Connection.new(@mock_socket)
|
24
|
+
@connection = Converge::DB::Connection.new(@mock_db)
|
25
|
+
|
26
|
+
@mock_config = mock
|
27
|
+
|
28
|
+
@mock_config.expects(:columns).at_least_once.returns(@columns)
|
29
|
+
|
30
|
+
@mock_model = mock
|
31
|
+
@mock_model.expects(:table_name).at_least_once.returns(@target_name)
|
32
|
+
|
33
|
+
@model = Converge::Model.new(@mock_config, @mock_model)
|
34
|
+
|
35
|
+
@mock_table_object_collection = mock
|
36
|
+
|
37
|
+
@concrete_staging_table = mock
|
38
|
+
@concrete_target_table = mock
|
39
|
+
end
|
40
|
+
|
41
|
+
test 'standard table load' do
|
42
|
+
progress = states('progress').starts_as('start')
|
43
|
+
|
44
|
+
staging_table = Converge::DB::Table::Staging.new(@model)
|
45
|
+
conn_staging = @connection.table(staging_table)
|
46
|
+
|
47
|
+
target_table = Converge::DB::Table::Target.new(@model)
|
48
|
+
|
49
|
+
@mock_socket
|
50
|
+
.expects(:run_command)
|
51
|
+
.when(progress.is('start'))
|
52
|
+
.with(
|
53
|
+
:cmd => "copy",
|
54
|
+
:objects => @mock_table_object_collection
|
55
|
+
).then(progress.is('loaded'))
|
56
|
+
|
57
|
+
@mock_socket.expects(:run_command)
|
58
|
+
.when(progress.is('loaded'))
|
59
|
+
.with(
|
60
|
+
:cmd => "update",
|
61
|
+
:source => @staging_name,
|
62
|
+
:destination => @target_name,
|
63
|
+
:columns => @columns
|
64
|
+
).then(progress.is('updated'))
|
65
|
+
|
66
|
+
@mock_socket.expects(:run_command)
|
67
|
+
.when(progress.is('updated'))
|
68
|
+
.with(
|
69
|
+
:cmd => "insert",
|
70
|
+
:source => @staging_name,
|
71
|
+
:destination => @target_name,
|
72
|
+
:columns => @columns
|
73
|
+
).then(progress.is('inserted'))
|
74
|
+
|
75
|
+
@mock_socket.expects(:run_command)
|
76
|
+
.when(progress.is('inserted'))
|
77
|
+
.with(
|
78
|
+
:cmd => "finalize"
|
79
|
+
).then(progress.is('finalized'))
|
80
|
+
|
81
|
+
|
82
|
+
conn_staging.copy(@mock_table_object_collection)
|
83
|
+
|
84
|
+
merger = Converge::DB::Table::Merger.new(staging_table, @connection)
|
85
|
+
merger.merge(target_table)
|
86
|
+
|
87
|
+
assert progress.is('finalized')
|
88
|
+
end
|
89
|
+
end
|
@@ -0,0 +1,12 @@
|
|
1
|
+
module Converge
|
2
|
+
module DBTestHelper
|
3
|
+
def db_command_order_matches(lines, expected)
|
4
|
+
lines = lines.split(/;\n/).map(&:strip).reject(&:empty?)
|
5
|
+
expected.length == lines.length and expected.zip(lines).all? do |elt|
|
6
|
+
line = elt[0].split(/\n/).map(&:strip).join(" ")
|
7
|
+
expected = elt[1].split(/\n/).map(&:strip).join(" ")
|
8
|
+
line == expected
|
9
|
+
end
|
10
|
+
end
|
11
|
+
end
|
12
|
+
end
|
@@ -0,0 +1,20 @@
|
|
1
|
+
module Converge
|
2
|
+
module Mock
|
3
|
+
class Connection
|
4
|
+
def initialize(conn)
|
5
|
+
@conn = conn
|
6
|
+
end
|
7
|
+
|
8
|
+
def run_command(cmd)
|
9
|
+
@conn.run_command(cmd)
|
10
|
+
end
|
11
|
+
|
12
|
+
def table(name)
|
13
|
+
Table.new(self, name)
|
14
|
+
end
|
15
|
+
|
16
|
+
attr_reader :conn
|
17
|
+
end
|
18
|
+
end
|
19
|
+
end
|
20
|
+
|
@@ -0,0 +1,22 @@
|
|
1
|
+
module Converge
|
2
|
+
module Mock
|
3
|
+
class Table
|
4
|
+
def initialize(conn, dest)
|
5
|
+
@conn = conn
|
6
|
+
@dest = dest
|
7
|
+
end
|
8
|
+
def copy(objects)
|
9
|
+
@conn.run_command(cmd: "copy", objects: objects)
|
10
|
+
end
|
11
|
+
def update(table)
|
12
|
+
@conn.run_command(cmd: "update", source: table.name, destination: @dest.name, columns: table.columns)
|
13
|
+
end
|
14
|
+
def insert(table)
|
15
|
+
@conn.run_command(cmd: "insert", source: table.name, destination: @dest.name, columns: table.columns)
|
16
|
+
end
|
17
|
+
def finalize
|
18
|
+
@conn.run_command(cmd: "finalize")
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
@@ -0,0 +1,23 @@
|
|
1
|
+
require 'coveralls'
|
2
|
+
|
3
|
+
if ENV["COVERAGE"]
|
4
|
+
require 'coveralls'
|
5
|
+
require 'codeclimate-test-reporter'
|
6
|
+
require 'simplecov'
|
7
|
+
SimpleCov.start do
|
8
|
+
add_group "Lib", "lib"
|
9
|
+
add_filter "/test/"
|
10
|
+
command_name "Integration Tests"
|
11
|
+
formatter SimpleCov::Formatter::MultiFormatter[
|
12
|
+
SimpleCov::Formatter::HTMLFormatter,
|
13
|
+
Coveralls::SimpleCov::Formatter,
|
14
|
+
CodeClimate::TestReporter::Formatter
|
15
|
+
]
|
16
|
+
end
|
17
|
+
end
|
18
|
+
|
19
|
+
require 'test/unit'
|
20
|
+
|
21
|
+
ENV["QUASAR_ENV"] = "test"
|
22
|
+
|
23
|
+
require 'mocha/setup'
|
@@ -0,0 +1,24 @@
|
|
1
|
+
require File.expand_path('../../../../test_helper', __FILE__)
|
2
|
+
|
3
|
+
require 'converge/db/table/target'
|
4
|
+
|
5
|
+
class Converge::DB::Table::BaseTest < Test::Unit::TestCase
|
6
|
+
setup do
|
7
|
+
@mock_model = mock
|
8
|
+
|
9
|
+
@base = Class.new do
|
10
|
+
include Converge::DB::Table::Base
|
11
|
+
end
|
12
|
+
|
13
|
+
@table = @base.new(@mock_model)
|
14
|
+
end
|
15
|
+
|
16
|
+
test '#name' do
|
17
|
+
mock_columns = mock
|
18
|
+
@mock_model.expects(:columns).returns(mock_columns)
|
19
|
+
|
20
|
+
res = @table.columns
|
21
|
+
assert_equal res, mock_columns
|
22
|
+
end
|
23
|
+
end
|
24
|
+
|
@@ -0,0 +1,75 @@
|
|
1
|
+
require File.expand_path('../../../../test_helper', __FILE__)
|
2
|
+
|
3
|
+
require 'converge/db/table/loader'
|
4
|
+
|
5
|
+
class Converge::DB::Table::LoaderTest < Test::Unit::TestCase
|
6
|
+
setup do
|
7
|
+
@test_host = "redshift-test.sjc.carrel.org"
|
8
|
+
@test_table = "mock_table"
|
9
|
+
@test_staging_table = "#{@test_table}_staging"
|
10
|
+
|
11
|
+
@mock_connection = mock
|
12
|
+
@mock_table = mock
|
13
|
+
#@mock_table.expects(:name).at_least_once.returns(@test_table)
|
14
|
+
#@mock_table.expects(:columns).at_least_once.returns(%w{id mock_col_1 mock_col_2})
|
15
|
+
|
16
|
+
@loader = Converge::DB::Table::Loader.new(@mock_table, @mock_connection)
|
17
|
+
end
|
18
|
+
|
19
|
+
test '#load' do
|
20
|
+
mock_object_collection = mock
|
21
|
+
|
22
|
+
@mock_table.expects(:truncate)
|
23
|
+
@mock_table.expects(:copy).with(mock_object_collection)
|
24
|
+
|
25
|
+
@loader.load(mock_object_collection)
|
26
|
+
end
|
27
|
+
|
28
|
+
test '#finalize once' do
|
29
|
+
state = states('loader_state').starts_as('unclean')
|
30
|
+
|
31
|
+
mock_object_collection = mock
|
32
|
+
@mock_table.expects(:truncate).
|
33
|
+
when(state.is('unclean')).
|
34
|
+
then(state.is('clean'))
|
35
|
+
|
36
|
+
@mock_table.expects(:copy).
|
37
|
+
when(state.is('clean')).
|
38
|
+
with(mock_object_collection).
|
39
|
+
then(state.is('copied'))
|
40
|
+
|
41
|
+
@mock_table.expects(:truncate).
|
42
|
+
when(state.is('copied')).
|
43
|
+
then(state.is('done'))
|
44
|
+
|
45
|
+
@loader.load(mock_object_collection)
|
46
|
+
@loader.finalize
|
47
|
+
end
|
48
|
+
|
49
|
+
test '#finalize twice should only run once' do
|
50
|
+
state = states('loader_state').starts_as('unclean')
|
51
|
+
mock_object_collection = mock
|
52
|
+
|
53
|
+
@mock_table.expects(:truncate).
|
54
|
+
when(state.is('unclean')).
|
55
|
+
then(state.is('clean'))
|
56
|
+
|
57
|
+
@mock_table.expects(:copy).
|
58
|
+
when(state.is('clean')).
|
59
|
+
with(mock_object_collection).
|
60
|
+
then(state.is('copied'))
|
61
|
+
|
62
|
+
@mock_table.expects(:truncate).
|
63
|
+
when(state.is('copied')).
|
64
|
+
then(state.is('done'))
|
65
|
+
|
66
|
+
@loader.load(mock_object_collection)
|
67
|
+
@loader.finalize
|
68
|
+
@loader.finalize
|
69
|
+
end
|
70
|
+
|
71
|
+
test '#finalize should be allowed if the table has not be touched yet' do
|
72
|
+
@mock_table.expects(:truncate)
|
73
|
+
@loader.finalize
|
74
|
+
end
|
75
|
+
end
|
@@ -0,0 +1,28 @@
|
|
1
|
+
require File.expand_path('../../../../test_helper', __FILE__)
|
2
|
+
|
3
|
+
require 'converge/db/table/merger'
|
4
|
+
|
5
|
+
class Converge::DB::Table::MergerTest < Test::Unit::TestCase
|
6
|
+
setup do
|
7
|
+
@connection = mock
|
8
|
+
end
|
9
|
+
test '#merger' do
|
10
|
+
mock_date_range = mock
|
11
|
+
|
12
|
+
mock_target_table = mock
|
13
|
+
mock_staging_table = mock
|
14
|
+
|
15
|
+
mock_conn_staging_table = mock
|
16
|
+
mock_conn_target_table = mock
|
17
|
+
|
18
|
+
mock_conn_target_table.expects(:update).with(mock_conn_staging_table)
|
19
|
+
mock_conn_target_table.expects(:insert).with(mock_conn_staging_table)
|
20
|
+
mock_conn_target_table.expects(:finalize)
|
21
|
+
|
22
|
+
@connection.expects(:table).with(mock_staging_table).returns(mock_conn_staging_table)
|
23
|
+
@connection.expects(:table).with(mock_target_table).returns(mock_conn_target_table)
|
24
|
+
|
25
|
+
merger = Converge::DB::Table::Merger.new(mock_staging_table, @connection)
|
26
|
+
merger.merge(mock_target_table)
|
27
|
+
end
|
28
|
+
end
|
@@ -0,0 +1,10 @@
|
|
1
|
+
require File.expand_path('../../../../test_helper', __FILE__)
|
2
|
+
|
3
|
+
require 'converge/db/table/target'
|
4
|
+
|
5
|
+
class Converge::DB::Table::TargetTest < Test::Unit::TestCase
|
6
|
+
setup do
|
7
|
+
@mock_model = mock
|
8
|
+
@target = Converge::DB::Table::Target.new(@mock_model)
|
9
|
+
end
|
10
|
+
end
|
@@ -0,0 +1,21 @@
|
|
1
|
+
if ENV["COVERAGE"]
|
2
|
+
require 'coveralls'
|
3
|
+
require 'codeclimate-test-reporter'
|
4
|
+
require 'simplecov'
|
5
|
+
SimpleCov.start do
|
6
|
+
add_group "Lib", "lib"
|
7
|
+
add_filter "/test/"
|
8
|
+
command_name "Unit Tests"
|
9
|
+
formatter SimpleCov::Formatter::MultiFormatter[
|
10
|
+
SimpleCov::Formatter::HTMLFormatter,
|
11
|
+
Coveralls::SimpleCov::Formatter,
|
12
|
+
CodeClimate::TestReporter::Formatter
|
13
|
+
]
|
14
|
+
end
|
15
|
+
end
|
16
|
+
|
17
|
+
require 'test/unit'
|
18
|
+
|
19
|
+
ENV["QUASAR_ENV"] = "test"
|
20
|
+
|
21
|
+
require 'mocha/setup'
|
metadata
ADDED
@@ -0,0 +1,187 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: converge
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.0.1.pre.alpha.2
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Ed Carrel
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
date: 2014-10-14 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: hydrogen
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - "~>"
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: '0'
|
20
|
+
type: :runtime
|
21
|
+
prerelease: false
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - "~>"
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: '0'
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: activesupport
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - "~>"
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: '4'
|
34
|
+
type: :runtime
|
35
|
+
prerelease: false
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - "~>"
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: '4'
|
41
|
+
- !ruby/object:Gem::Dependency
|
42
|
+
name: test-unit
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
44
|
+
requirements:
|
45
|
+
- - "~>"
|
46
|
+
- !ruby/object:Gem::Version
|
47
|
+
version: '3'
|
48
|
+
type: :development
|
49
|
+
prerelease: false
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
51
|
+
requirements:
|
52
|
+
- - "~>"
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: '3'
|
55
|
+
- !ruby/object:Gem::Dependency
|
56
|
+
name: mocha
|
57
|
+
requirement: !ruby/object:Gem::Requirement
|
58
|
+
requirements:
|
59
|
+
- - "~>"
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
version: '1'
|
62
|
+
type: :development
|
63
|
+
prerelease: false
|
64
|
+
version_requirements: !ruby/object:Gem::Requirement
|
65
|
+
requirements:
|
66
|
+
- - "~>"
|
67
|
+
- !ruby/object:Gem::Version
|
68
|
+
version: '1'
|
69
|
+
- !ruby/object:Gem::Dependency
|
70
|
+
name: simplecov
|
71
|
+
requirement: !ruby/object:Gem::Requirement
|
72
|
+
requirements:
|
73
|
+
- - "~>"
|
74
|
+
- !ruby/object:Gem::Version
|
75
|
+
version: '0'
|
76
|
+
type: :development
|
77
|
+
prerelease: false
|
78
|
+
version_requirements: !ruby/object:Gem::Requirement
|
79
|
+
requirements:
|
80
|
+
- - "~>"
|
81
|
+
- !ruby/object:Gem::Version
|
82
|
+
version: '0'
|
83
|
+
- !ruby/object:Gem::Dependency
|
84
|
+
name: bundler
|
85
|
+
requirement: !ruby/object:Gem::Requirement
|
86
|
+
requirements:
|
87
|
+
- - "~>"
|
88
|
+
- !ruby/object:Gem::Version
|
89
|
+
version: '1.6'
|
90
|
+
type: :development
|
91
|
+
prerelease: false
|
92
|
+
version_requirements: !ruby/object:Gem::Requirement
|
93
|
+
requirements:
|
94
|
+
- - "~>"
|
95
|
+
- !ruby/object:Gem::Version
|
96
|
+
version: '1.6'
|
97
|
+
- !ruby/object:Gem::Dependency
|
98
|
+
name: rake
|
99
|
+
requirement: !ruby/object:Gem::Requirement
|
100
|
+
requirements:
|
101
|
+
- - "~>"
|
102
|
+
- !ruby/object:Gem::Version
|
103
|
+
version: '10.0'
|
104
|
+
type: :development
|
105
|
+
prerelease: false
|
106
|
+
version_requirements: !ruby/object:Gem::Requirement
|
107
|
+
requirements:
|
108
|
+
- - "~>"
|
109
|
+
- !ruby/object:Gem::Version
|
110
|
+
version: '10.0'
|
111
|
+
description: Data merger
|
112
|
+
email:
|
113
|
+
- edward@carrel.org
|
114
|
+
executables: []
|
115
|
+
extensions: []
|
116
|
+
extra_rdoc_files: []
|
117
|
+
files:
|
118
|
+
- ".gitignore"
|
119
|
+
- ".travis.yml"
|
120
|
+
- CONTRIBUTING.md
|
121
|
+
- Gemfile
|
122
|
+
- LICENSE
|
123
|
+
- README.md
|
124
|
+
- Rakefile
|
125
|
+
- converge.gemspec
|
126
|
+
- lib/converge.rb
|
127
|
+
- lib/converge/db.rb
|
128
|
+
- lib/converge/db/connection.rb
|
129
|
+
- lib/converge/db/connection/table.rb
|
130
|
+
- lib/converge/db/table.rb
|
131
|
+
- lib/converge/db/table/base.rb
|
132
|
+
- lib/converge/db/table/loader.rb
|
133
|
+
- lib/converge/db/table/merger.rb
|
134
|
+
- lib/converge/db/table/partitioner/bookend.rb
|
135
|
+
- lib/converge/db/table/staging.rb
|
136
|
+
- lib/converge/db/table/target.rb
|
137
|
+
- lib/converge/model.rb
|
138
|
+
- lib/converge/version.rb
|
139
|
+
- test/integration/converge_test.rb
|
140
|
+
- test/integration/db_test_helper.rb
|
141
|
+
- test/integration/mock/connection.rb
|
142
|
+
- test/integration/mock/table.rb
|
143
|
+
- test/integration/mock_adapter.rb
|
144
|
+
- test/integration/test_helper.rb
|
145
|
+
- test/unit/converge/db/table/base_test.rb
|
146
|
+
- test/unit/converge/db/table/loader_test.rb
|
147
|
+
- test/unit/converge/db/table/merger_test.rb
|
148
|
+
- test/unit/converge/db/table/staging_test.rb
|
149
|
+
- test/unit/converge/db/table/target_test.rb
|
150
|
+
- test/unit/test_helper.rb
|
151
|
+
homepage: https://github.com/azanar/converge
|
152
|
+
licenses:
|
153
|
+
- MIT
|
154
|
+
metadata: {}
|
155
|
+
post_install_message:
|
156
|
+
rdoc_options: []
|
157
|
+
require_paths:
|
158
|
+
- lib
|
159
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
160
|
+
requirements:
|
161
|
+
- - ">="
|
162
|
+
- !ruby/object:Gem::Version
|
163
|
+
version: '0'
|
164
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
165
|
+
requirements:
|
166
|
+
- - ">"
|
167
|
+
- !ruby/object:Gem::Version
|
168
|
+
version: 1.3.1
|
169
|
+
requirements: []
|
170
|
+
rubyforge_project:
|
171
|
+
rubygems_version: 2.2.2
|
172
|
+
signing_key:
|
173
|
+
specification_version: 4
|
174
|
+
summary: Data merger for Redshift
|
175
|
+
test_files:
|
176
|
+
- test/integration/converge_test.rb
|
177
|
+
- test/integration/db_test_helper.rb
|
178
|
+
- test/integration/mock/connection.rb
|
179
|
+
- test/integration/mock/table.rb
|
180
|
+
- test/integration/mock_adapter.rb
|
181
|
+
- test/integration/test_helper.rb
|
182
|
+
- test/unit/converge/db/table/base_test.rb
|
183
|
+
- test/unit/converge/db/table/loader_test.rb
|
184
|
+
- test/unit/converge/db/table/merger_test.rb
|
185
|
+
- test/unit/converge/db/table/staging_test.rb
|
186
|
+
- test/unit/converge/db/table/target_test.rb
|
187
|
+
- test/unit/test_helper.rb
|