converge 0.0.1.pre.alpha.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.gitignore +14 -0
- data/.travis.yml +17 -0
- data/CONTRIBUTING.md +33 -0
- data/Gemfile +6 -0
- data/LICENSE +22 -0
- data/README.md +40 -0
- data/Rakefile +32 -0
- data/converge.gemspec +29 -0
- data/lib/converge.rb +32 -0
- data/lib/converge/db.rb +6 -0
- data/lib/converge/db/connection.rb +16 -0
- data/lib/converge/db/connection/table.rb +59 -0
- data/lib/converge/db/table.rb +4 -0
- data/lib/converge/db/table/base.rb +24 -0
- data/lib/converge/db/table/loader.rb +26 -0
- data/lib/converge/db/table/merger.rb +19 -0
- data/lib/converge/db/table/partitioner/bookend.rb +23 -0
- data/lib/converge/db/table/staging.rb +22 -0
- data/lib/converge/db/table/target.rb +15 -0
- data/lib/converge/model.rb +21 -0
- data/lib/converge/version.rb +3 -0
- data/test/integration/converge_test.rb +89 -0
- data/test/integration/db_test_helper.rb +12 -0
- data/test/integration/mock/connection.rb +20 -0
- data/test/integration/mock/table.rb +22 -0
- data/test/integration/mock_adapter.rb +4 -0
- data/test/integration/test_helper.rb +23 -0
- data/test/unit/converge/db/table/base_test.rb +24 -0
- data/test/unit/converge/db/table/loader_test.rb +75 -0
- data/test/unit/converge/db/table/merger_test.rb +28 -0
- data/test/unit/converge/db/table/staging_test.rb +7 -0
- data/test/unit/converge/db/table/target_test.rb +10 -0
- data/test/unit/test_helper.rb +21 -0
- metadata +187 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: 4844beacb6a4e9fd06ab828198fb6be4eb63b4f6
|
4
|
+
data.tar.gz: 83806209ddeb54f71971a1260a7516afedcc7dc1
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: f9a22f9924a3bad3a90f3ed2a855b9273b76c39f3fab185111b8b3002441b89286d06d00b89ca35d8c31dec32b12f898a97e681ff6f8086fccf76e0f6f362935
|
7
|
+
data.tar.gz: 724cf8af3aa27b277713cf3e6985293a3aabf684f436db536dec5d0849fe965b089fe07570634157dfc9926cac92c647d54c2fe3d364deb6704b0121cdae5122
|
data/.gitignore
ADDED
data/.travis.yml
ADDED
data/CONTRIBUTING.md
ADDED
@@ -0,0 +1,33 @@
|
|
1
|
+
Contributing to Converge
|
2
|
+
========================
|
3
|
+
|
4
|
+
First of all, thanks for your interest in contributing to Converge!
|
5
|
+
|
6
|
+
Below are a list of guidelines to help keep this project sane in the face of open source entropy.
|
7
|
+
|
8
|
+
How To Submit A Patch
|
9
|
+
---------------------
|
10
|
+
|
11
|
+
Any patch you feel would make this a better project is welcome, including but not limited to: bug fixes, new features, refactoring, and better documentation.
|
12
|
+
|
13
|
+
Make a fork of this project, and hack to your heart's content. Once you have something you think would be great to include into master, feel free to open a pull request. Please keep in mind the following guidelines, which will make it more likely for your pull request to be accepted:
|
14
|
+
|
15
|
+
1. Please have test cases, both in the unit suite and the integration suite, that fail when your code is not present.
|
16
|
+
1. Please open a topic branch in your fork, and create your pull request against that.
|
17
|
+
* Use a separate topic for each feature and/or bug fix you want to submit.
|
18
|
+
1. Be logically consistent in your commits. Make them as small as possible, so they can tell a story the maintainer can understand.
|
19
|
+
1. Keeping a consistent style in the codebase is important for maintainability. When possible, please stick to the guidelines within [this guide](https://github.com/bbatsov/ruby-style-guide).
|
20
|
+
1. Pay attention to things like code climate and test coverage. A drop in these metrics will not necessarily preclude a patch from being accepted, but we'd much prefer to keep these numbers high.
|
21
|
+
|
22
|
+
Please keep in mind your contributions will be licensed the same as the rest of Converge, under the MIT license.
|
23
|
+
|
24
|
+
It may take the developer a day or two to get to your pull request. If your request has not been responded to in a few days, though, feel free to ping the developer team, preferably through a comment on the pull.
|
25
|
+
|
26
|
+
How To Report An Issue
|
27
|
+
----------------------
|
28
|
+
|
29
|
+
Issue reports are welcome, even if not accompanied by a pull request. Keep in mind that submitting a pull request will mean your issue likely gets resolved sooner, because less work will be left to do.
|
30
|
+
|
31
|
+
The most important thing an issue can do is get a developer as close as possible to seeing what you are seeing, in the case of a bug report. In the case of a feature request, you want to get them to the point where they don't see what you want to see, so they can make a decision about whether and how to include the proposed feature.
|
32
|
+
|
33
|
+
More about good bug reports [here](http://www.chiark.greenend.org.uk/~sgtatham/bugs.html).
|
data/Gemfile
ADDED
data/LICENSE
ADDED
@@ -0,0 +1,22 @@
|
|
1
|
+
Copyright (c) 2014 Ed Carrel
|
2
|
+
|
3
|
+
MIT License
|
4
|
+
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining
|
6
|
+
a copy of this software and associated documentation files (the
|
7
|
+
"Software"), to deal in the Software without restriction, including
|
8
|
+
without limitation the rights to use, copy, modify, merge, publish,
|
9
|
+
distribute, sublicense, and/or sell copies of the Software, and to
|
10
|
+
permit persons to whom the Software is furnished to do so, subject to
|
11
|
+
the following conditions:
|
12
|
+
|
13
|
+
The above copyright notice and this permission notice shall be
|
14
|
+
included in all copies or substantial portions of the Software.
|
15
|
+
|
16
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
17
|
+
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
18
|
+
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
19
|
+
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
20
|
+
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
21
|
+
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
22
|
+
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
data/README.md
ADDED
@@ -0,0 +1,40 @@
|
|
1
|
+
# Converge
|
2
|
+
|
3
|
+
Data merger for Redshift
|
4
|
+
|
5
|
+
## Installation
|
6
|
+
|
7
|
+
Add this line to your application's Gemfile:
|
8
|
+
|
9
|
+
```ruby
|
10
|
+
gem 'converge'
|
11
|
+
```
|
12
|
+
|
13
|
+
And then execute:
|
14
|
+
|
15
|
+
$ bundle
|
16
|
+
|
17
|
+
Or install it yourself as:
|
18
|
+
|
19
|
+
$ gem install converge
|
20
|
+
|
21
|
+
## Usage
|
22
|
+
|
23
|
+
TODO: Write usage instructions here
|
24
|
+
|
25
|
+
API Documentation
|
26
|
+
-------------
|
27
|
+
|
28
|
+
See [RubyDoc](http://rubydoc.info/github/azanar/converge/index)
|
29
|
+
|
30
|
+
Contributors
|
31
|
+
------------
|
32
|
+
|
33
|
+
See [Contributing](CONTRIBUTING.md) for details.
|
34
|
+
|
35
|
+
License
|
36
|
+
-------
|
37
|
+
|
38
|
+
©2014 Ed Carrel. Released under the MIT License.
|
39
|
+
|
40
|
+
See [License](LICENSE) for details.
|
data/Rakefile
ADDED
@@ -0,0 +1,32 @@
|
|
1
|
+
require "bundler/gem_tasks"
|
2
|
+
|
3
|
+
require 'coveralls/rake/task'
|
4
|
+
|
5
|
+
require 'rake/testtask'
|
6
|
+
|
7
|
+
task :test do
|
8
|
+
Rake::Task['test:unit'].invoke
|
9
|
+
Rake::Task['test:integration'].invoke
|
10
|
+
end
|
11
|
+
|
12
|
+
namespace :test do
|
13
|
+
Rake::TestTask.new("integration") do |t|
|
14
|
+
t.libs << "test"
|
15
|
+
t.libs << "config"
|
16
|
+
t.test_files = FileList['test/integration/**/*_test.rb']
|
17
|
+
t.verbose = true
|
18
|
+
end
|
19
|
+
|
20
|
+
Rake::TestTask.new("unit") do |t|
|
21
|
+
t.libs << "test"
|
22
|
+
t.libs << "config"
|
23
|
+
t.test_files = FileList['test/unit/**/*_test.rb']
|
24
|
+
t.verbose = true
|
25
|
+
end
|
26
|
+
end
|
27
|
+
|
28
|
+
Coveralls::RakeTask.new
|
29
|
+
|
30
|
+
task :test_with_coveralls => ['test:unit', 'test:integration', 'coveralls:push']
|
31
|
+
|
32
|
+
task :default => :test_with_coveralls
|
data/converge.gemspec
ADDED
@@ -0,0 +1,29 @@
|
|
1
|
+
# coding: utf-8
|
2
|
+
lib = File.expand_path('../lib', __FILE__)
|
3
|
+
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
4
|
+
require 'converge/version'
|
5
|
+
|
6
|
+
Gem::Specification.new do |spec|
|
7
|
+
spec.name = "converge"
|
8
|
+
spec.version = Converge::VERSION
|
9
|
+
spec.authors = ["Ed Carrel"]
|
10
|
+
spec.email = ["edward@carrel.org"]
|
11
|
+
spec.summary = %q{Data merger for Redshift}
|
12
|
+
spec.description = %q{Data merger}
|
13
|
+
spec.homepage = "https://github.com/azanar/converge"
|
14
|
+
spec.license = "MIT"
|
15
|
+
|
16
|
+
spec.files = `git ls-files -z`.split("\x0")
|
17
|
+
spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
|
18
|
+
spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
|
19
|
+
spec.require_paths = ["lib"]
|
20
|
+
|
21
|
+
spec.add_runtime_dependency 'hydrogen', '~> 0'
|
22
|
+
spec.add_runtime_dependency 'activesupport', '~> 4'
|
23
|
+
|
24
|
+
spec.add_development_dependency 'test-unit', '~> 3'
|
25
|
+
spec.add_development_dependency 'mocha', '~> 1'
|
26
|
+
spec.add_development_dependency 'simplecov', '~> 0'
|
27
|
+
spec.add_development_dependency "bundler", "~> 1.6"
|
28
|
+
spec.add_development_dependency "rake", "~> 10.0"
|
29
|
+
end
|
data/lib/converge.rb
ADDED
@@ -0,0 +1,32 @@
|
|
1
|
+
require 'logger'
|
2
|
+
|
3
|
+
require "hydrogen"
|
4
|
+
|
5
|
+
require "converge/db"
|
6
|
+
require "converge/model"
|
7
|
+
require "converge/version"
|
8
|
+
|
9
|
+
module Converge
|
10
|
+
module_function
|
11
|
+
def logger
|
12
|
+
return @logger if @logger
|
13
|
+
|
14
|
+
@logger = Logger.new(STDOUT)
|
15
|
+
@logger.formatter = proc { |severity, datetime, progname, msg|
|
16
|
+
"[#{datetime}, #{severity}] #{msg}\n"
|
17
|
+
}
|
18
|
+
@logger
|
19
|
+
end
|
20
|
+
|
21
|
+
def logger=(logger)
|
22
|
+
@logger = logger
|
23
|
+
end
|
24
|
+
|
25
|
+
def env
|
26
|
+
@_env ||= ActiveSupport::StringInquirer.new(ENV["QUASAR_ENV"] || ENV["RAILS_ENV"] || "development")
|
27
|
+
end
|
28
|
+
|
29
|
+
def env=(environment)
|
30
|
+
@_env = ActiveSupport::StringInquirer.new(environment)
|
31
|
+
end
|
32
|
+
end
|
data/lib/converge/db.rb
ADDED
@@ -0,0 +1,59 @@
|
|
1
|
+
module Converge
|
2
|
+
module DB
|
3
|
+
class Connection
|
4
|
+
class Table
|
5
|
+
def initialize(conn, table, opts = {})
|
6
|
+
@conn = conn
|
7
|
+
@table = table
|
8
|
+
@stale = false
|
9
|
+
end
|
10
|
+
|
11
|
+
def name
|
12
|
+
@table.name
|
13
|
+
end
|
14
|
+
|
15
|
+
def columns
|
16
|
+
@table.columns
|
17
|
+
end
|
18
|
+
|
19
|
+
def key
|
20
|
+
@table.key
|
21
|
+
end
|
22
|
+
|
23
|
+
def update(table)
|
24
|
+
conn_table.update(table)
|
25
|
+
@stale = true
|
26
|
+
end
|
27
|
+
|
28
|
+
def insert(table)
|
29
|
+
conn_table.insert(table)
|
30
|
+
@stale = true
|
31
|
+
end
|
32
|
+
|
33
|
+
def copy(object_collection)
|
34
|
+
conn_table.copy(object_collection)
|
35
|
+
@stale = true
|
36
|
+
end
|
37
|
+
|
38
|
+
def truncate
|
39
|
+
conn_table.truncate
|
40
|
+
end
|
41
|
+
|
42
|
+
def finalize
|
43
|
+
if @stale
|
44
|
+
conn_table.finalize
|
45
|
+
@stale = false
|
46
|
+
else
|
47
|
+
Converge.logger.warn "Called finalize on a finalized table"
|
48
|
+
end
|
49
|
+
end
|
50
|
+
|
51
|
+
private
|
52
|
+
|
53
|
+
def conn_table
|
54
|
+
@conn.table(self)
|
55
|
+
end
|
56
|
+
end
|
57
|
+
end
|
58
|
+
end
|
59
|
+
end
|
@@ -0,0 +1,24 @@
|
|
1
|
+
module Converge
|
2
|
+
module DB
|
3
|
+
class Table
|
4
|
+
module Base
|
5
|
+
def initialize(model)
|
6
|
+
@model = model
|
7
|
+
@stale = false
|
8
|
+
end
|
9
|
+
|
10
|
+
def columns
|
11
|
+
@model.columns
|
12
|
+
end
|
13
|
+
|
14
|
+
def key
|
15
|
+
@model.key
|
16
|
+
end
|
17
|
+
|
18
|
+
def loader(connection)
|
19
|
+
Loader.new(self, connection)
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
@@ -0,0 +1,26 @@
|
|
1
|
+
module Converge
|
2
|
+
module DB
|
3
|
+
class Table
|
4
|
+
class Loader
|
5
|
+
def initialize(table, conn)
|
6
|
+
@table = table
|
7
|
+
@conn = conn
|
8
|
+
@truncated = false #assume we aren't truncated.
|
9
|
+
end
|
10
|
+
|
11
|
+
def load(object_collection)
|
12
|
+
@table.truncate
|
13
|
+
@table.copy(object_collection)
|
14
|
+
@truncated = false
|
15
|
+
end
|
16
|
+
|
17
|
+
def finalize
|
18
|
+
unless @truncated
|
19
|
+
@table.truncate
|
20
|
+
@truncated = true
|
21
|
+
end
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
@@ -0,0 +1,19 @@
|
|
1
|
+
module Converge
|
2
|
+
module DB
|
3
|
+
class Table
|
4
|
+
class Merger
|
5
|
+
def initialize(source, conn)
|
6
|
+
@conn = conn
|
7
|
+
@conn_source = @conn.table(source)
|
8
|
+
end
|
9
|
+
|
10
|
+
def merge(destination)
|
11
|
+
conn_destination = @conn.table(destination)
|
12
|
+
conn_destination.update(@conn_source)
|
13
|
+
conn_destination.insert(@conn_source)
|
14
|
+
conn_destination.finalize
|
15
|
+
end
|
16
|
+
end
|
17
|
+
end
|
18
|
+
end
|
19
|
+
end
|
@@ -0,0 +1,23 @@
|
|
1
|
+
require 'converge/db/table/partitioner'
|
2
|
+
module Converge
|
3
|
+
module DB
|
4
|
+
class Table
|
5
|
+
module Partitioner
|
6
|
+
class Bookend
|
7
|
+
include Partitioner
|
8
|
+
def initialize(source)
|
9
|
+
@source = source
|
10
|
+
end
|
11
|
+
|
12
|
+
def matches?
|
13
|
+
true
|
14
|
+
end
|
15
|
+
|
16
|
+
def partition_for(dest)
|
17
|
+
@source
|
18
|
+
end
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
23
|
+
end
|
@@ -0,0 +1,22 @@
|
|
1
|
+
require 'converge/db/table/base'
|
2
|
+
|
3
|
+
require 'converge/db/table/loader'
|
4
|
+
require 'converge/db/table/merger'
|
5
|
+
|
6
|
+
module Converge
|
7
|
+
module DB
|
8
|
+
class Table
|
9
|
+
class Staging
|
10
|
+
include Converge::DB::Table::Base
|
11
|
+
|
12
|
+
def name
|
13
|
+
"#{@model.name}_staging"
|
14
|
+
end
|
15
|
+
|
16
|
+
def merger(connection)
|
17
|
+
Merger.new(self, connection)
|
18
|
+
end
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
@@ -0,0 +1,21 @@
|
|
1
|
+
module Converge
|
2
|
+
class Model
|
3
|
+
def initialize(config, model)
|
4
|
+
@config = config
|
5
|
+
@model = model
|
6
|
+
end
|
7
|
+
|
8
|
+
def name
|
9
|
+
@model.table_name
|
10
|
+
end
|
11
|
+
|
12
|
+
def columns
|
13
|
+
@config.columns || @model.columns
|
14
|
+
end
|
15
|
+
|
16
|
+
|
17
|
+
def remove_quotes?
|
18
|
+
@config.remove_quotes?
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
@@ -0,0 +1,89 @@
|
|
1
|
+
require File.expand_path('../test_helper', __FILE__)
|
2
|
+
|
3
|
+
require File.expand_path('../db_test_helper', __FILE__)
|
4
|
+
|
5
|
+
require File.expand_path('../mock_adapter', __FILE__)
|
6
|
+
|
7
|
+
require 'converge'
|
8
|
+
|
9
|
+
require 'active_support'
|
10
|
+
require 'active_support/core_ext/integer/time'
|
11
|
+
require 'active_support/core_ext/date/calculations'
|
12
|
+
require 'active_support/core_ext/time/calculations'
|
13
|
+
|
14
|
+
class Converge::IntegrationTest < Test::Unit::TestCase
|
15
|
+
include Converge::DBTestHelper
|
16
|
+
|
17
|
+
setup do
|
18
|
+
@columns= %w{id mock_col_1 mock_col_2}
|
19
|
+
@target_name = "mock_models"
|
20
|
+
@staging_name = "mock_models_staging"
|
21
|
+
|
22
|
+
@mock_socket = mock
|
23
|
+
@mock_db = Converge::Mock::Connection.new(@mock_socket)
|
24
|
+
@connection = Converge::DB::Connection.new(@mock_db)
|
25
|
+
|
26
|
+
@mock_config = mock
|
27
|
+
|
28
|
+
@mock_config.expects(:columns).at_least_once.returns(@columns)
|
29
|
+
|
30
|
+
@mock_model = mock
|
31
|
+
@mock_model.expects(:table_name).at_least_once.returns(@target_name)
|
32
|
+
|
33
|
+
@model = Converge::Model.new(@mock_config, @mock_model)
|
34
|
+
|
35
|
+
@mock_table_object_collection = mock
|
36
|
+
|
37
|
+
@concrete_staging_table = mock
|
38
|
+
@concrete_target_table = mock
|
39
|
+
end
|
40
|
+
|
41
|
+
test 'standard table load' do
|
42
|
+
progress = states('progress').starts_as('start')
|
43
|
+
|
44
|
+
staging_table = Converge::DB::Table::Staging.new(@model)
|
45
|
+
conn_staging = @connection.table(staging_table)
|
46
|
+
|
47
|
+
target_table = Converge::DB::Table::Target.new(@model)
|
48
|
+
|
49
|
+
@mock_socket
|
50
|
+
.expects(:run_command)
|
51
|
+
.when(progress.is('start'))
|
52
|
+
.with(
|
53
|
+
:cmd => "copy",
|
54
|
+
:objects => @mock_table_object_collection
|
55
|
+
).then(progress.is('loaded'))
|
56
|
+
|
57
|
+
@mock_socket.expects(:run_command)
|
58
|
+
.when(progress.is('loaded'))
|
59
|
+
.with(
|
60
|
+
:cmd => "update",
|
61
|
+
:source => @staging_name,
|
62
|
+
:destination => @target_name,
|
63
|
+
:columns => @columns
|
64
|
+
).then(progress.is('updated'))
|
65
|
+
|
66
|
+
@mock_socket.expects(:run_command)
|
67
|
+
.when(progress.is('updated'))
|
68
|
+
.with(
|
69
|
+
:cmd => "insert",
|
70
|
+
:source => @staging_name,
|
71
|
+
:destination => @target_name,
|
72
|
+
:columns => @columns
|
73
|
+
).then(progress.is('inserted'))
|
74
|
+
|
75
|
+
@mock_socket.expects(:run_command)
|
76
|
+
.when(progress.is('inserted'))
|
77
|
+
.with(
|
78
|
+
:cmd => "finalize"
|
79
|
+
).then(progress.is('finalized'))
|
80
|
+
|
81
|
+
|
82
|
+
conn_staging.copy(@mock_table_object_collection)
|
83
|
+
|
84
|
+
merger = Converge::DB::Table::Merger.new(staging_table, @connection)
|
85
|
+
merger.merge(target_table)
|
86
|
+
|
87
|
+
assert progress.is('finalized')
|
88
|
+
end
|
89
|
+
end
|
@@ -0,0 +1,12 @@
|
|
1
|
+
module Converge
|
2
|
+
module DBTestHelper
|
3
|
+
def db_command_order_matches(lines, expected)
|
4
|
+
lines = lines.split(/;\n/).map(&:strip).reject(&:empty?)
|
5
|
+
expected.length == lines.length and expected.zip(lines).all? do |elt|
|
6
|
+
line = elt[0].split(/\n/).map(&:strip).join(" ")
|
7
|
+
expected = elt[1].split(/\n/).map(&:strip).join(" ")
|
8
|
+
line == expected
|
9
|
+
end
|
10
|
+
end
|
11
|
+
end
|
12
|
+
end
|
@@ -0,0 +1,20 @@
|
|
1
|
+
module Converge
|
2
|
+
module Mock
|
3
|
+
class Connection
|
4
|
+
def initialize(conn)
|
5
|
+
@conn = conn
|
6
|
+
end
|
7
|
+
|
8
|
+
def run_command(cmd)
|
9
|
+
@conn.run_command(cmd)
|
10
|
+
end
|
11
|
+
|
12
|
+
def table(name)
|
13
|
+
Table.new(self, name)
|
14
|
+
end
|
15
|
+
|
16
|
+
attr_reader :conn
|
17
|
+
end
|
18
|
+
end
|
19
|
+
end
|
20
|
+
|
@@ -0,0 +1,22 @@
|
|
1
|
+
module Converge
|
2
|
+
module Mock
|
3
|
+
class Table
|
4
|
+
def initialize(conn, dest)
|
5
|
+
@conn = conn
|
6
|
+
@dest = dest
|
7
|
+
end
|
8
|
+
def copy(objects)
|
9
|
+
@conn.run_command(cmd: "copy", objects: objects)
|
10
|
+
end
|
11
|
+
def update(table)
|
12
|
+
@conn.run_command(cmd: "update", source: table.name, destination: @dest.name, columns: table.columns)
|
13
|
+
end
|
14
|
+
def insert(table)
|
15
|
+
@conn.run_command(cmd: "insert", source: table.name, destination: @dest.name, columns: table.columns)
|
16
|
+
end
|
17
|
+
def finalize
|
18
|
+
@conn.run_command(cmd: "finalize")
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
@@ -0,0 +1,23 @@
|
|
1
|
+
require 'coveralls'
|
2
|
+
|
3
|
+
if ENV["COVERAGE"]
|
4
|
+
require 'coveralls'
|
5
|
+
require 'codeclimate-test-reporter'
|
6
|
+
require 'simplecov'
|
7
|
+
SimpleCov.start do
|
8
|
+
add_group "Lib", "lib"
|
9
|
+
add_filter "/test/"
|
10
|
+
command_name "Integration Tests"
|
11
|
+
formatter SimpleCov::Formatter::MultiFormatter[
|
12
|
+
SimpleCov::Formatter::HTMLFormatter,
|
13
|
+
Coveralls::SimpleCov::Formatter,
|
14
|
+
CodeClimate::TestReporter::Formatter
|
15
|
+
]
|
16
|
+
end
|
17
|
+
end
|
18
|
+
|
19
|
+
require 'test/unit'
|
20
|
+
|
21
|
+
ENV["QUASAR_ENV"] = "test"
|
22
|
+
|
23
|
+
require 'mocha/setup'
|
@@ -0,0 +1,24 @@
|
|
1
|
+
require File.expand_path('../../../../test_helper', __FILE__)
|
2
|
+
|
3
|
+
require 'converge/db/table/target'
|
4
|
+
|
5
|
+
class Converge::DB::Table::BaseTest < Test::Unit::TestCase
|
6
|
+
setup do
|
7
|
+
@mock_model = mock
|
8
|
+
|
9
|
+
@base = Class.new do
|
10
|
+
include Converge::DB::Table::Base
|
11
|
+
end
|
12
|
+
|
13
|
+
@table = @base.new(@mock_model)
|
14
|
+
end
|
15
|
+
|
16
|
+
test '#name' do
|
17
|
+
mock_columns = mock
|
18
|
+
@mock_model.expects(:columns).returns(mock_columns)
|
19
|
+
|
20
|
+
res = @table.columns
|
21
|
+
assert_equal res, mock_columns
|
22
|
+
end
|
23
|
+
end
|
24
|
+
|
@@ -0,0 +1,75 @@
|
|
1
|
+
require File.expand_path('../../../../test_helper', __FILE__)
|
2
|
+
|
3
|
+
require 'converge/db/table/loader'
|
4
|
+
|
5
|
+
class Converge::DB::Table::LoaderTest < Test::Unit::TestCase
|
6
|
+
setup do
|
7
|
+
@test_host = "redshift-test.sjc.carrel.org"
|
8
|
+
@test_table = "mock_table"
|
9
|
+
@test_staging_table = "#{@test_table}_staging"
|
10
|
+
|
11
|
+
@mock_connection = mock
|
12
|
+
@mock_table = mock
|
13
|
+
#@mock_table.expects(:name).at_least_once.returns(@test_table)
|
14
|
+
#@mock_table.expects(:columns).at_least_once.returns(%w{id mock_col_1 mock_col_2})
|
15
|
+
|
16
|
+
@loader = Converge::DB::Table::Loader.new(@mock_table, @mock_connection)
|
17
|
+
end
|
18
|
+
|
19
|
+
test '#load' do
|
20
|
+
mock_object_collection = mock
|
21
|
+
|
22
|
+
@mock_table.expects(:truncate)
|
23
|
+
@mock_table.expects(:copy).with(mock_object_collection)
|
24
|
+
|
25
|
+
@loader.load(mock_object_collection)
|
26
|
+
end
|
27
|
+
|
28
|
+
test '#finalize once' do
|
29
|
+
state = states('loader_state').starts_as('unclean')
|
30
|
+
|
31
|
+
mock_object_collection = mock
|
32
|
+
@mock_table.expects(:truncate).
|
33
|
+
when(state.is('unclean')).
|
34
|
+
then(state.is('clean'))
|
35
|
+
|
36
|
+
@mock_table.expects(:copy).
|
37
|
+
when(state.is('clean')).
|
38
|
+
with(mock_object_collection).
|
39
|
+
then(state.is('copied'))
|
40
|
+
|
41
|
+
@mock_table.expects(:truncate).
|
42
|
+
when(state.is('copied')).
|
43
|
+
then(state.is('done'))
|
44
|
+
|
45
|
+
@loader.load(mock_object_collection)
|
46
|
+
@loader.finalize
|
47
|
+
end
|
48
|
+
|
49
|
+
test '#finalize twice should only run once' do
|
50
|
+
state = states('loader_state').starts_as('unclean')
|
51
|
+
mock_object_collection = mock
|
52
|
+
|
53
|
+
@mock_table.expects(:truncate).
|
54
|
+
when(state.is('unclean')).
|
55
|
+
then(state.is('clean'))
|
56
|
+
|
57
|
+
@mock_table.expects(:copy).
|
58
|
+
when(state.is('clean')).
|
59
|
+
with(mock_object_collection).
|
60
|
+
then(state.is('copied'))
|
61
|
+
|
62
|
+
@mock_table.expects(:truncate).
|
63
|
+
when(state.is('copied')).
|
64
|
+
then(state.is('done'))
|
65
|
+
|
66
|
+
@loader.load(mock_object_collection)
|
67
|
+
@loader.finalize
|
68
|
+
@loader.finalize
|
69
|
+
end
|
70
|
+
|
71
|
+
test '#finalize should be allowed if the table has not be touched yet' do
|
72
|
+
@mock_table.expects(:truncate)
|
73
|
+
@loader.finalize
|
74
|
+
end
|
75
|
+
end
|
@@ -0,0 +1,28 @@
|
|
1
|
+
require File.expand_path('../../../../test_helper', __FILE__)
|
2
|
+
|
3
|
+
require 'converge/db/table/merger'
|
4
|
+
|
5
|
+
class Converge::DB::Table::MergerTest < Test::Unit::TestCase
|
6
|
+
setup do
|
7
|
+
@connection = mock
|
8
|
+
end
|
9
|
+
test '#merger' do
|
10
|
+
mock_date_range = mock
|
11
|
+
|
12
|
+
mock_target_table = mock
|
13
|
+
mock_staging_table = mock
|
14
|
+
|
15
|
+
mock_conn_staging_table = mock
|
16
|
+
mock_conn_target_table = mock
|
17
|
+
|
18
|
+
mock_conn_target_table.expects(:update).with(mock_conn_staging_table)
|
19
|
+
mock_conn_target_table.expects(:insert).with(mock_conn_staging_table)
|
20
|
+
mock_conn_target_table.expects(:finalize)
|
21
|
+
|
22
|
+
@connection.expects(:table).with(mock_staging_table).returns(mock_conn_staging_table)
|
23
|
+
@connection.expects(:table).with(mock_target_table).returns(mock_conn_target_table)
|
24
|
+
|
25
|
+
merger = Converge::DB::Table::Merger.new(mock_staging_table, @connection)
|
26
|
+
merger.merge(mock_target_table)
|
27
|
+
end
|
28
|
+
end
|
@@ -0,0 +1,10 @@
|
|
1
|
+
require File.expand_path('../../../../test_helper', __FILE__)
|
2
|
+
|
3
|
+
require 'converge/db/table/target'
|
4
|
+
|
5
|
+
class Converge::DB::Table::TargetTest < Test::Unit::TestCase
|
6
|
+
setup do
|
7
|
+
@mock_model = mock
|
8
|
+
@target = Converge::DB::Table::Target.new(@mock_model)
|
9
|
+
end
|
10
|
+
end
|
@@ -0,0 +1,21 @@
|
|
1
|
+
if ENV["COVERAGE"]
|
2
|
+
require 'coveralls'
|
3
|
+
require 'codeclimate-test-reporter'
|
4
|
+
require 'simplecov'
|
5
|
+
SimpleCov.start do
|
6
|
+
add_group "Lib", "lib"
|
7
|
+
add_filter "/test/"
|
8
|
+
command_name "Unit Tests"
|
9
|
+
formatter SimpleCov::Formatter::MultiFormatter[
|
10
|
+
SimpleCov::Formatter::HTMLFormatter,
|
11
|
+
Coveralls::SimpleCov::Formatter,
|
12
|
+
CodeClimate::TestReporter::Formatter
|
13
|
+
]
|
14
|
+
end
|
15
|
+
end
|
16
|
+
|
17
|
+
require 'test/unit'
|
18
|
+
|
19
|
+
ENV["QUASAR_ENV"] = "test"
|
20
|
+
|
21
|
+
require 'mocha/setup'
|
metadata
ADDED
@@ -0,0 +1,187 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: converge
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.0.1.pre.alpha.2
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Ed Carrel
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
date: 2014-10-14 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: hydrogen
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - "~>"
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: '0'
|
20
|
+
type: :runtime
|
21
|
+
prerelease: false
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - "~>"
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: '0'
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: activesupport
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - "~>"
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: '4'
|
34
|
+
type: :runtime
|
35
|
+
prerelease: false
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - "~>"
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: '4'
|
41
|
+
- !ruby/object:Gem::Dependency
|
42
|
+
name: test-unit
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
44
|
+
requirements:
|
45
|
+
- - "~>"
|
46
|
+
- !ruby/object:Gem::Version
|
47
|
+
version: '3'
|
48
|
+
type: :development
|
49
|
+
prerelease: false
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
51
|
+
requirements:
|
52
|
+
- - "~>"
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: '3'
|
55
|
+
- !ruby/object:Gem::Dependency
|
56
|
+
name: mocha
|
57
|
+
requirement: !ruby/object:Gem::Requirement
|
58
|
+
requirements:
|
59
|
+
- - "~>"
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
version: '1'
|
62
|
+
type: :development
|
63
|
+
prerelease: false
|
64
|
+
version_requirements: !ruby/object:Gem::Requirement
|
65
|
+
requirements:
|
66
|
+
- - "~>"
|
67
|
+
- !ruby/object:Gem::Version
|
68
|
+
version: '1'
|
69
|
+
- !ruby/object:Gem::Dependency
|
70
|
+
name: simplecov
|
71
|
+
requirement: !ruby/object:Gem::Requirement
|
72
|
+
requirements:
|
73
|
+
- - "~>"
|
74
|
+
- !ruby/object:Gem::Version
|
75
|
+
version: '0'
|
76
|
+
type: :development
|
77
|
+
prerelease: false
|
78
|
+
version_requirements: !ruby/object:Gem::Requirement
|
79
|
+
requirements:
|
80
|
+
- - "~>"
|
81
|
+
- !ruby/object:Gem::Version
|
82
|
+
version: '0'
|
83
|
+
- !ruby/object:Gem::Dependency
|
84
|
+
name: bundler
|
85
|
+
requirement: !ruby/object:Gem::Requirement
|
86
|
+
requirements:
|
87
|
+
- - "~>"
|
88
|
+
- !ruby/object:Gem::Version
|
89
|
+
version: '1.6'
|
90
|
+
type: :development
|
91
|
+
prerelease: false
|
92
|
+
version_requirements: !ruby/object:Gem::Requirement
|
93
|
+
requirements:
|
94
|
+
- - "~>"
|
95
|
+
- !ruby/object:Gem::Version
|
96
|
+
version: '1.6'
|
97
|
+
- !ruby/object:Gem::Dependency
|
98
|
+
name: rake
|
99
|
+
requirement: !ruby/object:Gem::Requirement
|
100
|
+
requirements:
|
101
|
+
- - "~>"
|
102
|
+
- !ruby/object:Gem::Version
|
103
|
+
version: '10.0'
|
104
|
+
type: :development
|
105
|
+
prerelease: false
|
106
|
+
version_requirements: !ruby/object:Gem::Requirement
|
107
|
+
requirements:
|
108
|
+
- - "~>"
|
109
|
+
- !ruby/object:Gem::Version
|
110
|
+
version: '10.0'
|
111
|
+
description: Data merger
|
112
|
+
email:
|
113
|
+
- edward@carrel.org
|
114
|
+
executables: []
|
115
|
+
extensions: []
|
116
|
+
extra_rdoc_files: []
|
117
|
+
files:
|
118
|
+
- ".gitignore"
|
119
|
+
- ".travis.yml"
|
120
|
+
- CONTRIBUTING.md
|
121
|
+
- Gemfile
|
122
|
+
- LICENSE
|
123
|
+
- README.md
|
124
|
+
- Rakefile
|
125
|
+
- converge.gemspec
|
126
|
+
- lib/converge.rb
|
127
|
+
- lib/converge/db.rb
|
128
|
+
- lib/converge/db/connection.rb
|
129
|
+
- lib/converge/db/connection/table.rb
|
130
|
+
- lib/converge/db/table.rb
|
131
|
+
- lib/converge/db/table/base.rb
|
132
|
+
- lib/converge/db/table/loader.rb
|
133
|
+
- lib/converge/db/table/merger.rb
|
134
|
+
- lib/converge/db/table/partitioner/bookend.rb
|
135
|
+
- lib/converge/db/table/staging.rb
|
136
|
+
- lib/converge/db/table/target.rb
|
137
|
+
- lib/converge/model.rb
|
138
|
+
- lib/converge/version.rb
|
139
|
+
- test/integration/converge_test.rb
|
140
|
+
- test/integration/db_test_helper.rb
|
141
|
+
- test/integration/mock/connection.rb
|
142
|
+
- test/integration/mock/table.rb
|
143
|
+
- test/integration/mock_adapter.rb
|
144
|
+
- test/integration/test_helper.rb
|
145
|
+
- test/unit/converge/db/table/base_test.rb
|
146
|
+
- test/unit/converge/db/table/loader_test.rb
|
147
|
+
- test/unit/converge/db/table/merger_test.rb
|
148
|
+
- test/unit/converge/db/table/staging_test.rb
|
149
|
+
- test/unit/converge/db/table/target_test.rb
|
150
|
+
- test/unit/test_helper.rb
|
151
|
+
homepage: https://github.com/azanar/converge
|
152
|
+
licenses:
|
153
|
+
- MIT
|
154
|
+
metadata: {}
|
155
|
+
post_install_message:
|
156
|
+
rdoc_options: []
|
157
|
+
require_paths:
|
158
|
+
- lib
|
159
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
160
|
+
requirements:
|
161
|
+
- - ">="
|
162
|
+
- !ruby/object:Gem::Version
|
163
|
+
version: '0'
|
164
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
165
|
+
requirements:
|
166
|
+
- - ">"
|
167
|
+
- !ruby/object:Gem::Version
|
168
|
+
version: 1.3.1
|
169
|
+
requirements: []
|
170
|
+
rubyforge_project:
|
171
|
+
rubygems_version: 2.2.2
|
172
|
+
signing_key:
|
173
|
+
specification_version: 4
|
174
|
+
summary: Data merger for Redshift
|
175
|
+
test_files:
|
176
|
+
- test/integration/converge_test.rb
|
177
|
+
- test/integration/db_test_helper.rb
|
178
|
+
- test/integration/mock/connection.rb
|
179
|
+
- test/integration/mock/table.rb
|
180
|
+
- test/integration/mock_adapter.rb
|
181
|
+
- test/integration/test_helper.rb
|
182
|
+
- test/unit/converge/db/table/base_test.rb
|
183
|
+
- test/unit/converge/db/table/loader_test.rb
|
184
|
+
- test/unit/converge/db/table/merger_test.rb
|
185
|
+
- test/unit/converge/db/table/staging_test.rb
|
186
|
+
- test/unit/converge/db/table/target_test.rb
|
187
|
+
- test/unit/test_helper.rb
|