ethel 0.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/.gitignore +18 -0
- data/Gemfile +11 -0
- data/Guardfile +8 -0
- data/LICENSE.txt +22 -0
- data/README.md +29 -0
- data/Rakefile +9 -0
- data/ethel.gemspec +19 -0
- data/lib/ethel.rb +11 -0
- data/lib/ethel/field.rb +10 -0
- data/lib/ethel/migration.rb +30 -0
- data/lib/ethel/operation.rb +29 -0
- data/lib/ethel/operations/add_field.rb +15 -0
- data/lib/ethel/operations/cast.rb +28 -0
- data/lib/ethel/operations/copy.rb +10 -0
- data/lib/ethel/source.rb +30 -0
- data/lib/ethel/sources/csv.rb +29 -0
- data/lib/ethel/target.rb +26 -0
- data/lib/ethel/targets/csv.rb +46 -0
- data/lib/ethel/version.rb +3 -0
- data/test/helper.rb +29 -0
- data/test/integration/test_cast_migration.rb +13 -0
- data/test/integration/test_copy_migration.rb +13 -0
- data/test/unit/operations/test_add_field.rb +27 -0
- data/test/unit/operations/test_cast.rb +52 -0
- data/test/unit/operations/test_copy.rb +35 -0
- data/test/unit/sources/test_csv.rb +49 -0
- data/test/unit/targets/test_csv.rb +39 -0
- data/test/unit/test_field.rb +13 -0
- data/test/unit/test_migration.rb +47 -0
- data/test/unit/test_operation.rb +35 -0
- data/test/unit/test_source.rb +60 -0
- data/test/unit/test_target.rb +37 -0
- metadata +90 -0
data/.gitignore
ADDED
data/Gemfile
ADDED
data/Guardfile
ADDED
data/LICENSE.txt
ADDED
@@ -0,0 +1,22 @@
|
|
1
|
+
Copyright (c) 2013 Jeremy Stephens
|
2
|
+
|
3
|
+
MIT License
|
4
|
+
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining
|
6
|
+
a copy of this software and associated documentation files (the
|
7
|
+
"Software"), to deal in the Software without restriction, including
|
8
|
+
without limitation the rights to use, copy, modify, merge, publish,
|
9
|
+
distribute, sublicense, and/or sell copies of the Software, and to
|
10
|
+
permit persons to whom the Software is furnished to do so, subject to
|
11
|
+
the following conditions:
|
12
|
+
|
13
|
+
The above copyright notice and this permission notice shall be
|
14
|
+
included in all copies or substantial portions of the Software.
|
15
|
+
|
16
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
17
|
+
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
18
|
+
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
19
|
+
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
20
|
+
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
21
|
+
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
22
|
+
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
data/README.md
ADDED
@@ -0,0 +1,29 @@
|
|
1
|
+
# Ethel
|
2
|
+
|
3
|
+
TODO: Write a gem description
|
4
|
+
|
5
|
+
## Installation
|
6
|
+
|
7
|
+
Add this line to your application's Gemfile:
|
8
|
+
|
9
|
+
gem 'ethel'
|
10
|
+
|
11
|
+
And then execute:
|
12
|
+
|
13
|
+
$ bundle
|
14
|
+
|
15
|
+
Or install it yourself as:
|
16
|
+
|
17
|
+
$ gem install ethel
|
18
|
+
|
19
|
+
## Usage
|
20
|
+
|
21
|
+
TODO: Write usage instructions here
|
22
|
+
|
23
|
+
## Contributing
|
24
|
+
|
25
|
+
1. Fork it
|
26
|
+
2. Create your feature branch (`git checkout -b my-new-feature`)
|
27
|
+
3. Commit your changes (`git commit -am 'Add some feature'`)
|
28
|
+
4. Push to the branch (`git push origin my-new-feature`)
|
29
|
+
5. Create new Pull Request
|
data/Rakefile
ADDED
data/ethel.gemspec
ADDED
@@ -0,0 +1,19 @@
|
|
1
|
+
# -*- encoding: utf-8 -*-
|
2
|
+
lib = File.expand_path('../lib', __FILE__)
|
3
|
+
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
4
|
+
require 'ethel/version'
|
5
|
+
|
6
|
+
Gem::Specification.new do |gem|
|
7
|
+
gem.name = "ethel"
|
8
|
+
gem.version = Ethel::VERSION
|
9
|
+
gem.authors = ["Jeremy Stephens"]
|
10
|
+
gem.email = ["jeremy.f.stephens@vanderbilt.edu"]
|
11
|
+
gem.description = %q{Ethel is an ORM-agnostic library of ETL (extract-transform-load) utilities}
|
12
|
+
gem.summary = %q{ORM-agnostic ETL (extract-transform-load) utilities}
|
13
|
+
gem.homepage = "https://github.com/coupler/ethel"
|
14
|
+
|
15
|
+
gem.files = `git ls-files`.split($/)
|
16
|
+
gem.executables = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
|
17
|
+
gem.test_files = gem.files.grep(%r{^(test|spec|features)/})
|
18
|
+
gem.require_paths = ["lib"]
|
19
|
+
end
|
data/lib/ethel.rb
ADDED
data/lib/ethel/field.rb
ADDED
@@ -0,0 +1,30 @@
|
|
1
|
+
module Ethel
|
2
|
+
class Migration
|
3
|
+
def initialize(source, target)
|
4
|
+
@source = source
|
5
|
+
@target = target
|
6
|
+
@operations = []
|
7
|
+
end
|
8
|
+
|
9
|
+
def copy(field)
|
10
|
+
@operations << Operations::Copy.new(field)
|
11
|
+
end
|
12
|
+
|
13
|
+
def cast(field, type)
|
14
|
+
@operations << Operations::Cast.new(field, type)
|
15
|
+
end
|
16
|
+
|
17
|
+
def run
|
18
|
+
@operations.each do |operation|
|
19
|
+
operation.before_transform(@source, @target)
|
20
|
+
end
|
21
|
+
@target.prepare
|
22
|
+
|
23
|
+
@source.each do |row|
|
24
|
+
row = @operations.inject(row) { |r, op| op.transform(r) }
|
25
|
+
@target.add_row(row)
|
26
|
+
end
|
27
|
+
@target.flush
|
28
|
+
end
|
29
|
+
end
|
30
|
+
end
|
@@ -0,0 +1,29 @@
|
|
1
|
+
module Ethel
|
2
|
+
class Operation
|
3
|
+
def initialize(*args)
|
4
|
+
@child_operations = []
|
5
|
+
end
|
6
|
+
|
7
|
+
def before_transform(source, target)
|
8
|
+
@child_operations.each do |child_operation|
|
9
|
+
child_operation.before_transform(source, target)
|
10
|
+
end
|
11
|
+
end
|
12
|
+
|
13
|
+
def transform(row)
|
14
|
+
@child_operations.inject(row) do |row, child_operation|
|
15
|
+
child_operation.transform(row)
|
16
|
+
end
|
17
|
+
end
|
18
|
+
|
19
|
+
protected
|
20
|
+
|
21
|
+
def add_child_operation(operation)
|
22
|
+
@child_operations << operation
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
26
|
+
|
27
|
+
require 'ethel/operations/add_field'
|
28
|
+
require 'ethel/operations/copy'
|
29
|
+
require 'ethel/operations/cast'
|
@@ -0,0 +1,28 @@
|
|
1
|
+
module Ethel
|
2
|
+
module Operations
|
3
|
+
class Cast < Operation
|
4
|
+
def initialize(field, new_type)
|
5
|
+
super
|
6
|
+
@original_field = field
|
7
|
+
@field_name = field.name
|
8
|
+
@new_type = new_type
|
9
|
+
@new_field = Field.new(@field_name, :type => @new_type)
|
10
|
+
add_child_operation(AddField.new(@new_field))
|
11
|
+
end
|
12
|
+
|
13
|
+
def transform(row)
|
14
|
+
row = super(row)
|
15
|
+
|
16
|
+
row[@field_name] =
|
17
|
+
case @new_type
|
18
|
+
when :integer
|
19
|
+
row[@field_name].to_i
|
20
|
+
when :string
|
21
|
+
row[@field_name].to_s
|
22
|
+
end
|
23
|
+
|
24
|
+
row
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
data/lib/ethel/source.rb
ADDED
@@ -0,0 +1,30 @@
|
|
1
|
+
module Ethel
|
2
|
+
class Source
|
3
|
+
include Enumerable
|
4
|
+
|
5
|
+
def schema
|
6
|
+
raise NotImplementedError
|
7
|
+
end
|
8
|
+
|
9
|
+
def each
|
10
|
+
raise NotImplementedError
|
11
|
+
end
|
12
|
+
|
13
|
+
def field_names
|
14
|
+
schema.collect(&:first)
|
15
|
+
end
|
16
|
+
|
17
|
+
def fields
|
18
|
+
@fields ||= schema.inject({}) do |hash, (name, options)|
|
19
|
+
hash[name] = Field.new(name, options)
|
20
|
+
hash
|
21
|
+
end
|
22
|
+
end
|
23
|
+
|
24
|
+
def all
|
25
|
+
to_a
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
29
|
+
|
30
|
+
require 'ethel/sources/csv'
|
@@ -0,0 +1,29 @@
|
|
1
|
+
module Ethel
|
2
|
+
module Sources
|
3
|
+
class CSV < Source
|
4
|
+
def initialize(options = {})
|
5
|
+
if options[:string]
|
6
|
+
@data = ::CSV.parse(options[:string], :headers => true)
|
7
|
+
elsif options[:file]
|
8
|
+
@data = ::CSV.read(options[:file], :headers => true)
|
9
|
+
end
|
10
|
+
end
|
11
|
+
|
12
|
+
def schema
|
13
|
+
if @schema.nil?
|
14
|
+
@schema = []
|
15
|
+
@data.headers.each do |name|
|
16
|
+
@schema << [name, {:type => :string}]
|
17
|
+
end
|
18
|
+
end
|
19
|
+
@schema
|
20
|
+
end
|
21
|
+
|
22
|
+
def each
|
23
|
+
@data.each do |row|
|
24
|
+
yield row.to_hash
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
29
|
+
end
|
data/lib/ethel/target.rb
ADDED
@@ -0,0 +1,26 @@
|
|
1
|
+
module Ethel
|
2
|
+
class Target
|
3
|
+
def initialize(*args)
|
4
|
+
end
|
5
|
+
|
6
|
+
def add_field(*args)
|
7
|
+
raise NotImplementedError
|
8
|
+
end
|
9
|
+
|
10
|
+
def prepare
|
11
|
+
end
|
12
|
+
|
13
|
+
def add_row(*args)
|
14
|
+
raise NotImplementedError
|
15
|
+
end
|
16
|
+
|
17
|
+
def flush
|
18
|
+
end
|
19
|
+
|
20
|
+
def data
|
21
|
+
nil
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
25
|
+
|
26
|
+
require 'ethel/targets/csv'
|
@@ -0,0 +1,46 @@
|
|
1
|
+
module Ethel
|
2
|
+
module Targets
|
3
|
+
class CSV < Target
|
4
|
+
def initialize(options)
|
5
|
+
super
|
6
|
+
|
7
|
+
@options = options
|
8
|
+
@fields = []
|
9
|
+
@rows = []
|
10
|
+
end
|
11
|
+
|
12
|
+
def add_field(field)
|
13
|
+
@fields << field
|
14
|
+
end
|
15
|
+
|
16
|
+
def add_row(row)
|
17
|
+
@rows << row
|
18
|
+
end
|
19
|
+
|
20
|
+
def flush
|
21
|
+
headers = @fields.collect(&:name)
|
22
|
+
csv_options = {
|
23
|
+
:headers => headers,
|
24
|
+
:write_headers => true
|
25
|
+
}
|
26
|
+
csv =
|
27
|
+
if @options[:file]
|
28
|
+
::CSV.open(@options[:file], 'wb', csv_options)
|
29
|
+
elsif @options[:string]
|
30
|
+
@data = ""
|
31
|
+
::CSV.new(@data, csv_options)
|
32
|
+
end
|
33
|
+
@rows.each do |row|
|
34
|
+
csv << row.values_at(*headers)
|
35
|
+
end
|
36
|
+
csv.close
|
37
|
+
end
|
38
|
+
|
39
|
+
def data
|
40
|
+
if @options[:string]
|
41
|
+
@data
|
42
|
+
end
|
43
|
+
end
|
44
|
+
end
|
45
|
+
end
|
46
|
+
end
|
data/test/helper.rb
ADDED
@@ -0,0 +1,29 @@
|
|
1
|
+
require 'rubygems'
|
2
|
+
require 'bundler'
|
3
|
+
begin
|
4
|
+
Bundler.setup(:default, :development)
|
5
|
+
rescue Bundler::BundlerError => e
|
6
|
+
$stderr.puts e.message
|
7
|
+
$stderr.puts "Run `bundle install` to install missing gems"
|
8
|
+
exit e.status_code
|
9
|
+
end
|
10
|
+
|
11
|
+
require 'test/unit'
|
12
|
+
require 'mocha/setup'
|
13
|
+
require 'tempfile'
|
14
|
+
|
15
|
+
$LOAD_PATH.unshift(File.dirname(__FILE__))
|
16
|
+
$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
|
17
|
+
require 'ethel'
|
18
|
+
|
19
|
+
class SequenceHelper
|
20
|
+
include Mocha::API
|
21
|
+
|
22
|
+
def initialize(name)
|
23
|
+
@seq = sequence(name)
|
24
|
+
end
|
25
|
+
|
26
|
+
def <<(expectation)
|
27
|
+
expectation.in_sequence(@seq)
|
28
|
+
end
|
29
|
+
end
|
@@ -0,0 +1,13 @@
|
|
1
|
+
require 'helper'
|
2
|
+
|
3
|
+
class TestCastMigration < Test::Unit::TestCase
|
4
|
+
test "casting integer from csv to csv" do
|
5
|
+
source = Ethel::Sources::CSV.new(:string => "foo,bar\nstuff,123")
|
6
|
+
target = Ethel::Targets::CSV.new(:string => true)
|
7
|
+
migration = Ethel::Migration.new(source, target)
|
8
|
+
migration.cast(source.fields['foo'], :integer)
|
9
|
+
migration.cast(source.fields['bar'], :integer)
|
10
|
+
migration.run
|
11
|
+
assert_equal "foo,bar\n0,123\n", target.data
|
12
|
+
end
|
13
|
+
end
|
@@ -0,0 +1,13 @@
|
|
1
|
+
require 'helper'
|
2
|
+
|
3
|
+
class TestCopyMigration < Test::Unit::TestCase
|
4
|
+
test "copying from csv to csv" do
|
5
|
+
source = Ethel::Sources::CSV.new(:string => "foo,bar\nstuff,123")
|
6
|
+
target = Ethel::Targets::CSV.new(:string => true)
|
7
|
+
migration = Ethel::Migration.new(source, target)
|
8
|
+
migration.copy(source.fields['foo'])
|
9
|
+
migration.copy(source.fields['bar'])
|
10
|
+
migration.run
|
11
|
+
assert_equal "foo,bar\nstuff,123\n", target.data
|
12
|
+
end
|
13
|
+
end
|
@@ -0,0 +1,27 @@
|
|
1
|
+
require 'helper'
|
2
|
+
|
3
|
+
module TestOperations
|
4
|
+
class TestAddField < Test::Unit::TestCase
|
5
|
+
def self.const_missing(name)
|
6
|
+
if Ethel.const_defined?(name)
|
7
|
+
Ethel.const_get(name)
|
8
|
+
else
|
9
|
+
super
|
10
|
+
end
|
11
|
+
end
|
12
|
+
|
13
|
+
test "subclass of Operation" do
|
14
|
+
assert_equal Operation, Operations::AddField.superclass
|
15
|
+
end
|
16
|
+
|
17
|
+
test "#before_transform calls Target#add_field" do
|
18
|
+
field = stub('field')
|
19
|
+
op = Operations::AddField.new(field)
|
20
|
+
|
21
|
+
source = stub('source')
|
22
|
+
target = stub('target')
|
23
|
+
target.expects(:add_field).with(field)
|
24
|
+
op.before_transform(source, target)
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
@@ -0,0 +1,52 @@
|
|
1
|
+
require 'helper'
|
2
|
+
|
3
|
+
module TestOperations
|
4
|
+
class TestCast < Test::Unit::TestCase
|
5
|
+
def self.const_missing(name)
|
6
|
+
if Ethel.const_defined?(name)
|
7
|
+
Ethel.const_get(name)
|
8
|
+
else
|
9
|
+
super
|
10
|
+
end
|
11
|
+
end
|
12
|
+
|
13
|
+
def setup
|
14
|
+
@original_field = stub('original field', :name => 'foo')
|
15
|
+
@new_field = stub('new field')
|
16
|
+
Field.stubs(:new).returns(@new_field)
|
17
|
+
@child_operation = stub('child operation')
|
18
|
+
Operations::AddField.stubs(:new).returns(@child_operation)
|
19
|
+
end
|
20
|
+
|
21
|
+
test "subclass of Operation" do
|
22
|
+
assert_equal Operation, Operations::Cast.superclass
|
23
|
+
end
|
24
|
+
|
25
|
+
test "has AddField child operation with new field" do
|
26
|
+
Field.expects(:new).with('foo', :type => :integer).returns(@new_field)
|
27
|
+
Operations::AddField.expects(:new).with(@new_field).
|
28
|
+
returns(@child_operation)
|
29
|
+
|
30
|
+
op = Operations::Cast.new(@original_field, :integer)
|
31
|
+
|
32
|
+
source = stub('source')
|
33
|
+
target = stub('target')
|
34
|
+
@child_operation.expects(:before_transform).with(source, target)
|
35
|
+
op.before_transform(source, target)
|
36
|
+
end
|
37
|
+
|
38
|
+
test "uses to_i when casting to integer" do
|
39
|
+
row = {'foo' => '123'}
|
40
|
+
op = Operations::Cast.new(@original_field, :integer)
|
41
|
+
@child_operation.stubs(:transform).with(row).returns(row)
|
42
|
+
assert_equal({'foo' => 123}, op.transform(row))
|
43
|
+
end
|
44
|
+
|
45
|
+
test "uses to_s when casting to string" do
|
46
|
+
row = {'foo' => 123}
|
47
|
+
op = Operations::Cast.new(@original_field, :string)
|
48
|
+
@child_operation.stubs(:transform).with(row).returns(row)
|
49
|
+
assert_equal({'foo' => '123'}, op.transform(row))
|
50
|
+
end
|
51
|
+
end
|
52
|
+
end
|
@@ -0,0 +1,35 @@
|
|
1
|
+
require 'helper'
|
2
|
+
|
3
|
+
module TestOperations
|
4
|
+
class TestCopy < Test::Unit::TestCase
|
5
|
+
def self.const_missing(name)
|
6
|
+
if Ethel.const_defined?(name)
|
7
|
+
Ethel.const_get(name)
|
8
|
+
else
|
9
|
+
super
|
10
|
+
end
|
11
|
+
end
|
12
|
+
|
13
|
+
def setup
|
14
|
+
@field = stub('field', :name => 'foo')
|
15
|
+
@child_operation = stub('child operation')
|
16
|
+
Operations::AddField.stubs(:new).returns(@child_operation)
|
17
|
+
end
|
18
|
+
|
19
|
+
test "subclass of Operation" do
|
20
|
+
assert_equal Operation, Operations::Copy.superclass
|
21
|
+
end
|
22
|
+
|
23
|
+
test "has AddField child operation with same field" do
|
24
|
+
Operations::AddField.expects(:new).with(@field).
|
25
|
+
returns(@child_operation)
|
26
|
+
|
27
|
+
op = Operations::Copy.new(@field)
|
28
|
+
|
29
|
+
source = stub('source')
|
30
|
+
target = stub('target')
|
31
|
+
@child_operation.expects(:before_transform).with(source, target)
|
32
|
+
op.before_transform(source, target)
|
33
|
+
end
|
34
|
+
end
|
35
|
+
end
|
@@ -0,0 +1,49 @@
|
|
1
|
+
require 'helper'
|
2
|
+
|
3
|
+
module TestSources
|
4
|
+
class TestCSV < Test::Unit::TestCase
|
5
|
+
def self.const_missing(name)
|
6
|
+
if Ethel.const_defined?(name)
|
7
|
+
Ethel.const_get(name)
|
8
|
+
else
|
9
|
+
super
|
10
|
+
end
|
11
|
+
end
|
12
|
+
|
13
|
+
test "subclass of Source" do
|
14
|
+
assert_equal Source, Sources::CSV.superclass
|
15
|
+
end
|
16
|
+
|
17
|
+
test "load from string" do
|
18
|
+
csv = Sources::CSV.new(:string => "foo,bar\n1,2")
|
19
|
+
expected_schema = [
|
20
|
+
['foo', { :type => :string }],
|
21
|
+
['bar', { :type => :string }]
|
22
|
+
]
|
23
|
+
assert_equal expected_schema, csv.schema
|
24
|
+
end
|
25
|
+
|
26
|
+
test "load from file" do
|
27
|
+
file = Tempfile.new('csv')
|
28
|
+
file.write("foo,bar\n1,2")
|
29
|
+
file.close
|
30
|
+
|
31
|
+
csv = Sources::CSV.new(:file => file.path)
|
32
|
+
expected_schema = [
|
33
|
+
['foo', { :type => :string }],
|
34
|
+
['bar', { :type => :string }]
|
35
|
+
]
|
36
|
+
assert_equal expected_schema, csv.schema
|
37
|
+
end
|
38
|
+
|
39
|
+
test "each" do
|
40
|
+
csv = Sources::CSV.new(:string => "foo,bar\n1,2\na,b")
|
41
|
+
rows = []
|
42
|
+
csv.each do |row|
|
43
|
+
rows << row
|
44
|
+
end
|
45
|
+
assert_equal [{'foo' => '1', 'bar' => '2'}, {'foo' => 'a', 'bar' => 'b'}],
|
46
|
+
rows
|
47
|
+
end
|
48
|
+
end
|
49
|
+
end
|
@@ -0,0 +1,39 @@
|
|
1
|
+
require 'helper'
|
2
|
+
|
3
|
+
module TestTargets
|
4
|
+
class TestCSV < Test::Unit::TestCase
|
5
|
+
def self.const_missing(name)
|
6
|
+
if Ethel.const_defined?(name)
|
7
|
+
Ethel.const_get(name)
|
8
|
+
else
|
9
|
+
super
|
10
|
+
end
|
11
|
+
end
|
12
|
+
|
13
|
+
test "subclass of Target" do
|
14
|
+
assert_equal Target, Targets::CSV.superclass
|
15
|
+
end
|
16
|
+
|
17
|
+
test "output to file" do
|
18
|
+
file = Tempfile.new('csv')
|
19
|
+
field = stub('field', :name => 'foo', :type => :string)
|
20
|
+
csv = Targets::CSV.new(:file => file.path)
|
21
|
+
csv.add_field(field)
|
22
|
+
csv.add_row({'foo' => 'bar'})
|
23
|
+
csv.flush
|
24
|
+
|
25
|
+
file.rewind
|
26
|
+
assert_equal "foo\nbar\n", file.read
|
27
|
+
end
|
28
|
+
|
29
|
+
test "output to string" do
|
30
|
+
field = stub('field', :name => 'foo', :type => :string)
|
31
|
+
csv = Targets::CSV.new(:string => true)
|
32
|
+
csv.add_field(field)
|
33
|
+
csv.add_row({'foo' => 'bar'})
|
34
|
+
csv.flush
|
35
|
+
|
36
|
+
assert_equal "foo\nbar\n", csv.data
|
37
|
+
end
|
38
|
+
end
|
39
|
+
end
|
@@ -0,0 +1,13 @@
|
|
1
|
+
require 'helper'
|
2
|
+
|
3
|
+
class TestField < Test::Unit::TestCase
|
4
|
+
test "name" do
|
5
|
+
field = Ethel::Field.new('foo', {:type => :string})
|
6
|
+
assert_equal 'foo', field.name
|
7
|
+
end
|
8
|
+
|
9
|
+
test "type" do
|
10
|
+
field = Ethel::Field.new('foo', {:type => :string})
|
11
|
+
assert_equal :string, field.type
|
12
|
+
end
|
13
|
+
end
|
@@ -0,0 +1,47 @@
|
|
1
|
+
require 'helper'
|
2
|
+
|
3
|
+
class TestMigration < Test::Unit::TestCase
|
4
|
+
def setup
|
5
|
+
@source = stub('source')
|
6
|
+
@target = stub('target')
|
7
|
+
end
|
8
|
+
|
9
|
+
test "copying a field" do
|
10
|
+
m = Ethel::Migration.new(@source, @target)
|
11
|
+
|
12
|
+
field = stub('field')
|
13
|
+
copy_operation = stub('copy operation')
|
14
|
+
Ethel::Operations::Copy.expects(:new).with(field).returns(copy_operation)
|
15
|
+
m.copy(field)
|
16
|
+
|
17
|
+
seq = SequenceHelper.new('run sequence')
|
18
|
+
row = stub('row')
|
19
|
+
seq << copy_operation.expects(:before_transform).with(@source, @target)
|
20
|
+
seq << @target.expects(:prepare)
|
21
|
+
seq << @source.expects(:each).yields(row)
|
22
|
+
seq << copy_operation.expects(:transform).with(row).returns(row)
|
23
|
+
seq << @target.expects(:add_row).with(row)
|
24
|
+
seq << @target.expects(:flush)
|
25
|
+
m.run
|
26
|
+
end
|
27
|
+
|
28
|
+
test "casting a field" do
|
29
|
+
m = Ethel::Migration.new(@source, @target)
|
30
|
+
|
31
|
+
field = stub('field')
|
32
|
+
cast_operation = stub('cast operation')
|
33
|
+
Ethel::Operations::Cast.expects(:new).with(field, :integer).
|
34
|
+
returns(cast_operation)
|
35
|
+
m.cast(field, :integer)
|
36
|
+
|
37
|
+
seq = SequenceHelper.new('run sequence')
|
38
|
+
row = stub('row')
|
39
|
+
seq << cast_operation.expects(:before_transform).with(@source, @target)
|
40
|
+
seq << @target.expects(:prepare)
|
41
|
+
seq << @source.expects(:each).yields(row)
|
42
|
+
seq << cast_operation.expects(:transform).with(row).returns(row)
|
43
|
+
seq << @target.expects(:add_row).with(row)
|
44
|
+
seq << @target.expects(:flush)
|
45
|
+
m.run
|
46
|
+
end
|
47
|
+
end
|
@@ -0,0 +1,35 @@
|
|
1
|
+
require 'helper'
|
2
|
+
|
3
|
+
class TestOperation < Test::Unit::TestCase
|
4
|
+
def new_subclass(&block)
|
5
|
+
Class.new(Ethel::Operation, &block)
|
6
|
+
end
|
7
|
+
|
8
|
+
test "#before_transform chains child operations" do
|
9
|
+
child = stub('child operation')
|
10
|
+
klass = new_subclass do
|
11
|
+
define_method(:initialize) do |*args|
|
12
|
+
super(*args)
|
13
|
+
add_child_operation(child)
|
14
|
+
end
|
15
|
+
end
|
16
|
+
op = klass.new
|
17
|
+
|
18
|
+
child.expects(:before_transform).with('foo', 'bar')
|
19
|
+
op.before_transform('foo', 'bar')
|
20
|
+
end
|
21
|
+
|
22
|
+
test "#transform chains child operations" do
|
23
|
+
child = stub('child operation')
|
24
|
+
klass = new_subclass do
|
25
|
+
define_method(:initialize) do |*args|
|
26
|
+
super(*args)
|
27
|
+
add_child_operation(child)
|
28
|
+
end
|
29
|
+
end
|
30
|
+
op = klass.new
|
31
|
+
|
32
|
+
child.expects(:transform).with({'foo' => 'bar'}).returns({'foo' => 123})
|
33
|
+
assert_equal({'foo' => 123}, op.transform({'foo' => 'bar'}))
|
34
|
+
end
|
35
|
+
end
|
@@ -0,0 +1,60 @@
|
|
1
|
+
require 'helper'
|
2
|
+
|
3
|
+
class TestSource < Test::Unit::TestCase
|
4
|
+
def new_subclass(&block)
|
5
|
+
Class.new(Ethel::Source, &block)
|
6
|
+
end
|
7
|
+
|
8
|
+
test "schema raises NotImplementedError" do
|
9
|
+
klass = new_subclass
|
10
|
+
source = klass.new
|
11
|
+
assert_raises(NotImplementedError) { source.schema }
|
12
|
+
end
|
13
|
+
|
14
|
+
test "field_names" do
|
15
|
+
klass = new_subclass do
|
16
|
+
def schema
|
17
|
+
[['foo', {}], ['bar', {}]]
|
18
|
+
end
|
19
|
+
end
|
20
|
+
source = klass.new
|
21
|
+
assert_equal %w{foo bar}, source.field_names
|
22
|
+
end
|
23
|
+
|
24
|
+
test "fields" do
|
25
|
+
klass = new_subclass do
|
26
|
+
def schema
|
27
|
+
[['foo', {:type => :string}], ['bar', {:type => :string}]]
|
28
|
+
end
|
29
|
+
end
|
30
|
+
source = klass.new
|
31
|
+
|
32
|
+
field_1 = stub('field 1')
|
33
|
+
Ethel::Field.expects(:new).with('foo', {:type => :string}).returns(field_1)
|
34
|
+
field_2 = stub('field 2')
|
35
|
+
Ethel::Field.expects(:new).with('bar', {:type => :string}).returns(field_2)
|
36
|
+
assert_equal({'foo' => field_1, 'bar' => field_2}, source.fields)
|
37
|
+
end
|
38
|
+
|
39
|
+
test "each raises NotImplementedError" do
|
40
|
+
klass = new_subclass
|
41
|
+
source = klass.new
|
42
|
+
assert_raises(NotImplementedError) { source.each }
|
43
|
+
end
|
44
|
+
|
45
|
+
test "includes Enumerable" do
|
46
|
+
assert_include Ethel::Source.included_modules, Enumerable
|
47
|
+
end
|
48
|
+
|
49
|
+
test "all" do
|
50
|
+
klass = new_subclass do
|
51
|
+
def each
|
52
|
+
yield({'foo' => 1, 'bar' => 2})
|
53
|
+
yield({'foo' => 3, 'bar' => 4})
|
54
|
+
end
|
55
|
+
end
|
56
|
+
source = klass.new
|
57
|
+
assert_equal [{'foo' => 1, 'bar' => 2}, {'foo' => 3, 'bar' => 4}],
|
58
|
+
source.all
|
59
|
+
end
|
60
|
+
end
|
@@ -0,0 +1,37 @@
|
|
1
|
+
require 'helper'
|
2
|
+
|
3
|
+
class TestTarget < Test::Unit::TestCase
|
4
|
+
def new_subclass(&block)
|
5
|
+
Class.new(Ethel::Target, &block)
|
6
|
+
end
|
7
|
+
|
8
|
+
test "#add_field raises NotImplementedError" do
|
9
|
+
klass = new_subclass
|
10
|
+
target = klass.new
|
11
|
+
assert_raises(NotImplementedError) { target.add_field('foo') }
|
12
|
+
end
|
13
|
+
|
14
|
+
test "#add_row raises NotImplementedError" do
|
15
|
+
klass = new_subclass
|
16
|
+
target = klass.new
|
17
|
+
assert_raises(NotImplementedError) { target.add_row('foo') }
|
18
|
+
end
|
19
|
+
|
20
|
+
test "#flush is a no-op" do
|
21
|
+
klass = new_subclass
|
22
|
+
target = klass.new
|
23
|
+
assert_nothing_raised { target.flush }
|
24
|
+
end
|
25
|
+
|
26
|
+
test "#data returns nil" do
|
27
|
+
klass = new_subclass
|
28
|
+
target = klass.new
|
29
|
+
assert_nil target.data
|
30
|
+
end
|
31
|
+
|
32
|
+
test "#prepare is a no-op" do
|
33
|
+
klass = new_subclass
|
34
|
+
target = klass.new
|
35
|
+
assert_nothing_raised { target.prepare }
|
36
|
+
end
|
37
|
+
end
|
metadata
ADDED
@@ -0,0 +1,90 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: ethel
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.0.1
|
5
|
+
prerelease:
|
6
|
+
platform: ruby
|
7
|
+
authors:
|
8
|
+
- Jeremy Stephens
|
9
|
+
autorequire:
|
10
|
+
bindir: bin
|
11
|
+
cert_chain: []
|
12
|
+
date: 2013-02-06 00:00:00.000000000 Z
|
13
|
+
dependencies: []
|
14
|
+
description: Ethel is an ORM-agnostic library of ETL (extract-transform-load) utilities
|
15
|
+
email:
|
16
|
+
- jeremy.f.stephens@vanderbilt.edu
|
17
|
+
executables: []
|
18
|
+
extensions: []
|
19
|
+
extra_rdoc_files: []
|
20
|
+
files:
|
21
|
+
- .gitignore
|
22
|
+
- Gemfile
|
23
|
+
- Guardfile
|
24
|
+
- LICENSE.txt
|
25
|
+
- README.md
|
26
|
+
- Rakefile
|
27
|
+
- ethel.gemspec
|
28
|
+
- lib/ethel.rb
|
29
|
+
- lib/ethel/field.rb
|
30
|
+
- lib/ethel/migration.rb
|
31
|
+
- lib/ethel/operation.rb
|
32
|
+
- lib/ethel/operations/add_field.rb
|
33
|
+
- lib/ethel/operations/cast.rb
|
34
|
+
- lib/ethel/operations/copy.rb
|
35
|
+
- lib/ethel/source.rb
|
36
|
+
- lib/ethel/sources/csv.rb
|
37
|
+
- lib/ethel/target.rb
|
38
|
+
- lib/ethel/targets/csv.rb
|
39
|
+
- lib/ethel/version.rb
|
40
|
+
- test/helper.rb
|
41
|
+
- test/integration/test_cast_migration.rb
|
42
|
+
- test/integration/test_copy_migration.rb
|
43
|
+
- test/unit/operations/test_add_field.rb
|
44
|
+
- test/unit/operations/test_cast.rb
|
45
|
+
- test/unit/operations/test_copy.rb
|
46
|
+
- test/unit/sources/test_csv.rb
|
47
|
+
- test/unit/targets/test_csv.rb
|
48
|
+
- test/unit/test_field.rb
|
49
|
+
- test/unit/test_migration.rb
|
50
|
+
- test/unit/test_operation.rb
|
51
|
+
- test/unit/test_source.rb
|
52
|
+
- test/unit/test_target.rb
|
53
|
+
homepage: https://github.com/coupler/ethel
|
54
|
+
licenses: []
|
55
|
+
post_install_message:
|
56
|
+
rdoc_options: []
|
57
|
+
require_paths:
|
58
|
+
- lib
|
59
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
60
|
+
none: false
|
61
|
+
requirements:
|
62
|
+
- - ! '>='
|
63
|
+
- !ruby/object:Gem::Version
|
64
|
+
version: '0'
|
65
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
66
|
+
none: false
|
67
|
+
requirements:
|
68
|
+
- - ! '>='
|
69
|
+
- !ruby/object:Gem::Version
|
70
|
+
version: '0'
|
71
|
+
requirements: []
|
72
|
+
rubyforge_project:
|
73
|
+
rubygems_version: 1.8.23
|
74
|
+
signing_key:
|
75
|
+
specification_version: 3
|
76
|
+
summary: ORM-agnostic ETL (extract-transform-load) utilities
|
77
|
+
test_files:
|
78
|
+
- test/helper.rb
|
79
|
+
- test/integration/test_cast_migration.rb
|
80
|
+
- test/integration/test_copy_migration.rb
|
81
|
+
- test/unit/operations/test_add_field.rb
|
82
|
+
- test/unit/operations/test_cast.rb
|
83
|
+
- test/unit/operations/test_copy.rb
|
84
|
+
- test/unit/sources/test_csv.rb
|
85
|
+
- test/unit/targets/test_csv.rb
|
86
|
+
- test/unit/test_field.rb
|
87
|
+
- test/unit/test_migration.rb
|
88
|
+
- test/unit/test_operation.rb
|
89
|
+
- test/unit/test_source.rb
|
90
|
+
- test/unit/test_target.rb
|