schlepp 0.0.1.pre.alpha.1 → 0.0.1.pre.alpha.2
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.travis.yml +6 -5
- data/Gemfile +2 -1
- data/lib/schlepp/sink/filter/chunker.rb +40 -0
- data/lib/schlepp/sink/filter/compressor/stream.rb +44 -0
- data/lib/schlepp/sink/filter/compressor/writer.rb +36 -0
- data/lib/schlepp/sink/filter/compressor.rb +20 -0
- data/lib/schlepp/sink/filter/formatter/csv/writer.rb +25 -0
- data/lib/schlepp/sink/filter/formatter/csv.rb +21 -0
- data/lib/schlepp/sink/filter.rb +20 -0
- data/lib/schlepp/sink/table_object/carosel.rb +65 -0
- data/lib/schlepp/sink/table_object/factory.rb +28 -0
- data/lib/schlepp/sink/table_object/filter/observer.rb +23 -0
- data/lib/schlepp/sink/table_object/filter/writer.rb +26 -0
- data/lib/schlepp/sink/table_object/filters.rb +22 -0
- data/lib/schlepp/sink/table_object/stream.rb +25 -0
- data/lib/schlepp/sink/table_object/writer/factory.rb +22 -0
- data/lib/schlepp/sink.rb +17 -2
- data/lib/schlepp/source/csv.rb +1 -1
- data/lib/schlepp/version.rb +1 -1
- data/test/integration/mock/sink.rb +51 -0
- data/test/integration/mock/source.rb +9 -0
- data/test/integration/schlepp_test.rb +27 -8
- data/test/integration/test_helper.rb +13 -6
- data/test/unit/schlepp/table_object/carosel_test.rb +104 -0
- data/test/unit/schlepp/table_object/factory_test.rb +55 -0
- data/test/unit/schlepp_test.rb +5 -4
- data/test/unit/test_helper.rb +13 -6
- metadata +27 -29
- data/bin/schlepp.rb +0 -29
- data/lib/schlepp/sink/chunker.rb +0 -30
- data/lib/schlepp/sink/data_object.rb +0 -26
- data/lib/schlepp/sink/data_stream.rb +0 -39
- data/lib/schlepp/sink/loader.rb +0 -35
- data/lib/schlepp/sink/sequencer.rb +0 -42
- data/lib/schlepp/sinks/fs/chunker.rb +0 -38
- data/lib/schlepp/sinks/fs/sequencer.rb +0 -21
- data/lib/schlepp/sinks/fs/table_object/collection.rb +0 -23
- data/lib/schlepp/sinks/fs/table_object/writer.rb +0 -33
- data/lib/schlepp/sinks/fs/table_object.rb +0 -31
- data/lib/schlepp/sinks/fs.rb +0 -2
- data/lib/schlepp/table_object/chunker.rb +0 -33
- data/test/unit/schlepp/sink/data_object_test.rb +0 -0
- data/test/unit/schlepp/sink/data_stream_test.rb +0 -0
- data/test/unit/schlepp/sink/loader_test.rb +0 -29
- data/test/unit/schlepp/sink/sequencer_test.rb +0 -0
- data/test/unit/schlepp/table_object/chunker_test.rb +0 -0
- /data/test/unit/schlepp/{sink/chunker_test.rb → table_object/writer/factory_test.rb} +0 -0
|
@@ -1,33 +0,0 @@
|
|
|
1
|
-
require 'hydrogen/table_object/part'
|
|
2
|
-
require 'hydrogen/table_object/part/sequence'
|
|
3
|
-
|
|
4
|
-
module Schlepp
|
|
5
|
-
class TableObject
|
|
6
|
-
class Chunker
|
|
7
|
-
def initialize(table_object, opts = {})
|
|
8
|
-
@table_object = table_object
|
|
9
|
-
@chunk = 0
|
|
10
|
-
@parts = []
|
|
11
|
-
end
|
|
12
|
-
|
|
13
|
-
attr_reader :parts
|
|
14
|
-
|
|
15
|
-
def sequence
|
|
16
|
-
Hydrogen::TableObject::Part::Sequence.new(@table_object, parts)
|
|
17
|
-
end
|
|
18
|
-
|
|
19
|
-
def parts
|
|
20
|
-
@parts
|
|
21
|
-
end
|
|
22
|
-
|
|
23
|
-
def next
|
|
24
|
-
@chunk += 1
|
|
25
|
-
part = Hydrogen::TableObject::Part.new(@chunk, @table_object)
|
|
26
|
-
|
|
27
|
-
@parts << part
|
|
28
|
-
|
|
29
|
-
part
|
|
30
|
-
end
|
|
31
|
-
end
|
|
32
|
-
end
|
|
33
|
-
end
|
|
File without changes
|
|
File without changes
|
|
@@ -1,29 +0,0 @@
|
|
|
1
|
-
require File.expand_path('../../../test_helper', __FILE__)
|
|
2
|
-
|
|
3
|
-
require 'schlepp/sink/loader'
|
|
4
|
-
require 'schlepp/sink/data_object'
|
|
5
|
-
|
|
6
|
-
class Schlepp::Sink::LoaderTest < Test::Unit::TestCase
|
|
7
|
-
include TestHelper
|
|
8
|
-
setup do
|
|
9
|
-
@mock_table_object = mock
|
|
10
|
-
|
|
11
|
-
@mock_data_object = mock
|
|
12
|
-
Schlepp::Sink::DataObject.expects(:new).with(@mock_table_object).returns(@mock_data_object)
|
|
13
|
-
|
|
14
|
-
@s3_loader = Schlepp::Sink::Loader.new(@mock_table_object)
|
|
15
|
-
end
|
|
16
|
-
|
|
17
|
-
test '#write' do
|
|
18
|
-
@mock_data_object.expects(:write).with("FOO|BAR|BAZ")
|
|
19
|
-
@s3_loader.write(["FOO|BAR|BAZ"])
|
|
20
|
-
end
|
|
21
|
-
|
|
22
|
-
test '#finalize' do
|
|
23
|
-
@mock_data_object.expects(:finalize)
|
|
24
|
-
|
|
25
|
-
@s3_loader.finalize
|
|
26
|
-
end
|
|
27
|
-
|
|
28
|
-
end
|
|
29
|
-
|
|
File without changes
|
|
File without changes
|
|
File without changes
|