schlepp 0.0.1.pre.alpha.1 → 0.0.1.pre.alpha.2

data/test/unit/schlepp/table_object/carosel_test.rb

@@ -0,0 +1,104 @@
+require File.expand_path('../../../test_helper', __FILE__)
+
+require 'schlepp/sink/table_object/carosel'
+
+class Schlepp::Sink::TableObject::CaroselTest < Test::Unit::TestCase
+  include TestHelper 
+
+  setup do
+    @mock_factory = mock
+
+    @mock_table_object = mock
+
+    @mock_filters = 5.times.map do |wf|
+      mock
+    end
+
+    @mock_table_object_factory = mock
+    Schlepp::Sink::TableObject::Factory.expects(:new).with(@mock_factory, @mock_filters).returns(@mock_table_object_factory)
+
+    @mock_table_object_factory.expects(:new).returns(@mock_table_object)
+
+    mock_table_object_writer_factory = mock
+    Schlepp::Sink::TableObject::Writer::Factory.expects(:new).with(@mock_factory, @mock_filters, @mock_table_object).returns(mock_table_object_writer_factory)
+
+    @mock_writer = mock
+    mock_table_object_writer_factory.expects(:new).returns(@mock_writer)
+
+    @carosel = Schlepp::Sink::TableObject::Carosel.new(@mock_factory, @mock_filters)
+  end
+
+  test "#write" do
+    mock_data = mock
+
+    @mock_filters.each do |f|
+      f.expects(:should_rotate?).returns(false)
+    end
+
+    @mock_writer.expects(:write).with(mock_data)
+
+    @carosel.write(mock_data)
+  end
+
+  test "#write with rotation" do
+    mock_data = mock
+
+    rotation_state = states('rotation').starts_as('unrotated')
+
+    @mock_filters.first.expects(:should_rotate?).
+                        when(rotation_state.is('unrotated')).
+                        then(rotation_state.is('rotated')).
+                        returns(true)
+
+
+    @mock_filters.each do |f|
+      f.expects(:should_rotate?).when(rotation_state.is('rotated')).returns(false)
+    end
+
+    @mock_writer.expects(:finalize)
+    
+    mock_second_table_object = mock
+    @mock_table_object_factory.expects(:new).returns(mock_second_table_object)
+
+    mock_table_object_writer_factory = mock
+    Schlepp::Sink::TableObject::Writer::Factory.expects(:new).with(@mock_factory, @mock_filters, mock_second_table_object).returns(mock_table_object_writer_factory)
+
+    mock_replacement_writer = mock
+    mock_table_object_writer_factory.expects(:new).returns(mock_replacement_writer)
+
+    mock_replacement_writer.expects(:write).with(mock_data)
+
+    @carosel.write(mock_data)
+  end
+  test "#write with endless rotation should fail" do
+    mock_data = mock
+
+    rotation_state = states('rotation').starts_as('unrotated')
+
+    @mock_filters.first.expects(:should_rotate?).
+                        when(rotation_state.is('unrotated')).
+                        then(rotation_state.is('rotated')).
+                        returns(true)
+
+    @mock_filters.first.expects(:should_rotate?).
+                        when(rotation_state.is('rotated')).
+                        returns(true)
+
+
+    @mock_writer.expects(:finalize)
+    
+    mock_second_table_object = mock
+    @mock_table_object_factory.expects(:new).returns(mock_second_table_object)
+
+    mock_table_object_writer_factory = mock
+    Schlepp::Sink::TableObject::Writer::Factory.expects(:new).with(@mock_factory, @mock_filters, mock_second_table_object).returns(mock_table_object_writer_factory)
+
+    mock_replacement_writer = mock
+    mock_table_object_writer_factory.expects(:new).returns(mock_replacement_writer)
+
+    assert_raises do
+      @carosel.write(mock_data)
+    end
+  end
+end
+

data/test/unit/schlepp/table_object/factory_test.rb

@@ -0,0 +1,55 @@
+require File.expand_path('../../../test_helper', __FILE__)
+
+require 'schlepp/sink/table_object/factory'
+
+class Schlepp::Sink::TableObject::FactoryTest < Test::Unit::TestCase
+  include TestHelper 
+  setup do
+    @mock_factory = mock
+
+    url = URI("file:///tmp/example")
+
+    @mock_factory.expects(:url).returns(url)
+
+    mock_model = mock
+
+    @mock_factory.expects(:model).returns(mock_model)
+
+
+    @mock_filters = 5.times.map do |wf|
+      mock_filter = mock
+      mock_filter.expects(:rotate)
+
+      mock_filter
+    end
+
+    @mock_filters.each_with_index do |f, idx|
+      f.expects(:extension).returns(idx)
+    end
+
+    @mock_table_object = mock
+    expected_url = URI("file:///tmp/example.0.1.2.3.4")
+    Hydrogen::TableObject.expects(:new).with(mock_model, expected_url).returns(@mock_table_object)
+
+    @factory = Schlepp::Sink::TableObject::Factory.new(@mock_factory, @mock_filters)
+  end
+  test "#new" do
+    res = @factory.new
+
+    assert_equal res, @mock_table_object
+  end
+
+  test "#writer" do
+
+    mock_table_object_writer_factory = mock
+    Schlepp::Sink::TableObject::Writer::Factory.expects(:new).with(@mock_factory, @mock_filters, @mock_table_object).returns(mock_table_object_writer_factory)
+    
+    mock_writer = mock
+    mock_table_object_writer_factory.expects(:new).returns(mock_writer)
+
+
+    res = @factory.writer
+
+    assert_equal res, mock_writer
+  end
+end

data/test/unit/schlepp_test.rb

@@ -5,8 +5,6 @@ require 'schlepp'
 class SchleppTest < Test::Unit::TestCase
   include TestHelper 
   test '.schlepp' do
-    table_object = Hydrogen::TableObject.new(@mock_model)
-
     mock_source = mock
 
     mock_source.expects(:each).multiple_yields("foo", "bar", "baz")
@@ -17,8 +15,11 @@ class SchleppTest < Test::Unit::TestCase
     mock_sink.expects(:write).with("bar")
     mock_sink.expects(:write).with("baz")
 
-    mock_sink.expects(:finalize)
+    mock_parts = mock
+    mock_sink.expects(:finalize).returns(mock_parts)
+
+    parts = Schlepp.schlepp(mock_source, mock_sink)
 
-    Schlepp.schlepp(mock_source, mock_sink)
+    assert_equal parts, mock_parts
   end
 end

data/test/unit/test_helper.rb

@@ -1,10 +1,17 @@
-require 'coveralls'
-Coveralls.wear!
-
-if ENV["ENABLE_SIMPLE_COV"]
+if ENV["COVERAGE"]
+  require 'coveralls'
+  require 'codeclimate-test-reporter'
   require 'simplecov'
-  require File.expand_path('../../simplecov_helper', __FILE__)
-  SimpleCov.start 'pocketchange'
+  SimpleCov.start do
+    add_group "Lib", "lib"
+    add_filter "/test/"
+    command_name "Unit Tests"
+    formatter SimpleCov::Formatter::MultiFormatter[
+      SimpleCov::Formatter::HTMLFormatter,
+      Coveralls::SimpleCov::Formatter,
+      CodeClimate::TestReporter::Formatter
+    ]
+  end
 end
 
 require 'test/unit'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: schlepp
 version: !ruby/object:Gem::Version
-  version: 0.0.1.pre.alpha.1
+  version: 0.0.1.pre.alpha.2
 platform: ruby
 authors:
 - Ed Carrel
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-09-21 00:00:00.000000000 Z
+date: 2014-10-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -111,8 +111,7 @@ dependencies:
 description: An extensible chunking file transferer
 email:
 - edward@carrel.org
-executables:
-- schlepp.rb
+executables: []
 extensions: []
 extra_rdoc_files: []
 files:
@@ -123,34 +122,34 @@ files:
 - LICENSE
 - README.md
 - Rakefile
-- bin/schlepp.rb
 - lib/schlepp.rb
 - lib/schlepp/env.rb
 - lib/schlepp/sink.rb
-- lib/schlepp/sink/chunker.rb
-- lib/schlepp/sink/data_object.rb
-- lib/schlepp/sink/data_stream.rb
-- lib/schlepp/sink/loader.rb
-- lib/schlepp/sink/sequencer.rb
-- lib/schlepp/sinks/fs.rb
-- lib/schlepp/sinks/fs/chunker.rb
-- lib/schlepp/sinks/fs/sequencer.rb
-- lib/schlepp/sinks/fs/table_object.rb
-- lib/schlepp/sinks/fs/table_object/collection.rb
-- lib/schlepp/sinks/fs/table_object/writer.rb
+- lib/schlepp/sink/filter.rb
+- lib/schlepp/sink/filter/chunker.rb
+- lib/schlepp/sink/filter/compressor.rb
+- lib/schlepp/sink/filter/compressor/stream.rb
+- lib/schlepp/sink/filter/compressor/writer.rb
+- lib/schlepp/sink/filter/formatter/csv.rb
+- lib/schlepp/sink/filter/formatter/csv/writer.rb
+- lib/schlepp/sink/table_object/carosel.rb
+- lib/schlepp/sink/table_object/factory.rb
+- lib/schlepp/sink/table_object/filter/observer.rb
+- lib/schlepp/sink/table_object/filter/writer.rb
+- lib/schlepp/sink/table_object/filters.rb
+- lib/schlepp/sink/table_object/stream.rb
+- lib/schlepp/sink/table_object/writer/factory.rb
 - lib/schlepp/source.rb
 - lib/schlepp/source/csv.rb
-- lib/schlepp/table_object/chunker.rb
 - lib/schlepp/version.rb
 - schlepp.gemspec
+- test/integration/mock/sink.rb
+- test/integration/mock/source.rb
 - test/integration/schlepp_test.rb
 - test/integration/test_helper.rb
-- test/unit/schlepp/sink/chunker_test.rb
-- test/unit/schlepp/sink/data_object_test.rb
-- test/unit/schlepp/sink/data_stream_test.rb
-- test/unit/schlepp/sink/loader_test.rb
-- test/unit/schlepp/sink/sequencer_test.rb
-- test/unit/schlepp/table_object/chunker_test.rb
+- test/unit/schlepp/table_object/carosel_test.rb
+- test/unit/schlepp/table_object/factory_test.rb
+- test/unit/schlepp/table_object/writer/factory_test.rb
 - test/unit/schlepp_test.rb
 - test/unit/test_helper.rb
 homepage: https://github.com/azanar/schlepp
@@ -178,13 +177,12 @@ signing_key:
 specification_version: 4
 summary: An extensible chunking file transferer
 test_files:
+- test/integration/mock/sink.rb
+- test/integration/mock/source.rb
 - test/integration/schlepp_test.rb
 - test/integration/test_helper.rb
-- test/unit/schlepp/sink/chunker_test.rb
-- test/unit/schlepp/sink/data_object_test.rb
-- test/unit/schlepp/sink/data_stream_test.rb
-- test/unit/schlepp/sink/loader_test.rb
-- test/unit/schlepp/sink/sequencer_test.rb
-- test/unit/schlepp/table_object/chunker_test.rb
+- test/unit/schlepp/table_object/carosel_test.rb
+- test/unit/schlepp/table_object/factory_test.rb
+- test/unit/schlepp/table_object/writer/factory_test.rb
 - test/unit/schlepp_test.rb
 - test/unit/test_helper.rb

data/bin/schlepp.rb

@@ -1,29 +0,0 @@
-#!/usr/bin/env ruby
-$LOAD_PATH.push(File.dirname(__FILE__) + '/../lib')
-
-require 'hydrogen'
-require 'schlepp/sinks/fs'
-
-config = {
-  :table_name => 'foo',
-  :source => {:file => 'data.csv'},
-  :columns => %w{foo bar baz}
-}
-
-AWS.config(
-  access_key_id: 'ACCESS_KEY_ID',
-  secret_access_key: 'SECRET_ACCESS_KEY',
-  stub_requests:     Schlepp.env.test?,
-)
-
-model = Hydrogen::Model.new(config)
-
-to = Hydrogen::TableObject.new(model)
-
-source = Schlepp::Source::CSV.new(File.new('data.csv','r'))
-
-l = Schlepp::Sink::Fs::Sequencer.new(to, :chunk_size => 40000)
-
-res = Schlepp.schlepp(source, l)
-
-puts res.inspect

data/lib/schlepp/sink/chunker.rb

@@ -1,30 +0,0 @@
-require 'schlepp/sink/loader'
-require 'schlepp/table_object/chunker'
-
-module Schlepp
-  module Sink
-    class Chunker
-      def initialize(table_object, opts = {})
-        @table_object = table_object
-      end
-
-      def sequence
-        chunker.sequence
-      end
-
-      def parts
-        chunker.parts
-      end
-
-      def next
-        chunker.next 
-      end
-
-      private
-
-      def chunker
-        @chunker ||= Schlepp::TableObject::Chunker.new(@table_object)
-      end
-    end
-  end
-end

data/lib/schlepp/sink/data_object.rb

@@ -1,26 +0,0 @@
-require 'schlepp/sink/data_stream'
-
-module Schlepp
-  module Sink
-    class DataObject
-      def initialize(table_object)
-        @stream = DataStream.new
-        @table_object = table_object
-      end
-
-      def write(data)
-        @stream.write(data)
-      end
-
-      def length
-        @stream.length
-      end
-
-      def finalize
-        @stream.finalize
-
-        @table_object.write(@stream.to_s)
-      end
-    end
-  end
-end

data/lib/schlepp/sink/data_stream.rb

@@ -1,39 +0,0 @@
-require 'zlib'
-
-module Schlepp
-  module Sink
-    class DataStream
-      def initialize
-        @buffer = StringIO.new("", "rb+")
-        @compressor = Zlib::GzipWriter.new(@buffer)
-        @dead = false
-      end
-
-      def write(data)
-        if @dead
-          raise "Stream has been dumped. No more writing permitted."
-        end
-        @compressor << data
-      end
-
-      def length
-        if @dead
-          raise "Stream has been dumped. No more writing permitted."
-        end
-        @compressor.pos
-      end
-
-      def finalize
-        if !@dead
-          @compressor.close
-          @dead = true
-        end
-      end
-
-      def to_s
-        finalize
-        @buffer.string
-      end
-    end
-  end
-end

data/lib/schlepp/sink/loader.rb

@@ -1,35 +0,0 @@
-require 'schlepp/sink/data_object'
-
-module Schlepp
-  module Sink
-    class Loader
-      def initialize(writer, opts = {})
-        @writer = writer
-
-        @file = DataObject.new(writer)
-      end
-
-      def name
-        @writer.name
-      end
-
-      def path
-        @writer.path
-      end
-
-      def write(rows)
-        Array(rows).each do |row|
-          @file.write(row)
-        end
-      end
-
-      def written
-        @file.length
-      end
-
-      def finalize
-        @file.finalize
-      end
-    end
-  end
-end

data/lib/schlepp/sink/sequencer.rb

@@ -1,42 +0,0 @@
-module Schlepp
-  module Sink
-    module Sequencer
-      DEFAULT_CHUNK_SIZE = 50 * 1024 * 1024 # 50 MB
-      def initialize(table_object, opts = {})
-        @table_object = table_object
-        @chunker = chunker.new(table_object)
-        @chunk_size = opts[:chunk_size] || DEFAULT_CHUNK_SIZE
-      end
-
-      def write(rows)
-        rows.each do |row|
-          if loader.written + row.length > @chunk_size
-            rotate
-          end
-          loader.write(row)
-        end
-      end
-
-      def parts
-        @chunker.parts
-      end
-
-      def finalize
-        loader.finalize
-      end
-
-      private
-
-      def loader
-        @loader ||= rotate
-      end
-
-      def rotate
-        if @loader
-          @loader.finalize
-        end
-        @loader = @chunker.next
-      end
-    end
-  end
-end

data/lib/schlepp/sinks/fs/chunker.rb

@@ -1,38 +0,0 @@
-require 'schlepp/sink/chunker'
-
-require 'schlepp/sinks/fs/table_object'
-require 'schlepp/sinks/fs/table_object/writer'
-
-module Schlepp
-  module Sink
-    module Fs
-      class Chunker
-        def initialize(table_object, opts = {})
-          @table_object = table_object
-        end
-
-        def sequence
-          chunker.sequence
-        end
-
-        def parts
-          chunker.parts
-        end
-
-        def next
-          part = chunker.next 
-
-          ts = Schlepp::Sink::Fs::TableObject.new(part)
-
-          Schlepp::Sink::Loader.new(ts)
-        end
-
-        private
-
-        def chunker
-          @chunker ||= Schlepp::Sink::Chunker.new(@table_object)
-        end
-      end
-    end
-  end
-end

data/lib/schlepp/sinks/fs/sequencer.rb

@@ -1,21 +0,0 @@
-require 'schlepp/sink/sequencer'
-require 'schlepp/sinks/fs/chunker'
-require 'schlepp/sinks/fs/table_object/collection'
-
-module Schlepp
-  module Sink
-    module Fs
-      class Sequencer
-        include Schlepp::Sink::Sequencer
-
-        def collection
-          Schlepp::Sink::Fs::TableObject::Collection
-        end
-
-        def chunker
-          Schlepp::Sink::Fs::Chunker
-        end
-      end
-    end
-  end
-end

data/lib/schlepp/sinks/fs/table_object/collection.rb

@@ -1,23 +0,0 @@
-require 'hydrogen/table_object/collection'
-
-module Schlepp
-  module Sink
-    module Fs
-      class TableObject
-        class Collection
-          include Hydrogen::TableObject::Collection
-
-          def url
-            "file:///#{path}"
-          end
-
-          def urls
-            parts.map do |p|
-              TableObject.new(p).url
-            end
-          end
-        end
-      end
-    end
-  end
-end

data/lib/schlepp/sinks/fs/table_object/writer.rb

@@ -1,33 +0,0 @@
-module Schlepp
-  module Sink
-    module Fs
-      class TableObject
-        class Writer
-          def initialize(table_object, opts = {})
-            @table_object = table_object
-            @written = 0
-          end
-
-          def file
-            @file ||= File.new("/tmp/#{@table_object.path}", 'w')
-          end
-
-          def write(rows)
-            Array(rows).each do |row|
-              @written += row.length
-              file.write(row)
-            end
-          end
-
-          def written
-            @written
-          end
-
-          def finalize
-            @file.close
-          end
-        end
-      end
-    end
-  end
-end

data/lib/schlepp/sinks/fs/table_object.rb

@@ -1,31 +0,0 @@
-module Schlepp
-  module Sink
-    module Fs
-      class TableObject
-        def initialize(table_object, opts = {})
-          @table_object = table_object
-          @written = 0
-        end
-
-        def write(rows)
-          Array(rows).each do |row|
-            @written += row.length
-            file.write(row)
-          end
-        end
-
-        def written
-          @written
-        end
-
-        def finalize
-          @file.close
-        end
-
-        def file
-          @file ||= File.new("/tmp/#{@table_object.path}", 'w')
-        end
-      end
-    end
-  end
-end

data/lib/schlepp/sinks/fs.rb

@@ -1,2 +0,0 @@
-require 'schlepp/sinks/fs/sequencer'
-require 'schlepp/sinks/fs/chunker'