schlepp 0.0.1.pre.alpha.1 → 0.0.1.pre.alpha.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 937caf4f2826b085928376c932e22556cb4fcb77
-  data.tar.gz: f3ac27937abb7605c6fc351751f15610faea6c4d
+  metadata.gz: 2b1ecbdd8608cf937d4aacec3d65899d9a89991c
+  data.tar.gz: e545caacec77ee3eafce431aa2f43b339852f3b1
 SHA512:
-  metadata.gz: 01d49195f259f64f44caa3e158953ecc914c5289ab1d15e6fc475bfff8f4aa6cafded7162909469280f41afccc01baca46c5ee65e19219563bad71ef1799c5d0
-  data.tar.gz: e503f9af56f2abf60ed774a42cefc31813058d535d97254566d2bf09e6621248c684b9e97b7b36837eb95d3f5afee39d2e72246fffa491f470f5bb139f7c5b06
+  metadata.gz: dac1ede8bb4e2ce790e1794d884451223fbc64633ffd2de5404da0d67ea58999e9a2dbe2dfb773f7ef2c9a048b228b93e56ca1fc297dd03ca6beabe022e52c15
+  data.tar.gz: 87cc9f384d060cfe89e67874e549927bb6b892a568f6542693851b1af4f2a59960145cad16e7f4df2d0b27ace85eca4e6b7070eb8f2f98bdfe415c022258a147

data/.travis.yml

@@ -1,16 +1,17 @@
 language: ruby
 
 env:
-  - TEST_SUITE=unit
-  - TEST_SUITE=integration
+  global:
+    - COVERAGE=true
+  matrix:
+    - TEST_SUITE=unit
+    - TEST_SUITE=integration
 
 script: "bundle exec rake test:$TEST_SUITE"
 
 rvm:
   - "1.9.3"
-  - "2.1.2"
+  - "2.1.3"
   - "ruby-head"
   - "jruby-19mode"
-  - "jruby-head"
   - "rbx-2"
-

data/Gemfile

@@ -1,5 +1,6 @@
 source 'https://rubygems.org'
 
-gem 'coveralls', require: false
+gem 'coveralls', group: :test, require: false
+gem 'codeclimate-test-reporter', group: :test, require: nil
 
 gemspec

data/lib/schlepp/sink/filter/chunker.rb

@@ -0,0 +1,40 @@
+require 'schlepp/sink/table_object/filter/observer'
+require 'schlepp/sink/table_object/filter/writer'
+module Schlepp
+  class Sink
+    module Filter
+      class Chunker
+        class Builder
+          def initialize(observer)
+            @observer = observer
+          end
+
+          def new(writer)
+            Schlepp::Sink::TableObject::Filter::Writer.new(writer, @observer)
+          end
+        end
+        def initialize(opts = {})
+          @chunk = 0
+          @chunk_size = opts[:chunk_size] || 100000
+          @observer = Schlepp::Sink::TableObject::Filter::Observer.new
+        end
+
+        def extension
+          @chunk.to_s
+        end
+
+        def should_rotate?
+          @observer.count > @chunk_size
+        end
+
+        def rotate
+          @chunk += 1
+        end
+
+        def writer
+          @builder ||= Builder.new(@observer)
+        end
+      end
+    end
+  end
+end

data/lib/schlepp/sink/filter/compressor/stream.rb

@@ -0,0 +1,44 @@
+require 'zlib'
+
+module Schlepp
+  class Sink
+    class TableObject
+      class Compressor
+        class Stream
+          def initialize
+            @buffer = StringIO.new("", "rb+")
+            @compressor = Zlib::GzipWriter.new(@buffer)
+            @dead = false
+          end
+
+          def write(data)
+            if @dead
+              raise "Stream has been dumped. No more writing permitted."
+            end
+            @compressor << data
+          end
+
+          def length
+            if @dead
+              raise "Stream has been dumped. No more writing permitted."
+            end
+            @compressor.pos
+          end
+
+          def finalize
+            if !@dead
+              @compressor.close
+              @dead = true
+            end
+          end
+
+          def to_s
+            finalize
+            @buffer.string
+          end
+        end
+      end
+    end
+
+  end
+end

data/lib/schlepp/sink/filter/compressor/writer.rb

@@ -0,0 +1,36 @@
+require 'schlepp/sink/filter/compressor/stream'
+
+module Schlepp
+  class Sink
+    class TableObject
+      class Compressor
+        class Writer
+          def initialize(writer, opts = {})
+            @stream = Stream.new
+            @writer = writer
+            @written = 0
+          end
+
+          def write(rows)
+            Array(rows).each do |row|
+              @written += row.length
+              @stream.write(row)
+            end
+          end
+
+
+          def rotate?
+            false
+          end
+
+          def finalize
+            bits = @stream.to_s
+
+            @writer.write(bits)
+            @writer.finalize
+          end
+        end
+      end
+    end
+  end
+end

data/lib/schlepp/sink/filter/compressor.rb

@@ -0,0 +1,20 @@
+require 'schlepp/sink/filter'
+require 'schlepp/sink/filter/compressor/writer'
+
+module Schlepp
+  class Sink
+    module Filter
+      class Compressor
+        include Schlepp::Sink::Filter::Static
+        def extension
+          'gz'
+        end
+
+        def writer
+          Schlepp::Sink::TableObject::Compressor::Writer
+        end
+      end
+    end
+  end
+end
+

data/lib/schlepp/sink/filter/formatter/csv/writer.rb

@@ -0,0 +1,25 @@
+require 'csv'
+
+module Schlepp
+  class Sink 
+    class TableObject
+      class Formatter
+        class Csv
+          class Writer
+            def initialize(writer, opts = {})
+              @writer = writer
+            end
+
+            def write(rows)
+              @writer.write(rows.to_csv)
+            end
+
+            def finalize
+              @writer.finalize
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/schlepp/sink/filter/formatter/csv.rb

@@ -0,0 +1,21 @@
+require 'schlepp/sink/filter'
+require 'schlepp/sink/table_object/formatter/csv/writer'
+
+module Schlepp
+  class Sink
+    module Filter
+      class Csv
+        include Schlepp::Sink::Filter::Static
+        def extension
+          'csv'
+        end
+
+        def writer
+          Schlepp::Sink::TableObject::Formatter::Csv::Writer
+        end
+      end
+    end
+  end
+end
+
+

data/lib/schlepp/sink/filter.rb

@@ -0,0 +1,20 @@
+module Schlepp
+  class Sink
+    module Filter
+      module Static
+        def should_rotate?
+          false
+        end
+
+        def rotate
+          nil
+        end
+      end
+      module Simple
+        def writer
+          Schlepp::Sink::TableObject::Writer
+        end
+      end
+    end
+  end
+end

data/lib/schlepp/sink/table_object/carosel.rb

@@ -0,0 +1,65 @@
+require 'schlepp/sink/table_object/factory'
+require 'schlepp/sink/table_object/writer/factory'
+
+module Schlepp
+  class Sink
+    class TableObject
+      class Carosel
+        def initialize(factory, chain)
+          @factory = factory
+          @chain = chain
+          @parts = []
+          @current = rotate
+          @finalized = false
+        end
+
+        def write(data)
+          raise_if_finalized
+          if rotate?(data)
+            rotate
+            if rotate?(data)
+              raise
+            end
+          end
+          Array(data).each do |r|
+            @current.write(r)
+          end
+        end
+
+        def finalize
+          @current.finalize
+          @finalized = true
+          @parts
+        end
+
+        private
+
+        def raise_if_finalized
+          @finalized && raise
+        end
+
+        def rotate?(data)
+          @chain.any?(&:should_rotate?)
+        end
+
+        def rotate
+          @current.finalize if @current
+
+          to = table_object_factory.new
+
+          @parts << to
+
+          builder = Schlepp::Sink::TableObject::Writer::Factory.new(@factory, @chain, to)
+
+          @current = builder.new
+        end
+
+        private
+
+        def table_object_factory
+          @table_object_factory ||= Schlepp::Sink::TableObject::Factory.new(@factory, @chain)
+        end
+      end
+    end
+  end
+end

data/lib/schlepp/sink/table_object/factory.rb

@@ -0,0 +1,28 @@
+require 'schlepp/sink/table_object/writer/factory'
+
+module Schlepp
+  class Sink
+    class TableObject
+      class Factory
+        def initialize(factory, components)
+          @factory = factory
+          @components = components
+        end
+
+        def new
+          @components.each(&:rotate)
+
+          url = @components.inject(@factory.url) do |u, f|
+            u.merge("#{u.path}.#{f.extension}")
+          end
+
+          Hydrogen::TableObject.new(@factory.model, url)
+        end
+
+        def writer
+          Writer::Factory.new(@factory, @components, self.new).new
+        end
+      end
+    end
+  end
+end

data/lib/schlepp/sink/table_object/filter/observer.rb

@@ -0,0 +1,23 @@
+module Schlepp
+  class Sink
+    class TableObject
+      class Filter
+        class Observer
+          def initialize
+            reset
+          end
+
+          attr_reader :count
+
+          def add(data)
+            @count += data.length
+          end
+
+          def reset
+            @count = 0
+          end
+        end
+      end
+    end
+  end
+end

data/lib/schlepp/sink/table_object/filter/writer.rb

@@ -0,0 +1,26 @@
+module Schlepp
+  class Sink
+    class TableObject
+      class Filter
+        class Writer
+          def initialize(writer, observer, opts = {})
+            @writer = writer
+            @observer = observer
+          end
+
+          def write(rows)
+            Array(rows).each do |row|
+              @writer.write(row)
+              @observer.add(row)
+            end
+          end
+
+          def finalize
+            @writer.finalize
+            @observer.reset
+          end
+        end
+      end
+    end
+  end
+end

data/lib/schlepp/sink/table_object/filters.rb

@@ -0,0 +1,22 @@
+module Schlepp
+  class Sink
+    class TableObject
+      class Filters
+        def initialize
+          @filters = []
+        end
+
+        def add(filter)
+          @filters << filter
+        end
+
+        def decorate(writer)
+          @filters.reverse.inject(writer) do |base, f| 
+            f.decorate(base) 
+          end
+        end
+      end
+    end
+  end
+end
+

data/lib/schlepp/sink/table_object/stream.rb

@@ -0,0 +1,25 @@
+module Schlepp
+  class Sink
+    class TableObject
+      class Stream
+        def initialize(table_object_writer)
+          @table_object_writer = table_object_writer
+        end
+
+        def write(data)
+          @stream.write(data)
+        end
+
+        def length
+          @stream.length
+        end
+
+        def finalize
+          @stream.finalize
+
+          @table_object_writer.write(@stream.to_s)
+        end
+      end
+    end
+  end
+end

data/lib/schlepp/sink/table_object/writer/factory.rb

@@ -0,0 +1,22 @@
+module Schlepp
+  class Sink
+    class TableObject
+      class Writer
+        class Factory
+          def initialize(factory, components, table_object)
+            @factory = factory
+            @components = components
+            @table_object = table_object
+          end
+
+          def new
+            writer = @factory.writer(@table_object)
+            @components.reverse.inject(writer) {|base, f|
+              f.writer.new(base)
+            }
+          end
+        end
+      end
+    end
+  end
+end

data/lib/schlepp/sink.rb

@@ -1,8 +1,23 @@
-require 'schlepp/sink/loader'
+require 'schlepp/sink/table_object/carosel'
 
 module Schlepp
-  module Sink
+  class Sink
+    def initialize(model, factory, filters)
+      @model = model
+      @carosel = TableObject::Carosel.new(factory, filters)
+    end
 
+    def name
+      @model.name
+    end
+
+    def write(data)
+      @carosel.write(data)
+    end
+
+    def finalize
+      @carosel.finalize
+    end
   end
 end
 

data/lib/schlepp/source/csv.rb

@@ -17,7 +17,7 @@ module Schlepp
 
       def each
         @csv.each do |r|
-          yield r.fields
+          yield [r.fields]
         end
       end
     end

data/lib/schlepp/version.rb

@@ -1,3 +1,3 @@
 module Schlepp
-  VERSION = "0.0.1-alpha.1"
+  VERSION = "0.0.1-alpha.2"
 end

data/test/integration/mock/sink.rb

@@ -0,0 +1,51 @@
+require 'schlepp/sink/filter/chunker'
+
+module Schlepp::IntegrationTest::Mock
+  class Factory
+    def initialize(model, verifier)
+      @model = model
+      @verifier = verifier
+    end
+
+    def url
+      URI("http://www.example.com/")
+    end
+    
+    def model
+      @model
+    end
+
+    def writer(to)
+      Writer.new(to, @verifier)
+    end
+  end
+  class Writer
+    def initialize(table_object, verifier)
+      @table_object = table_object
+      @verifier = verifier
+    end
+
+    def write(data)
+      @verifier.write(data)
+    end
+    def finalize
+      @verifier.finalize
+    end
+  end
+  class Sink
+    def initialize(model, verifier)
+      @model = model
+      @factory = Factory.new(@model, verifier)
+      @sink = Schlepp::Sink.new(@model, @factory, [Schlepp::Sink::Filter::Chunker.new])
+    end
+
+    def write(data)
+      @sink.write(data)
+    end
+
+    def finalize
+      @sink.finalize
+    end
+  end
+end
+

data/test/integration/mock/source.rb

@@ -0,0 +1,9 @@
+module Schlepp::IntegrationTest::Mock
+  class Source
+    def each
+      1.upto(100).each do |x|
+        yield [x.to_s]
+      end
+    end
+  end
+end

data/test/integration/schlepp_test.rb

@@ -3,23 +3,42 @@ require File.expand_path('../test_helper', __FILE__)
 require 'pp'
 require 'schlepp'
 
+$LOAD_PATH.unshift(File.expand_path('../', __FILE__))
+
+
 class Schlepp::IntegrationTest < Test::Unit::TestCase
   setup do
+    @model= Hydrogen::Model.new({
+      :table_name => "foo",
+      :key => "key",
+      :columns => ["baz", "blarg"]
+    })
+
+    @verifier = mock
+
     @mock_writer = mock
+    require 'mock/source'
+    @mock_source = Mock::Source.new
 
-    #@mock_model = mock
-    #@mock_model.expects(:name).at_least_once.returns("mock_models")
+    require 'mock/sink'
+    @mock_sink = Mock::Sink.new(@model, @verifier)
   end
 
-  test 'stuff' do
-    table_object = Hydrogen::TableObject.new(@mock_model)
+  test '.schlepp' do
+    1.upto(100).each do |x|
+      @verifier.expects(:write).with(x.to_s)
+    end
+
+    @verifier.expects(:finalize)
+
+    res = Schlepp.schlepp(@mock_source, @mock_sink)
 
-    loader = Schlepp::Sink::Loader.new(@mock_writer)
+    object = res.first
 
-    @mock_writer.expects(:write)
-    loader.write(["FOO|BAR|BAZ"] * 10)
+    puts res.inspect
 
-    loader.finalize
+    assert_equal object.url, URI("http://www.example.com/.1")
+    assert_equal object.name, "foo"
   end
 end
 

data/test/integration/test_helper.rb

@@ -1,10 +1,17 @@
-require 'coveralls'
-Coveralls.wear!
-
-if ENV["ENABLE_SIMPLE_COV"]
+if ENV["COVERAGE"]
+  require 'coveralls'
+  require 'codeclimate-test-reporter'
   require 'simplecov'
-  require File.expand_path('../../simplecov_helper', __FILE__)
-  SimpleCov.start 'pocketchange'
+  SimpleCov.start do
+    add_group "Lib", "lib"
+    add_filter "/test/"
+    command_name "Integration Tests"
+    formatter SimpleCov::Formatter::MultiFormatter[
+      SimpleCov::Formatter::HTMLFormatter,
+      Coveralls::SimpleCov::Formatter,
+      CodeClimate::TestReporter::Formatter
+    ]
+  end
 end
 
 require 'test/unit'