schlepp 0.0.1.pre.alpha.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/.gitignore +34 -0
- data/.travis.yml +16 -0
- data/CONTRIBUTING.md +33 -0
- data/Gemfile +5 -0
- data/LICENSE +22 -0
- data/README.md +40 -0
- data/Rakefile +32 -0
- data/bin/schlepp.rb +29 -0
- data/lib/schlepp/env.rb +27 -0
- data/lib/schlepp/sink/chunker.rb +30 -0
- data/lib/schlepp/sink/data_object.rb +26 -0
- data/lib/schlepp/sink/data_stream.rb +39 -0
- data/lib/schlepp/sink/loader.rb +35 -0
- data/lib/schlepp/sink/sequencer.rb +42 -0
- data/lib/schlepp/sink.rb +8 -0
- data/lib/schlepp/sinks/fs/chunker.rb +38 -0
- data/lib/schlepp/sinks/fs/sequencer.rb +21 -0
- data/lib/schlepp/sinks/fs/table_object/collection.rb +23 -0
- data/lib/schlepp/sinks/fs/table_object/writer.rb +33 -0
- data/lib/schlepp/sinks/fs/table_object.rb +31 -0
- data/lib/schlepp/sinks/fs.rb +2 -0
- data/lib/schlepp/source/csv.rb +26 -0
- data/lib/schlepp/source.rb +9 -0
- data/lib/schlepp/table_object/chunker.rb +33 -0
- data/lib/schlepp/version.rb +3 -0
- data/lib/schlepp.rb +17 -0
- data/schlepp.gemspec +29 -0
- data/test/integration/schlepp_test.rb +25 -0
- data/test/integration/test_helper.rb +17 -0
- data/test/unit/schlepp/sink/chunker_test.rb +0 -0
- data/test/unit/schlepp/sink/data_object_test.rb +0 -0
- data/test/unit/schlepp/sink/data_stream_test.rb +0 -0
- data/test/unit/schlepp/sink/loader_test.rb +29 -0
- data/test/unit/schlepp/sink/sequencer_test.rb +0 -0
- data/test/unit/schlepp/table_object/chunker_test.rb +0 -0
- data/test/unit/schlepp_test.rb +24 -0
- data/test/unit/test_helper.rb +17 -0
- metadata +190 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: 937caf4f2826b085928376c932e22556cb4fcb77
|
4
|
+
data.tar.gz: f3ac27937abb7605c6fc351751f15610faea6c4d
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: 01d49195f259f64f44caa3e158953ecc914c5289ab1d15e6fc475bfff8f4aa6cafded7162909469280f41afccc01baca46c5ee65e19219563bad71ef1799c5d0
|
7
|
+
data.tar.gz: e503f9af56f2abf60ed774a42cefc31813058d535d97254566d2bf09e6621248c684b9e97b7b36837eb95d3f5afee39d2e72246fffa491f470f5bb139f7c5b06
|
data/.gitignore
ADDED
@@ -0,0 +1,34 @@
|
|
1
|
+
*.gem
|
2
|
+
*.rbc
|
3
|
+
/.config
|
4
|
+
/coverage/
|
5
|
+
/InstalledFiles
|
6
|
+
/pkg/
|
7
|
+
/spec/reports/
|
8
|
+
/test/tmp/
|
9
|
+
/test/version_tmp/
|
10
|
+
/tmp/
|
11
|
+
|
12
|
+
## Specific to RubyMotion:
|
13
|
+
.dat*
|
14
|
+
.repl_history
|
15
|
+
build/
|
16
|
+
|
17
|
+
## Documentation cache and generated files:
|
18
|
+
/.yardoc/
|
19
|
+
/_yardoc/
|
20
|
+
/doc/
|
21
|
+
/rdoc/
|
22
|
+
|
23
|
+
## Environment normalisation:
|
24
|
+
/.bundle/
|
25
|
+
/lib/bundler/man/
|
26
|
+
|
27
|
+
# for a library or gem, you might want to ignore these files since the code is
|
28
|
+
# intended to run in multiple environments; otherwise, check them in:
|
29
|
+
Gemfile.lock
|
30
|
+
.ruby-version
|
31
|
+
.ruby-gemset
|
32
|
+
|
33
|
+
# unless supporting rvm < 1.11.0 or doing something fancy, ignore this:
|
34
|
+
.rvmrc
|
data/.travis.yml
ADDED
data/CONTRIBUTING.md
ADDED
@@ -0,0 +1,33 @@
|
|
1
|
+
Contributing to Quasar
|
2
|
+
========================
|
3
|
+
|
4
|
+
First of all, thanks for your interest in contributing to Quasar!
|
5
|
+
|
6
|
+
Below are a list of guidelines to help keep this project sane in the face of open source entropy.
|
7
|
+
|
8
|
+
How To Submit A Patch
|
9
|
+
---------------------
|
10
|
+
|
11
|
+
Any patch you feel would make this a better project is welcome, including but not limited to: bug fixes, new features, refactoring, and better documentation.
|
12
|
+
|
13
|
+
Make a fork of this project, and hack to your heart's content. Once you have something you think would be great to include into master, feel free to open a pull request. Please keep in mind the following guidelines, which will make it more likely for your pull request to be accepted:
|
14
|
+
|
15
|
+
1. Please have test cases, both in the unit suite and the integration suite, that fail when your code is not present.
|
16
|
+
1. Please open a topic branch in your fork, and create your pull request against that.
|
17
|
+
* Use a separate topic for each feature and/or bug fix you want to submit.
|
18
|
+
1. Be logically consistent in your commits. Make them as small as possible, so they can tell a story the maintainer can understand.
|
19
|
+
1. Keeping a consistent style in the codebase is important for maintainability. When possible, please stick to the guidelines within [this guide](https://github.com/bbatsov/ruby-style-guide).
|
20
|
+
1. Pay attention to things like code climate and test coverage. A drop in these metrics will not necessarily preclude a patch from being accepted, but we'd much prefer to keep these numbers high.
|
21
|
+
|
22
|
+
Please keep in mind your contributions will be licensed the same as the rest of Quasar, under the MIT license.
|
23
|
+
|
24
|
+
It may take the developer a day or two to get to your pull request. If your request has not been responded to in a few days, though, feel free to ping the developer team, preferably through a comment on the pull.
|
25
|
+
|
26
|
+
How To Report An Issue
|
27
|
+
----------------------
|
28
|
+
|
29
|
+
Issue reports are welcome, even if not accompanied by a pull request. Keep in mind that submitting a pull request will mean your issue likely gets resolved sooner, because less work will be left to do.
|
30
|
+
|
31
|
+
The most important thing an issue can do is get a developer as close as possible to seeing what you are seeing, in the case of a bug report. In the case of a feature request, you want to get them to the point where they don't see what you want to see, so they can make a decision about whether and how to include the proposed feature.
|
32
|
+
|
33
|
+
More about good bug reports [here](http://www.chiark.greenend.org.uk/~sgtatham/bugs.html).
|
data/Gemfile
ADDED
data/LICENSE
ADDED
@@ -0,0 +1,22 @@
|
|
1
|
+
Copyright (c) 2014 Ed Carrel
|
2
|
+
|
3
|
+
MIT License
|
4
|
+
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining
|
6
|
+
a copy of this software and associated documentation files (the
|
7
|
+
"Software"), to deal in the Software without restriction, including
|
8
|
+
without limitation the rights to use, copy, modify, merge, publish,
|
9
|
+
distribute, sublicense, and/or sell copies of the Software, and to
|
10
|
+
permit persons to whom the Software is furnished to do so, subject to
|
11
|
+
the following conditions:
|
12
|
+
|
13
|
+
The above copyright notice and this permission notice shall be
|
14
|
+
included in all copies or substantial portions of the Software.
|
15
|
+
|
16
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
17
|
+
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
18
|
+
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
19
|
+
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
20
|
+
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
21
|
+
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
22
|
+
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
data/README.md
ADDED
@@ -0,0 +1,40 @@
|
|
1
|
+
# Schlepp
|
2
|
+
|
3
|
+
An extensible chunking file transferer
|
4
|
+
|
5
|
+
## Installation
|
6
|
+
|
7
|
+
Add this line to your application's Gemfile:
|
8
|
+
|
9
|
+
```ruby
|
10
|
+
gem 'schlepp'
|
11
|
+
```
|
12
|
+
|
13
|
+
And then execute:
|
14
|
+
|
15
|
+
$ bundle
|
16
|
+
|
17
|
+
Or install it yourself as:
|
18
|
+
|
19
|
+
$ gem install schlepp
|
20
|
+
|
21
|
+
## Usage
|
22
|
+
|
23
|
+
TODO: Write usage instructions here
|
24
|
+
|
25
|
+
API Documentation
|
26
|
+
-------------
|
27
|
+
|
28
|
+
See [RubyDoc](http://rubydoc.info/github/azanar/schlepp/index)
|
29
|
+
|
30
|
+
Contributors
|
31
|
+
------------
|
32
|
+
|
33
|
+
See [Contributing](CONTRIBUTING.md) for details.
|
34
|
+
|
35
|
+
License
|
36
|
+
-------
|
37
|
+
|
38
|
+
©2014 Ed Carrel. Released under the MIT License.
|
39
|
+
|
40
|
+
See [License](LICENSE) for details.
|
data/Rakefile
ADDED
@@ -0,0 +1,32 @@
|
|
1
|
+
require "bundler/gem_tasks"
|
2
|
+
|
3
|
+
require 'coveralls/rake/task'
|
4
|
+
|
5
|
+
require 'rake/testtask'
|
6
|
+
|
7
|
+
task :test do
|
8
|
+
Rake::Task['test:unit'].invoke
|
9
|
+
Rake::Task['test:integration'].invoke
|
10
|
+
end
|
11
|
+
|
12
|
+
namespace :test do
|
13
|
+
Rake::TestTask.new("integration") do |t|
|
14
|
+
t.libs << "test"
|
15
|
+
t.libs << "config"
|
16
|
+
t.test_files = FileList['test/integration/**/*_test.rb']
|
17
|
+
t.verbose = true
|
18
|
+
end
|
19
|
+
|
20
|
+
Rake::TestTask.new("unit") do |t|
|
21
|
+
t.libs << "test"
|
22
|
+
t.libs << "config"
|
23
|
+
t.test_files = FileList['test/unit/**/*_test.rb']
|
24
|
+
t.verbose = true
|
25
|
+
end
|
26
|
+
end
|
27
|
+
|
28
|
+
Coveralls::RakeTask.new
|
29
|
+
|
30
|
+
task :test_with_coveralls => ['test:unit', 'test:integration', 'coveralls:push']
|
31
|
+
|
32
|
+
task :default => :test_with_coveralls
|
data/bin/schlepp.rb
ADDED
@@ -0,0 +1,29 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
$LOAD_PATH.push(File.dirname(__FILE__) + '/../lib')
|
3
|
+
|
4
|
+
require 'hydrogen'
|
5
|
+
require 'schlepp/sinks/fs'
|
6
|
+
|
7
|
+
config = {
|
8
|
+
:table_name => 'foo',
|
9
|
+
:source => {:file => 'data.csv'},
|
10
|
+
:columns => %w{foo bar baz}
|
11
|
+
}
|
12
|
+
|
13
|
+
AWS.config(
|
14
|
+
access_key_id: 'ACCESS_KEY_ID',
|
15
|
+
secret_access_key: 'SECRET_ACCESS_KEY',
|
16
|
+
stub_requests: Schlepp.env.test?,
|
17
|
+
)
|
18
|
+
|
19
|
+
model = Hydrogen::Model.new(config)
|
20
|
+
|
21
|
+
to = Hydrogen::TableObject.new(model)
|
22
|
+
|
23
|
+
source = Schlepp::Source::CSV.new(File.new('data.csv','r'))
|
24
|
+
|
25
|
+
l = Schlepp::Sink::Fs::Sequencer.new(to, :chunk_size => 40000)
|
26
|
+
|
27
|
+
res = Schlepp.schlepp(source, l)
|
28
|
+
|
29
|
+
puts res.inspect
|
data/lib/schlepp/env.rb
ADDED
@@ -0,0 +1,27 @@
|
|
1
|
+
require 'active_support/string_inquirer'
|
2
|
+
|
3
|
+
module Schlepp
|
4
|
+
module_function
|
5
|
+
def logger
|
6
|
+
return @logger if @logger
|
7
|
+
|
8
|
+
@logger = Logger.new(STDOUT)
|
9
|
+
@logger.formatter = proc { |severity, datetime, progname, msg|
|
10
|
+
"[#{datetime}, #{severity}] #{msg}\n"
|
11
|
+
}
|
12
|
+
@logger
|
13
|
+
end
|
14
|
+
|
15
|
+
def logger=(logger)
|
16
|
+
@logger = logger
|
17
|
+
end
|
18
|
+
|
19
|
+
def env
|
20
|
+
@_env ||= ActiveSupport::StringInquirer.new(ENV["QUASAR_ENV"] || ENV["RAILS_ENV"] || "development")
|
21
|
+
end
|
22
|
+
|
23
|
+
def env=(environment)
|
24
|
+
@_env = ActiveSupport::StringInquirer.new(environment)
|
25
|
+
end
|
26
|
+
|
27
|
+
end
|
@@ -0,0 +1,30 @@
|
|
1
|
+
require 'schlepp/sink/loader'
|
2
|
+
require 'schlepp/table_object/chunker'
|
3
|
+
|
4
|
+
module Schlepp
|
5
|
+
module Sink
|
6
|
+
class Chunker
|
7
|
+
def initialize(table_object, opts = {})
|
8
|
+
@table_object = table_object
|
9
|
+
end
|
10
|
+
|
11
|
+
def sequence
|
12
|
+
chunker.sequence
|
13
|
+
end
|
14
|
+
|
15
|
+
def parts
|
16
|
+
chunker.parts
|
17
|
+
end
|
18
|
+
|
19
|
+
def next
|
20
|
+
chunker.next
|
21
|
+
end
|
22
|
+
|
23
|
+
private
|
24
|
+
|
25
|
+
def chunker
|
26
|
+
@chunker ||= Schlepp::TableObject::Chunker.new(@table_object)
|
27
|
+
end
|
28
|
+
end
|
29
|
+
end
|
30
|
+
end
|
@@ -0,0 +1,26 @@
|
|
1
|
+
require 'schlepp/sink/data_stream'
|
2
|
+
|
3
|
+
module Schlepp
|
4
|
+
module Sink
|
5
|
+
class DataObject
|
6
|
+
def initialize(table_object)
|
7
|
+
@stream = DataStream.new
|
8
|
+
@table_object = table_object
|
9
|
+
end
|
10
|
+
|
11
|
+
def write(data)
|
12
|
+
@stream.write(data)
|
13
|
+
end
|
14
|
+
|
15
|
+
def length
|
16
|
+
@stream.length
|
17
|
+
end
|
18
|
+
|
19
|
+
def finalize
|
20
|
+
@stream.finalize
|
21
|
+
|
22
|
+
@table_object.write(@stream.to_s)
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
@@ -0,0 +1,39 @@
|
|
1
|
+
require 'zlib'
|
2
|
+
|
3
|
+
module Schlepp
|
4
|
+
module Sink
|
5
|
+
class DataStream
|
6
|
+
def initialize
|
7
|
+
@buffer = StringIO.new("", "rb+")
|
8
|
+
@compressor = Zlib::GzipWriter.new(@buffer)
|
9
|
+
@dead = false
|
10
|
+
end
|
11
|
+
|
12
|
+
def write(data)
|
13
|
+
if @dead
|
14
|
+
raise "Stream has been dumped. No more writing permitted."
|
15
|
+
end
|
16
|
+
@compressor << data
|
17
|
+
end
|
18
|
+
|
19
|
+
def length
|
20
|
+
if @dead
|
21
|
+
raise "Stream has been dumped. No more writing permitted."
|
22
|
+
end
|
23
|
+
@compressor.pos
|
24
|
+
end
|
25
|
+
|
26
|
+
def finalize
|
27
|
+
if !@dead
|
28
|
+
@compressor.close
|
29
|
+
@dead = true
|
30
|
+
end
|
31
|
+
end
|
32
|
+
|
33
|
+
def to_s
|
34
|
+
finalize
|
35
|
+
@buffer.string
|
36
|
+
end
|
37
|
+
end
|
38
|
+
end
|
39
|
+
end
|
@@ -0,0 +1,35 @@
|
|
1
|
+
require 'schlepp/sink/data_object'
|
2
|
+
|
3
|
+
module Schlepp
|
4
|
+
module Sink
|
5
|
+
class Loader
|
6
|
+
def initialize(writer, opts = {})
|
7
|
+
@writer = writer
|
8
|
+
|
9
|
+
@file = DataObject.new(writer)
|
10
|
+
end
|
11
|
+
|
12
|
+
def name
|
13
|
+
@writer.name
|
14
|
+
end
|
15
|
+
|
16
|
+
def path
|
17
|
+
@writer.path
|
18
|
+
end
|
19
|
+
|
20
|
+
def write(rows)
|
21
|
+
Array(rows).each do |row|
|
22
|
+
@file.write(row)
|
23
|
+
end
|
24
|
+
end
|
25
|
+
|
26
|
+
def written
|
27
|
+
@file.length
|
28
|
+
end
|
29
|
+
|
30
|
+
def finalize
|
31
|
+
@file.finalize
|
32
|
+
end
|
33
|
+
end
|
34
|
+
end
|
35
|
+
end
|
@@ -0,0 +1,42 @@
|
|
1
|
+
module Schlepp
|
2
|
+
module Sink
|
3
|
+
module Sequencer
|
4
|
+
DEFAULT_CHUNK_SIZE = 50 * 1024 * 1024 # 50 MB
|
5
|
+
def initialize(table_object, opts = {})
|
6
|
+
@table_object = table_object
|
7
|
+
@chunker = chunker.new(table_object)
|
8
|
+
@chunk_size = opts[:chunk_size] || DEFAULT_CHUNK_SIZE
|
9
|
+
end
|
10
|
+
|
11
|
+
def write(rows)
|
12
|
+
rows.each do |row|
|
13
|
+
if loader.written + row.length > @chunk_size
|
14
|
+
rotate
|
15
|
+
end
|
16
|
+
loader.write(row)
|
17
|
+
end
|
18
|
+
end
|
19
|
+
|
20
|
+
def parts
|
21
|
+
@chunker.parts
|
22
|
+
end
|
23
|
+
|
24
|
+
def finalize
|
25
|
+
loader.finalize
|
26
|
+
end
|
27
|
+
|
28
|
+
private
|
29
|
+
|
30
|
+
def loader
|
31
|
+
@loader ||= rotate
|
32
|
+
end
|
33
|
+
|
34
|
+
def rotate
|
35
|
+
if @loader
|
36
|
+
@loader.finalize
|
37
|
+
end
|
38
|
+
@loader = @chunker.next
|
39
|
+
end
|
40
|
+
end
|
41
|
+
end
|
42
|
+
end
|
data/lib/schlepp/sink.rb
ADDED
@@ -0,0 +1,38 @@
|
|
1
|
+
require 'schlepp/sink/chunker'
|
2
|
+
|
3
|
+
require 'schlepp/sinks/fs/table_object'
|
4
|
+
require 'schlepp/sinks/fs/table_object/writer'
|
5
|
+
|
6
|
+
module Schlepp
|
7
|
+
module Sink
|
8
|
+
module Fs
|
9
|
+
class Chunker
|
10
|
+
def initialize(table_object, opts = {})
|
11
|
+
@table_object = table_object
|
12
|
+
end
|
13
|
+
|
14
|
+
def sequence
|
15
|
+
chunker.sequence
|
16
|
+
end
|
17
|
+
|
18
|
+
def parts
|
19
|
+
chunker.parts
|
20
|
+
end
|
21
|
+
|
22
|
+
def next
|
23
|
+
part = chunker.next
|
24
|
+
|
25
|
+
ts = Schlepp::Sink::Fs::TableObject.new(part)
|
26
|
+
|
27
|
+
Schlepp::Sink::Loader.new(ts)
|
28
|
+
end
|
29
|
+
|
30
|
+
private
|
31
|
+
|
32
|
+
def chunker
|
33
|
+
@chunker ||= Schlepp::Sink::Chunker.new(@table_object)
|
34
|
+
end
|
35
|
+
end
|
36
|
+
end
|
37
|
+
end
|
38
|
+
end
|
@@ -0,0 +1,21 @@
|
|
1
|
+
require 'schlepp/sink/sequencer'
|
2
|
+
require 'schlepp/sinks/fs/chunker'
|
3
|
+
require 'schlepp/sinks/fs/table_object/collection'
|
4
|
+
|
5
|
+
module Schlepp
|
6
|
+
module Sink
|
7
|
+
module Fs
|
8
|
+
class Sequencer
|
9
|
+
include Schlepp::Sink::Sequencer
|
10
|
+
|
11
|
+
def collection
|
12
|
+
Schlepp::Sink::Fs::TableObject::Collection
|
13
|
+
end
|
14
|
+
|
15
|
+
def chunker
|
16
|
+
Schlepp::Sink::Fs::Chunker
|
17
|
+
end
|
18
|
+
end
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
@@ -0,0 +1,23 @@
|
|
1
|
+
require 'hydrogen/table_object/collection'
|
2
|
+
|
3
|
+
module Schlepp
|
4
|
+
module Sink
|
5
|
+
module Fs
|
6
|
+
class TableObject
|
7
|
+
class Collection
|
8
|
+
include Hydrogen::TableObject::Collection
|
9
|
+
|
10
|
+
def url
|
11
|
+
"file:///#{path}"
|
12
|
+
end
|
13
|
+
|
14
|
+
def urls
|
15
|
+
parts.map do |p|
|
16
|
+
TableObject.new(p).url
|
17
|
+
end
|
18
|
+
end
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
23
|
+
end
|
@@ -0,0 +1,33 @@
|
|
1
|
+
module Schlepp
|
2
|
+
module Sink
|
3
|
+
module Fs
|
4
|
+
class TableObject
|
5
|
+
class Writer
|
6
|
+
def initialize(table_object, opts = {})
|
7
|
+
@table_object = table_object
|
8
|
+
@written = 0
|
9
|
+
end
|
10
|
+
|
11
|
+
def file
|
12
|
+
@file ||= File.new("/tmp/#{@table_object.path}", 'w')
|
13
|
+
end
|
14
|
+
|
15
|
+
def write(rows)
|
16
|
+
Array(rows).each do |row|
|
17
|
+
@written += row.length
|
18
|
+
file.write(row)
|
19
|
+
end
|
20
|
+
end
|
21
|
+
|
22
|
+
def written
|
23
|
+
@written
|
24
|
+
end
|
25
|
+
|
26
|
+
def finalize
|
27
|
+
@file.close
|
28
|
+
end
|
29
|
+
end
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|
33
|
+
end
|
@@ -0,0 +1,31 @@
|
|
1
|
+
module Schlepp
|
2
|
+
module Sink
|
3
|
+
module Fs
|
4
|
+
class TableObject
|
5
|
+
def initialize(table_object, opts = {})
|
6
|
+
@table_object = table_object
|
7
|
+
@written = 0
|
8
|
+
end
|
9
|
+
|
10
|
+
def write(rows)
|
11
|
+
Array(rows).each do |row|
|
12
|
+
@written += row.length
|
13
|
+
file.write(row)
|
14
|
+
end
|
15
|
+
end
|
16
|
+
|
17
|
+
def written
|
18
|
+
@written
|
19
|
+
end
|
20
|
+
|
21
|
+
def finalize
|
22
|
+
@file.close
|
23
|
+
end
|
24
|
+
|
25
|
+
def file
|
26
|
+
@file ||= File.new("/tmp/#{@table_object.path}", 'w')
|
27
|
+
end
|
28
|
+
end
|
29
|
+
end
|
30
|
+
end
|
31
|
+
end
|
@@ -0,0 +1,26 @@
|
|
1
|
+
require 'csv'
|
2
|
+
|
3
|
+
require 'schlepp/source'
|
4
|
+
|
5
|
+
module Schlepp
|
6
|
+
module Source
|
7
|
+
class CSV
|
8
|
+
def initialize(fp)
|
9
|
+
@csv = ::CSV.new(fp, {:headers => true})
|
10
|
+
|
11
|
+
@csv.shift
|
12
|
+
end
|
13
|
+
|
14
|
+
def columns
|
15
|
+
@csv.headers
|
16
|
+
end
|
17
|
+
|
18
|
+
def each
|
19
|
+
@csv.each do |r|
|
20
|
+
yield r.fields
|
21
|
+
end
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
26
|
+
|
@@ -0,0 +1,33 @@
|
|
1
|
+
require 'hydrogen/table_object/part'
|
2
|
+
require 'hydrogen/table_object/part/sequence'
|
3
|
+
|
4
|
+
module Schlepp
|
5
|
+
class TableObject
|
6
|
+
class Chunker
|
7
|
+
def initialize(table_object, opts = {})
|
8
|
+
@table_object = table_object
|
9
|
+
@chunk = 0
|
10
|
+
@parts = []
|
11
|
+
end
|
12
|
+
|
13
|
+
attr_reader :parts
|
14
|
+
|
15
|
+
def sequence
|
16
|
+
Hydrogen::TableObject::Part::Sequence.new(@table_object, parts)
|
17
|
+
end
|
18
|
+
|
19
|
+
def parts
|
20
|
+
@parts
|
21
|
+
end
|
22
|
+
|
23
|
+
def next
|
24
|
+
@chunk += 1
|
25
|
+
part = Hydrogen::TableObject::Part.new(@chunk, @table_object)
|
26
|
+
|
27
|
+
@parts << part
|
28
|
+
|
29
|
+
part
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|
33
|
+
end
|
data/lib/schlepp.rb
ADDED
@@ -0,0 +1,17 @@
|
|
1
|
+
require 'hydrogen'
|
2
|
+
|
3
|
+
require "schlepp/env"
|
4
|
+
require "schlepp/sink"
|
5
|
+
require "schlepp/source/csv"
|
6
|
+
require "schlepp/version"
|
7
|
+
|
8
|
+
module Schlepp
|
9
|
+
module_function
|
10
|
+
|
11
|
+
def schlepp(source, sink)
|
12
|
+
source.each do |s|
|
13
|
+
sink.write(s)
|
14
|
+
end
|
15
|
+
sink.finalize
|
16
|
+
end
|
17
|
+
end
|
data/schlepp.gemspec
ADDED
@@ -0,0 +1,29 @@
|
|
1
|
+
# coding: utf-8
|
2
|
+
lib = File.expand_path('../lib', __FILE__)
|
3
|
+
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
4
|
+
require 'schlepp/version'
|
5
|
+
|
6
|
+
Gem::Specification.new do |spec|
|
7
|
+
spec.name = "schlepp"
|
8
|
+
spec.version = Schlepp::VERSION
|
9
|
+
spec.authors = ["Ed Carrel"]
|
10
|
+
spec.email = ["edward@carrel.org"]
|
11
|
+
spec.summary = %q{An extensible chunking file transferer}
|
12
|
+
spec.description = %q{An extensible chunking file transferer}
|
13
|
+
spec.homepage = "https://github.com/azanar/schlepp"
|
14
|
+
spec.license = "MIT"
|
15
|
+
|
16
|
+
spec.files = `git ls-files -z`.split("\x0")
|
17
|
+
spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
|
18
|
+
spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
|
19
|
+
spec.require_paths = ["lib"]
|
20
|
+
|
21
|
+
spec.add_runtime_dependency 'activesupport', '~> 4'
|
22
|
+
spec.add_runtime_dependency 'hydrogen', '~> 0'
|
23
|
+
|
24
|
+
spec.add_development_dependency 'test-unit', '~> 3'
|
25
|
+
spec.add_development_dependency 'mocha', '~> 1'
|
26
|
+
spec.add_development_dependency 'simplecov', '~> 0'
|
27
|
+
spec.add_development_dependency "bundler", "~> 1.6"
|
28
|
+
spec.add_development_dependency "rake", "~> 10.0"
|
29
|
+
end
|
@@ -0,0 +1,25 @@
|
|
1
|
+
require File.expand_path('../test_helper', __FILE__)
|
2
|
+
|
3
|
+
require 'pp'
|
4
|
+
require 'schlepp'
|
5
|
+
|
6
|
+
class Schlepp::IntegrationTest < Test::Unit::TestCase
|
7
|
+
setup do
|
8
|
+
@mock_writer = mock
|
9
|
+
|
10
|
+
#@mock_model = mock
|
11
|
+
#@mock_model.expects(:name).at_least_once.returns("mock_models")
|
12
|
+
end
|
13
|
+
|
14
|
+
test 'stuff' do
|
15
|
+
table_object = Hydrogen::TableObject.new(@mock_model)
|
16
|
+
|
17
|
+
loader = Schlepp::Sink::Loader.new(@mock_writer)
|
18
|
+
|
19
|
+
@mock_writer.expects(:write)
|
20
|
+
loader.write(["FOO|BAR|BAZ"] * 10)
|
21
|
+
|
22
|
+
loader.finalize
|
23
|
+
end
|
24
|
+
end
|
25
|
+
|
@@ -0,0 +1,17 @@
|
|
1
|
+
require 'coveralls'
|
2
|
+
Coveralls.wear!
|
3
|
+
|
4
|
+
if ENV["ENABLE_SIMPLE_COV"]
|
5
|
+
require 'simplecov'
|
6
|
+
require File.expand_path('../../simplecov_helper', __FILE__)
|
7
|
+
SimpleCov.start 'pocketchange'
|
8
|
+
end
|
9
|
+
|
10
|
+
require 'test/unit'
|
11
|
+
|
12
|
+
ENV["QUASAR_ENV"] = "test"
|
13
|
+
|
14
|
+
require 'mocha/setup'
|
15
|
+
|
16
|
+
module TestHelper
|
17
|
+
end
|
File without changes
|
File without changes
|
File without changes
|
@@ -0,0 +1,29 @@
|
|
1
|
+
require File.expand_path('../../../test_helper', __FILE__)
|
2
|
+
|
3
|
+
require 'schlepp/sink/loader'
|
4
|
+
require 'schlepp/sink/data_object'
|
5
|
+
|
6
|
+
class Schlepp::Sink::LoaderTest < Test::Unit::TestCase
|
7
|
+
include TestHelper
|
8
|
+
setup do
|
9
|
+
@mock_table_object = mock
|
10
|
+
|
11
|
+
@mock_data_object = mock
|
12
|
+
Schlepp::Sink::DataObject.expects(:new).with(@mock_table_object).returns(@mock_data_object)
|
13
|
+
|
14
|
+
@s3_loader = Schlepp::Sink::Loader.new(@mock_table_object)
|
15
|
+
end
|
16
|
+
|
17
|
+
test '#write' do
|
18
|
+
@mock_data_object.expects(:write).with("FOO|BAR|BAZ")
|
19
|
+
@s3_loader.write(["FOO|BAR|BAZ"])
|
20
|
+
end
|
21
|
+
|
22
|
+
test '#finalize' do
|
23
|
+
@mock_data_object.expects(:finalize)
|
24
|
+
|
25
|
+
@s3_loader.finalize
|
26
|
+
end
|
27
|
+
|
28
|
+
end
|
29
|
+
|
File without changes
|
File without changes
|
@@ -0,0 +1,24 @@
|
|
1
|
+
require File.expand_path('../test_helper', __FILE__)
|
2
|
+
|
3
|
+
require 'schlepp'
|
4
|
+
|
5
|
+
class SchleppTest < Test::Unit::TestCase
|
6
|
+
include TestHelper
|
7
|
+
test '.schlepp' do
|
8
|
+
table_object = Hydrogen::TableObject.new(@mock_model)
|
9
|
+
|
10
|
+
mock_source = mock
|
11
|
+
|
12
|
+
mock_source.expects(:each).multiple_yields("foo", "bar", "baz")
|
13
|
+
|
14
|
+
mock_sink = mock
|
15
|
+
|
16
|
+
mock_sink.expects(:write).with("foo")
|
17
|
+
mock_sink.expects(:write).with("bar")
|
18
|
+
mock_sink.expects(:write).with("baz")
|
19
|
+
|
20
|
+
mock_sink.expects(:finalize)
|
21
|
+
|
22
|
+
Schlepp.schlepp(mock_source, mock_sink)
|
23
|
+
end
|
24
|
+
end
|
@@ -0,0 +1,17 @@
|
|
1
|
+
require 'coveralls'
|
2
|
+
Coveralls.wear!
|
3
|
+
|
4
|
+
if ENV["ENABLE_SIMPLE_COV"]
|
5
|
+
require 'simplecov'
|
6
|
+
require File.expand_path('../../simplecov_helper', __FILE__)
|
7
|
+
SimpleCov.start 'pocketchange'
|
8
|
+
end
|
9
|
+
|
10
|
+
require 'test/unit'
|
11
|
+
|
12
|
+
ENV["QUASAR_ENV"] = "test"
|
13
|
+
|
14
|
+
require 'mocha/setup'
|
15
|
+
|
16
|
+
module TestHelper
|
17
|
+
end
|
metadata
ADDED
@@ -0,0 +1,190 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: schlepp
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.0.1.pre.alpha.1
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Ed Carrel
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
date: 2014-09-21 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: activesupport
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - "~>"
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: '4'
|
20
|
+
type: :runtime
|
21
|
+
prerelease: false
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - "~>"
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: '4'
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: hydrogen
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - "~>"
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: '0'
|
34
|
+
type: :runtime
|
35
|
+
prerelease: false
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - "~>"
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: '0'
|
41
|
+
- !ruby/object:Gem::Dependency
|
42
|
+
name: test-unit
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
44
|
+
requirements:
|
45
|
+
- - "~>"
|
46
|
+
- !ruby/object:Gem::Version
|
47
|
+
version: '3'
|
48
|
+
type: :development
|
49
|
+
prerelease: false
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
51
|
+
requirements:
|
52
|
+
- - "~>"
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: '3'
|
55
|
+
- !ruby/object:Gem::Dependency
|
56
|
+
name: mocha
|
57
|
+
requirement: !ruby/object:Gem::Requirement
|
58
|
+
requirements:
|
59
|
+
- - "~>"
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
version: '1'
|
62
|
+
type: :development
|
63
|
+
prerelease: false
|
64
|
+
version_requirements: !ruby/object:Gem::Requirement
|
65
|
+
requirements:
|
66
|
+
- - "~>"
|
67
|
+
- !ruby/object:Gem::Version
|
68
|
+
version: '1'
|
69
|
+
- !ruby/object:Gem::Dependency
|
70
|
+
name: simplecov
|
71
|
+
requirement: !ruby/object:Gem::Requirement
|
72
|
+
requirements:
|
73
|
+
- - "~>"
|
74
|
+
- !ruby/object:Gem::Version
|
75
|
+
version: '0'
|
76
|
+
type: :development
|
77
|
+
prerelease: false
|
78
|
+
version_requirements: !ruby/object:Gem::Requirement
|
79
|
+
requirements:
|
80
|
+
- - "~>"
|
81
|
+
- !ruby/object:Gem::Version
|
82
|
+
version: '0'
|
83
|
+
- !ruby/object:Gem::Dependency
|
84
|
+
name: bundler
|
85
|
+
requirement: !ruby/object:Gem::Requirement
|
86
|
+
requirements:
|
87
|
+
- - "~>"
|
88
|
+
- !ruby/object:Gem::Version
|
89
|
+
version: '1.6'
|
90
|
+
type: :development
|
91
|
+
prerelease: false
|
92
|
+
version_requirements: !ruby/object:Gem::Requirement
|
93
|
+
requirements:
|
94
|
+
- - "~>"
|
95
|
+
- !ruby/object:Gem::Version
|
96
|
+
version: '1.6'
|
97
|
+
- !ruby/object:Gem::Dependency
|
98
|
+
name: rake
|
99
|
+
requirement: !ruby/object:Gem::Requirement
|
100
|
+
requirements:
|
101
|
+
- - "~>"
|
102
|
+
- !ruby/object:Gem::Version
|
103
|
+
version: '10.0'
|
104
|
+
type: :development
|
105
|
+
prerelease: false
|
106
|
+
version_requirements: !ruby/object:Gem::Requirement
|
107
|
+
requirements:
|
108
|
+
- - "~>"
|
109
|
+
- !ruby/object:Gem::Version
|
110
|
+
version: '10.0'
|
111
|
+
description: An extensible chunking file transferer
|
112
|
+
email:
|
113
|
+
- edward@carrel.org
|
114
|
+
executables:
|
115
|
+
- schlepp.rb
|
116
|
+
extensions: []
|
117
|
+
extra_rdoc_files: []
|
118
|
+
files:
|
119
|
+
- ".gitignore"
|
120
|
+
- ".travis.yml"
|
121
|
+
- CONTRIBUTING.md
|
122
|
+
- Gemfile
|
123
|
+
- LICENSE
|
124
|
+
- README.md
|
125
|
+
- Rakefile
|
126
|
+
- bin/schlepp.rb
|
127
|
+
- lib/schlepp.rb
|
128
|
+
- lib/schlepp/env.rb
|
129
|
+
- lib/schlepp/sink.rb
|
130
|
+
- lib/schlepp/sink/chunker.rb
|
131
|
+
- lib/schlepp/sink/data_object.rb
|
132
|
+
- lib/schlepp/sink/data_stream.rb
|
133
|
+
- lib/schlepp/sink/loader.rb
|
134
|
+
- lib/schlepp/sink/sequencer.rb
|
135
|
+
- lib/schlepp/sinks/fs.rb
|
136
|
+
- lib/schlepp/sinks/fs/chunker.rb
|
137
|
+
- lib/schlepp/sinks/fs/sequencer.rb
|
138
|
+
- lib/schlepp/sinks/fs/table_object.rb
|
139
|
+
- lib/schlepp/sinks/fs/table_object/collection.rb
|
140
|
+
- lib/schlepp/sinks/fs/table_object/writer.rb
|
141
|
+
- lib/schlepp/source.rb
|
142
|
+
- lib/schlepp/source/csv.rb
|
143
|
+
- lib/schlepp/table_object/chunker.rb
|
144
|
+
- lib/schlepp/version.rb
|
145
|
+
- schlepp.gemspec
|
146
|
+
- test/integration/schlepp_test.rb
|
147
|
+
- test/integration/test_helper.rb
|
148
|
+
- test/unit/schlepp/sink/chunker_test.rb
|
149
|
+
- test/unit/schlepp/sink/data_object_test.rb
|
150
|
+
- test/unit/schlepp/sink/data_stream_test.rb
|
151
|
+
- test/unit/schlepp/sink/loader_test.rb
|
152
|
+
- test/unit/schlepp/sink/sequencer_test.rb
|
153
|
+
- test/unit/schlepp/table_object/chunker_test.rb
|
154
|
+
- test/unit/schlepp_test.rb
|
155
|
+
- test/unit/test_helper.rb
|
156
|
+
homepage: https://github.com/azanar/schlepp
|
157
|
+
licenses:
|
158
|
+
- MIT
|
159
|
+
metadata: {}
|
160
|
+
post_install_message:
|
161
|
+
rdoc_options: []
|
162
|
+
require_paths:
|
163
|
+
- lib
|
164
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
165
|
+
requirements:
|
166
|
+
- - ">="
|
167
|
+
- !ruby/object:Gem::Version
|
168
|
+
version: '0'
|
169
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
170
|
+
requirements:
|
171
|
+
- - ">"
|
172
|
+
- !ruby/object:Gem::Version
|
173
|
+
version: 1.3.1
|
174
|
+
requirements: []
|
175
|
+
rubyforge_project:
|
176
|
+
rubygems_version: 2.2.2
|
177
|
+
signing_key:
|
178
|
+
specification_version: 4
|
179
|
+
summary: An extensible chunking file transferer
|
180
|
+
test_files:
|
181
|
+
- test/integration/schlepp_test.rb
|
182
|
+
- test/integration/test_helper.rb
|
183
|
+
- test/unit/schlepp/sink/chunker_test.rb
|
184
|
+
- test/unit/schlepp/sink/data_object_test.rb
|
185
|
+
- test/unit/schlepp/sink/data_stream_test.rb
|
186
|
+
- test/unit/schlepp/sink/loader_test.rb
|
187
|
+
- test/unit/schlepp/sink/sequencer_test.rb
|
188
|
+
- test/unit/schlepp/table_object/chunker_test.rb
|
189
|
+
- test/unit/schlepp_test.rb
|
190
|
+
- test/unit/test_helper.rb
|