metacrunch 2.2.1 → 2.2.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/{bin → exe}/metacrunch +0 -0
- data/lib/metacrunch/version.rb +1 -1
- data/metacrunch.gemspec +24 -27
- metadata +7 -53
- data/spec/assets/archive.tar +0 -0
- data/spec/assets/archive.tar.gz +0 -0
- data/spec/assets/file/some_file +0 -1
- data/spec/assets/file/some_file.gz +0 -0
- data/spec/assets/file/tar_archive.tar +0 -0
- data/spec/assets/file/tar_gz_archive.tar.gz +0 -0
- data/spec/assets/file/tgz_archive.tgz +0 -0
- data/spec/assets/file/zip_archive.zip +0 -0
- data/spec/assets/regular_file.txt +0 -1
- data/spec/assets/regular_file.txt.gz +0 -0
- data/spec/assets/sql_lite_config.rb +0 -17
- data/spec/file_reader_spec.rb +0 -73
- data/spec/file_writer_spec.rb +0 -46
- data/spec/metacrunch/file/reader/plain_file_reader_spec.rb +0 -11
- data/spec/metacrunch/file/reader/tar_reader_spec.rb +0 -15
- data/spec/metacrunch/file/reader/zip_reader_spec.rb +0 -13
- data/spec/metacrunch/file/reader_spec.rb +0 -47
- data/spec/metacrunch/file_spec.rb +0 -15
- data/spec/metacrunch/hash_spec.rb +0 -93
- data/spec/metacrunch/transformator/transformation_spec.rb +0 -28
- data/spec/snr_spec.rb +0 -73
- data/spec/spec_helper.rb +0 -43
- data/spec/tar_writer_spec.rb +0 -33
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA1:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 897ee8027975194e69d06b916f4602317eddf075
|
|
4
|
+
data.tar.gz: 31311e170a4a84e6968e14bd189c36fc1cb8890f
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: a92ccefea01550a5f664f744c760092a4e6165480684d3eba6833a9e62b74f638f1e5f588a6b666a1be238705a59f9dd1a0f3ee2afe13bfea404428613298dce
|
|
7
|
+
data.tar.gz: cf6854080223578ff1819a4c802257b6a7fcdc74dcd2aa4c91df3adb90c2492d601b0d71bcce5c49fc318d4005eb02dda30110ced80cb89a8bd808f234450b1d
|
data/{bin → exe}/metacrunch
RENAMED
|
File without changes
|
data/lib/metacrunch/version.rb
CHANGED
data/metacrunch.gemspec
CHANGED
|
@@ -1,31 +1,28 @@
|
|
|
1
|
-
|
|
1
|
+
# coding: utf-8
|
|
2
|
+
lib = File.expand_path("../lib", __FILE__)
|
|
3
|
+
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
|
4
|
+
require "metacrunch/version"
|
|
2
5
|
|
|
3
|
-
Gem::Specification.new do |
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
6
|
+
Gem::Specification.new do |spec|
|
|
7
|
+
spec.name = "metacrunch"
|
|
8
|
+
spec.version = Metacrunch::VERSION
|
|
9
|
+
spec.authors = ["René Sprotte", "Michael Sievers", "Marcel Otto"]
|
|
10
|
+
spec.summary = %q{Data processing toolkit for Ruby}
|
|
11
|
+
spec.homepage = "http://github.com/ubpb/metacrunch"
|
|
12
|
+
spec.license = "MIT"
|
|
10
13
|
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
s.require_paths = ["lib"]
|
|
16
|
-
s.version = Metacrunch::VERSION
|
|
17
|
-
s.bindir = "bin"
|
|
18
|
-
s.executables = ["metacrunch"]
|
|
14
|
+
spec.files = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
|
|
15
|
+
spec.bindir = "exe"
|
|
16
|
+
spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
|
|
17
|
+
spec.require_paths = ["lib"]
|
|
19
18
|
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
s.add_dependency "thor", "~> 0.19"
|
|
30
|
-
s.add_dependency "ox", "~> 2.1"
|
|
19
|
+
spec.add_dependency "activesupport", "~> 4.2", ">= 4.2.0"
|
|
20
|
+
spec.add_dependency "builder", "~> 3.2", ">= 3.2.2"
|
|
21
|
+
spec.add_dependency "highline", "~> 1.7"
|
|
22
|
+
spec.add_dependency "net-scp", "~> 1.2"
|
|
23
|
+
spec.add_dependency "net-ssh", "~> 2.9"
|
|
24
|
+
spec.add_dependency "parallel", "~> 1.6", ">= 1.6.0"
|
|
25
|
+
spec.add_dependency "rubyzip", ">= 1.0.0"
|
|
26
|
+
spec.add_dependency "thor", "~> 0.19"
|
|
27
|
+
spec.add_dependency "ox", "~> 2.1"
|
|
31
28
|
end
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: metacrunch
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 2.2.
|
|
4
|
+
version: 2.2.2
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- René Sprotte
|
|
8
8
|
- Michael Sievers
|
|
9
9
|
- Marcel Otto
|
|
10
10
|
autorequire:
|
|
11
|
-
bindir:
|
|
11
|
+
bindir: exe
|
|
12
12
|
cert_chain: []
|
|
13
13
|
date: 2015-09-25 00:00:00.000000000 Z
|
|
14
14
|
dependencies:
|
|
@@ -156,8 +156,8 @@ dependencies:
|
|
|
156
156
|
- - "~>"
|
|
157
157
|
- !ruby/object:Gem::Version
|
|
158
158
|
version: '2.1'
|
|
159
|
-
description:
|
|
160
|
-
email:
|
|
159
|
+
description:
|
|
160
|
+
email:
|
|
161
161
|
executables:
|
|
162
162
|
- metacrunch
|
|
163
163
|
extensions: []
|
|
@@ -171,7 +171,7 @@ files:
|
|
|
171
171
|
- Rakefile
|
|
172
172
|
- Readme.md
|
|
173
173
|
- bin/console
|
|
174
|
-
-
|
|
174
|
+
- exe/metacrunch
|
|
175
175
|
- lib/metacrunch.rb
|
|
176
176
|
- lib/metacrunch/cli.rb
|
|
177
177
|
- lib/metacrunch/cli/base.rb
|
|
@@ -204,29 +204,6 @@ files:
|
|
|
204
204
|
- lib/metacrunch/transformer/step.rb
|
|
205
205
|
- lib/metacrunch/version.rb
|
|
206
206
|
- metacrunch.gemspec
|
|
207
|
-
- spec/assets/archive.tar
|
|
208
|
-
- spec/assets/archive.tar.gz
|
|
209
|
-
- spec/assets/file/some_file
|
|
210
|
-
- spec/assets/file/some_file.gz
|
|
211
|
-
- spec/assets/file/tar_archive.tar
|
|
212
|
-
- spec/assets/file/tar_gz_archive.tar.gz
|
|
213
|
-
- spec/assets/file/tgz_archive.tgz
|
|
214
|
-
- spec/assets/file/zip_archive.zip
|
|
215
|
-
- spec/assets/regular_file.txt
|
|
216
|
-
- spec/assets/regular_file.txt.gz
|
|
217
|
-
- spec/assets/sql_lite_config.rb
|
|
218
|
-
- spec/file_reader_spec.rb
|
|
219
|
-
- spec/file_writer_spec.rb
|
|
220
|
-
- spec/metacrunch/file/reader/plain_file_reader_spec.rb
|
|
221
|
-
- spec/metacrunch/file/reader/tar_reader_spec.rb
|
|
222
|
-
- spec/metacrunch/file/reader/zip_reader_spec.rb
|
|
223
|
-
- spec/metacrunch/file/reader_spec.rb
|
|
224
|
-
- spec/metacrunch/file_spec.rb
|
|
225
|
-
- spec/metacrunch/hash_spec.rb
|
|
226
|
-
- spec/metacrunch/transformator/transformation_spec.rb
|
|
227
|
-
- spec/snr_spec.rb
|
|
228
|
-
- spec/spec_helper.rb
|
|
229
|
-
- spec/tar_writer_spec.rb
|
|
230
207
|
homepage: http://github.com/ubpb/metacrunch
|
|
231
208
|
licenses:
|
|
232
209
|
- MIT
|
|
@@ -239,7 +216,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
|
239
216
|
requirements:
|
|
240
217
|
- - ">="
|
|
241
218
|
- !ruby/object:Gem::Version
|
|
242
|
-
version:
|
|
219
|
+
version: '0'
|
|
243
220
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
|
244
221
|
requirements:
|
|
245
222
|
- - ">="
|
|
@@ -251,27 +228,4 @@ rubygems_version: 2.4.8
|
|
|
251
228
|
signing_key:
|
|
252
229
|
specification_version: 4
|
|
253
230
|
summary: Data processing toolkit for Ruby
|
|
254
|
-
test_files:
|
|
255
|
-
- spec/assets/archive.tar
|
|
256
|
-
- spec/assets/archive.tar.gz
|
|
257
|
-
- spec/assets/file/some_file
|
|
258
|
-
- spec/assets/file/some_file.gz
|
|
259
|
-
- spec/assets/file/tar_archive.tar
|
|
260
|
-
- spec/assets/file/tar_gz_archive.tar.gz
|
|
261
|
-
- spec/assets/file/tgz_archive.tgz
|
|
262
|
-
- spec/assets/file/zip_archive.zip
|
|
263
|
-
- spec/assets/regular_file.txt
|
|
264
|
-
- spec/assets/regular_file.txt.gz
|
|
265
|
-
- spec/assets/sql_lite_config.rb
|
|
266
|
-
- spec/file_reader_spec.rb
|
|
267
|
-
- spec/file_writer_spec.rb
|
|
268
|
-
- spec/metacrunch/file/reader/plain_file_reader_spec.rb
|
|
269
|
-
- spec/metacrunch/file/reader/tar_reader_spec.rb
|
|
270
|
-
- spec/metacrunch/file/reader/zip_reader_spec.rb
|
|
271
|
-
- spec/metacrunch/file/reader_spec.rb
|
|
272
|
-
- spec/metacrunch/file_spec.rb
|
|
273
|
-
- spec/metacrunch/hash_spec.rb
|
|
274
|
-
- spec/metacrunch/transformator/transformation_spec.rb
|
|
275
|
-
- spec/snr_spec.rb
|
|
276
|
-
- spec/spec_helper.rb
|
|
277
|
-
- spec/tar_writer_spec.rb
|
|
231
|
+
test_files: []
|
data/spec/assets/archive.tar
DELETED
|
Binary file
|
data/spec/assets/archive.tar.gz
DELETED
|
Binary file
|
data/spec/assets/file/some_file
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
Some text
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
THIS IS A TEST
|
|
Binary file
|
|
@@ -1,17 +0,0 @@
|
|
|
1
|
-
pipeline do
|
|
2
|
-
reader "Metacrunch::Archive::Reader", {
|
|
3
|
-
bulk_size: 2,
|
|
4
|
-
filenames: "foo.tar.gz"
|
|
5
|
-
}
|
|
6
|
-
|
|
7
|
-
processor -> (items, pipeline) {
|
|
8
|
-
items.map! do |_item|
|
|
9
|
-
|
|
10
|
-
end
|
|
11
|
-
}
|
|
12
|
-
#processor "Metacrunch::Sqlite::Writer", {
|
|
13
|
-
# database: "mab_xml.db"
|
|
14
|
-
#}
|
|
15
|
-
|
|
16
|
-
processor "Metacrunch::Debugger"
|
|
17
|
-
end
|
data/spec/file_reader_spec.rb
DELETED
|
@@ -1,73 +0,0 @@
|
|
|
1
|
-
describe Metacrunch::FileReader do
|
|
2
|
-
|
|
3
|
-
let(:regular_file) { File.join(RSpec.root, "assets", "regular_file.txt") }
|
|
4
|
-
let(:compressed_regular_file) { File.join(RSpec.root, "assets", "regular_file.txt.gz") }
|
|
5
|
-
let(:archive_file) { File.join(RSpec.root, "assets", "archive.tar") }
|
|
6
|
-
let(:compressed_archive_file) { File.join(RSpec.root, "assets", "archive.tar.gz") }
|
|
7
|
-
|
|
8
|
-
it "can read a single file" do
|
|
9
|
-
reader = Metacrunch::FileReader.new(regular_file)
|
|
10
|
-
expect(reader.each.count).to be(1)
|
|
11
|
-
end
|
|
12
|
-
|
|
13
|
-
it "can read multiple files" do
|
|
14
|
-
reader = Metacrunch::FileReader.new([regular_file, regular_file])
|
|
15
|
-
expect(reader.each.count).to be(2)
|
|
16
|
-
end
|
|
17
|
-
|
|
18
|
-
it "can read regular files" do
|
|
19
|
-
reader = Metacrunch::FileReader.new(regular_file)
|
|
20
|
-
reader.each do |entry|
|
|
21
|
-
expect(entry).to be_instance_of(Metacrunch::FileReader::Entry)
|
|
22
|
-
expect(entry.filename).to eq(regular_file)
|
|
23
|
-
expect(entry.archive_filename).to be_nil
|
|
24
|
-
expect(entry.from_archive?).to be false
|
|
25
|
-
expect(entry.contents).to eq("THIS IS A TEST\n")
|
|
26
|
-
end
|
|
27
|
-
end
|
|
28
|
-
|
|
29
|
-
it "can read compressed regular files" do
|
|
30
|
-
reader = Metacrunch::FileReader.new(compressed_regular_file)
|
|
31
|
-
reader.each do |entry|
|
|
32
|
-
expect(entry).to be_instance_of(Metacrunch::FileReader::Entry)
|
|
33
|
-
expect(entry.filename).to eq(compressed_regular_file)
|
|
34
|
-
expect(entry.archive_filename).to be_nil
|
|
35
|
-
expect(entry.from_archive?).to be false
|
|
36
|
-
expect(entry.contents).to eq("THIS IS A TEST\n")
|
|
37
|
-
end
|
|
38
|
-
end
|
|
39
|
-
|
|
40
|
-
it "can read TAR archives" do
|
|
41
|
-
reader = Metacrunch::FileReader.new(archive_file)
|
|
42
|
-
reader.each do |entry|
|
|
43
|
-
expect(entry).not_to be_nil
|
|
44
|
-
expect(entry).to be_instance_of(Metacrunch::FileReader::Entry)
|
|
45
|
-
expect(entry.filename).to eq(archive_file)
|
|
46
|
-
expect(entry.archive_filename).to eq("regular_file.txt")
|
|
47
|
-
expect(entry.from_archive?).to be true
|
|
48
|
-
expect(entry.contents).to eq("THIS IS A TEST\n")
|
|
49
|
-
end
|
|
50
|
-
end
|
|
51
|
-
|
|
52
|
-
it "can read compressed TAR archives" do
|
|
53
|
-
reader = Metacrunch::FileReader.new(compressed_archive_file)
|
|
54
|
-
reader.each do |entry|
|
|
55
|
-
expect(entry).not_to be_nil
|
|
56
|
-
expect(entry).to be_instance_of(Metacrunch::FileReader::Entry)
|
|
57
|
-
expect(entry.filename).to eq(compressed_archive_file)
|
|
58
|
-
expect(entry.archive_filename).to eq("regular_file.txt")
|
|
59
|
-
expect(entry.from_archive?).to be true
|
|
60
|
-
expect(entry.contents).to eq("THIS IS A TEST\n")
|
|
61
|
-
end
|
|
62
|
-
end
|
|
63
|
-
|
|
64
|
-
describe ".each()" do
|
|
65
|
-
context "when called without a block" do
|
|
66
|
-
it "returns an enumerator" do
|
|
67
|
-
reader = Metacrunch::FileReader.new(regular_file)
|
|
68
|
-
expect(reader.each).to be_instance_of(Enumerator)
|
|
69
|
-
end
|
|
70
|
-
end
|
|
71
|
-
end
|
|
72
|
-
|
|
73
|
-
end
|
data/spec/file_writer_spec.rb
DELETED
|
@@ -1,46 +0,0 @@
|
|
|
1
|
-
describe Metacrunch::FileWriter do
|
|
2
|
-
|
|
3
|
-
let(:regular_file) { "/tmp/metacrunch_spec_regular_file.txt" }
|
|
4
|
-
let(:compressed_file) { "/tmp/metacrunch_spec_regular_file.txt.gz" }
|
|
5
|
-
|
|
6
|
-
before do
|
|
7
|
-
::File.unlink(regular_file) if ::File.exist?(regular_file)
|
|
8
|
-
::File.unlink(compressed_file) if ::File.exist?(compressed_file)
|
|
9
|
-
end
|
|
10
|
-
|
|
11
|
-
it "can write a file" do
|
|
12
|
-
writer = Metacrunch::FileWriter.new(regular_file)
|
|
13
|
-
writer.write("FOO")
|
|
14
|
-
writer.write("BAR")
|
|
15
|
-
writer.close
|
|
16
|
-
|
|
17
|
-
content = File.read(regular_file)
|
|
18
|
-
expect(content).to eq("FOOBAR")
|
|
19
|
-
end
|
|
20
|
-
|
|
21
|
-
it "can write a compressed file" do
|
|
22
|
-
writer = Metacrunch::FileWriter.new(compressed_file, compress: true)
|
|
23
|
-
writer.write("FOO")
|
|
24
|
-
writer.write("BAR")
|
|
25
|
-
writer.close
|
|
26
|
-
|
|
27
|
-
io = Zlib::GzipReader.open(compressed_file)
|
|
28
|
-
content = io.read
|
|
29
|
-
io.close
|
|
30
|
-
|
|
31
|
-
expect(content).to eq("FOOBAR")
|
|
32
|
-
end
|
|
33
|
-
|
|
34
|
-
it "can override existing file" do
|
|
35
|
-
::File.write(regular_file, "FOO")
|
|
36
|
-
|
|
37
|
-
expect {
|
|
38
|
-
Metacrunch::FileWriter.new(regular_file)
|
|
39
|
-
}.to raise_error(Metacrunch::FileWriter::FileExistError)
|
|
40
|
-
|
|
41
|
-
expect {
|
|
42
|
-
Metacrunch::FileWriter.new(regular_file, override: true)
|
|
43
|
-
}.not_to raise_error
|
|
44
|
-
end
|
|
45
|
-
|
|
46
|
-
end
|
|
@@ -1,11 +0,0 @@
|
|
|
1
|
-
describe Metacrunch::File::Reader::PlainFileReader do
|
|
2
|
-
describe ".accepts?" do
|
|
3
|
-
it "accepts all filenames" do
|
|
4
|
-
expect(described_class.accepts?("foo")).to equal(true)
|
|
5
|
-
expect(described_class.accepts?("foo.gz")).to equal(true)
|
|
6
|
-
expect(described_class.accepts?("foo.tar.gz")).to equal(true)
|
|
7
|
-
expect(described_class.accepts?("foo.xml")).to equal(true)
|
|
8
|
-
expect(described_class.accepts?("foo.zip")).to equal(true)
|
|
9
|
-
end
|
|
10
|
-
end
|
|
11
|
-
end
|
|
@@ -1,15 +0,0 @@
|
|
|
1
|
-
describe Metacrunch::File::Reader::TarFileReader do
|
|
2
|
-
describe ".accepts?" do
|
|
3
|
-
it "accepts filenames this reader can process" do
|
|
4
|
-
expect(described_class.accepts?("foo.tar")).to equal(true)
|
|
5
|
-
expect(described_class.accepts?("foo.tar.gz")).to equal(true)
|
|
6
|
-
expect(described_class.accepts?("foo.tgz")).to equal(true)
|
|
7
|
-
end
|
|
8
|
-
|
|
9
|
-
it "does not accept filenames this reader cannot process" do
|
|
10
|
-
expect(described_class.accepts?("plain_file")).to equal(false)
|
|
11
|
-
expect(described_class.accepts?("plain_file.gz")).to equal(false)
|
|
12
|
-
expect(described_class.accepts?("tatar.gz")).to equal(false)
|
|
13
|
-
end
|
|
14
|
-
end
|
|
15
|
-
end
|
|
@@ -1,13 +0,0 @@
|
|
|
1
|
-
describe Metacrunch::File::Reader::ZipFileReader do
|
|
2
|
-
describe ".accepts?" do
|
|
3
|
-
it "accepts filenames this reader can process" do
|
|
4
|
-
expect(described_class.accepts?("foo.zip")).to equal(true)
|
|
5
|
-
end
|
|
6
|
-
|
|
7
|
-
it "does not accept filenames this reader cannot process" do
|
|
8
|
-
expect(described_class.accepts?("plain_file")).to equal(false)
|
|
9
|
-
expect(described_class.accepts?("plain.zip.file")).to equal(false)
|
|
10
|
-
expect(described_class.accepts?("foozip")).to equal(false)
|
|
11
|
-
end
|
|
12
|
-
end
|
|
13
|
-
end
|
|
@@ -1,47 +0,0 @@
|
|
|
1
|
-
describe Metacrunch::File::Reader do
|
|
2
|
-
def pipeline_factory
|
|
3
|
-
Class.new do
|
|
4
|
-
def terminate!; @terminated = true; end
|
|
5
|
-
def terminated?; !!@terminated; end
|
|
6
|
-
end
|
|
7
|
-
.new
|
|
8
|
-
end
|
|
9
|
-
|
|
10
|
-
def reader_factory(filename)
|
|
11
|
-
described_class.new({
|
|
12
|
-
bulk_size: 10,
|
|
13
|
-
filenames: [filename],
|
|
14
|
-
force_content_encoding: "utf-8"
|
|
15
|
-
})
|
|
16
|
-
end
|
|
17
|
-
|
|
18
|
-
# filenames
|
|
19
|
-
let(:plain_file_name) { File.join(asset_dir, "file/some_file") }
|
|
20
|
-
let(:plain_gz_file_name) { File.join(asset_dir, "file/some_file.gz") }
|
|
21
|
-
let(:tar_file_name) { File.join(asset_dir, "file/tar_archive.tar") }
|
|
22
|
-
let(:tar_gz_file_name) { File.join(asset_dir, "file/tar_gz_archive.tar.gz") }
|
|
23
|
-
let(:tgz_file_name) { File.join(asset_dir, "file/tgz_archive.tgz") }
|
|
24
|
-
let(:zip_file_name) { File.join(asset_dir, "file/zip_archive.zip") }
|
|
25
|
-
|
|
26
|
-
# readers
|
|
27
|
-
let(:plain_file_reader) { reader_factory(plain_file_name) }
|
|
28
|
-
let(:plain_gz_file_reader) { reader_factory(plain_gz_file_name) }
|
|
29
|
-
let(:tar_reader) { reader_factory(tar_file_name) }
|
|
30
|
-
let(:tar_gz_reader) { reader_factory(tar_gz_file_name) }
|
|
31
|
-
let(:tgz_reader) { reader_factory(tgz_file_name) }
|
|
32
|
-
let(:zip_reader) { reader_factory(zip_file_name) }
|
|
33
|
-
|
|
34
|
-
describe "#call" do
|
|
35
|
-
it "fills items with #{_file_type = Metacrunch::File} objects" do
|
|
36
|
-
[plain_file_reader, plain_gz_file_reader, tar_reader, tar_gz_reader,tgz_reader, zip_reader].each do |_reader|
|
|
37
|
-
pipeline = pipeline_factory
|
|
38
|
-
|
|
39
|
-
_reader.call(items = [], pipeline)
|
|
40
|
-
expect(items.length).not_to eq(0)
|
|
41
|
-
expect(items).to all(be_a(_file_type))
|
|
42
|
-
|
|
43
|
-
_reader.call(items, pipeline) until pipeline.terminated?
|
|
44
|
-
end
|
|
45
|
-
end
|
|
46
|
-
end
|
|
47
|
-
end
|
|
@@ -1,15 +0,0 @@
|
|
|
1
|
-
describe Metacrunch::File do
|
|
2
|
-
describe "#to_h" do
|
|
3
|
-
let(:file) do
|
|
4
|
-
described_class.new({
|
|
5
|
-
content: "content",
|
|
6
|
-
entry_name: "entry_name",
|
|
7
|
-
file_name: "file_name"
|
|
8
|
-
})
|
|
9
|
-
end
|
|
10
|
-
|
|
11
|
-
it "returns a hash representation of the file object" do
|
|
12
|
-
expect(file.to_h).to be_a(Hash)
|
|
13
|
-
end
|
|
14
|
-
end
|
|
15
|
-
end
|
|
@@ -1,93 +0,0 @@
|
|
|
1
|
-
describe Metacrunch::Hash do
|
|
2
|
-
describe ".add" do
|
|
3
|
-
it "adds the given key/value pair" do
|
|
4
|
-
described_class.add(hash = {}, :key, "value")
|
|
5
|
-
expect(hash[:key]).to eq("value")
|
|
6
|
-
end
|
|
7
|
-
|
|
8
|
-
it "respects the type of the key" do
|
|
9
|
-
hash = {}
|
|
10
|
-
|
|
11
|
-
described_class.add(hash, :symbol_key1, "value")
|
|
12
|
-
described_class.add(hash, ":symbol_key2", "value")
|
|
13
|
-
described_class.add(hash, "string_key", "value")
|
|
14
|
-
|
|
15
|
-
expect(hash[:symbol_key1]).to eq("value")
|
|
16
|
-
expect(hash[:symbol_key2]).to eq("value")
|
|
17
|
-
expect(hash["string_key"]).to eq("value")
|
|
18
|
-
end
|
|
19
|
-
|
|
20
|
-
context "if wrong arguments are given" do
|
|
21
|
-
it "raises an #{exception_type = ArgumentError}" do
|
|
22
|
-
expect { described_class.add(hash) }.to raise_error(exception_type)
|
|
23
|
-
end
|
|
24
|
-
end
|
|
25
|
-
|
|
26
|
-
context "if key includes forward dashes" do
|
|
27
|
-
it "splits the key by forward dashes" do
|
|
28
|
-
described_class.add(hash = {}, "key/subkey", "value")
|
|
29
|
-
expect(hash["key"]["subkey"]).to eq("value")
|
|
30
|
-
end
|
|
31
|
-
|
|
32
|
-
it "respects the type of the keys" do
|
|
33
|
-
described_class.add(hash = {}, "key/:subkey/sub_sub_key", "value")
|
|
34
|
-
expect(hash["key"][:subkey]["sub_sub_key"]).to eq("value")
|
|
35
|
-
end
|
|
36
|
-
end
|
|
37
|
-
|
|
38
|
-
context "if multiple keys are given" do
|
|
39
|
-
it "dives into the object and adds the key/value pair" do
|
|
40
|
-
hash = { "key" => { "subkey" => "former_value" } }
|
|
41
|
-
described_class.add(hash, "key", "subkey", "value")
|
|
42
|
-
expect(hash["key"]["subkey"]).to eq(["former_value", "value"])
|
|
43
|
-
end
|
|
44
|
-
end
|
|
45
|
-
|
|
46
|
-
context "if there already exists a value for the given key" do
|
|
47
|
-
context "if the value is an array" do
|
|
48
|
-
it "adds the value to the array" do
|
|
49
|
-
hash = { "key" => ["former_value1", "former_value2"] }
|
|
50
|
-
described_class.add(hash, "key", "value")
|
|
51
|
-
expect(hash["key"]).to eq(["former_value1", "former_value2", "value"])
|
|
52
|
-
end
|
|
53
|
-
|
|
54
|
-
it "returns aggregated value" do
|
|
55
|
-
hash = { "key" => ["former_value1", "former_value2"] }
|
|
56
|
-
expect(described_class.add(hash, "key", "value")).to eq(["former_value1", "former_value2", "value"])
|
|
57
|
-
end
|
|
58
|
-
end
|
|
59
|
-
|
|
60
|
-
it "makes the existing value an array and adds the given value" do
|
|
61
|
-
hash = { "key" => "former_value" }
|
|
62
|
-
described_class.add(hash, "key", "value")
|
|
63
|
-
expect(hash["key"]).to eq(["former_value", "value"])
|
|
64
|
-
end
|
|
65
|
-
|
|
66
|
-
it "returns aggregated value" do
|
|
67
|
-
hash = { "key" => "former_value" }
|
|
68
|
-
expect(described_class.add(hash, "key", "value")).to eq(["former_value", "value"])
|
|
69
|
-
end
|
|
70
|
-
end
|
|
71
|
-
|
|
72
|
-
context "if the given value is nil" do
|
|
73
|
-
it "does not add anything" do
|
|
74
|
-
described_class.add(hash = {}, "key/subkey", nil)
|
|
75
|
-
expect(hash).to eq({})
|
|
76
|
-
end
|
|
77
|
-
end
|
|
78
|
-
|
|
79
|
-
context "if the given value is empty" do
|
|
80
|
-
it "does not add anything" do
|
|
81
|
-
hash = {}
|
|
82
|
-
described_class.add(hash, "key/subkey1", "")
|
|
83
|
-
described_class.add(hash, "key/subkey2", [])
|
|
84
|
-
described_class.add(hash, "key/subkey3", Class.new { def empty?; true; end }.new)
|
|
85
|
-
expect(hash).to eq({})
|
|
86
|
-
end
|
|
87
|
-
end
|
|
88
|
-
|
|
89
|
-
it "returns the value" do
|
|
90
|
-
expect(described_class.add({}, "key", "value")).to eq("value")
|
|
91
|
-
end
|
|
92
|
-
end
|
|
93
|
-
end
|
|
@@ -1,28 +0,0 @@
|
|
|
1
|
-
describe Metacrunch::Transformator::Transformation do
|
|
2
|
-
class SomeStep < described_class::Step
|
|
3
|
-
def call
|
|
4
|
-
self.target = { muff: 1 }
|
|
5
|
-
end
|
|
6
|
-
end
|
|
7
|
-
|
|
8
|
-
class SomeTransformation < described_class
|
|
9
|
-
sequence [
|
|
10
|
-
SomeStep
|
|
11
|
-
]
|
|
12
|
-
end
|
|
13
|
-
|
|
14
|
-
it "foo" do
|
|
15
|
-
result = SomeTransformation.call({
|
|
16
|
-
my: {
|
|
17
|
-
awesome: {
|
|
18
|
-
source: {
|
|
19
|
-
title: "foo",
|
|
20
|
-
description: "bar"
|
|
21
|
-
}
|
|
22
|
-
}
|
|
23
|
-
}
|
|
24
|
-
})
|
|
25
|
-
|
|
26
|
-
result
|
|
27
|
-
end
|
|
28
|
-
end
|
data/spec/snr_spec.rb
DELETED
|
@@ -1,73 +0,0 @@
|
|
|
1
|
-
describe Metacrunch::SNR do
|
|
2
|
-
|
|
3
|
-
let(:snr) { Metacrunch::SNR.new }
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
describe ".add" do
|
|
7
|
-
it "adds a field and creates a section if the section doesn't exists" do
|
|
8
|
-
expect {
|
|
9
|
-
snr.add("mysection", "myfield", "My Value")
|
|
10
|
-
}.to change { snr.sections.count }.by(1)
|
|
11
|
-
end
|
|
12
|
-
|
|
13
|
-
it "adds a field to an existsing section" do
|
|
14
|
-
snr.add_section(Metacrunch::SNR::Section.new("mysection"))
|
|
15
|
-
|
|
16
|
-
expect {
|
|
17
|
-
snr.add("mysection", "myfield", "My Value")
|
|
18
|
-
}.not_to change { snr.sections.count }
|
|
19
|
-
end
|
|
20
|
-
end
|
|
21
|
-
|
|
22
|
-
describe ".sections" do
|
|
23
|
-
let(:section1) { Metacrunch::SNR::Section.new("section1") }
|
|
24
|
-
let(:section2) { Metacrunch::SNR::Section.new("section2") }
|
|
25
|
-
|
|
26
|
-
before {
|
|
27
|
-
snr.add_section(section1)
|
|
28
|
-
snr.add_section(section2)
|
|
29
|
-
}
|
|
30
|
-
|
|
31
|
-
subject { snr.sections }
|
|
32
|
-
|
|
33
|
-
it "returns all sections" do
|
|
34
|
-
expect(snr.sections.count).to be(2)
|
|
35
|
-
end
|
|
36
|
-
|
|
37
|
-
it { is_expected.not_to be_empty }
|
|
38
|
-
it { is_expected.to include(section1) }
|
|
39
|
-
it { is_expected.to include(section2) }
|
|
40
|
-
end
|
|
41
|
-
|
|
42
|
-
describe ".section(name)" do
|
|
43
|
-
context "when section exists" do
|
|
44
|
-
let(:section1) { Metacrunch::SNR::Section.new("section1") }
|
|
45
|
-
before { snr.add_section(section1) }
|
|
46
|
-
subject { snr.section("section1") }
|
|
47
|
-
|
|
48
|
-
it { is_expected.not_to be_nil }
|
|
49
|
-
it { is_expected.to be(section1) }
|
|
50
|
-
end
|
|
51
|
-
|
|
52
|
-
context "when section doesn't exists" do
|
|
53
|
-
subject { snr.section("no_such_section") }
|
|
54
|
-
|
|
55
|
-
it { is_expected.to be_nil }
|
|
56
|
-
end
|
|
57
|
-
end
|
|
58
|
-
|
|
59
|
-
describe ".to_xml" do
|
|
60
|
-
before {
|
|
61
|
-
snr.add("section1", "title", "Foo Bar")
|
|
62
|
-
snr.add("section1", "artist", ["Sievers, Michael", "Sprotte, René"])
|
|
63
|
-
snr.add("section2", "link", {label: "Click here", url: "http://example.com"})
|
|
64
|
-
}
|
|
65
|
-
|
|
66
|
-
subject { snr.to_xml }
|
|
67
|
-
|
|
68
|
-
it { is_expected.not_to be_nil }
|
|
69
|
-
it { is_expected.not_to be_empty }
|
|
70
|
-
it { is_expected.to eq("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<snr>\n <section1>\n <title>Foo Bar</title>\n <artist>Sievers, Michael</artist>\n <artist>Sprotte, René</artist>\n </section1>\n <section2>\n <link>\n <label>Click here</label>\n <url>http://example.com</url>\n </link>\n </section2>\n</snr>\n") }
|
|
71
|
-
end
|
|
72
|
-
|
|
73
|
-
end
|
data/spec/spec_helper.rb
DELETED
|
@@ -1,43 +0,0 @@
|
|
|
1
|
-
if ENV["CODECLIMATE_REPO_TOKEN"]
|
|
2
|
-
require "codeclimate-test-reporter"
|
|
3
|
-
CodeClimate::TestReporter.start
|
|
4
|
-
else
|
|
5
|
-
require "simplecov"
|
|
6
|
-
SimpleCov.start
|
|
7
|
-
end
|
|
8
|
-
|
|
9
|
-
require "metacrunch"
|
|
10
|
-
|
|
11
|
-
begin
|
|
12
|
-
require "pry"
|
|
13
|
-
require "yaml"
|
|
14
|
-
rescue LoadError
|
|
15
|
-
end
|
|
16
|
-
|
|
17
|
-
RSpec.configure do |config|
|
|
18
|
-
# begin --- rspec 3.1 generator
|
|
19
|
-
config.expect_with :rspec do |expectations|
|
|
20
|
-
expectations.include_chain_clauses_in_custom_matcher_descriptions = true
|
|
21
|
-
end
|
|
22
|
-
|
|
23
|
-
config.mock_with :rspec do |mocks|
|
|
24
|
-
mocks.verify_partial_doubles = true
|
|
25
|
-
end
|
|
26
|
-
# end --- rspec 3.1 generator
|
|
27
|
-
end
|
|
28
|
-
|
|
29
|
-
# Helper to provide RSpec.root
|
|
30
|
-
module ::RSpec
|
|
31
|
-
module_function
|
|
32
|
-
def root
|
|
33
|
-
@spec_root ||= Pathname.new(__dir__)
|
|
34
|
-
end
|
|
35
|
-
end
|
|
36
|
-
|
|
37
|
-
def asset_dir
|
|
38
|
-
File.expand_path(File.join(File.dirname(__FILE__), "assets"))
|
|
39
|
-
end
|
|
40
|
-
|
|
41
|
-
def read_asset(path_to_file)
|
|
42
|
-
File.read(File.expand_path(File.join(asset_dir, path_to_file)))
|
|
43
|
-
end
|
data/spec/tar_writer_spec.rb
DELETED
|
@@ -1,33 +0,0 @@
|
|
|
1
|
-
describe Metacrunch::TarWriter do
|
|
2
|
-
|
|
3
|
-
let(:regular_tar_file) { "/tmp/metacrunch_spec_regular_file.tar" }
|
|
4
|
-
let(:compressed_tar_file) { "/tmp/metacrunch_spec_regular_file.tar.gz" }
|
|
5
|
-
|
|
6
|
-
before do
|
|
7
|
-
File.unlink(regular_tar_file) if File.exist?(regular_tar_file)
|
|
8
|
-
File.unlink(compressed_tar_file) if File.exist?(compressed_tar_file)
|
|
9
|
-
end
|
|
10
|
-
|
|
11
|
-
it "can write a file" do
|
|
12
|
-
writer = Metacrunch::TarWriter.new(regular_tar_file)
|
|
13
|
-
writer.write("FOO", filename: "foo.txt")
|
|
14
|
-
writer.write("BAR", filename: "bar.txt")
|
|
15
|
-
writer.close
|
|
16
|
-
|
|
17
|
-
files = Metacrunch::FileReader.new(regular_tar_file).each
|
|
18
|
-
expect(files.count).to eq(2)
|
|
19
|
-
expect(files.first.contents).to eq("FOO")
|
|
20
|
-
end
|
|
21
|
-
|
|
22
|
-
it "can write a compressed file" do
|
|
23
|
-
writer = Metacrunch::TarWriter.new(compressed_tar_file)
|
|
24
|
-
writer.write("FOO", filename: "foo.txt")
|
|
25
|
-
writer.write("BAR", filename: "bar.txt")
|
|
26
|
-
writer.close
|
|
27
|
-
|
|
28
|
-
files = Metacrunch::FileReader.new(compressed_tar_file).each
|
|
29
|
-
expect(files.count).to eq(2)
|
|
30
|
-
expect(files.first.contents).to eq("FOO")
|
|
31
|
-
end
|
|
32
|
-
|
|
33
|
-
end
|