transformator 1.0.0.pre1 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Gemfile +13 -8
- data/bin/console +10 -0
- data/bin/setup +7 -0
- data/braindump.md +32 -0
- data/lib/transformator/transformation/step.rb +45 -0
- data/lib/transformator/transformation.rb +48 -0
- data/lib/transformator/version.rb +1 -1
- data/lib/transformator.rb +3 -13
- data/spec/spec_helper.rb +13 -1
- data/spec/transformator/transformation_spec.rb +32 -0
- data/transformator.gemspec +2 -4
- metadata +17 -42
- data/bin/transformator +0 -4
- data/lib/transformator/cli.rb +0 -42
- data/lib/transformator/dispatcher.rb +0 -43
- data/lib/transformator/filesystem_pattern_evaluator.rb +0 -14
- data/lib/transformator/gzip_reader.rb +0 -0
- data/lib/transformator/gzip_writer.rb +0 -0
- data/lib/transformator/null_processor.rb +0 -9
- data/lib/transformator/oga_xml_dumper.rb +0 -10
- data/lib/transformator/oga_xml_parser.rb +0 -10
- data/lib/transformator/ox_xml_dumper.rb +0 -10
- data/lib/transformator/ox_xml_parser.rb +0 -10
- data/lib/transformator/parallel_processor.rb +0 -39
- data/lib/transformator/processor.rb +0 -4
- data/lib/transformator/tar_reader/patched_rubygems_tar_reader.rb +0 -41
- data/lib/transformator/tar_reader.rb +0 -51
- data/lib/transformator/tar_writer.rb +0 -0
- data/spec/transformator_spec.rb +0 -2
- data/ubpb.yml.erb +0 -25
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 4833b5ce9ba4c2b367a97b9bc745050098603911
|
4
|
+
data.tar.gz: 212cd88f4bfb6e2cc8b43ec8b81b20e46a6ee191
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 65f9078502c79144bb44b3216a05b71a567e93297d8fe35091cdd1e53ab08dafff637777a7a6a1adb96d12e0ef51881ae8bb47ab53847f9b2a53c629967ccb08
|
7
|
+
data.tar.gz: 96090ba140428d845d58cd822a443f25cb5d6fcce4b80c1166da4ff0f3b17203719cf2e79482934e851ae9645926f04d7099f9d4c33cdbe4273983814fa1c773
|
data/Gemfile
CHANGED
@@ -1,14 +1,19 @@
|
|
1
1
|
source "https://rubygems.org"
|
2
2
|
|
3
|
-
# Specify your gem's dependencies in
|
3
|
+
# Specify your gem's dependencies in your gemspec
|
4
4
|
gemspec
|
5
5
|
|
6
|
-
|
7
|
-
|
6
|
+
if !ENV["CI"]
|
7
|
+
group :development do
|
8
|
+
gem "hashdiff"
|
9
|
+
gem "pry", "~> 0.9.12.6"
|
10
|
+
gem "pry-byebug", "<= 1.3.2"
|
11
|
+
gem "pry-rescue", "~> 1.4.1"
|
12
|
+
gem "pry-stack_explorer", "~> 0.4.9.1"
|
13
|
+
gem "pry-syntax-hacks", "~> 0.0.6"
|
14
|
+
end
|
15
|
+
end
|
8
16
|
|
9
|
-
|
10
|
-
gem "
|
11
|
-
gem "pry-stack_explorer", "~> 0.4.9.1"
|
12
|
-
else
|
13
|
-
#gem "pry-nav", "~> 0.2.4"
|
17
|
+
group :test do
|
18
|
+
gem "codeclimate-test-reporter", require: nil
|
14
19
|
end
|
data/bin/console
ADDED
@@ -0,0 +1,10 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
require "bundler/setup"
|
4
|
+
require "transformator"
|
5
|
+
|
6
|
+
# You can add fixtures and/or initialization code here to make experimenting
|
7
|
+
# with your gem easier. You can also use a different console, if you like.
|
8
|
+
|
9
|
+
require "pry"
|
10
|
+
Pry.start
|
data/bin/setup
ADDED
data/braindump.md
ADDED
@@ -0,0 +1,32 @@
|
|
1
|
+
- Mehrstufiger Lademechanismus (konkret 2 stufig)
|
2
|
+
- implizierter cli Parameter --bundle-transformations
|
3
|
+
- um das original Gemfile nicht zu ändern, könnte man es nach Gemfile_transformations kopieren
|
4
|
+
- Gemfile_transformations(.lock) müssen in .gitignore
|
5
|
+
- in Gemfile_transformations werden nun die Transformations Gems/Pfade/gits angehängt
|
6
|
+
- darauf folgende wird gebundelt
|
7
|
+
- anschließend wird der aktuelle Prozess durch einen Aufruf folgender Form *ersetzt*
|
8
|
+
BUNDLE_GEMFILE=Gemfile.transformations bundle exec bin/transformator --no-bundle-transformations ... (oder so ähnlich)
|
9
|
+
|
10
|
+
- Möglichkeit mehrere Transformationen in Reihe zu schalten
|
11
|
+
- dazu müsste lediglich innerhalb des ForkingEnumProzessors mehrere Transformationsscritte nacheinander durchgeführt werden
|
12
|
+
|
13
|
+
- da jede Transformation auf die source/target URI zugreifen kann, könnten hierrüber auch Reader/writer realisiert werden
|
14
|
+
- die Frage ist, inwieweit man die einzelnen Reader/writer in einzelne Gems aufsplittet
|
15
|
+
- ein pragmatischer Kompromiss wäre, dass die Reader/Writer die oft benutzt werden (Elasticsearch, Postgres, Tar) Bestandteil
|
16
|
+
des Transformator Gems sind/bleiben
|
17
|
+
- man hat darüber hinaus aber die Möglichkeit, Reader/Writer über "Transformationen" hinzuzufügen
|
18
|
+
|
19
|
+
- cool wäre, wenn man Files (Tar oder plain Files) über ssh URIs spezifizieren könnte
|
20
|
+
- transformator -s ssh://msievers@node1/~/mab/archive1.tgz
|
21
|
+
- die frage ist, wie man dies intern realisiert
|
22
|
+
- die einfache möglichkeit wäre zunächst einmal, den TarReader aufzubohren, damit dieser diesen Fall kennt/untersützt
|
23
|
+
- das hätte jedoch den nachteil, dass jeder reader den ssh Kram erneut für sich (implementieren) bzw. behandeln müsste
|
24
|
+
- die zweite Möglichkeit wäre, einen SSH Reader zu haben, welcher vor den TarReader geschaltet wird
|
25
|
+
- dieser könnte bsp. das File nach /tmp Laden und anschließend den TarReader damit starten
|
26
|
+
- Schwierig hierbei wäre, die Kontinuität des TarReaders (Enumerators) sicherzustellen, wenn diesem ein anderer Reader vorgeschaltet wäre
|
27
|
+
- dies könnte wieder über Enumeratoren gelingen
|
28
|
+
- ssh_reader.each -> gibt dateinamen im lokalen fs zurück
|
29
|
+
- tar_reader call ss_reader.each, liest jeweils ein file und enumeriert über die darin befindlichen einträge
|
30
|
+
- sobald keine einträge mehr vorhanden sind, bekommt er den nächsten dateinamen vom ssh reader usw.
|
31
|
+
|
32
|
+
- neben den CLI Parametern sollte es möglich sein, ein YAML Config File anzugeben, welches vorallem bei koomplexeren Settings das Handling erleichtert
|
@@ -0,0 +1,45 @@
|
|
1
|
+
require_relative "../transformation"
|
2
|
+
|
3
|
+
class Transformator::Transformation::Step
|
4
|
+
attr_accessor :transformation
|
5
|
+
|
6
|
+
def initialize(transformation = nil, options = {})
|
7
|
+
if transformation.is_a?(Hash)
|
8
|
+
options = transformation
|
9
|
+
transformation = nil
|
10
|
+
end
|
11
|
+
|
12
|
+
if transformation
|
13
|
+
@transformation = transformation
|
14
|
+
else
|
15
|
+
@transformation = Struct.new(:source, :target).new.tap do |_struct|
|
16
|
+
_struct.source = options[:source]
|
17
|
+
_struct.target = options[:target]
|
18
|
+
end
|
19
|
+
end
|
20
|
+
end
|
21
|
+
|
22
|
+
def call
|
23
|
+
end
|
24
|
+
|
25
|
+
#
|
26
|
+
# Each step has transparent access to all methods of it's transformation
|
27
|
+
#
|
28
|
+
def method_missing(method_name, *args, &block)
|
29
|
+
if @transformation.respond_to?(method_name)
|
30
|
+
@transformation.send(method_name, *args, &block)
|
31
|
+
else
|
32
|
+
super
|
33
|
+
end
|
34
|
+
end
|
35
|
+
|
36
|
+
def respond_to_missing?(method_name, include_private = false)
|
37
|
+
@transformation.respond_to?(method_name) || super
|
38
|
+
end
|
39
|
+
|
40
|
+
# avoid method_missing penalty for the most used transformation methods
|
41
|
+
def source; @transformation.source; end
|
42
|
+
def source=(value); @transformation.source=(value); end
|
43
|
+
def target; @transformation.target; end
|
44
|
+
def target=(value); @transformation.target=(value); end
|
45
|
+
end
|
@@ -0,0 +1,48 @@
|
|
1
|
+
require_relative "../transformator"
|
2
|
+
|
3
|
+
class Transformator::Transformation
|
4
|
+
require_relative "./transformation/step"
|
5
|
+
|
6
|
+
attr_accessor :source
|
7
|
+
attr_accessor :target
|
8
|
+
|
9
|
+
class << self
|
10
|
+
def steps(value = nil)
|
11
|
+
unless value
|
12
|
+
@steps
|
13
|
+
else
|
14
|
+
@steps = value
|
15
|
+
end
|
16
|
+
end
|
17
|
+
alias_method :sequence, :steps
|
18
|
+
end
|
19
|
+
|
20
|
+
def self.call(*args)
|
21
|
+
new.call(*args)
|
22
|
+
end
|
23
|
+
|
24
|
+
# since a transformation can have many steps, writing a "require" for each is tedious
|
25
|
+
def self.require_directory(directory)
|
26
|
+
Dir.glob("#{File.expand_path(directory)}/*.rb").each do |_filename|
|
27
|
+
require _filename
|
28
|
+
end
|
29
|
+
end
|
30
|
+
|
31
|
+
def initialize
|
32
|
+
# steps are instanced once, which means that instance variables retain
|
33
|
+
@steps = self.class.steps.flatten.map do |_step|
|
34
|
+
_step.is_a?(Class) ? _step.new(self) : _step
|
35
|
+
end
|
36
|
+
end
|
37
|
+
|
38
|
+
def call(source, options = {})
|
39
|
+
@source = source
|
40
|
+
@target = options[:target]
|
41
|
+
|
42
|
+
@steps.each do |_step|
|
43
|
+
_step.is_a?(Proc) ? instance_exec(&_step) : _step.call
|
44
|
+
end
|
45
|
+
|
46
|
+
return @target
|
47
|
+
end
|
48
|
+
end
|
data/lib/transformator.rb
CHANGED
@@ -1,17 +1,7 @@
|
|
1
|
+
require "active_support"
|
2
|
+
require "active_support/core_ext"
|
1
3
|
require "transformator/version"
|
2
4
|
|
3
5
|
module Transformator
|
4
|
-
require_relative "./transformator/
|
5
|
-
require_relative "./transformator/dispatcher"
|
6
|
-
require_relative "./transformator/filesystem_pattern_evaluator"
|
7
|
-
#require_relative "./transformator/gzip_reader"
|
8
|
-
#require_relative "./transformator/gzip_writer"
|
9
|
-
require_relative "./transformator/oga_xml_dumper"
|
10
|
-
require_relative "./transformator/oga_xml_parser"
|
11
|
-
#require_relative "./transformator/ox_xml_dumper"
|
12
|
-
#require_relative "./transformator/ox_xml_parser"
|
13
|
-
require_relative "./transformator/parallel_processor"
|
14
|
-
require_relative "./transformator/null_processor"
|
15
|
-
require_relative "./transformator/tar_reader"
|
16
|
-
#require_relative "./transformator/tar_writer"
|
6
|
+
require_relative "./transformator/transformation"
|
17
7
|
end
|
data/spec/spec_helper.rb
CHANGED
@@ -1,6 +1,14 @@
|
|
1
|
+
if ENV["CODECLIMATE_REPO_TOKEN"]
|
2
|
+
require "codeclimate-test-reporter"
|
3
|
+
CodeClimate::TestReporter.start
|
4
|
+
else
|
5
|
+
require "simplecov"
|
6
|
+
SimpleCov.start
|
7
|
+
end
|
8
|
+
|
1
9
|
require "transformator"
|
10
|
+
require "yaml"
|
2
11
|
|
3
|
-
# require pry if possible
|
4
12
|
begin
|
5
13
|
require "pry"
|
6
14
|
rescue LoadError
|
@@ -17,3 +25,7 @@ RSpec.configure do |config|
|
|
17
25
|
end
|
18
26
|
# end --- rspec 3.1 generator
|
19
27
|
end
|
28
|
+
|
29
|
+
def read_asset(path_to_file)
|
30
|
+
File.read(File.expand_path(File.join(File.dirname(__FILE__), "assets", path_to_file)))
|
31
|
+
end
|
@@ -0,0 +1,32 @@
|
|
1
|
+
require "transformator/transformation"
|
2
|
+
require "transformator/transformation/step"
|
3
|
+
|
4
|
+
describe Transformator::Transformation do
|
5
|
+
class SomeStep < Transformator::Transformation::Step
|
6
|
+
def call
|
7
|
+
self.target = { muff: 1 }
|
8
|
+
end
|
9
|
+
end
|
10
|
+
|
11
|
+
class SomeTransformation < Transformator::Transformation
|
12
|
+
sequence [
|
13
|
+
SomeStep
|
14
|
+
]
|
15
|
+
end
|
16
|
+
|
17
|
+
it "foo" do
|
18
|
+
result = SomeTransformation.call({
|
19
|
+
my: {
|
20
|
+
awesome: {
|
21
|
+
source: {
|
22
|
+
title: "foo",
|
23
|
+
description: "bar"
|
24
|
+
}
|
25
|
+
}
|
26
|
+
}
|
27
|
+
})
|
28
|
+
|
29
|
+
binding.pry
|
30
|
+
result
|
31
|
+
end
|
32
|
+
end
|
data/transformator.gemspec
CHANGED
@@ -16,10 +16,8 @@ Gem::Specification.new do |spec|
|
|
16
16
|
spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
|
17
17
|
spec.require_paths = ["lib"]
|
18
18
|
|
19
|
-
spec.add_dependency "
|
20
|
-
|
21
|
-
spec.add_dependency "rubysl", "~> 2.1.0" if RUBY_ENGINE == "rbx"
|
22
|
-
spec.add_dependency "thread", "~> 0.1.5"
|
19
|
+
spec.add_dependency "activesupport"
|
20
|
+
spec.add_dependency "rubysl", ">= 2.1.0" if RUBY_ENGINE == "rbx"
|
23
21
|
|
24
22
|
spec.add_development_dependency "benchmark-ips", ">= 2.0.0"
|
25
23
|
spec.add_development_dependency "bundler", "~> 1.7"
|
metadata
CHANGED
@@ -1,43 +1,29 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: transformator
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.0.0
|
4
|
+
version: 1.0.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Michael Sievers
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2015-
|
11
|
+
date: 2015-10-02 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
|
-
name:
|
14
|
+
name: activesupport
|
15
15
|
requirement: !ruby/object:Gem::Requirement
|
16
16
|
requirements:
|
17
17
|
- - ">="
|
18
18
|
- !ruby/object:Gem::Version
|
19
|
-
version: 0
|
19
|
+
version: '0'
|
20
20
|
type: :runtime
|
21
21
|
prerelease: false
|
22
22
|
version_requirements: !ruby/object:Gem::Requirement
|
23
23
|
requirements:
|
24
24
|
- - ">="
|
25
25
|
- !ruby/object:Gem::Version
|
26
|
-
version: 0
|
27
|
-
- !ruby/object:Gem::Dependency
|
28
|
-
name: thread
|
29
|
-
requirement: !ruby/object:Gem::Requirement
|
30
|
-
requirements:
|
31
|
-
- - "~>"
|
32
|
-
- !ruby/object:Gem::Version
|
33
|
-
version: 0.1.5
|
34
|
-
type: :runtime
|
35
|
-
prerelease: false
|
36
|
-
version_requirements: !ruby/object:Gem::Requirement
|
37
|
-
requirements:
|
38
|
-
- - "~>"
|
39
|
-
- !ruby/object:Gem::Version
|
40
|
-
version: 0.1.5
|
26
|
+
version: '0'
|
41
27
|
- !ruby/object:Gem::Dependency
|
42
28
|
name: benchmark-ips
|
43
29
|
requirement: !ruby/object:Gem::Requirement
|
@@ -117,7 +103,8 @@ dependencies:
|
|
117
103
|
description:
|
118
104
|
email:
|
119
105
|
executables:
|
120
|
-
-
|
106
|
+
- console
|
107
|
+
- setup
|
121
108
|
extensions: []
|
122
109
|
extra_rdoc_files: []
|
123
110
|
files:
|
@@ -127,28 +114,16 @@ files:
|
|
127
114
|
- LICENSE.txt
|
128
115
|
- README.md
|
129
116
|
- Rakefile
|
130
|
-
- bin/
|
117
|
+
- bin/console
|
118
|
+
- bin/setup
|
119
|
+
- braindump.md
|
131
120
|
- lib/transformator.rb
|
132
|
-
- lib/transformator/
|
133
|
-
- lib/transformator/
|
134
|
-
- lib/transformator/filesystem_pattern_evaluator.rb
|
135
|
-
- lib/transformator/gzip_reader.rb
|
136
|
-
- lib/transformator/gzip_writer.rb
|
137
|
-
- lib/transformator/null_processor.rb
|
138
|
-
- lib/transformator/oga_xml_dumper.rb
|
139
|
-
- lib/transformator/oga_xml_parser.rb
|
140
|
-
- lib/transformator/ox_xml_dumper.rb
|
141
|
-
- lib/transformator/ox_xml_parser.rb
|
142
|
-
- lib/transformator/parallel_processor.rb
|
143
|
-
- lib/transformator/processor.rb
|
144
|
-
- lib/transformator/tar_reader.rb
|
145
|
-
- lib/transformator/tar_reader/patched_rubygems_tar_reader.rb
|
146
|
-
- lib/transformator/tar_writer.rb
|
121
|
+
- lib/transformator/transformation.rb
|
122
|
+
- lib/transformator/transformation/step.rb
|
147
123
|
- lib/transformator/version.rb
|
148
124
|
- spec/spec_helper.rb
|
149
|
-
- spec/
|
125
|
+
- spec/transformator/transformation_spec.rb
|
150
126
|
- transformator.gemspec
|
151
|
-
- ubpb.yml.erb
|
152
127
|
homepage: https://github.com/ubpb/transformator
|
153
128
|
licenses:
|
154
129
|
- MIT
|
@@ -164,15 +139,15 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
164
139
|
version: '0'
|
165
140
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
166
141
|
requirements:
|
167
|
-
- - "
|
142
|
+
- - ">="
|
168
143
|
- !ruby/object:Gem::Version
|
169
|
-
version:
|
144
|
+
version: '0'
|
170
145
|
requirements: []
|
171
146
|
rubyforge_project:
|
172
|
-
rubygems_version: 2.4.
|
147
|
+
rubygems_version: 2.4.8
|
173
148
|
signing_key:
|
174
149
|
specification_version: 4
|
175
150
|
summary: A data transformation framework
|
176
151
|
test_files:
|
177
152
|
- spec/spec_helper.rb
|
178
|
-
- spec/
|
153
|
+
- spec/transformator/transformation_spec.rb
|
data/bin/transformator
DELETED
data/lib/transformator/cli.rb
DELETED
@@ -1,42 +0,0 @@
|
|
1
|
-
require "erb"
|
2
|
-
require "optparse"
|
3
|
-
require "transformator"
|
4
|
-
require "yaml"
|
5
|
-
|
6
|
-
require "pry" # TODO: remove
|
7
|
-
|
8
|
-
class Transformator::Cli
|
9
|
-
def initialize(argv = [])
|
10
|
-
if argv.empty?
|
11
|
-
puts options_parser.help
|
12
|
-
else
|
13
|
-
@options = parse_argv(argv) || {}
|
14
|
-
end
|
15
|
-
|
16
|
-
Transformator::Dispatcher.new(
|
17
|
-
YAML.load(
|
18
|
-
ERB.new(
|
19
|
-
File.read(
|
20
|
-
File.expand_path(@options[:config_file_name])
|
21
|
-
)
|
22
|
-
).result
|
23
|
-
)
|
24
|
-
).call
|
25
|
-
end
|
26
|
-
|
27
|
-
def parse_argv(argv)
|
28
|
-
presence({}.tap do |result|
|
29
|
-
OptionParser.new do |opts|
|
30
|
-
opts.banner = "Usage: transformator [options]"
|
31
|
-
|
32
|
-
opts.on( "-c", "--config-file FILE", "Configuration file in yaml format" ) do |config_file_name|
|
33
|
-
result[:config_file_name] = config_file_name
|
34
|
-
end
|
35
|
-
end.parse(argv)
|
36
|
-
end)
|
37
|
-
end
|
38
|
-
|
39
|
-
def presence(object)
|
40
|
-
(object.empty? rescue false) ? nil : object
|
41
|
-
end
|
42
|
-
end
|
@@ -1,43 +0,0 @@
|
|
1
|
-
require "transformator"
|
2
|
-
|
3
|
-
class Transformator::Dispatcher
|
4
|
-
class State
|
5
|
-
attr_accessor :finished
|
6
|
-
attr_accessor :result
|
7
|
-
|
8
|
-
def initialize(initial_result)
|
9
|
-
@initial_result = initial_result
|
10
|
-
reset_result!
|
11
|
-
end
|
12
|
-
|
13
|
-
def finished?
|
14
|
-
@finished == true
|
15
|
-
end
|
16
|
-
|
17
|
-
def reset_result!
|
18
|
-
@result = @initial_result.dup
|
19
|
-
end
|
20
|
-
end
|
21
|
-
|
22
|
-
def initialize(processors = [])
|
23
|
-
@processors = processors.map do |processor|
|
24
|
-
if processor["options"]
|
25
|
-
processor["class"].new(processor["options"])
|
26
|
-
else
|
27
|
-
processor["class"].new
|
28
|
-
end
|
29
|
-
end
|
30
|
-
|
31
|
-
@state = State.new([])
|
32
|
-
end
|
33
|
-
|
34
|
-
def call
|
35
|
-
until @state.finished?
|
36
|
-
@processors.each do |processor|
|
37
|
-
processor.call!(@state)
|
38
|
-
end
|
39
|
-
|
40
|
-
@state.reset_result!
|
41
|
-
end
|
42
|
-
end
|
43
|
-
end
|
@@ -1,14 +0,0 @@
|
|
1
|
-
require "transformator/processor"
|
2
|
-
|
3
|
-
class Transformator::FilesystemPatternEvaluator < Transformator::Processor
|
4
|
-
def initialize(options = {})
|
5
|
-
@patterns = options["patterns"]
|
6
|
-
end
|
7
|
-
|
8
|
-
def call!(state)
|
9
|
-
state.result = @patterns.map do |pattern|
|
10
|
-
Dir.glob(File.expand_path(pattern))
|
11
|
-
end.flatten
|
12
|
-
binding.pry
|
13
|
-
end
|
14
|
-
end
|
File without changes
|
File without changes
|
@@ -1,39 +0,0 @@
|
|
1
|
-
require "transformator/processor"
|
2
|
-
require "thread/pool"
|
3
|
-
|
4
|
-
class Transformator::ParallelProcessor < Transformator::Processor
|
5
|
-
def initialize(options = {})
|
6
|
-
@processors = options["processors"].map do |processor|
|
7
|
-
if processor["options"]
|
8
|
-
processor["class"].new(processor["options"])
|
9
|
-
else
|
10
|
-
processor["class"].new
|
11
|
-
end
|
12
|
-
end
|
13
|
-
|
14
|
-
@number_of_workers = options["workers"]
|
15
|
-
@thread_pool = Thread.pool(@number_of_workers)
|
16
|
-
end
|
17
|
-
|
18
|
-
def call!(state)
|
19
|
-
if state.result.length > 0
|
20
|
-
chunk_size_per_worker = state.result.length.fdiv(@number_of_workers).ceil
|
21
|
-
results_enumerator = state.result.each_slice(chunk_size_per_worker)
|
22
|
-
|
23
|
-
chunks = Array.new(@number_of_workers).map do |_|
|
24
|
-
results_enumerator.next
|
25
|
-
end
|
26
|
-
|
27
|
-
@number_of_workers.times do |index|
|
28
|
-
@thread_pool.process do
|
29
|
-
@processors.each do |processor|
|
30
|
-
processor.call!(Struct.new(:result).new(chunks[index]))
|
31
|
-
end
|
32
|
-
end
|
33
|
-
end
|
34
|
-
|
35
|
-
@thread_pool.wait_done
|
36
|
-
state.result = chunks.inject(&:concat)
|
37
|
-
end
|
38
|
-
end
|
39
|
-
end
|
@@ -1,41 +0,0 @@
|
|
1
|
-
#--
|
2
|
-
# Copyright (C) 2004 Mauricio Julio Fernández Pradier
|
3
|
-
# See LICENSE.txt for additional licensing information.
|
4
|
-
#++
|
5
|
-
require "rubygems/package"
|
6
|
-
|
7
|
-
class Transformator::TarReader::PatchedRubygemsTarReader < Gem::Package::TarReader
|
8
|
-
def each
|
9
|
-
return enum_for __method__ unless block_given?
|
10
|
-
|
11
|
-
until @io.eof? do
|
12
|
-
header = Gem::Package::TarHeader.from @io
|
13
|
-
return if header.empty?
|
14
|
-
|
15
|
-
entry = Gem::Package::TarReader::Entry.new header, @io
|
16
|
-
size = entry.header.size
|
17
|
-
|
18
|
-
yield entry.read
|
19
|
-
|
20
|
-
skip = (512 - (size % 512)) % 512
|
21
|
-
pending = size - entry.bytes_read
|
22
|
-
|
23
|
-
begin
|
24
|
-
# avoid reading...
|
25
|
-
@io.seek pending, IO::SEEK_CUR
|
26
|
-
pending = 0
|
27
|
-
rescue Errno::EINVAL, NameError
|
28
|
-
while pending > 0 do
|
29
|
-
bytes_read = @io.read([pending, 4096].min).size
|
30
|
-
raise UnexpectedEOF if @io.eof?
|
31
|
-
pending -= bytes_read
|
32
|
-
end
|
33
|
-
end
|
34
|
-
|
35
|
-
@io.read skip # discard trailing zeros
|
36
|
-
|
37
|
-
# make sure nobody can use #read, #getc or #rewind anymore
|
38
|
-
entry.close
|
39
|
-
end
|
40
|
-
end
|
41
|
-
end
|
@@ -1,51 +0,0 @@
|
|
1
|
-
require "transformator/processor"
|
2
|
-
|
3
|
-
class Transformator::TarReader < Transformator::Processor
|
4
|
-
require_relative "./tar_reader/patched_rubygems_tar_reader"
|
5
|
-
|
6
|
-
attr_accessor :files
|
7
|
-
|
8
|
-
def initialize(options = {})
|
9
|
-
@bulk_size = options["bulk_size"]
|
10
|
-
@current_archive_enumerator = nil
|
11
|
-
|
12
|
-
if options["files"]
|
13
|
-
@files = ensure_array(options["files"]).map do |filename|
|
14
|
-
Dir.glob(File.expand_path(filename))
|
15
|
-
end.flatten
|
16
|
-
end
|
17
|
-
end
|
18
|
-
|
19
|
-
def call!(state)
|
20
|
-
@files = state.result.slice!(0..-1) unless @files
|
21
|
-
|
22
|
-
if @current_archive_enumerator.nil?
|
23
|
-
if !@files.empty?
|
24
|
-
@current_archive_enumerator = PatchedRubygemsTarReader.new(
|
25
|
-
Zlib::GzipReader.open(
|
26
|
-
File.expand_path(@files.pop)
|
27
|
-
)
|
28
|
-
).each
|
29
|
-
else
|
30
|
-
state.finished = true
|
31
|
-
end
|
32
|
-
end
|
33
|
-
|
34
|
-
@bulk_size.times do
|
35
|
-
begin
|
36
|
-
if @current_archive_enumerator
|
37
|
-
state.result.push @current_archive_enumerator.next
|
38
|
-
end
|
39
|
-
rescue StopIteration
|
40
|
-
@current_archive_enumerator = nil
|
41
|
-
end
|
42
|
-
end
|
43
|
-
end
|
44
|
-
|
45
|
-
#
|
46
|
-
private
|
47
|
-
#
|
48
|
-
def ensure_array(object)
|
49
|
-
object.is_a?(Array) ? object : [object]
|
50
|
-
end
|
51
|
-
end
|
File without changes
|
data/spec/transformator_spec.rb
DELETED
data/ubpb.yml.erb
DELETED
@@ -1,25 +0,0 @@
|
|
1
|
-
---
|
2
|
-
#- class: !ruby/class "Transformator::FilesystemPatternEvaluator"
|
3
|
-
# options:
|
4
|
-
# patterns:
|
5
|
-
# - ~/github/mab*.tar.gz
|
6
|
-
|
7
|
-
- class: !ruby/class "Transformator::TarReader"
|
8
|
-
options:
|
9
|
-
bulk_size: 320
|
10
|
-
files:
|
11
|
-
- ~/github/mab*.tar.gz
|
12
|
-
|
13
|
-
- class: !ruby/class "Transformator::ParallelProcessor"
|
14
|
-
options:
|
15
|
-
workers: 2
|
16
|
-
processors:
|
17
|
-
- class: !ruby/class "Transformator::OgaXmlParser"
|
18
|
-
- class: !ruby/class "Transformator::OgaXmlDumper"
|
19
|
-
|
20
|
-
- class: !ruby/class "Transformator::NullProcessor"
|
21
|
-
#- class: !ruby/class "Transformator::TarWriter"
|
22
|
-
# options:
|
23
|
-
# archive_size: 5000
|
24
|
-
# gzip_archive: true
|
25
|
-
# filename: /tmp/out.tar.gz
|