embulk 0.1.0 → 0.2.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +8 -8
- data/COPYING +14 -0
- data/ChangeLog +11 -0
- data/README.md +5 -3
- data/build.gradle +1 -1
- data/embulk-cli/pom.xml +1 -1
- data/embulk-core/pom.xml +1 -1
- data/embulk-standards/pom.xml +1 -1
- data/lib/embulk/command/embulk_example.rb +1 -1
- data/lib/embulk/data/bundle/embulk/input_example.rb +9 -5
- data/lib/embulk/guess_gzip.rb +8 -10
- data/lib/embulk/input_plugin.rb +10 -3
- data/lib/embulk/version.rb +1 -1
- data/pom.xml +1 -1
- metadata +6 -4
checksums.yaml
CHANGED
@@ -1,15 +1,15 @@
|
|
1
1
|
---
|
2
2
|
!binary "U0hBMQ==":
|
3
3
|
metadata.gz: !binary |-
|
4
|
-
|
4
|
+
NDBmNzE4N2NiODA5MmM5NzBlMjFjMTg5ZDNkMjIxM2VmYzUzM2I5NA==
|
5
5
|
data.tar.gz: !binary |-
|
6
|
-
|
6
|
+
ZmRmZDUwNTU0ZmE1NjNhZDc5NzQyMzEwMGM2ODBmNjM1ZTY5ZmY3NA==
|
7
7
|
SHA512:
|
8
8
|
metadata.gz: !binary |-
|
9
|
-
|
10
|
-
|
11
|
-
|
9
|
+
Njk2ZDU4MDNkODFmZTJiNmEyNmNhNWIwOWYzZjViYWI0YmYxNmMwMjBhNDQ5
|
10
|
+
ODMwY2YxOGI0ZWFiNjExNTdjMGE5ZmVlMDA2YWUwYTY2YWUxZDRjYTQ0OTY5
|
11
|
+
NzI0MGVkODQwYWI1YjUzNzdhNmNkMmVhMGRmZjRlZWIxZmQxYzU=
|
12
12
|
data.tar.gz: !binary |-
|
13
|
-
|
14
|
-
|
15
|
-
|
13
|
+
N2Q3NTEwN2NlMDg5OGQ1NmRlNmVkMzcyNzRiMjBhODY3MDExODA1ZDI5YTYy
|
14
|
+
ZmNjNTM3YjU5NGE4ZmM4NGFlNWE4ZmEwZWI2YTIzNjM3YmVmODE5MDc2OTQ1
|
15
|
+
NWMxNzM1OGUzZTExNTY0ZmM0MTBmYWRmNDJhZTc4MjhlNTZkNmQ=
|
data/COPYING
ADDED
@@ -0,0 +1,14 @@
|
|
1
|
+
Copyright (C) 2015 Sadayuki Furuhashi
|
2
|
+
|
3
|
+
Licensed under the Apache License, Version 2.0 (the "License");
|
4
|
+
you may not use this file except in compliance with the License.
|
5
|
+
You may obtain a copy of the License at
|
6
|
+
|
7
|
+
http://www.apache.org/licenses/LICENSE-2.0
|
8
|
+
|
9
|
+
Unless required by applicable law or agreed to in writing, software
|
10
|
+
distributed under the License is distributed on an "AS IS" BASIS,
|
11
|
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
12
|
+
See the License for the specific language governing permissions and
|
13
|
+
limitations under the License.
|
14
|
+
|
data/ChangeLog
ADDED
data/README.md
CHANGED
@@ -1,4 +1,6 @@
|
|
1
|
-
# Embulk
|
1
|
+
# Embulk
|
2
|
+
|
3
|
+
A plugin-based parallel bulk data loader that makes painful data integration works relaxed.
|
2
4
|
|
3
5
|
## What's Embulk?
|
4
6
|
|
@@ -6,10 +8,10 @@ TODO
|
|
6
8
|
|
7
9
|
## Quick Start
|
8
10
|
|
9
|
-
The single-file package is the simplest way to try Embulk. You can download the latest embulk.jar from [releases]() and run it with java:
|
11
|
+
The single-file package is the simplest way to try Embulk. You can download the latest embulk-VERSION.jar from [the releases page](https://bintray.com/embulk/maven/embulk/view#files) and run it with java:
|
10
12
|
|
11
13
|
```
|
12
|
-
wget https://
|
14
|
+
wget https://bintray.com/artifact/download/embulk/maven/embulk-0.2.0.jar -O embulk.jar
|
13
15
|
java -jar embulk.jar --help
|
14
16
|
```
|
15
17
|
|
data/build.gradle
CHANGED
data/embulk-cli/pom.xml
CHANGED
data/embulk-core/pom.xml
CHANGED
data/embulk-standards/pom.xml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
module Embulk
|
2
2
|
|
3
3
|
class InputExample < InputPlugin
|
4
|
-
# input plugin file name must be: embulk/
|
4
|
+
# input plugin file name must be: embulk/input_<name>.rb
|
5
5
|
Plugin.register_input('example', self)
|
6
6
|
|
7
7
|
def self.transaction(config, &control)
|
@@ -23,13 +23,17 @@ module Embulk
|
|
23
23
|
return {}
|
24
24
|
end
|
25
25
|
|
26
|
-
def
|
27
|
-
|
26
|
+
def initialize(task, schema, index, page_builder)
|
27
|
+
super
|
28
|
+
end
|
29
|
+
|
30
|
+
def run
|
31
|
+
puts "Example input thread #{@index}..."
|
28
32
|
|
29
33
|
10.times do |i|
|
30
|
-
page_builder.add([i, 10.0, "example"])
|
34
|
+
@page_builder.add([i, 10.0, "example"])
|
31
35
|
end
|
32
|
-
page_builder.finish # don't forget to call finish :-)
|
36
|
+
@page_builder.finish # don't forget to call finish :-)
|
33
37
|
|
34
38
|
commit_report = {
|
35
39
|
}
|
data/lib/embulk/guess_gzip.rb
CHANGED
@@ -1,18 +1,16 @@
|
|
1
1
|
module Embulk
|
2
|
-
module Guess
|
3
2
|
|
4
|
-
|
5
|
-
|
3
|
+
class GzipGuess < GuessPlugin
|
4
|
+
Plugin.register_guess('gzip', self)
|
6
5
|
|
7
|
-
|
6
|
+
GZIP_HEADER = "\x1f\x8b".force_encoding('ASCII-8BIT').freeze
|
8
7
|
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
end
|
13
|
-
return {}
|
8
|
+
def guess(config, sample_buffer)
|
9
|
+
if sample_buffer[0,2] == GZIP_HEADER
|
10
|
+
return {"decoders" => [{"type" => "gzip"}]}
|
14
11
|
end
|
12
|
+
return {}
|
15
13
|
end
|
16
|
-
|
17
14
|
end
|
15
|
+
|
18
16
|
end
|
data/lib/embulk/input_plugin.rb
CHANGED
@@ -9,8 +9,15 @@ module Embulk
|
|
9
9
|
raise NotImplementedError, "InputPlugin.transaction(config, &control) must be implemented"
|
10
10
|
end
|
11
11
|
|
12
|
-
def
|
13
|
-
|
12
|
+
def initialize(task, schema, index, page_builder)
|
13
|
+
@task = task
|
14
|
+
@schema = schema
|
15
|
+
@index = index
|
16
|
+
@page_builder = page_builder
|
17
|
+
end
|
18
|
+
|
19
|
+
def run
|
20
|
+
raise NotImplementedError, "InputPlugin#run must be implemented"
|
14
21
|
end
|
15
22
|
|
16
23
|
if Embulk.java?
|
@@ -43,7 +50,7 @@ module Embulk
|
|
43
50
|
task_source = DataSource.from_java_object(java_task_source)
|
44
51
|
schema = Schema.from_java_object(java_schema)
|
45
52
|
page_builder = PageBuilder.new(schema, java_output)
|
46
|
-
commit_report_hash = @ruby_class.
|
53
|
+
commit_report_hash = @ruby_class.new(task_source, schema, processor_index, page_builder).run
|
47
54
|
return DataSource.from_ruby_hash(commit_report_hash).java_object
|
48
55
|
end
|
49
56
|
end
|
data/lib/embulk/version.rb
CHANGED
data/pom.xml
CHANGED
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: embulk
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.2.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Sadayuki Furuhashi
|
@@ -105,6 +105,8 @@ extensions: []
|
|
105
105
|
extra_rdoc_files: []
|
106
106
|
files:
|
107
107
|
- .gitignore
|
108
|
+
- COPYING
|
109
|
+
- ChangeLog
|
108
110
|
- Gemfile
|
109
111
|
- Gemfile.lock
|
110
112
|
- README.md
|
@@ -119,9 +121,9 @@ files:
|
|
119
121
|
- classpath/commons-codec-1.3.jar
|
120
122
|
- classpath/commons-lang3-3.1.jar
|
121
123
|
- classpath/commons-logging-1.2.jar
|
122
|
-
- classpath/embulk-cli-0.
|
123
|
-
- classpath/embulk-core-0.
|
124
|
-
- classpath/embulk-standards-0.
|
124
|
+
- classpath/embulk-cli-0.2.0-SNAPSHOT.jar
|
125
|
+
- classpath/embulk-core-0.2.0-SNAPSHOT.jar
|
126
|
+
- classpath/embulk-standards-0.2.0-SNAPSHOT.jar
|
125
127
|
- classpath/guava-17.0.jar
|
126
128
|
- classpath/guice-3.0.jar
|
127
129
|
- classpath/guice-multibindings-3.0.jar
|