embulk 0.6.16 → 0.6.17
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +33 -45
- data/build.gradle +3 -3
- data/embulk-core/src/main/java/org/embulk/spi/Exec.java +6 -0
- data/embulk-core/src/main/java/org/embulk/spi/util/InputStreamFileInput.java +73 -1
- data/embulk-core/src/main/java/org/embulk/spi/util/InputStreamTransactionalFileInput.java +25 -0
- data/embulk-core/src/main/java/org/embulk/spi/util/Timestamps.java +53 -0
- data/embulk-docs/src/_static/embulk-logo.svg +133 -0
- data/embulk-docs/src/release.rst +1 -0
- data/embulk-docs/src/release/release-0.6.17.rst +39 -0
- data/embulk-standards/src/main/java/org/embulk/standards/CsvFormatterPlugin.java +2 -17
- data/embulk-standards/src/main/java/org/embulk/standards/CsvParserPlugin.java +3 -22
- data/embulk-standards/src/main/java/org/embulk/standards/GzipFileDecoderPlugin.java +2 -2
- data/embulk-standards/src/main/java/org/embulk/standards/GzipFileEncoderPlugin.java +0 -1
- data/embulk-standards/src/main/java/org/embulk/standards/LocalFileInputPlugin.java +18 -42
- data/embulk-standards/src/main/java/org/embulk/standards/LocalFileOutputPlugin.java +3 -3
- data/lib/embulk/command/embulk_new_plugin.rb +40 -14
- data/lib/embulk/command/embulk_run.rb +3 -3
- data/lib/embulk/data/new/README.md.erb +18 -17
- data/lib/embulk/data/new/java/build.gradle.erb +1 -1
- data/lib/embulk/data/new/java/decoder.java.erb +53 -9
- data/lib/embulk/data/new/java/encoder.java.erb +54 -8
- data/lib/embulk/data/new/java/file_input.java.erb +91 -12
- data/lib/embulk/data/new/java/file_output.java.erb +35 -8
- data/lib/embulk/data/new/java/filter.java.erb +16 -7
- data/lib/embulk/data/new/java/formatter.java.erb +16 -7
- data/lib/embulk/data/new/java/input.java.erb +18 -14
- data/lib/embulk/data/new/java/output.java.erb +16 -8
- data/lib/embulk/data/new/java/parser.java.erb +17 -8
- data/lib/embulk/data/new/java/plugin_loader.rb.erb +1 -1
- data/lib/embulk/data/new/java/test.java.erb +1 -1
- data/lib/embulk/data/new/ruby/filter.rb.erb +6 -4
- data/lib/embulk/data/new/ruby/formatter.rb.erb +6 -4
- data/lib/embulk/data/new/ruby/gemspec.erb +2 -2
- data/lib/embulk/data/new/ruby/input.rb.erb +6 -4
- data/lib/embulk/data/new/ruby/output.rb.erb +6 -4
- data/lib/embulk/data/new/ruby/parser.rb.erb +6 -4
- data/lib/embulk/file_input.rb +4 -0
- data/lib/embulk/file_output.rb +4 -0
- data/lib/embulk/version.rb +1 -1
- metadata +8 -4
@@ -1,6 +1,7 @@
|
|
1
|
-
package
|
1
|
+
package <%= java_package_name %>;
|
2
2
|
|
3
3
|
import java.util.List;
|
4
|
+
import com.google.common.base.Optional;
|
4
5
|
import org.embulk.config.CommitReport;
|
5
6
|
import org.embulk.config.Config;
|
6
7
|
import org.embulk.config.ConfigDefault;
|
@@ -20,14 +21,21 @@ public class <%= java_class_name %>
|
|
20
21
|
public interface PluginTask
|
21
22
|
extends Task
|
22
23
|
{
|
23
|
-
|
24
|
-
|
24
|
+
// configuration option 1 (required integer)
|
25
|
+
@Config("option1")
|
26
|
+
public int getOption1();
|
25
27
|
|
26
|
-
|
27
|
-
@
|
28
|
-
|
28
|
+
// configuration option 2 (optional string, null is not allowed)
|
29
|
+
@Config("optoin2")
|
30
|
+
@ConfigDefault("\"myvalue\"")
|
31
|
+
public String getOption2();
|
29
32
|
|
30
|
-
//
|
33
|
+
// configuration option 3 (optional string, null is allowed)
|
34
|
+
@Config("optoin3")
|
35
|
+
@ConfigDefault("null")
|
36
|
+
public Optional<String> getOption3();
|
37
|
+
|
38
|
+
// if you get schema from config
|
31
39
|
@Config("columns")
|
32
40
|
public SchemaConfig getColumns();
|
33
41
|
}
|
@@ -67,17 +75,13 @@ public class <%= java_class_name %>
|
|
67
75
|
{
|
68
76
|
PluginTask task = taskSource.loadTask(PluginTask.class);
|
69
77
|
|
70
|
-
//
|
71
|
-
throw new UnsupportedOperationException("
|
78
|
+
// Write your code here :)
|
79
|
+
throw new UnsupportedOperationException("<%= java_class_name %>.run method is not implemented yet");
|
72
80
|
}
|
73
81
|
|
74
82
|
@Override
|
75
83
|
public ConfigDiff guess(ConfigSource config)
|
76
84
|
{
|
77
|
-
|
78
|
-
throw new UnsupportedOperationException("'<%= name %>' input plugin does not support guessing.");
|
79
|
-
//ConfigDiff diff = Exec.newConfigDiff();
|
80
|
-
//diff.set("property1", "value");
|
81
|
-
//return diff;
|
85
|
+
return Exec.newConfigDiff();
|
82
86
|
}
|
83
87
|
}
|
@@ -1,6 +1,7 @@
|
|
1
|
-
package
|
1
|
+
package <%= java_package_name %>;
|
2
2
|
|
3
3
|
import java.util.List;
|
4
|
+
import com.google.common.base.Optional;
|
4
5
|
import org.embulk.config.CommitReport;
|
5
6
|
import org.embulk.config.Config;
|
6
7
|
import org.embulk.config.ConfigDefault;
|
@@ -20,12 +21,19 @@ public class <%= java_class_name %>
|
|
20
21
|
public interface PluginTask
|
21
22
|
extends Task
|
22
23
|
{
|
23
|
-
|
24
|
-
|
24
|
+
// configuration option 1 (required integer)
|
25
|
+
@Config("option1")
|
26
|
+
public int getOption1();
|
25
27
|
|
26
|
-
|
27
|
-
@
|
28
|
-
|
28
|
+
// configuration option 2 (optional string, null is not allowed)
|
29
|
+
@Config("optoin2")
|
30
|
+
@ConfigDefault("\"myvalue\"")
|
31
|
+
public String getOption2();
|
32
|
+
|
33
|
+
// configuration option 3 (optional string, null is allowed)
|
34
|
+
@Config("optoin3")
|
35
|
+
@ConfigDefault("null")
|
36
|
+
public Optional<String> getOption3();
|
29
37
|
}
|
30
38
|
|
31
39
|
@Override
|
@@ -63,7 +71,7 @@ public class <%= java_class_name %>
|
|
63
71
|
{
|
64
72
|
PluginTask task = taskSource.loadTask(PluginTask.class);
|
65
73
|
|
66
|
-
//
|
67
|
-
throw new UnsupportedOperationException("
|
74
|
+
// Write your code here :)
|
75
|
+
throw new UnsupportedOperationException("<%= java_class_name %>.run method is not implemented yet");
|
68
76
|
}
|
69
77
|
}
|
@@ -1,5 +1,6 @@
|
|
1
|
-
package
|
1
|
+
package <%= java_package_name %>;
|
2
2
|
|
3
|
+
import com.google.common.base.Optional;
|
3
4
|
import org.embulk.config.Config;
|
4
5
|
import org.embulk.config.ConfigDefault;
|
5
6
|
import org.embulk.config.ConfigDiff;
|
@@ -18,14 +19,21 @@ public class <%= java_class_name %>
|
|
18
19
|
public interface PluginTask
|
19
20
|
extends Task
|
20
21
|
{
|
21
|
-
|
22
|
-
|
22
|
+
// configuration option 1 (required integer)
|
23
|
+
@Config("option1")
|
24
|
+
public int getOption1();
|
23
25
|
|
24
|
-
|
25
|
-
@
|
26
|
-
|
26
|
+
// configuration option 2 (optional string, null is not allowed)
|
27
|
+
@Config("optoin2")
|
28
|
+
@ConfigDefault("\"myvalue\"")
|
29
|
+
public String getOption2();
|
27
30
|
|
28
|
-
//
|
31
|
+
// configuration option 3 (optional string, null is allowed)
|
32
|
+
@Config("optoin3")
|
33
|
+
@ConfigDefault("null")
|
34
|
+
public Optional<String> getOption3();
|
35
|
+
|
36
|
+
// if you get schema from config or data source
|
29
37
|
@Config("columns")
|
30
38
|
public SchemaConfig getColumns();
|
31
39
|
}
|
@@ -46,6 +54,7 @@ public class <%= java_class_name %>
|
|
46
54
|
{
|
47
55
|
PluginTask task = taskSource.loadTask(PluginTask.class);
|
48
56
|
|
49
|
-
//
|
57
|
+
// Write your code here :)
|
58
|
+
throw new UnsupportedOperationException("<%= java_class_name %>.run method is not implemented yet");
|
50
59
|
}
|
51
60
|
}
|
@@ -7,8 +7,9 @@ module Embulk
|
|
7
7
|
def self.transaction(config, in_schema, &control)
|
8
8
|
# configuration code:
|
9
9
|
task = {
|
10
|
-
"
|
11
|
-
"
|
10
|
+
"option1" => config.param("option1", :integer), # integer, required
|
11
|
+
"option2" => config.param("option2", :string, default: "myvalue"), # string, optional
|
12
|
+
"option3" => config.param("option3", :string, default: nil), # string, optional
|
12
13
|
}
|
13
14
|
|
14
15
|
yield(task, out_columns)
|
@@ -16,8 +17,9 @@ module Embulk
|
|
16
17
|
|
17
18
|
def init
|
18
19
|
# initialization code:
|
19
|
-
@
|
20
|
-
@
|
20
|
+
@option1 = task["option1"]
|
21
|
+
@option2 = task["option2"]
|
22
|
+
@option3 = task["option3"]
|
21
23
|
end
|
22
24
|
|
23
25
|
def close
|
@@ -7,8 +7,9 @@ module Embulk
|
|
7
7
|
def self.transaction(config, schema, &control)
|
8
8
|
# configuration code:
|
9
9
|
task = {
|
10
|
-
"
|
11
|
-
"
|
10
|
+
"option1" => config.param("option1", :integer), # integer, required
|
11
|
+
"option2" => config.param("option2", :string, default: "myvalue"), # string, optional
|
12
|
+
"option3" => config.param("option3", :string, default: nil), # string, optional
|
12
13
|
}
|
13
14
|
|
14
15
|
yield(task)
|
@@ -16,8 +17,9 @@ module Embulk
|
|
16
17
|
|
17
18
|
def init
|
18
19
|
# initialization code:
|
19
|
-
@
|
20
|
-
@
|
20
|
+
@option1 = task["option1"]
|
21
|
+
@option2 = task["option2"]
|
22
|
+
@option3 = task["option3"]
|
21
23
|
|
22
24
|
# your data
|
23
25
|
@current_file == nil
|
@@ -1,13 +1,13 @@
|
|
1
1
|
|
2
2
|
Gem::Specification.new do |spec|
|
3
|
-
spec.name = "<%=
|
3
|
+
spec.name = "<%= full_project_name %>"
|
4
4
|
spec.version = "0.1.0"
|
5
5
|
spec.authors = [<%= author.dump %>]
|
6
6
|
spec.summary = <%= "#{display_name} #{display_category} plugin for Embulk".dump %>
|
7
7
|
spec.description = <%= "#{description}".dump %>
|
8
8
|
spec.email = [<%= email.dump %>]
|
9
9
|
spec.licenses = ["MIT"]
|
10
|
-
# TODO set this: spec.homepage = "https://github.com/<%= email[/([^@]*)/] %>/<%=
|
10
|
+
# TODO set this: spec.homepage = "https://github.com/<%= email[/([^@]*)/] %>/<%= full_project_name %>"
|
11
11
|
|
12
12
|
spec.files = `git ls-files`.split("\n") + Dir["classpath/*.jar"]
|
13
13
|
spec.test_files = spec.files.grep(%r{^(test|spec)/})
|
@@ -7,8 +7,9 @@ module Embulk
|
|
7
7
|
def self.transaction(config, &control)
|
8
8
|
# configuration code:
|
9
9
|
task = {
|
10
|
-
"
|
11
|
-
"
|
10
|
+
"option1" => config.param("option1", :integer), # integer, required
|
11
|
+
"option2" => config.param("option2", :string, default: "myvalue"), # string, optional
|
12
|
+
"option3" => config.param("option3", :string, default: nil), # string, optional
|
12
13
|
}
|
13
14
|
|
14
15
|
columns = [
|
@@ -39,8 +40,9 @@ module Embulk
|
|
39
40
|
|
40
41
|
def init
|
41
42
|
# initialization code:
|
42
|
-
@
|
43
|
-
@
|
43
|
+
@option1 = task["option1"]
|
44
|
+
@option2 = task["option2"]
|
45
|
+
@option3 = task["option3"]
|
44
46
|
end
|
45
47
|
|
46
48
|
def run
|
@@ -7,8 +7,9 @@ module Embulk
|
|
7
7
|
def self.transaction(config, schema, count, &control)
|
8
8
|
# configuration code:
|
9
9
|
task = {
|
10
|
-
"
|
11
|
-
"
|
10
|
+
"option1" => config.param("option1", :integer), # integer, required
|
11
|
+
"option2" => config.param("option2", :string, default: "myvalue"), # string, optional
|
12
|
+
"option3" => config.param("option3", :string, default: nil), # string, optional
|
12
13
|
}
|
13
14
|
|
14
15
|
# resumable output:
|
@@ -29,8 +30,9 @@ module Embulk
|
|
29
30
|
|
30
31
|
def init
|
31
32
|
# initialization code:
|
32
|
-
@
|
33
|
-
@
|
33
|
+
@option1 = task["option1"]
|
34
|
+
@option2 = task["option2"]
|
35
|
+
@option3 = task["option3"]
|
34
36
|
end
|
35
37
|
|
36
38
|
def close
|
@@ -7,8 +7,9 @@ module Embulk
|
|
7
7
|
def self.transaction(config, &control)
|
8
8
|
# configuration code:
|
9
9
|
task = {
|
10
|
-
"
|
11
|
-
"
|
10
|
+
"option1" => config.param("option1", :integer), # integer, required
|
11
|
+
"option2" => config.param("option2", :string, default: "myvalue"), # string, optional
|
12
|
+
"option3" => config.param("option3", :string, default: nil), # string, optional
|
12
13
|
}
|
13
14
|
|
14
15
|
columns = [
|
@@ -22,8 +23,9 @@ module Embulk
|
|
22
23
|
|
23
24
|
def init
|
24
25
|
# initialization code:
|
25
|
-
@
|
26
|
-
@
|
26
|
+
@option1 = task["option1"]
|
27
|
+
@option2 = task["option2"]
|
28
|
+
@option3 = task["option3"]
|
27
29
|
end
|
28
30
|
|
29
31
|
def run(file_input)
|
data/lib/embulk/file_input.rb
CHANGED
data/lib/embulk/file_output.rb
CHANGED
data/lib/embulk/version.rb
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: embulk
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.6.
|
4
|
+
version: 0.6.17
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Sadayuki Furuhashi
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2015-07-
|
11
|
+
date: 2015-07-17 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|
@@ -234,6 +234,7 @@ files:
|
|
234
234
|
- embulk-core/src/main/java/org/embulk/spi/util/FileOutputOutputStream.java
|
235
235
|
- embulk-core/src/main/java/org/embulk/spi/util/Filters.java
|
236
236
|
- embulk-core/src/main/java/org/embulk/spi/util/InputStreamFileInput.java
|
237
|
+
- embulk-core/src/main/java/org/embulk/spi/util/InputStreamTransactionalFileInput.java
|
237
238
|
- embulk-core/src/main/java/org/embulk/spi/util/Inputs.java
|
238
239
|
- embulk-core/src/main/java/org/embulk/spi/util/LineDecoder.java
|
239
240
|
- embulk-core/src/main/java/org/embulk/spi/util/LineEncoder.java
|
@@ -244,6 +245,7 @@ files:
|
|
244
245
|
- embulk-core/src/main/java/org/embulk/spi/util/Pages.java
|
245
246
|
- embulk-core/src/main/java/org/embulk/spi/util/ResumableInputStream.java
|
246
247
|
- embulk-core/src/main/java/org/embulk/spi/util/RetryExecutor.java
|
248
|
+
- embulk-core/src/main/java/org/embulk/spi/util/Timestamps.java
|
247
249
|
- embulk-core/src/main/java/org/embulk/spi/util/dynamic/AbstractDynamicColumnSetter.java
|
248
250
|
- embulk-core/src/main/java/org/embulk/spi/util/dynamic/BooleanColumnSetter.java
|
249
251
|
- embulk-core/src/main/java/org/embulk/spi/util/dynamic/DefaultValueSetter.java
|
@@ -290,6 +292,7 @@ files:
|
|
290
292
|
- embulk-docs/push-gh-pages.sh
|
291
293
|
- embulk-docs/src/_static/embulk-architecture.png
|
292
294
|
- embulk-docs/src/_static/embulk-logo.png
|
295
|
+
- embulk-docs/src/_static/embulk-logo.svg
|
293
296
|
- embulk-docs/src/built-in.rst
|
294
297
|
- embulk-docs/src/conf.py
|
295
298
|
- embulk-docs/src/customization.rst
|
@@ -329,6 +332,7 @@ files:
|
|
329
332
|
- embulk-docs/src/release/release-0.6.14.rst
|
330
333
|
- embulk-docs/src/release/release-0.6.15.rst
|
331
334
|
- embulk-docs/src/release/release-0.6.16.rst
|
335
|
+
- embulk-docs/src/release/release-0.6.17.rst
|
332
336
|
- embulk-docs/src/release/release-0.6.2.rst
|
333
337
|
- embulk-docs/src/release/release-0.6.3.rst
|
334
338
|
- embulk-docs/src/release/release-0.6.4.rst
|
@@ -445,8 +449,8 @@ files:
|
|
445
449
|
- classpath/bval-jsr303-0.5.jar
|
446
450
|
- classpath/commons-beanutils-core-1.8.3.jar
|
447
451
|
- classpath/commons-lang3-3.1.jar
|
448
|
-
- classpath/embulk-core-0.6.
|
449
|
-
- classpath/embulk-standards-0.6.
|
452
|
+
- classpath/embulk-core-0.6.17.jar
|
453
|
+
- classpath/embulk-standards-0.6.17.jar
|
450
454
|
- classpath/guava-18.0.jar
|
451
455
|
- classpath/guice-4.0.jar
|
452
456
|
- classpath/guice-multibindings-4.0.jar
|