embulk 0.6.16 → 0.6.17
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +33 -45
- data/build.gradle +3 -3
- data/embulk-core/src/main/java/org/embulk/spi/Exec.java +6 -0
- data/embulk-core/src/main/java/org/embulk/spi/util/InputStreamFileInput.java +73 -1
- data/embulk-core/src/main/java/org/embulk/spi/util/InputStreamTransactionalFileInput.java +25 -0
- data/embulk-core/src/main/java/org/embulk/spi/util/Timestamps.java +53 -0
- data/embulk-docs/src/_static/embulk-logo.svg +133 -0
- data/embulk-docs/src/release.rst +1 -0
- data/embulk-docs/src/release/release-0.6.17.rst +39 -0
- data/embulk-standards/src/main/java/org/embulk/standards/CsvFormatterPlugin.java +2 -17
- data/embulk-standards/src/main/java/org/embulk/standards/CsvParserPlugin.java +3 -22
- data/embulk-standards/src/main/java/org/embulk/standards/GzipFileDecoderPlugin.java +2 -2
- data/embulk-standards/src/main/java/org/embulk/standards/GzipFileEncoderPlugin.java +0 -1
- data/embulk-standards/src/main/java/org/embulk/standards/LocalFileInputPlugin.java +18 -42
- data/embulk-standards/src/main/java/org/embulk/standards/LocalFileOutputPlugin.java +3 -3
- data/lib/embulk/command/embulk_new_plugin.rb +40 -14
- data/lib/embulk/command/embulk_run.rb +3 -3
- data/lib/embulk/data/new/README.md.erb +18 -17
- data/lib/embulk/data/new/java/build.gradle.erb +1 -1
- data/lib/embulk/data/new/java/decoder.java.erb +53 -9
- data/lib/embulk/data/new/java/encoder.java.erb +54 -8
- data/lib/embulk/data/new/java/file_input.java.erb +91 -12
- data/lib/embulk/data/new/java/file_output.java.erb +35 -8
- data/lib/embulk/data/new/java/filter.java.erb +16 -7
- data/lib/embulk/data/new/java/formatter.java.erb +16 -7
- data/lib/embulk/data/new/java/input.java.erb +18 -14
- data/lib/embulk/data/new/java/output.java.erb +16 -8
- data/lib/embulk/data/new/java/parser.java.erb +17 -8
- data/lib/embulk/data/new/java/plugin_loader.rb.erb +1 -1
- data/lib/embulk/data/new/java/test.java.erb +1 -1
- data/lib/embulk/data/new/ruby/filter.rb.erb +6 -4
- data/lib/embulk/data/new/ruby/formatter.rb.erb +6 -4
- data/lib/embulk/data/new/ruby/gemspec.erb +2 -2
- data/lib/embulk/data/new/ruby/input.rb.erb +6 -4
- data/lib/embulk/data/new/ruby/output.rb.erb +6 -4
- data/lib/embulk/data/new/ruby/parser.rb.erb +6 -4
- data/lib/embulk/file_input.rb +4 -0
- data/lib/embulk/file_output.rb +4 -0
- data/lib/embulk/version.rb +1 -1
- metadata +8 -4
@@ -1,6 +1,7 @@
|
|
1
|
-
package
|
1
|
+
package <%= java_package_name %>;
|
2
2
|
|
3
3
|
import java.util.List;
|
4
|
+
import com.google.common.base.Optional;
|
4
5
|
import org.embulk.config.CommitReport;
|
5
6
|
import org.embulk.config.Config;
|
6
7
|
import org.embulk.config.ConfigDefault;
|
@@ -20,14 +21,21 @@ public class <%= java_class_name %>
|
|
20
21
|
public interface PluginTask
|
21
22
|
extends Task
|
22
23
|
{
|
23
|
-
|
24
|
-
|
24
|
+
// configuration option 1 (required integer)
|
25
|
+
@Config("option1")
|
26
|
+
public int getOption1();
|
25
27
|
|
26
|
-
|
27
|
-
@
|
28
|
-
|
28
|
+
// configuration option 2 (optional string, null is not allowed)
|
29
|
+
@Config("optoin2")
|
30
|
+
@ConfigDefault("\"myvalue\"")
|
31
|
+
public String getOption2();
|
29
32
|
|
30
|
-
//
|
33
|
+
// configuration option 3 (optional string, null is allowed)
|
34
|
+
@Config("optoin3")
|
35
|
+
@ConfigDefault("null")
|
36
|
+
public Optional<String> getOption3();
|
37
|
+
|
38
|
+
// if you get schema from config
|
31
39
|
@Config("columns")
|
32
40
|
public SchemaConfig getColumns();
|
33
41
|
}
|
@@ -67,17 +75,13 @@ public class <%= java_class_name %>
|
|
67
75
|
{
|
68
76
|
PluginTask task = taskSource.loadTask(PluginTask.class);
|
69
77
|
|
70
|
-
//
|
71
|
-
throw new UnsupportedOperationException("
|
78
|
+
// Write your code here :)
|
79
|
+
throw new UnsupportedOperationException("<%= java_class_name %>.run method is not implemented yet");
|
72
80
|
}
|
73
81
|
|
74
82
|
@Override
|
75
83
|
public ConfigDiff guess(ConfigSource config)
|
76
84
|
{
|
77
|
-
|
78
|
-
throw new UnsupportedOperationException("'<%= name %>' input plugin does not support guessing.");
|
79
|
-
//ConfigDiff diff = Exec.newConfigDiff();
|
80
|
-
//diff.set("property1", "value");
|
81
|
-
//return diff;
|
85
|
+
return Exec.newConfigDiff();
|
82
86
|
}
|
83
87
|
}
|
@@ -1,6 +1,7 @@
|
|
1
|
-
package
|
1
|
+
package <%= java_package_name %>;
|
2
2
|
|
3
3
|
import java.util.List;
|
4
|
+
import com.google.common.base.Optional;
|
4
5
|
import org.embulk.config.CommitReport;
|
5
6
|
import org.embulk.config.Config;
|
6
7
|
import org.embulk.config.ConfigDefault;
|
@@ -20,12 +21,19 @@ public class <%= java_class_name %>
|
|
20
21
|
public interface PluginTask
|
21
22
|
extends Task
|
22
23
|
{
|
23
|
-
|
24
|
-
|
24
|
+
// configuration option 1 (required integer)
|
25
|
+
@Config("option1")
|
26
|
+
public int getOption1();
|
25
27
|
|
26
|
-
|
27
|
-
@
|
28
|
-
|
28
|
+
// configuration option 2 (optional string, null is not allowed)
|
29
|
+
@Config("optoin2")
|
30
|
+
@ConfigDefault("\"myvalue\"")
|
31
|
+
public String getOption2();
|
32
|
+
|
33
|
+
// configuration option 3 (optional string, null is allowed)
|
34
|
+
@Config("optoin3")
|
35
|
+
@ConfigDefault("null")
|
36
|
+
public Optional<String> getOption3();
|
29
37
|
}
|
30
38
|
|
31
39
|
@Override
|
@@ -63,7 +71,7 @@ public class <%= java_class_name %>
|
|
63
71
|
{
|
64
72
|
PluginTask task = taskSource.loadTask(PluginTask.class);
|
65
73
|
|
66
|
-
//
|
67
|
-
throw new UnsupportedOperationException("
|
74
|
+
// Write your code here :)
|
75
|
+
throw new UnsupportedOperationException("<%= java_class_name %>.run method is not implemented yet");
|
68
76
|
}
|
69
77
|
}
|
@@ -1,5 +1,6 @@
|
|
1
|
-
package
|
1
|
+
package <%= java_package_name %>;
|
2
2
|
|
3
|
+
import com.google.common.base.Optional;
|
3
4
|
import org.embulk.config.Config;
|
4
5
|
import org.embulk.config.ConfigDefault;
|
5
6
|
import org.embulk.config.ConfigDiff;
|
@@ -18,14 +19,21 @@ public class <%= java_class_name %>
|
|
18
19
|
public interface PluginTask
|
19
20
|
extends Task
|
20
21
|
{
|
21
|
-
|
22
|
-
|
22
|
+
// configuration option 1 (required integer)
|
23
|
+
@Config("option1")
|
24
|
+
public int getOption1();
|
23
25
|
|
24
|
-
|
25
|
-
@
|
26
|
-
|
26
|
+
// configuration option 2 (optional string, null is not allowed)
|
27
|
+
@Config("optoin2")
|
28
|
+
@ConfigDefault("\"myvalue\"")
|
29
|
+
public String getOption2();
|
27
30
|
|
28
|
-
//
|
31
|
+
// configuration option 3 (optional string, null is allowed)
|
32
|
+
@Config("optoin3")
|
33
|
+
@ConfigDefault("null")
|
34
|
+
public Optional<String> getOption3();
|
35
|
+
|
36
|
+
// if you get schema from config or data source
|
29
37
|
@Config("columns")
|
30
38
|
public SchemaConfig getColumns();
|
31
39
|
}
|
@@ -46,6 +54,7 @@ public class <%= java_class_name %>
|
|
46
54
|
{
|
47
55
|
PluginTask task = taskSource.loadTask(PluginTask.class);
|
48
56
|
|
49
|
-
//
|
57
|
+
// Write your code here :)
|
58
|
+
throw new UnsupportedOperationException("<%= java_class_name %>.run method is not implemented yet");
|
50
59
|
}
|
51
60
|
}
|
@@ -7,8 +7,9 @@ module Embulk
|
|
7
7
|
def self.transaction(config, in_schema, &control)
|
8
8
|
# configuration code:
|
9
9
|
task = {
|
10
|
-
"
|
11
|
-
"
|
10
|
+
"option1" => config.param("option1", :integer), # integer, required
|
11
|
+
"option2" => config.param("option2", :string, default: "myvalue"), # string, optional
|
12
|
+
"option3" => config.param("option3", :string, default: nil), # string, optional
|
12
13
|
}
|
13
14
|
|
14
15
|
yield(task, out_columns)
|
@@ -16,8 +17,9 @@ module Embulk
|
|
16
17
|
|
17
18
|
def init
|
18
19
|
# initialization code:
|
19
|
-
@
|
20
|
-
@
|
20
|
+
@option1 = task["option1"]
|
21
|
+
@option2 = task["option2"]
|
22
|
+
@option3 = task["option3"]
|
21
23
|
end
|
22
24
|
|
23
25
|
def close
|
@@ -7,8 +7,9 @@ module Embulk
|
|
7
7
|
def self.transaction(config, schema, &control)
|
8
8
|
# configuration code:
|
9
9
|
task = {
|
10
|
-
"
|
11
|
-
"
|
10
|
+
"option1" => config.param("option1", :integer), # integer, required
|
11
|
+
"option2" => config.param("option2", :string, default: "myvalue"), # string, optional
|
12
|
+
"option3" => config.param("option3", :string, default: nil), # string, optional
|
12
13
|
}
|
13
14
|
|
14
15
|
yield(task)
|
@@ -16,8 +17,9 @@ module Embulk
|
|
16
17
|
|
17
18
|
def init
|
18
19
|
# initialization code:
|
19
|
-
@
|
20
|
-
@
|
20
|
+
@option1 = task["option1"]
|
21
|
+
@option2 = task["option2"]
|
22
|
+
@option3 = task["option3"]
|
21
23
|
|
22
24
|
# your data
|
23
25
|
@current_file == nil
|
@@ -1,13 +1,13 @@
|
|
1
1
|
|
2
2
|
Gem::Specification.new do |spec|
|
3
|
-
spec.name = "<%=
|
3
|
+
spec.name = "<%= full_project_name %>"
|
4
4
|
spec.version = "0.1.0"
|
5
5
|
spec.authors = [<%= author.dump %>]
|
6
6
|
spec.summary = <%= "#{display_name} #{display_category} plugin for Embulk".dump %>
|
7
7
|
spec.description = <%= "#{description}".dump %>
|
8
8
|
spec.email = [<%= email.dump %>]
|
9
9
|
spec.licenses = ["MIT"]
|
10
|
-
# TODO set this: spec.homepage = "https://github.com/<%= email[/([^@]*)/] %>/<%=
|
10
|
+
# TODO set this: spec.homepage = "https://github.com/<%= email[/([^@]*)/] %>/<%= full_project_name %>"
|
11
11
|
|
12
12
|
spec.files = `git ls-files`.split("\n") + Dir["classpath/*.jar"]
|
13
13
|
spec.test_files = spec.files.grep(%r{^(test|spec)/})
|
@@ -7,8 +7,9 @@ module Embulk
|
|
7
7
|
def self.transaction(config, &control)
|
8
8
|
# configuration code:
|
9
9
|
task = {
|
10
|
-
"
|
11
|
-
"
|
10
|
+
"option1" => config.param("option1", :integer), # integer, required
|
11
|
+
"option2" => config.param("option2", :string, default: "myvalue"), # string, optional
|
12
|
+
"option3" => config.param("option3", :string, default: nil), # string, optional
|
12
13
|
}
|
13
14
|
|
14
15
|
columns = [
|
@@ -39,8 +40,9 @@ module Embulk
|
|
39
40
|
|
40
41
|
def init
|
41
42
|
# initialization code:
|
42
|
-
@
|
43
|
-
@
|
43
|
+
@option1 = task["option1"]
|
44
|
+
@option2 = task["option2"]
|
45
|
+
@option3 = task["option3"]
|
44
46
|
end
|
45
47
|
|
46
48
|
def run
|
@@ -7,8 +7,9 @@ module Embulk
|
|
7
7
|
def self.transaction(config, schema, count, &control)
|
8
8
|
# configuration code:
|
9
9
|
task = {
|
10
|
-
"
|
11
|
-
"
|
10
|
+
"option1" => config.param("option1", :integer), # integer, required
|
11
|
+
"option2" => config.param("option2", :string, default: "myvalue"), # string, optional
|
12
|
+
"option3" => config.param("option3", :string, default: nil), # string, optional
|
12
13
|
}
|
13
14
|
|
14
15
|
# resumable output:
|
@@ -29,8 +30,9 @@ module Embulk
|
|
29
30
|
|
30
31
|
def init
|
31
32
|
# initialization code:
|
32
|
-
@
|
33
|
-
@
|
33
|
+
@option1 = task["option1"]
|
34
|
+
@option2 = task["option2"]
|
35
|
+
@option3 = task["option3"]
|
34
36
|
end
|
35
37
|
|
36
38
|
def close
|
@@ -7,8 +7,9 @@ module Embulk
|
|
7
7
|
def self.transaction(config, &control)
|
8
8
|
# configuration code:
|
9
9
|
task = {
|
10
|
-
"
|
11
|
-
"
|
10
|
+
"option1" => config.param("option1", :integer), # integer, required
|
11
|
+
"option2" => config.param("option2", :string, default: "myvalue"), # string, optional
|
12
|
+
"option3" => config.param("option3", :string, default: nil), # string, optional
|
12
13
|
}
|
13
14
|
|
14
15
|
columns = [
|
@@ -22,8 +23,9 @@ module Embulk
|
|
22
23
|
|
23
24
|
def init
|
24
25
|
# initialization code:
|
25
|
-
@
|
26
|
-
@
|
26
|
+
@option1 = task["option1"]
|
27
|
+
@option2 = task["option2"]
|
28
|
+
@option3 = task["option3"]
|
27
29
|
end
|
28
30
|
|
29
31
|
def run(file_input)
|
data/lib/embulk/file_input.rb
CHANGED
data/lib/embulk/file_output.rb
CHANGED
data/lib/embulk/version.rb
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: embulk
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.6.
|
4
|
+
version: 0.6.17
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Sadayuki Furuhashi
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2015-07-
|
11
|
+
date: 2015-07-17 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|
@@ -234,6 +234,7 @@ files:
|
|
234
234
|
- embulk-core/src/main/java/org/embulk/spi/util/FileOutputOutputStream.java
|
235
235
|
- embulk-core/src/main/java/org/embulk/spi/util/Filters.java
|
236
236
|
- embulk-core/src/main/java/org/embulk/spi/util/InputStreamFileInput.java
|
237
|
+
- embulk-core/src/main/java/org/embulk/spi/util/InputStreamTransactionalFileInput.java
|
237
238
|
- embulk-core/src/main/java/org/embulk/spi/util/Inputs.java
|
238
239
|
- embulk-core/src/main/java/org/embulk/spi/util/LineDecoder.java
|
239
240
|
- embulk-core/src/main/java/org/embulk/spi/util/LineEncoder.java
|
@@ -244,6 +245,7 @@ files:
|
|
244
245
|
- embulk-core/src/main/java/org/embulk/spi/util/Pages.java
|
245
246
|
- embulk-core/src/main/java/org/embulk/spi/util/ResumableInputStream.java
|
246
247
|
- embulk-core/src/main/java/org/embulk/spi/util/RetryExecutor.java
|
248
|
+
- embulk-core/src/main/java/org/embulk/spi/util/Timestamps.java
|
247
249
|
- embulk-core/src/main/java/org/embulk/spi/util/dynamic/AbstractDynamicColumnSetter.java
|
248
250
|
- embulk-core/src/main/java/org/embulk/spi/util/dynamic/BooleanColumnSetter.java
|
249
251
|
- embulk-core/src/main/java/org/embulk/spi/util/dynamic/DefaultValueSetter.java
|
@@ -290,6 +292,7 @@ files:
|
|
290
292
|
- embulk-docs/push-gh-pages.sh
|
291
293
|
- embulk-docs/src/_static/embulk-architecture.png
|
292
294
|
- embulk-docs/src/_static/embulk-logo.png
|
295
|
+
- embulk-docs/src/_static/embulk-logo.svg
|
293
296
|
- embulk-docs/src/built-in.rst
|
294
297
|
- embulk-docs/src/conf.py
|
295
298
|
- embulk-docs/src/customization.rst
|
@@ -329,6 +332,7 @@ files:
|
|
329
332
|
- embulk-docs/src/release/release-0.6.14.rst
|
330
333
|
- embulk-docs/src/release/release-0.6.15.rst
|
331
334
|
- embulk-docs/src/release/release-0.6.16.rst
|
335
|
+
- embulk-docs/src/release/release-0.6.17.rst
|
332
336
|
- embulk-docs/src/release/release-0.6.2.rst
|
333
337
|
- embulk-docs/src/release/release-0.6.3.rst
|
334
338
|
- embulk-docs/src/release/release-0.6.4.rst
|
@@ -445,8 +449,8 @@ files:
|
|
445
449
|
- classpath/bval-jsr303-0.5.jar
|
446
450
|
- classpath/commons-beanutils-core-1.8.3.jar
|
447
451
|
- classpath/commons-lang3-3.1.jar
|
448
|
-
- classpath/embulk-core-0.6.
|
449
|
-
- classpath/embulk-standards-0.6.
|
452
|
+
- classpath/embulk-core-0.6.17.jar
|
453
|
+
- classpath/embulk-standards-0.6.17.jar
|
450
454
|
- classpath/guava-18.0.jar
|
451
455
|
- classpath/guice-4.0.jar
|
452
456
|
- classpath/guice-multibindings-4.0.jar
|