embulk 0.8.18 → 0.8.19
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +10 -0
- data/build.gradle +10 -3
- data/embulk-cli/build.gradle +2 -0
- data/embulk-cli/src/main/bat/selfrun.bat +98 -0
- data/embulk-cli/src/main/java/org/embulk/cli/EmbulkExample.java +82 -0
- data/embulk-cli/src/main/java/org/embulk/cli/EmbulkMigrate.java +458 -0
- data/embulk-cli/src/main/java/org/embulk/cli/EmbulkNew.java +419 -0
- data/embulk-cli/src/main/java/org/embulk/cli/EmbulkSelfUpdate.java +248 -0
- data/embulk-cli/src/main/sh/selfrun.sh +0 -103
- data/embulk-cli/src/test/java/org/embulk/cli/SelfrunTest.java +158 -143
- data/embulk-core/build.gradle +2 -2
- data/embulk-core/src/main/java/org/embulk/EmbulkVersion.java +109 -0
- data/embulk-core/src/main/java/org/embulk/exec/GuessExecutor.java +11 -0
- data/embulk-core/src/main/java/org/embulk/exec/PreviewExecutor.java +29 -3
- data/embulk-core/src/main/java/org/embulk/exec/SamplingParserPlugin.java +47 -13
- data/embulk-core/src/main/java/org/embulk/spi/FileInputRunner.java +6 -3
- data/embulk-core/src/main/java/org/embulk/spi/PageBuilder.java +385 -64
- data/embulk-core/src/main/java/org/embulk/spi/TempFileSpace.java +2 -1
- data/embulk-core/src/test/java/org/embulk/spi/TestPageBuilderReader.java +62 -0
- data/embulk-docs/src/built-in.rst +59 -21
- data/embulk-docs/src/customization.rst +8 -8
- data/embulk-docs/src/developers/index.rst +45 -0
- data/embulk-docs/src/index.rst +11 -7
- data/embulk-docs/src/recipe.rst +1 -1
- data/embulk-docs/src/recipe/{scheduled-csv-load-to-elasticsearch-kibana4.rst → scheduled-csv-load-to-elasticsearch-kibana5.rst} +26 -24
- data/embulk-docs/src/release.rst +1 -0
- data/embulk-docs/src/release/release-0.4.0.rst +1 -1
- data/embulk-docs/src/release/release-0.5.0.rst +1 -1
- data/embulk-docs/src/release/release-0.6.0.rst +1 -1
- data/embulk-docs/src/release/release-0.6.20.rst +1 -1
- data/embulk-docs/src/release/release-0.8.19.rst +43 -0
- data/embulk-standards/src/main/java/org/embulk/standards/CsvParserPlugin.java +2 -2
- data/embulk-standards/src/main/java/org/embulk/standards/LocalFileInputPlugin.java +30 -1
- data/embulk-standards/src/test/java/org/embulk/standards/guess/TestCsvGuessPlugin.java +10 -0
- data/embulk-standards/src/test/java/org/embulk/standards/preview/TestFilePreview.java +73 -0
- data/embulk-standards/src/test/resources/org/embulk/standards/guess/csv/test/test_skip_suggest_if_empty_sample_records.csv +5 -0
- data/embulk-standards/src/test/resources/org/embulk/standards/guess/csv/test/test_skip_suggest_if_empty_sample_records_guessed.yml +2 -0
- data/embulk-standards/src/test/resources/org/embulk/standards/guess/csv/test/test_skip_suggest_if_empty_sample_records_seed.yml +1 -0
- data/embulk-standards/src/test/resources/org/embulk/standards/preview/file/test/test_sample_buffer_bytes.csv +5 -0
- data/embulk-standards/src/test/resources/org/embulk/standards/preview/file/test/test_sample_buffer_bytes_exec.yml +1 -0
- data/embulk-standards/src/test/resources/org/embulk/standards/preview/file/test/test_sample_buffer_bytes_load.yml +19 -0
- data/embulk-standards/src/test/resources/org/embulk/standards/preview/file/test/test_sample_buffer_bytes_previewed.csv +1 -0
- data/embulk-standards/src/test/resources/org/embulk/standards/preview/file/test/test_simple.csv +5 -0
- data/embulk-standards/src/test/resources/org/embulk/standards/preview/file/test/test_simple_load.yml +19 -0
- data/embulk-standards/src/test/resources/org/embulk/standards/preview/file/test/test_simple_previewed.csv +4 -0
- data/embulk-test/src/main/java/org/embulk/test/PreviewResultInputPlugin.java +65 -0
- data/embulk-test/src/main/java/org/embulk/test/TestingBulkLoader.java +5 -0
- data/embulk-test/src/main/java/org/embulk/test/TestingEmbulk.java +59 -2
- data/embulk.gemspec +2 -1
- data/lib/embulk/command/embulk_run.rb +11 -49
- data/lib/embulk/data/new/README.md.vm +106 -0
- data/lib/embulk/data/new/{gitignore.erb → gitignore.vm} +3 -3
- data/lib/embulk/data/new/java/{build.gradle.erb → build.gradle.vm} +8 -8
- data/lib/embulk/data/new/java/{decoder.java.erb → decoder.java.vm} +6 -4
- data/lib/embulk/data/new/java/{encoder.java.erb → encoder.java.vm} +7 -5
- data/lib/embulk/data/new/java/{file_input.java.erb → file_input.java.vm} +9 -7
- data/lib/embulk/data/new/java/{file_output.java.erb → file_output.java.vm} +7 -5
- data/lib/embulk/data/new/java/{filter.java.erb → filter.java.vm} +4 -3
- data/lib/embulk/data/new/java/{formatter.java.erb → formatter.java.vm} +5 -4
- data/lib/embulk/data/new/java/{input.java.erb → input.java.vm} +6 -4
- data/lib/embulk/data/new/java/{output.java.erb → output.java.vm} +7 -5
- data/lib/embulk/data/new/java/{parser.java.erb → parser.java.vm} +5 -4
- data/lib/embulk/data/new/java/plugin_loader.rb.vm +3 -0
- data/lib/embulk/data/new/java/test.java.vm +5 -0
- data/lib/embulk/data/new/ruby/decoder_guess.rb.vm +25 -0
- data/lib/embulk/data/new/ruby/{filter.rb.erb → filter.rb.vm} +2 -2
- data/lib/embulk/data/new/ruby/{formatter.rb.erb → formatter.rb.vm} +2 -2
- data/lib/embulk/data/new/ruby/gemspec.vm +20 -0
- data/lib/embulk/data/new/ruby/{input.rb.erb → input.rb.vm} +10 -10
- data/lib/embulk/data/new/ruby/{output.rb.erb → output.rb.vm} +7 -7
- data/lib/embulk/data/new/ruby/{parser.rb.erb → parser.rb.vm} +2 -2
- data/lib/embulk/data/new/ruby/parser_guess.rb.vm +65 -0
- data/lib/embulk/guess/csv.rb +5 -0
- data/lib/embulk/version.rb +22 -1
- metadata +55 -35
- data/lib/embulk/command/embulk_example.rb +0 -33
- data/lib/embulk/command/embulk_generate_bin.rb +0 -62
- data/lib/embulk/command/embulk_migrate_plugin.rb +0 -244
- data/lib/embulk/command/embulk_new_plugin.rb +0 -126
- data/lib/embulk/command/embulk_selfupdate.rb +0 -121
- data/lib/embulk/data/new/README.md.erb +0 -111
- data/lib/embulk/data/new/java/plugin_loader.rb.erb +0 -3
- data/lib/embulk/data/new/java/test.java.erb +0 -5
- data/lib/embulk/data/new/ruby/decoder_guess.rb.erb +0 -25
- data/lib/embulk/data/new/ruby/gemspec.erb +0 -20
- data/lib/embulk/data/new/ruby/parser_guess.rb.erb +0 -65
|
@@ -1,18 +1,19 @@
|
|
|
1
|
-
package
|
|
1
|
+
package ${javaPackageName};
|
|
2
2
|
|
|
3
3
|
import com.google.common.base.Optional;
|
|
4
|
+
|
|
4
5
|
import org.embulk.config.Config;
|
|
5
6
|
import org.embulk.config.ConfigDefault;
|
|
6
7
|
import org.embulk.config.ConfigDiff;
|
|
7
8
|
import org.embulk.config.ConfigSource;
|
|
8
9
|
import org.embulk.config.Task;
|
|
9
10
|
import org.embulk.config.TaskSource;
|
|
11
|
+
import org.embulk.spi.FileOutput;
|
|
10
12
|
import org.embulk.spi.FormatterPlugin;
|
|
11
13
|
import org.embulk.spi.PageOutput;
|
|
12
|
-
import org.embulk.spi.FileOutput;
|
|
13
14
|
import org.embulk.spi.Schema;
|
|
14
15
|
|
|
15
|
-
public class
|
|
16
|
+
public class ${javaClassName}
|
|
16
17
|
implements FormatterPlugin
|
|
17
18
|
{
|
|
18
19
|
public interface PluginTask
|
|
@@ -49,6 +50,6 @@ public class <%= java_class_name %>
|
|
|
49
50
|
PluginTask task = taskSource.loadTask(PluginTask.class);
|
|
50
51
|
|
|
51
52
|
// Write your code here :)
|
|
52
|
-
throw new UnsupportedOperationException("
|
|
53
|
+
throw new UnsupportedOperationException("${javaClassName}.open method is not implemented yet");
|
|
53
54
|
}
|
|
54
55
|
}
|
|
@@ -1,13 +1,15 @@
|
|
|
1
|
-
package
|
|
1
|
+
package ${javaPackageName};
|
|
2
2
|
|
|
3
3
|
import java.util.List;
|
|
4
|
+
|
|
4
5
|
import com.google.common.base.Optional;
|
|
5
|
-
|
|
6
|
+
|
|
6
7
|
import org.embulk.config.Config;
|
|
7
8
|
import org.embulk.config.ConfigDefault;
|
|
8
9
|
import org.embulk.config.ConfigDiff;
|
|
9
10
|
import org.embulk.config.ConfigSource;
|
|
10
11
|
import org.embulk.config.Task;
|
|
12
|
+
import org.embulk.config.TaskReport;
|
|
11
13
|
import org.embulk.config.TaskSource;
|
|
12
14
|
import org.embulk.spi.Exec;
|
|
13
15
|
import org.embulk.spi.InputPlugin;
|
|
@@ -15,7 +17,7 @@ import org.embulk.spi.PageOutput;
|
|
|
15
17
|
import org.embulk.spi.Schema;
|
|
16
18
|
import org.embulk.spi.SchemaConfig;
|
|
17
19
|
|
|
18
|
-
public class
|
|
20
|
+
public class ${javaClassName}
|
|
19
21
|
implements InputPlugin
|
|
20
22
|
{
|
|
21
23
|
public interface PluginTask
|
|
@@ -76,7 +78,7 @@ public class <%= java_class_name %>
|
|
|
76
78
|
PluginTask task = taskSource.loadTask(PluginTask.class);
|
|
77
79
|
|
|
78
80
|
// Write your code here :)
|
|
79
|
-
throw new UnsupportedOperationException("
|
|
81
|
+
throw new UnsupportedOperationException("${javaClassName}.run method is not implemented yet");
|
|
80
82
|
}
|
|
81
83
|
|
|
82
84
|
@Override
|
|
@@ -1,13 +1,15 @@
|
|
|
1
|
-
package
|
|
1
|
+
package ${javaPackageName};
|
|
2
2
|
|
|
3
3
|
import java.util.List;
|
|
4
|
+
|
|
4
5
|
import com.google.common.base.Optional;
|
|
5
|
-
|
|
6
|
+
|
|
6
7
|
import org.embulk.config.Config;
|
|
7
8
|
import org.embulk.config.ConfigDefault;
|
|
8
9
|
import org.embulk.config.ConfigDiff;
|
|
9
10
|
import org.embulk.config.ConfigSource;
|
|
10
11
|
import org.embulk.config.Task;
|
|
12
|
+
import org.embulk.config.TaskReport;
|
|
11
13
|
import org.embulk.config.TaskSource;
|
|
12
14
|
import org.embulk.spi.Exec;
|
|
13
15
|
import org.embulk.spi.OutputPlugin;
|
|
@@ -15,7 +17,7 @@ import org.embulk.spi.PageOutput;
|
|
|
15
17
|
import org.embulk.spi.Schema;
|
|
16
18
|
import org.embulk.spi.TransactionalPageOutput;
|
|
17
19
|
|
|
18
|
-
public class
|
|
20
|
+
public class ${javaClassName}
|
|
19
21
|
implements OutputPlugin
|
|
20
22
|
{
|
|
21
23
|
public interface PluginTask
|
|
@@ -56,7 +58,7 @@ public class <%= java_class_name %>
|
|
|
56
58
|
Schema schema, int taskCount,
|
|
57
59
|
OutputPlugin.Control control)
|
|
58
60
|
{
|
|
59
|
-
throw new UnsupportedOperationException("
|
|
61
|
+
throw new UnsupportedOperationException("${name} output plugin does not support resuming");
|
|
60
62
|
}
|
|
61
63
|
|
|
62
64
|
@Override
|
|
@@ -72,6 +74,6 @@ public class <%= java_class_name %>
|
|
|
72
74
|
PluginTask task = taskSource.loadTask(PluginTask.class);
|
|
73
75
|
|
|
74
76
|
// Write your code here :)
|
|
75
|
-
throw new UnsupportedOperationException("
|
|
77
|
+
throw new UnsupportedOperationException("${javaClassName}.run method is not implemented yet");
|
|
76
78
|
}
|
|
77
79
|
}
|
|
@@ -1,19 +1,20 @@
|
|
|
1
|
-
package
|
|
1
|
+
package ${javaPackageName};
|
|
2
2
|
|
|
3
3
|
import com.google.common.base.Optional;
|
|
4
|
+
|
|
4
5
|
import org.embulk.config.Config;
|
|
5
6
|
import org.embulk.config.ConfigDefault;
|
|
6
7
|
import org.embulk.config.ConfigDiff;
|
|
7
8
|
import org.embulk.config.ConfigSource;
|
|
8
9
|
import org.embulk.config.Task;
|
|
9
10
|
import org.embulk.config.TaskSource;
|
|
10
|
-
import org.embulk.spi.ParserPlugin;
|
|
11
11
|
import org.embulk.spi.FileInput;
|
|
12
12
|
import org.embulk.spi.PageOutput;
|
|
13
|
+
import org.embulk.spi.ParserPlugin;
|
|
13
14
|
import org.embulk.spi.Schema;
|
|
14
15
|
import org.embulk.spi.SchemaConfig;
|
|
15
16
|
|
|
16
|
-
public class
|
|
17
|
+
public class ${javaClassName}
|
|
17
18
|
implements ParserPlugin
|
|
18
19
|
{
|
|
19
20
|
public interface PluginTask
|
|
@@ -55,6 +56,6 @@ public class <%= java_class_name %>
|
|
|
55
56
|
PluginTask task = taskSource.loadTask(PluginTask.class);
|
|
56
57
|
|
|
57
58
|
// Write your code here :)
|
|
58
|
-
throw new UnsupportedOperationException("
|
|
59
|
+
throw new UnsupportedOperationException("${javaClassName}.run method is not implemented yet");
|
|
59
60
|
}
|
|
60
61
|
}
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
module Embulk
|
|
2
|
+
module Guess
|
|
3
|
+
|
|
4
|
+
# TODO implement guess plugin to make this command work:
|
|
5
|
+
# $ embulk guess -g "${name}" partial-config.yml
|
|
6
|
+
#if (${language} == "ruby")
|
|
7
|
+
|
|
8
|
+
# require "embulk/${embulkCategory}/${name}.rb"
|
|
9
|
+
#end
|
|
10
|
+
|
|
11
|
+
# class ${rubyGuessClassName} < GuessPlugin
|
|
12
|
+
# Plugin.register_guess("${name}", self)
|
|
13
|
+
#
|
|
14
|
+
# FOO_BAR_HEADER = "\x1f\x8b".force_encoding('ASCII-8BIT').freeze
|
|
15
|
+
#
|
|
16
|
+
# def guess(config, sample_buffer)
|
|
17
|
+
# if sample_buffer[0,2] == FOO_BAR_HEADER
|
|
18
|
+
# return {"decoders" => [{"type" => "${name}"}]}
|
|
19
|
+
# end
|
|
20
|
+
# return {}
|
|
21
|
+
# end
|
|
22
|
+
# end
|
|
23
|
+
|
|
24
|
+
end
|
|
25
|
+
end
|
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
module Embulk
|
|
2
2
|
module Filter
|
|
3
3
|
|
|
4
|
-
class
|
|
5
|
-
Plugin.register_filter(
|
|
4
|
+
class ${rubyClassName} < FilterPlugin
|
|
5
|
+
Plugin.register_filter("${name}", self)
|
|
6
6
|
|
|
7
7
|
def self.transaction(config, in_schema, &control)
|
|
8
8
|
# configuration code:
|
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
module Embulk
|
|
2
2
|
module Formatter
|
|
3
3
|
|
|
4
|
-
class
|
|
5
|
-
Plugin.register_formatter(
|
|
4
|
+
class ${rubyClassName} < FormatterPlugin
|
|
5
|
+
Plugin.register_formatter("${name}", self)
|
|
6
6
|
|
|
7
7
|
def self.transaction(config, schema, &control)
|
|
8
8
|
# configuration code:
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
|
|
2
|
+
Gem::Specification.new do |spec|
|
|
3
|
+
spec.name = "${fullProjectName}"
|
|
4
|
+
spec.version = "0.1.0"
|
|
5
|
+
spec.authors = ["${author}"]
|
|
6
|
+
spec.summary = "${displayName} ${displayCategory} plugin for Embulk"
|
|
7
|
+
spec.description = "${description}"
|
|
8
|
+
spec.email = ["${email}"]
|
|
9
|
+
spec.licenses = ["MIT"]
|
|
10
|
+
# TODO set this: spec.homepage = "https://github.com/${expectedGitHubAccount}/${fullProjectName}"
|
|
11
|
+
|
|
12
|
+
spec.files = `git ls-files`.split("\n") + Dir["classpath/*.jar"]
|
|
13
|
+
spec.test_files = spec.files.grep(%r{^(test|spec)/})
|
|
14
|
+
spec.require_paths = ["lib"]
|
|
15
|
+
|
|
16
|
+
#spec.add_dependency 'YOUR_GEM_DEPENDENCY', ['~> YOUR_GEM_DEPENDENCY_VERSION']
|
|
17
|
+
spec.add_development_dependency 'embulk', ['>= ${embulkVersion}']
|
|
18
|
+
spec.add_development_dependency 'bundler', ['>= 1.10.6']
|
|
19
|
+
spec.add_development_dependency 'rake', ['>= 10.0']
|
|
20
|
+
end
|
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
module Embulk
|
|
2
2
|
module Input
|
|
3
3
|
|
|
4
|
-
class
|
|
5
|
-
Plugin.register_input(
|
|
4
|
+
class ${rubyClassName} < InputPlugin
|
|
5
|
+
Plugin.register_input("${name}", self)
|
|
6
6
|
|
|
7
7
|
def self.transaction(config, &control)
|
|
8
8
|
# configuration code:
|
|
@@ -29,14 +29,14 @@ module Embulk
|
|
|
29
29
|
end
|
|
30
30
|
|
|
31
31
|
# TODO
|
|
32
|
-
#def self.guess(config)
|
|
33
|
-
#
|
|
34
|
-
#
|
|
35
|
-
#
|
|
36
|
-
#
|
|
37
|
-
#
|
|
38
|
-
#
|
|
39
|
-
#end
|
|
32
|
+
# def self.guess(config)
|
|
33
|
+
# sample_records = [
|
|
34
|
+
# {"example"=>"a", "column"=>1, "value"=>0.1},
|
|
35
|
+
# {"example"=>"a", "column"=>2, "value"=>0.2},
|
|
36
|
+
# ]
|
|
37
|
+
# columns = Guess::SchemaGuess.from_hash_records(sample_records)
|
|
38
|
+
# return {"columns" => columns}
|
|
39
|
+
# end
|
|
40
40
|
|
|
41
41
|
def init
|
|
42
42
|
# initialization code:
|
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
module Embulk
|
|
2
2
|
module Output
|
|
3
3
|
|
|
4
|
-
class
|
|
5
|
-
Plugin.register_output(
|
|
4
|
+
class ${rubyClassName} < OutputPlugin
|
|
5
|
+
Plugin.register_output("${name}", self)
|
|
6
6
|
|
|
7
7
|
def self.transaction(config, schema, count, &control)
|
|
8
8
|
# configuration code:
|
|
@@ -21,12 +21,12 @@ module Embulk
|
|
|
21
21
|
return next_config_diff
|
|
22
22
|
end
|
|
23
23
|
|
|
24
|
-
#def self.resume(task, schema, count, &control)
|
|
25
|
-
#
|
|
24
|
+
# def self.resume(task, schema, count, &control)
|
|
25
|
+
# task_reports = yield(task)
|
|
26
26
|
#
|
|
27
|
-
#
|
|
28
|
-
#
|
|
29
|
-
#end
|
|
27
|
+
# next_config_diff = {}
|
|
28
|
+
# return next_config_diff
|
|
29
|
+
# end
|
|
30
30
|
|
|
31
31
|
def init
|
|
32
32
|
# initialization code:
|
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
module Embulk
|
|
2
2
|
module Parser
|
|
3
3
|
|
|
4
|
-
class
|
|
5
|
-
Plugin.register_parser(
|
|
4
|
+
class ${rubyClassName} < ParserPlugin
|
|
5
|
+
Plugin.register_parser("${name}", self)
|
|
6
6
|
|
|
7
7
|
def self.transaction(config, &control)
|
|
8
8
|
# configuration code:
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
module Embulk
|
|
2
|
+
module Guess
|
|
3
|
+
|
|
4
|
+
# TODO implement guess plugin to make this command work:
|
|
5
|
+
# $ embulk guess -g "${name}" partial-config.yml
|
|
6
|
+
#
|
|
7
|
+
# Depending on the file format the plugin uses, you can use choose
|
|
8
|
+
# one of binary guess (GuessPlugin), text guess (TextGuessPlugin),
|
|
9
|
+
# or line guess (LineGuessPlugin).
|
|
10
|
+
#if (${language} == "ruby")
|
|
11
|
+
|
|
12
|
+
# require "embulk/${embulkCategory}/${name}.rb"
|
|
13
|
+
#end
|
|
14
|
+
|
|
15
|
+
# class ${rubyGuessClassName} < GuessPlugin
|
|
16
|
+
# Plugin.register_guess("${name}", self)
|
|
17
|
+
#
|
|
18
|
+
# def guess(config, sample_buffer)
|
|
19
|
+
# if sample_buffer[0,2] == GZIP_HEADER
|
|
20
|
+
# guessed = {}
|
|
21
|
+
# guessed["type"] = "${name}"
|
|
22
|
+
# guessed["property1"] = "guessed-value"
|
|
23
|
+
# return {"parser" => guessed}
|
|
24
|
+
# else
|
|
25
|
+
# return {}
|
|
26
|
+
# end
|
|
27
|
+
# end
|
|
28
|
+
# end
|
|
29
|
+
|
|
30
|
+
# class ${rubyGuessClassName} < TextGuessPlugin
|
|
31
|
+
# Plugin.register_guess("${name}", self)
|
|
32
|
+
#
|
|
33
|
+
# def guess_text(config, sample_text)
|
|
34
|
+
# js = JSON.parse(sample_text) rescue nil
|
|
35
|
+
# if js && js["mykeyword"] == "keyword"
|
|
36
|
+
# guessed = {}
|
|
37
|
+
# guessed["type"] = "${name}"
|
|
38
|
+
# guessed["property1"] = "guessed-value"
|
|
39
|
+
# return {"parser" => guessed}
|
|
40
|
+
# else
|
|
41
|
+
# return {}
|
|
42
|
+
# end
|
|
43
|
+
# end
|
|
44
|
+
# end
|
|
45
|
+
|
|
46
|
+
# class ${rubyGuessClassName} < LineGuessPlugin
|
|
47
|
+
# Plugin.register_guess("${name}", self)
|
|
48
|
+
#
|
|
49
|
+
# def guess_lines(config, sample_lines)
|
|
50
|
+
# all_line_matched = sample_lines.all? do |line|
|
|
51
|
+
# line =~ /mypattern/
|
|
52
|
+
# end
|
|
53
|
+
# if all_line_matched
|
|
54
|
+
# guessed = {}
|
|
55
|
+
# guessed["type"] = "${name}"
|
|
56
|
+
# guessed["property1"] = "guessed-value"
|
|
57
|
+
# return {"parser" => guessed}
|
|
58
|
+
# else
|
|
59
|
+
# return {}
|
|
60
|
+
# end
|
|
61
|
+
# end
|
|
62
|
+
# end
|
|
63
|
+
|
|
64
|
+
end
|
|
65
|
+
end
|
data/lib/embulk/guess/csv.rb
CHANGED
|
@@ -104,6 +104,11 @@ module Embulk
|
|
|
104
104
|
|
|
105
105
|
sample_records = split_lines(parser_guessed, true, sample_lines, delim, {})
|
|
106
106
|
|
|
107
|
+
# It should fail if CSV parser cannot parse sample_lines.
|
|
108
|
+
if sample_records.nil? || sample_records.empty?
|
|
109
|
+
return {}
|
|
110
|
+
end
|
|
111
|
+
|
|
107
112
|
if sample_lines.size == 1
|
|
108
113
|
# The file contains only 1 line. Assume that there are no header line.
|
|
109
114
|
header_line = false
|
data/lib/embulk/version.rb
CHANGED
|
@@ -1,3 +1,24 @@
|
|
|
1
|
+
# TODO(v2)[#562]: Remove this file.
|
|
2
|
+
# https://github.com/embulk/embulk/issues/562
|
|
1
3
|
module Embulk
|
|
2
|
-
|
|
4
|
+
@@warned = false
|
|
5
|
+
|
|
6
|
+
VERSION_INTERNAL = '0.8.19'
|
|
7
|
+
|
|
8
|
+
DEPRECATED_MESSAGE = 'Embulk::VERSION in (J)Ruby is deprecated. Use org.embulk.EmbulkVersion::VERSION instead. If this message is from a plugin, please tell this to the author of the plugin!'
|
|
9
|
+
def self.const_missing(name)
|
|
10
|
+
if name == :VERSION
|
|
11
|
+
unless @@warned
|
|
12
|
+
if Embulk.method_defined?(:logger)
|
|
13
|
+
Embulk.logger.warn(DEPRECATED_MESSAGE)
|
|
14
|
+
@@warned = true
|
|
15
|
+
else
|
|
16
|
+
STDERR.puts(DEPRECATED_MESSAGE)
|
|
17
|
+
end
|
|
18
|
+
end
|
|
19
|
+
return VERSION_INTERNAL
|
|
20
|
+
else
|
|
21
|
+
super
|
|
22
|
+
end
|
|
23
|
+
end
|
|
3
24
|
end
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: embulk
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.8.
|
|
4
|
+
version: 0.8.19
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Sadayuki Furuhashi
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: bin
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2017-
|
|
11
|
+
date: 2017-05-13 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: jruby-jars
|
|
@@ -106,11 +106,13 @@ files:
|
|
|
106
106
|
- classpath/bval-core-0.5.jar
|
|
107
107
|
- classpath/bval-jsr303-0.5.jar
|
|
108
108
|
- classpath/commons-beanutils-core-1.8.3.jar
|
|
109
|
+
- classpath/commons-collections-3.2.1.jar
|
|
109
110
|
- classpath/commons-compress-1.10.jar
|
|
110
|
-
- classpath/commons-
|
|
111
|
-
- classpath/
|
|
112
|
-
- classpath/embulk-
|
|
113
|
-
- classpath/embulk-
|
|
111
|
+
- classpath/commons-lang-2.4.jar
|
|
112
|
+
- classpath/commons-lang3-3.4.jar
|
|
113
|
+
- classpath/embulk-cli-0.8.19.jar
|
|
114
|
+
- classpath/embulk-core-0.8.19.jar
|
|
115
|
+
- classpath/embulk-standards-0.8.19.jar
|
|
114
116
|
- classpath/guava-18.0.jar
|
|
115
117
|
- classpath/guice-4.0.jar
|
|
116
118
|
- classpath/guice-bootstrap-0.1.1.jar
|
|
@@ -126,14 +128,22 @@ files:
|
|
|
126
128
|
- classpath/joda-time-2.9.2.jar
|
|
127
129
|
- classpath/logback-classic-1.1.3.jar
|
|
128
130
|
- classpath/logback-core-1.1.3.jar
|
|
131
|
+
- classpath/maven-artifact-3.3.9.jar
|
|
129
132
|
- classpath/msgpack-core-0.8.11.jar
|
|
130
133
|
- classpath/netty-buffer-4.0.44.Final.jar
|
|
131
134
|
- classpath/netty-common-4.0.44.Final.jar
|
|
135
|
+
- classpath/plexus-utils-3.0.22.jar
|
|
132
136
|
- classpath/slf4j-api-1.7.12.jar
|
|
133
137
|
- classpath/slice-0.9.jar
|
|
134
|
-
- classpath/snakeyaml-1.
|
|
138
|
+
- classpath/snakeyaml-1.18.jar
|
|
135
139
|
- classpath/validation-api-1.1.0.Final.jar
|
|
140
|
+
- classpath/velocity-1.7.jar
|
|
136
141
|
- embulk-cli/build.gradle
|
|
142
|
+
- embulk-cli/src/main/bat/selfrun.bat
|
|
143
|
+
- embulk-cli/src/main/java/org/embulk/cli/EmbulkExample.java
|
|
144
|
+
- embulk-cli/src/main/java/org/embulk/cli/EmbulkMigrate.java
|
|
145
|
+
- embulk-cli/src/main/java/org/embulk/cli/EmbulkNew.java
|
|
146
|
+
- embulk-cli/src/main/java/org/embulk/cli/EmbulkSelfUpdate.java
|
|
137
147
|
- embulk-cli/src/main/java/org/embulk/cli/Main.java
|
|
138
148
|
- embulk-cli/src/main/sh/selfrun.sh
|
|
139
149
|
- embulk-cli/src/test/java/org/embulk/cli/DummyMain.java
|
|
@@ -141,6 +151,7 @@ files:
|
|
|
141
151
|
- embulk-core/build.gradle
|
|
142
152
|
- embulk-core/src/main/java/org/embulk/EmbulkEmbed.java
|
|
143
153
|
- embulk-core/src/main/java/org/embulk/EmbulkService.java
|
|
154
|
+
- embulk-core/src/main/java/org/embulk/EmbulkVersion.java
|
|
144
155
|
- embulk-core/src/main/java/org/embulk/command/PreviewPrinter.java
|
|
145
156
|
- embulk-core/src/main/java/org/embulk/command/TablePreviewPrinter.java
|
|
146
157
|
- embulk-core/src/main/java/org/embulk/command/VerticalPreviewPrinter.java
|
|
@@ -355,9 +366,10 @@ files:
|
|
|
355
366
|
- embulk-docs/src/built-in.rst
|
|
356
367
|
- embulk-docs/src/conf.py
|
|
357
368
|
- embulk-docs/src/customization.rst
|
|
369
|
+
- embulk-docs/src/developers/index.rst
|
|
358
370
|
- embulk-docs/src/index.rst
|
|
359
371
|
- embulk-docs/src/recipe.rst
|
|
360
|
-
- embulk-docs/src/recipe/scheduled-csv-load-to-elasticsearch-
|
|
372
|
+
- embulk-docs/src/recipe/scheduled-csv-load-to-elasticsearch-kibana5.rst
|
|
361
373
|
- embulk-docs/src/release.rst
|
|
362
374
|
- embulk-docs/src/release/release-0.1.0.rst
|
|
363
375
|
- embulk-docs/src/release/release-0.2.0.rst
|
|
@@ -433,6 +445,7 @@ files:
|
|
|
433
445
|
- embulk-docs/src/release/release-0.8.16.rst
|
|
434
446
|
- embulk-docs/src/release/release-0.8.17.rst
|
|
435
447
|
- embulk-docs/src/release/release-0.8.18.rst
|
|
448
|
+
- embulk-docs/src/release/release-0.8.19.rst
|
|
436
449
|
- embulk-docs/src/release/release-0.8.2.rst
|
|
437
450
|
- embulk-docs/src/release/release-0.8.3.rst
|
|
438
451
|
- embulk-docs/src/release/release-0.8.4.rst
|
|
@@ -467,6 +480,7 @@ files:
|
|
|
467
480
|
- embulk-standards/src/test/java/org/embulk/standards/TestRenameFilterPlugin.java
|
|
468
481
|
- embulk-standards/src/test/java/org/embulk/standards/guess/TestCsvAllStringsGuessPlugin.java
|
|
469
482
|
- embulk-standards/src/test/java/org/embulk/standards/guess/TestCsvGuessPlugin.java
|
|
483
|
+
- embulk-standards/src/test/java/org/embulk/standards/preview/TestFilePreview.java
|
|
470
484
|
- embulk-standards/src/test/resources/org/embulk/standards/guess/csv/test/test_1_int_single_column_row.csv
|
|
471
485
|
- embulk-standards/src/test/resources/org/embulk/standards/guess/csv/test/test_1_int_single_column_row_and_header.csv
|
|
472
486
|
- embulk-standards/src/test/resources/org/embulk/standards/guess/csv/test/test_1_int_single_column_row_and_header_guessed.yml
|
|
@@ -524,6 +538,9 @@ files:
|
|
|
524
538
|
- embulk-standards/src/test/resources/org/embulk/standards/guess/csv/test/test_single_quote.csv
|
|
525
539
|
- embulk-standards/src/test/resources/org/embulk/standards/guess/csv/test/test_single_quote_guessed.yml
|
|
526
540
|
- embulk-standards/src/test/resources/org/embulk/standards/guess/csv/test/test_single_quote_seed.yml
|
|
541
|
+
- embulk-standards/src/test/resources/org/embulk/standards/guess/csv/test/test_skip_suggest_if_empty_sample_records.csv
|
|
542
|
+
- embulk-standards/src/test/resources/org/embulk/standards/guess/csv/test/test_skip_suggest_if_empty_sample_records_guessed.yml
|
|
543
|
+
- embulk-standards/src/test/resources/org/embulk/standards/guess/csv/test/test_skip_suggest_if_empty_sample_records_seed.yml
|
|
527
544
|
- embulk-standards/src/test/resources/org/embulk/standards/guess/csv/test/test_string_single_column.csv
|
|
528
545
|
- embulk-standards/src/test/resources/org/embulk/standards/guess/csv/test/test_string_single_column_guessed.yml
|
|
529
546
|
- embulk-standards/src/test/resources/org/embulk/standards/guess/csv/test/test_string_single_column_seed.yml
|
|
@@ -536,6 +553,13 @@ files:
|
|
|
536
553
|
- embulk-standards/src/test/resources/org/embulk/standards/guess/csv_all_strings/test/test_simple.csv
|
|
537
554
|
- embulk-standards/src/test/resources/org/embulk/standards/guess/csv_all_strings/test/test_simple_guessed.yml
|
|
538
555
|
- embulk-standards/src/test/resources/org/embulk/standards/guess/csv_all_strings/test/test_simple_seed.yml
|
|
556
|
+
- embulk-standards/src/test/resources/org/embulk/standards/preview/file/test/test_sample_buffer_bytes.csv
|
|
557
|
+
- embulk-standards/src/test/resources/org/embulk/standards/preview/file/test/test_sample_buffer_bytes_exec.yml
|
|
558
|
+
- embulk-standards/src/test/resources/org/embulk/standards/preview/file/test/test_sample_buffer_bytes_load.yml
|
|
559
|
+
- embulk-standards/src/test/resources/org/embulk/standards/preview/file/test/test_sample_buffer_bytes_previewed.csv
|
|
560
|
+
- embulk-standards/src/test/resources/org/embulk/standards/preview/file/test/test_simple.csv
|
|
561
|
+
- embulk-standards/src/test/resources/org/embulk/standards/preview/file/test/test_simple_load.yml
|
|
562
|
+
- embulk-standards/src/test/resources/org/embulk/standards/preview/file/test/test_simple_previewed.csv
|
|
539
563
|
- embulk-standards/src/test/resources/org/embulk/standards/remove_columns/test/test_keep.csv
|
|
540
564
|
- embulk-standards/src/test/resources/org/embulk/standards/remove_columns/test/test_keep_expected.csv
|
|
541
565
|
- embulk-standards/src/test/resources/org/embulk/standards/remove_columns/test/test_keep_filter.yml
|
|
@@ -554,6 +578,7 @@ files:
|
|
|
554
578
|
- embulk-standards/src/test/resources/org/embulk/standards/remove_columns/test/test_remove_without_unmatched_filter.yml
|
|
555
579
|
- embulk-test/build.gradle
|
|
556
580
|
- embulk-test/src/main/java/org/embulk/test/EmbulkTests.java
|
|
581
|
+
- embulk-test/src/main/java/org/embulk/test/PreviewResultInputPlugin.java
|
|
557
582
|
- embulk-test/src/main/java/org/embulk/test/TestingBulkLoader.java
|
|
558
583
|
- embulk-test/src/main/java/org/embulk/test/TestingEmbulk.java
|
|
559
584
|
- embulk.gemspec
|
|
@@ -565,13 +590,8 @@ files:
|
|
|
565
590
|
- lib/embulk/buffer.rb
|
|
566
591
|
- lib/embulk/column.rb
|
|
567
592
|
- lib/embulk/command/embulk_bundle.rb
|
|
568
|
-
- lib/embulk/command/embulk_example.rb
|
|
569
|
-
- lib/embulk/command/embulk_generate_bin.rb
|
|
570
593
|
- lib/embulk/command/embulk_main.rb
|
|
571
|
-
- lib/embulk/command/embulk_migrate_plugin.rb
|
|
572
|
-
- lib/embulk/command/embulk_new_plugin.rb
|
|
573
594
|
- lib/embulk/command/embulk_run.rb
|
|
574
|
-
- lib/embulk/command/embulk_selfupdate.rb
|
|
575
595
|
- lib/embulk/data/bundle/.bundle/config
|
|
576
596
|
- lib/embulk/data/bundle/.ruby-version
|
|
577
597
|
- lib/embulk/data/bundle/Gemfile
|
|
@@ -579,37 +599,37 @@ files:
|
|
|
579
599
|
- lib/embulk/data/bundle/embulk/input/example.rb
|
|
580
600
|
- lib/embulk/data/bundle/embulk/output/example.rb
|
|
581
601
|
- lib/embulk/data/new/LICENSE.txt
|
|
582
|
-
- lib/embulk/data/new/README.md.
|
|
583
|
-
- lib/embulk/data/new/gitignore.
|
|
584
|
-
- lib/embulk/data/new/java/build.gradle.
|
|
602
|
+
- lib/embulk/data/new/README.md.vm
|
|
603
|
+
- lib/embulk/data/new/gitignore.vm
|
|
604
|
+
- lib/embulk/data/new/java/build.gradle.vm
|
|
585
605
|
- lib/embulk/data/new/java/config/checkstyle/checkstyle.xml
|
|
586
606
|
- lib/embulk/data/new/java/config/checkstyle/default.xml
|
|
587
|
-
- lib/embulk/data/new/java/decoder.java.
|
|
588
|
-
- lib/embulk/data/new/java/encoder.java.
|
|
589
|
-
- lib/embulk/data/new/java/file_input.java.
|
|
590
|
-
- lib/embulk/data/new/java/file_output.java.
|
|
591
|
-
- lib/embulk/data/new/java/filter.java.
|
|
592
|
-
- lib/embulk/data/new/java/formatter.java.
|
|
607
|
+
- lib/embulk/data/new/java/decoder.java.vm
|
|
608
|
+
- lib/embulk/data/new/java/encoder.java.vm
|
|
609
|
+
- lib/embulk/data/new/java/file_input.java.vm
|
|
610
|
+
- lib/embulk/data/new/java/file_output.java.vm
|
|
611
|
+
- lib/embulk/data/new/java/filter.java.vm
|
|
612
|
+
- lib/embulk/data/new/java/formatter.java.vm
|
|
593
613
|
- lib/embulk/data/new/java/gradle/wrapper/gradle-wrapper.jar
|
|
594
614
|
- lib/embulk/data/new/java/gradle/wrapper/gradle-wrapper.properties
|
|
595
615
|
- lib/embulk/data/new/java/gradlew
|
|
596
616
|
- lib/embulk/data/new/java/gradlew.bat
|
|
597
|
-
- lib/embulk/data/new/java/input.java.
|
|
598
|
-
- lib/embulk/data/new/java/output.java.
|
|
599
|
-
- lib/embulk/data/new/java/parser.java.
|
|
600
|
-
- lib/embulk/data/new/java/plugin_loader.rb.
|
|
601
|
-
- lib/embulk/data/new/java/test.java.
|
|
617
|
+
- lib/embulk/data/new/java/input.java.vm
|
|
618
|
+
- lib/embulk/data/new/java/output.java.vm
|
|
619
|
+
- lib/embulk/data/new/java/parser.java.vm
|
|
620
|
+
- lib/embulk/data/new/java/plugin_loader.rb.vm
|
|
621
|
+
- lib/embulk/data/new/java/test.java.vm
|
|
602
622
|
- lib/embulk/data/new/ruby/.ruby-version
|
|
603
623
|
- lib/embulk/data/new/ruby/Gemfile
|
|
604
624
|
- lib/embulk/data/new/ruby/Rakefile
|
|
605
|
-
- lib/embulk/data/new/ruby/decoder_guess.rb.
|
|
606
|
-
- lib/embulk/data/new/ruby/filter.rb.
|
|
607
|
-
- lib/embulk/data/new/ruby/formatter.rb.
|
|
608
|
-
- lib/embulk/data/new/ruby/gemspec.
|
|
609
|
-
- lib/embulk/data/new/ruby/input.rb.
|
|
610
|
-
- lib/embulk/data/new/ruby/output.rb.
|
|
611
|
-
- lib/embulk/data/new/ruby/parser.rb.
|
|
612
|
-
- lib/embulk/data/new/ruby/parser_guess.rb.
|
|
625
|
+
- lib/embulk/data/new/ruby/decoder_guess.rb.vm
|
|
626
|
+
- lib/embulk/data/new/ruby/filter.rb.vm
|
|
627
|
+
- lib/embulk/data/new/ruby/formatter.rb.vm
|
|
628
|
+
- lib/embulk/data/new/ruby/gemspec.vm
|
|
629
|
+
- lib/embulk/data/new/ruby/input.rb.vm
|
|
630
|
+
- lib/embulk/data/new/ruby/output.rb.vm
|
|
631
|
+
- lib/embulk/data/new/ruby/parser.rb.vm
|
|
632
|
+
- lib/embulk/data/new/ruby/parser_guess.rb.vm
|
|
613
633
|
- lib/embulk/data/package_data.rb
|
|
614
634
|
- lib/embulk/data_source.rb
|
|
615
635
|
- lib/embulk/decoder_plugin.rb
|