embulk 0.6.25 → 0.6.26
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/build.gradle +1 -1
- data/embulk-docs/src/built-in.rst +2 -0
- data/embulk-docs/src/release.rst +1 -0
- data/embulk-docs/src/release/release-0.6.26.rst +17 -0
- data/embulk-standards/src/main/java/org/embulk/standards/CsvParserPlugin.java +8 -0
- data/lib/embulk/exec.rb +8 -0
- data/lib/embulk/plugin.rb +1 -0
- data/lib/embulk/version.rb +1 -1
- metadata +6 -4
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 99bafeaaf1880afd940f1fbc79d93a932755a25d
|
4
|
+
data.tar.gz: 47375952c1fe8131eee784eb0f99bc27768127b3
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 7f80c21126f7614b58b88fd3b5427bc1d8ca775c1ee1ccf5c00e887d54e199f215d4606df1c3cc14d670e973dfec362cbc505cf7e3e793a7978425dfb9f01e5f
|
7
|
+
data.tar.gz: 0c9261acf5b853b5f75efeaf13b3e9eb14f91c54f1d4b29b91721958427e3bb587820a0000679d79ba2df73613bdd872b04696a731b0f3814e544431f8074731
|
data/build.gradle
CHANGED
@@ -181,6 +181,8 @@ Options
|
|
181
181
|
+----------------------------+----------+----------------------------------------------------------------------------------------------------------------+------------------------+
|
182
182
|
| max\_quoted\_size\_limit | integer | Maximum number of bytes of a quoted value. If a value exceeds the limit, the row will be skipped | ``131072`` by default |
|
183
183
|
+----------------------------+----------+----------------------------------------------------------------------------------------------------------------+------------------------+
|
184
|
+
| stop\_on\_invalid\_record | boolean | Stop bulk load transaction if a file includes invalid record (such as invalid timestamp) | ``false`` by default |
|
185
|
+
+----------------------------+----------+----------------------------------------------------------------------------------------------------------------+------------------------+
|
184
186
|
| default\_timezone | string | Time zone of timestamp columns if the value itself doesn't include time zone description (eg. Asia/Tokyo) | ``UTC`` by default |
|
185
187
|
+----------------------------+----------+----------------------------------------------------------------------------------------------------------------+------------------------+
|
186
188
|
| newline | enum | Newline character (CRLF, LF or CR) | ``CRLF`` by default |
|
data/embulk-docs/src/release.rst
CHANGED
@@ -0,0 +1,17 @@
|
|
1
|
+
Release 0.6.26
|
2
|
+
==================================
|
3
|
+
|
4
|
+
Built-in plugins
|
5
|
+
------------------
|
6
|
+
|
7
|
+
* Added ``stop_on_invalid_record`` option to ``parser-csv`` plugin. This option stops bulkloading if it finds a broken record such as invalid timestamp or invalid integer format rathar than skipping it.
|
8
|
+
|
9
|
+
|
10
|
+
Ruby Plugin API
|
11
|
+
------------------
|
12
|
+
|
13
|
+
* Added ``Embulk::Exec.preview?`` which returns true if plugins are running in preview.
|
14
|
+
|
15
|
+
Release Date
|
16
|
+
------------------
|
17
|
+
2015-08-20
|
@@ -91,6 +91,10 @@ public class CsvParserPlugin
|
|
91
91
|
@Config("allow_extra_columns")
|
92
92
|
@ConfigDefault("false")
|
93
93
|
boolean getAllowExtraColumns();
|
94
|
+
|
95
|
+
@Config("stop_on_invalid_record")
|
96
|
+
@ConfigDefault("false")
|
97
|
+
boolean getStopOnInvalidRecord();
|
94
98
|
}
|
95
99
|
|
96
100
|
public static class QuoteCharacter
|
@@ -231,6 +235,7 @@ public class CsvParserPlugin
|
|
231
235
|
final String nullStringOrNull = task.getNullString().orNull();
|
232
236
|
final boolean allowOptionalColumns = task.getAllowOptionalColumns();
|
233
237
|
final boolean allowExtraColumns = task.getAllowExtraColumns();
|
238
|
+
final boolean stopOnInvalidRecord = task.getStopOnInvalidRecord();
|
234
239
|
int skipHeaderLines = task.getSkipHeaderLines();
|
235
240
|
|
236
241
|
try (final PageBuilder pageBuilder = new PageBuilder(Exec.getBufferAllocator(), schema, output)) {
|
@@ -354,6 +359,9 @@ public class CsvParserPlugin
|
|
354
359
|
} catch (CsvTokenizer.InvalidFormatException | CsvRecordValidateException e) {
|
355
360
|
long lineNumber = tokenizer.getCurrentLineNumber();
|
356
361
|
String skippedLine = tokenizer.skipCurrentLine();
|
362
|
+
if (stopOnInvalidRecord) {
|
363
|
+
throw new ConfigException(String.format("Invalid record at line %d: %s", lineNumber, skippedLine), e);
|
364
|
+
}
|
357
365
|
log.warn(String.format("Skipped line %d (%s): %s", lineNumber, e.getMessage(), skippedLine));
|
358
366
|
//exec.notice().skippedLine(skippedLine);
|
359
367
|
|
data/lib/embulk/exec.rb
ADDED
data/lib/embulk/plugin.rb
CHANGED
data/lib/embulk/version.rb
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: embulk
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.6.
|
4
|
+
version: 0.6.26
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Sadayuki Furuhashi
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2015-08-
|
11
|
+
date: 2015-08-20 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|
@@ -104,8 +104,8 @@ files:
|
|
104
104
|
- classpath/bval-jsr303-0.5.jar
|
105
105
|
- classpath/commons-beanutils-core-1.8.3.jar
|
106
106
|
- classpath/commons-lang3-3.1.jar
|
107
|
-
- classpath/embulk-core-0.6.
|
108
|
-
- classpath/embulk-standards-0.6.
|
107
|
+
- classpath/embulk-core-0.6.26.jar
|
108
|
+
- classpath/embulk-standards-0.6.26.jar
|
109
109
|
- classpath/guava-18.0.jar
|
110
110
|
- classpath/guice-4.0.jar
|
111
111
|
- classpath/guice-multibindings-4.0.jar
|
@@ -386,6 +386,7 @@ files:
|
|
386
386
|
- embulk-docs/src/release/release-0.6.23.rst
|
387
387
|
- embulk-docs/src/release/release-0.6.24.rst
|
388
388
|
- embulk-docs/src/release/release-0.6.25.rst
|
389
|
+
- embulk-docs/src/release/release-0.6.26.rst
|
389
390
|
- embulk-docs/src/release/release-0.6.3.rst
|
390
391
|
- embulk-docs/src/release/release-0.6.4.rst
|
391
392
|
- embulk-docs/src/release/release-0.6.5.rst
|
@@ -465,6 +466,7 @@ files:
|
|
465
466
|
- lib/embulk/decoder_plugin.rb
|
466
467
|
- lib/embulk/encoder_plugin.rb
|
467
468
|
- lib/embulk/error.rb
|
469
|
+
- lib/embulk/exec.rb
|
468
470
|
- lib/embulk/executor_plugin.rb
|
469
471
|
- lib/embulk/file_input.rb
|
470
472
|
- lib/embulk/file_input_plugin.rb
|