embulk-decoder-commons-compress 0.3.2 → 0.3.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 8c492e93cc8decc06fa54a952930d446748d1bc3
-  data.tar.gz: 3bc26ce03ac3fd8cbd450b12d5c798f1f0d88326
+  metadata.gz: d01788f5dc971e9ee19b2139c5079fc2d54ebc81
+  data.tar.gz: d7cd120f691677b626b5b758f665e2dafab083ce
 SHA512:
-  metadata.gz: a3e7102ce968c9d4c7b7733025d8b141ee55c9dd4f3e730c05a0e9ea0d11f0ddadab2fb9ca34b107eb30949108cdb6c3eb669a31eb5111c95eae52c65c6360dd
-  data.tar.gz: aa39557828feb9906a43d061b2edecaf558bdcae8ff6b00bd6f3623ff37c09d5d3e905f18789c5b8b170a91cb41058ea229fd2df376a6487e644aacefe36c45d
+  metadata.gz: bb6b7d5f8717bc3464ca3e6c5272a6eb7a9e3eee262f2effbfb57e0b8784151b417a9d3e8cabad624080eea8adbabb461f9536b051ddbd676335f783bba5f51a
+  data.tar.gz: 4b1c4d7d7b01ea23c388d858717f8762db2bc2b97651bfb26273e02ec2c307d47faade8143269c4b491edb1adf2acaa9204d86594e1c682cfc0d53dd0d3da7e9

data/README.md

@@ -17,6 +17,7 @@ This decoder plugin for Embulk supports various archive formats using [Apache Co
   - The format type is one of supported formats by by [Apache Commons Compress](http://commons.apache.org/proper/commons-compress/).
   - Auto detect is used when there is no configuration. This can use for a single format. If a file format is solid compression like tar.gz, please set format config explicitly.
   - Some listing formats in [Apache Commons Compress](http://commons.apache.org/proper/commons-compress/) may not work in your environment. I could confirm the following formats work well. Your environment may be able to use other formats listed in the site.
+- **decompress_concatenated**: gzip, bzip2, and xz formats support multiple concatenated streams. The default value of this parameter is true. If you want to disable it, then set to false. See [CompressorStreamFactory.setDecompressConcatenated()](https://commons.apache.org/proper/commons-compress/apidocs/org/apache/commons/compress/compressors/CompressorStreamFactory.html#setDecompressConcatenated(boolean)) in ver.1.9 for more details.
 
 ## Formats
 
@@ -29,7 +30,6 @@ This decoder plugin for Embulk supports various archive formats using [Apache Co
   - tbz, tbz2, tb2, tar.bz2
   - taz, tz, tar.Z
 
-If input files are concatenated gzip or bzip2 format, please set format parameter explicitly.
 
 ## Example
 
@@ -62,24 +62,18 @@ in:
       format: tgz
 ```
 
-- Set *format* parameter to handle concatenated gzip(or bzip2) file.
-```yaml
-in:
-  type: any input plugin type
-  decoders:
-    - type: commons-compress
-      format: gz
-```
+- Set decompress_concatenated to false if you would like to read only the first concatenated entry.
 
 ```yaml
 in:
   type: any input plugin type
   decoders:
     - type: commons-compress
-      format: bzip2
+      decompress_concatenated: false
 ```
 
 
+
 ## Build
 
 ```

data/build.gradle

@@ -18,7 +18,7 @@ configurations {
     provided
 }
 
-version = "0.3.2"
+version = "0.3.3"
 
 dependencies {
     compile  "org.embulk:embulk-core:0.7.0"

data/src/integration-test/java/org/embulk/filter/TestIntegration.java

@@ -111,6 +111,20 @@ public class TestIntegration {
                 getChecksumFromFiles("result_concatenated_bzip2_000.00.csv"));
     }
 
+    @Test
+    public void testNoConcatenatedGzip() throws Exception {
+        assertEquals("Verify input and output contents are identical.",
+                getChecksumFromFiles(SAMPLE_1_SRC_FILES),
+                getChecksumFromFiles("result_no_concatenated_gzip_000.00.csv"));
+    }
+
+    @Test
+    public void testNoConcatenatedBzip2() throws Exception {
+        assertEquals("Verify input and output contents are identical.",
+                getChecksumFromFiles(SAMPLE_1_SRC_FILES),
+                getChecksumFromFiles("result_no_concatenated_bzip2_000.00.csv"));
+    }
+
     private long getChecksumFromFiles(String ... files) throws IOException {
         Checksum cksum = new CRC32();
 

data/src/integration-test/resources/config_concatenated_bzip2.yml

@@ -3,7 +3,6 @@ in:
   path_prefix: ./concatenated.csv.bz2
   decoders:
     - type: commons-compress
-      format: bzip2
   parser:
     charset: UTF-8
     newline: CRLF

data/src/integration-test/resources/config_concatenated_gz.yml

@@ -3,7 +3,6 @@ in:
   path_prefix: ./concatenated.csv.gz
   decoders:
     - type: commons-compress
-      format: gz
   parser:
     charset: UTF-8
     newline: CRLF

data/src/integration-test/resources/config_no_concatenated_bzip2.yml

@@ -0,0 +1,27 @@
+in:
+  type: file
+  path_prefix: ./concatenated.csv.bz2
+  decoders:
+    - type: commons-compress
+      decompress_concatenated: false
+  parser:
+    charset: UTF-8
+    newline: CRLF
+    type: csv
+    delimiter: ','
+    quote: '"'
+    trim_if_not_quoted: false
+    skip_header_lines: 0
+    allow_extra_columns: false
+    allow_optional_columns: false
+    columns:
+    - {name: id, type: long}
+    - {name: comment, type: string}
+out:
+  type: file
+  path_prefix: ./result_no_concatenated_bzip2_
+  file_ext: csv
+  formatter:
+    type: csv
+    quote_policy: MINIMAL
+    newline: LF

data/src/integration-test/resources/config_no_concatenated_gzip.yml

@@ -0,0 +1,27 @@
+in:
+  type: file
+  path_prefix: ./concatenated.csv.gz
+  decoders:
+    - type: commons-compress
+      decompress_concatenated: false
+  parser:
+    charset: UTF-8
+    newline: CRLF
+    type: csv
+    delimiter: ','
+    quote: '"'
+    trim_if_not_quoted: false
+    skip_header_lines: 0
+    allow_extra_columns: false
+    allow_optional_columns: false
+    columns:
+    - {name: id, type: long}
+    - {name: comment, type: string}
+out:
+  type: file
+  path_prefix: ./result_no_concatenated_gzip_
+  file_ext: csv
+  formatter:
+    type: csv
+    quote_policy: MINIMAL
+    newline: LF

data/src/main/java/org/embulk/decoder/CommonsCompressDecoderPlugin.java

@@ -21,6 +21,10 @@ public class CommonsCompressDecoderPlugin
         @ConfigDefault("\"\"")
         public String getFormat();
 
+        @Config("decompress_concatenated")
+        @ConfigDefault("true")
+        public boolean getDecompressConcatenated();
+
         @ConfigInject
         public BufferAllocator getBufferAllocator();
     }

data/src/main/java/org/embulk/decoder/CommonsCompressProvider.java

@@ -26,6 +26,7 @@ class CommonsCompressProvider implements Provider {
     private final boolean formatAutoDetection;
     private Iterator<InputStream> inputStreamIterator;
     private String[] formats;
+    private final boolean decompressConcatenated;
 
     CommonsCompressProvider(PluginTask task, FileInputInputStream files) {
         this.files = files;
@@ -37,6 +38,8 @@ class CommonsCompressProvider implements Provider {
                 throw new RuntimeException("Failed to get a format.");
             }
         }
+        this.decompressConcatenated = task == null
+            || task.getDecompressConcatenated();
     }
 
     @Override
@@ -162,6 +165,7 @@ class CommonsCompressProvider implements Provider {
     CompressorInputStream createCompressorInputStream(String format,
             InputStream in) throws IOException, CompressorException {
         CompressorStreamFactory factory = new CompressorStreamFactory();
+        factory.setDecompressConcatenated(decompressConcatenated);
         if (CommonsCompressUtil.isAutoDetect(format)) {
             in = in.markSupported() ? in : new BufferedInputStream(in);
             try {
@@ -171,12 +175,6 @@ class CommonsCompressProvider implements Provider {
                         "Failed to detect a file format. Please try to set a format explicitly.",
                         e);
             }
-        }
-
-        if (CompressorStreamFactory.GZIP.equalsIgnoreCase(format)) {
-            return new GzipCompressorInputStream(in, true);
-        } else if (CompressorStreamFactory.BZIP2.equalsIgnoreCase(format)) {
-            return new BZip2CompressorInputStream(in, true);
         } else {
             return factory.createCompressorInputStream(format, in);
         }

data/src/test/java/org/embulk/decoder/TestCommonsCompressDecoderPlugin.java

@@ -59,6 +59,16 @@ public class TestCommonsCompressDecoderPlugin
         Assert.assertEquals("Verify the default config value.", DEFAULT_FORMAT_CONFIG, configDefault.value());
     }
 
+    @Test
+    public void testPluginTaskGetDecompressConcatenated() throws Exception {
+        Method method = CommonsCompressDecoderPlugin.PluginTask.class.getMethod("getDecompressConcatenated");
+        Config config = method.getAnnotation(Config.class);
+        ConfigDefault configDefault = method.getAnnotation(ConfigDefault.class);
+
+        Assert.assertEquals("Verify the config name.", "decompress_concatenated", config.value());
+        Assert.assertEquals("Verify the default config value.", "true", configDefault.value());
+    }
+
     @Test
     public void testTransaction(@Mocked final ConfigSource config, @Mocked final DecoderPlugin.Control control)
     {
@@ -552,9 +562,11 @@ public class TestCommonsCompressDecoderPlugin
 
     private class MockPluginTask implements CommonsCompressDecoderPlugin.PluginTask {
         private final String format;
+        private final boolean decompressConcatenated;
 
         MockPluginTask(String format) {
             this.format = format;
+            this.decompressConcatenated = true;
         }
 
         @Override
@@ -571,6 +583,11 @@ public class TestCommonsCompressDecoderPlugin
             return format;
         }
 
+        @Override
+        public boolean getDecompressConcatenated() {
+            return decompressConcatenated;
+        }
+
         @Override
         public BufferAllocator getBufferAllocator() {
             return newBufferAllocator();

data/src/test/java/org/embulk/decoder/TestCommonsCompressProvider.java

@@ -243,6 +243,10 @@ public class TestCommonsCompressProvider {
 
     @Test
     public void testCreateInputStreamConcatenatedGZ() throws Exception {
+        new NonStrictExpectations() {{
+            task.getDecompressConcatenated(); result = true;
+        }};
+
         try (CommonsCompressProvider provider = new CommonsCompressProvider(task, files)) {
             Iterator<InputStream> it = provider.createInputStreamIterator(
                     new String[]{CompressorStreamFactory.GZIP}, 0, getResourceInputStream("concatenated.csv.gz"));
@@ -252,6 +256,10 @@ public class TestCommonsCompressProvider {
 
     @Test
     public void testCreateInputStreamConcatenatedGZip() throws Exception {
+        new NonStrictExpectations() {{
+            task.getDecompressConcatenated(); result = true;
+        }};
+
         try (CommonsCompressProvider provider = new CommonsCompressProvider(task, files)) {
             Iterator<InputStream> it = provider.createInputStreamIterator(
                     CommonsCompressUtil.toFormats("gzip"), 0, getResourceInputStream("concatenated.csv.gz"));
@@ -261,6 +269,10 @@ public class TestCommonsCompressProvider {
 
     @Test
     public void testCreateInputStreamConcatenatedBZip2() throws Exception {
+        new NonStrictExpectations() {{
+            task.getDecompressConcatenated(); result = true;
+        }};
+
         try (CommonsCompressProvider provider = new CommonsCompressProvider(task, files)) {
             Iterator<InputStream> it = provider.createInputStreamIterator(
                     new String[]{CompressorStreamFactory.BZIP2}, 0, getResourceInputStream("concatenated.csv.bz2"));
@@ -270,6 +282,10 @@ public class TestCommonsCompressProvider {
 
     @Test
     public void testCreateInputStreamConcatenatedBZ2() throws Exception {
+        new NonStrictExpectations() {{
+            task.getDecompressConcatenated(); result = true;
+        }};
+
         try (CommonsCompressProvider provider = new CommonsCompressProvider(task, files)) {
             Iterator<InputStream> it = provider.createInputStreamIterator(
                     CommonsCompressUtil.toFormats("bz2"), 0, getResourceInputStream("concatenated.csv.bz2"));

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: embulk-decoder-commons-compress
 version: !ruby/object:Gem::Version
-  version: 0.3.2
+  version: 0.3.3
 platform: ruby
 authors:
 - hata
@@ -65,6 +65,8 @@ files:
 - src/integration-test/resources/config_concatenated_gz.yml
 - src/integration-test/resources/config_concatenated_gzip.yml
 - src/integration-test/resources/config_gz.yml
+- src/integration-test/resources/config_no_concatenated_bzip2.yml
+- src/integration-test/resources/config_no_concatenated_gzip.yml
 - src/integration-test/resources/config_tar.Z.yml
 - src/integration-test/resources/config_tar.bz2.yml
 - src/integration-test/resources/config_tar.gz.yml
@@ -111,7 +113,7 @@ files:
 - src/test/resources/org/embulk/decoder/samples.tgz
 - src/test/resources/org/embulk/decoder/samples.zip
 - classpath/commons-compress-1.9.jar
-- classpath/embulk-decoder-commons-compress-0.3.2.jar
+- classpath/embulk-decoder-commons-compress-0.3.3.jar
 homepage: https://github.com/hata/embulk-decoder-commons-compress
 licenses:
 - MIT