embulk-decoder-commons-compress 0.3.2 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 8c492e93cc8decc06fa54a952930d446748d1bc3
-  data.tar.gz: 3bc26ce03ac3fd8cbd450b12d5c798f1f0d88326
+  metadata.gz: e5f9220df5b031a29034546701f855a47ec8dd58
+  data.tar.gz: 096289231bfcfc5a70e67faf28bb5252a2f308c3
 SHA512:
-  metadata.gz: a3e7102ce968c9d4c7b7733025d8b141ee55c9dd4f3e730c05a0e9ea0d11f0ddadab2fb9ca34b107eb30949108cdb6c3eb669a31eb5111c95eae52c65c6360dd
-  data.tar.gz: aa39557828feb9906a43d061b2edecaf558bdcae8ff6b00bd6f3623ff37c09d5d3e905f18789c5b8b170a91cb41058ea229fd2df376a6487e644aacefe36c45d
+  metadata.gz: bd02b792a6d97303e70ba516a47ac2d49406426e6adcfa65be63d09accf89068a1d030967a7e0e3a013e4598257cbf9526aa3b5cb75ea66c519f150393026370
+  data.tar.gz: bc60b5fc44b7e7b9802e23f4868dffcb72d779f0233f36c39f2ab75934480a4e71109b82008e177f2c89935fb63b31450a1734ba5cb7b23e1d2bbfdc40979f6a

data/.travis.yml

@@ -1,3 +1,5 @@
 language: java
+dist: xenial
+jdk:
+  - openjdk8
 script: ./gradlew -DenableIntegrationTest=true gem
-

data/README.md

@@ -17,6 +17,8 @@ This decoder plugin for Embulk supports various archive formats using [Apache Co
   - The format type is one of supported formats by by [Apache Commons Compress](http://commons.apache.org/proper/commons-compress/).
   - Auto detect is used when there is no configuration. This can use for a single format. If a file format is solid compression like tar.gz, please set format config explicitly.
   - Some listing formats in [Apache Commons Compress](http://commons.apache.org/proper/commons-compress/) may not work in your environment. I could confirm the following formats work well. Your environment may be able to use other formats listed in the site.
+- **decompress_concatenated**: gzip, bzip2, and xz formats support multiple concatenated streams. The default value of this parameter is true. If you want to disable it, then set to false. See [CompressorStreamFactory.setDecompressConcatenated()](https://commons.apache.org/proper/commons-compress/apidocs/org/apache/commons/compress/compressors/CompressorStreamFactory.html#setDecompressConcatenated(boolean)) in ver.1.9 for more details.
+- **match_name**: Only the files in an archive which match to match_name are processed. match_name is set by regular expression.
 
 ## Formats
 
@@ -29,7 +31,6 @@ This decoder plugin for Embulk supports various archive formats using [Apache Co
   - tbz, tbz2, tb2, tar.bz2
   - taz, tz, tar.Z
 
-If input files are concatenated gzip or bzip2 format, please set format parameter explicitly.
 
 ## Example
 
@@ -62,24 +63,28 @@ in:
       format: tgz
 ```
 
-- Set *format* parameter to handle concatenated gzip(or bzip2) file.
+- Set decompress_concatenated to false if you would like to read the first concatenated gzip/bzip2 archive only.
+
 ```yaml
 in:
   type: any input plugin type
   decoders:
     - type: commons-compress
-      format: gz
+      decompress_concatenated: false
 ```
 
+- Set match_name to extract only the files whose suffix is '.csv' from an archive.
+
 ```yaml
 in:
   type: any input plugin type
   decoders:
     - type: commons-compress
-      format: bzip2
+      match_name: ".*\\.csv"
 ```
 
 
+
 ## Build
 
 ```

data/build.gradle

@@ -7,8 +7,8 @@ import com.github.jrubygradle.JRubyExec
 
 apply from: 'https://raw.githubusercontent.com/hata/gradle-plugins/master/embulk-integration-test.gradle'
 
-sourceCompatibility = '1.7'
-targetCompatibility = '1.7'
+sourceCompatibility = '1.8'
+targetCompatibility = '1.8'
 
 repositories {
     mavenCentral()
@@ -18,12 +18,12 @@ configurations {
     provided
 }
 
-version = "0.3.2"
+version = "0.5.0"
 
 dependencies {
-    compile  "org.embulk:embulk-core:0.7.0"
-    compile "org.apache.commons:commons-compress:1.9"
-    provided "org.embulk:embulk-core:0.7.0"
+    compile  "org.embulk:embulk-core:0.9.23"
+    compile "org.apache.commons:commons-compress:1.20"
+    provided "org.embulk:embulk-core:0.9.23"
     // compile "YOUR_JAR_DEPENDENCY_GROUP:YOUR_JAR_DEPENDENCY_MODULE:YOUR_JAR_DEPENDENCY_VERSION"
     testCompile "org.jmockit:jmockit:1.15"
     testCompile "junit:junit:4.+"

data/gradle/wrapper/gradle-wrapper.properties

@@ -1,6 +1,6 @@
-#Wed Feb 04 13:46:12 PST 2015
+#Fri Jun 05 07:10:25 JST 2020
+distributionUrl=https\://services.gradle.org/distributions/gradle-4.8.1-all.zip
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
-zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-2.2.1-bin.zip
+zipStoreBase=GRADLE_USER_HOME

data/gradlew

@@ -1,4 +1,4 @@
-#!/usr/bin/env bash
+#!/usr/bin/env sh
 
 ##############################################################################
 ##
@@ -6,20 +6,38 @@
 ##
 ##############################################################################
 
-# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
-DEFAULT_JVM_OPTS=""
+# Attempt to set APP_HOME
+# Resolve links: $0 may be a link
+PRG="$0"
+# Need this for relative symlinks.
+while [ -h "$PRG" ] ; do
+    ls=`ls -ld "$PRG"`
+    link=`expr "$ls" : '.*-> \(.*\)$'`
+    if expr "$link" : '/.*' > /dev/null; then
+        PRG="$link"
+    else
+        PRG=`dirname "$PRG"`"/$link"
+    fi
+done
+SAVED="`pwd`"
+cd "`dirname \"$PRG\"`/" >/dev/null
+APP_HOME="`pwd -P`"
+cd "$SAVED" >/dev/null
 
 APP_NAME="Gradle"
 APP_BASE_NAME=`basename "$0"`
 
+# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
+DEFAULT_JVM_OPTS=""
+
 # Use the maximum available, or set MAX_FD != -1 to use that value.
 MAX_FD="maximum"
 
-warn ( ) {
+warn () {
     echo "$*"
 }
 
-die ( ) {
+die () {
     echo
     echo "$*"
     echo
@@ -30,6 +48,7 @@ die ( ) {
 cygwin=false
 msys=false
 darwin=false
+nonstop=false
 case "`uname`" in
   CYGWIN* )
     cygwin=true
@@ -40,31 +59,11 @@ case "`uname`" in
   MINGW* )
     msys=true
     ;;
+  NONSTOP* )
+    nonstop=true
+    ;;
 esac
 
-# For Cygwin, ensure paths are in UNIX format before anything is touched.
-if $cygwin ; then
-    [ -n "$JAVA_HOME" ] && JAVA_HOME=`cygpath --unix "$JAVA_HOME"`
-fi
-
-# Attempt to set APP_HOME
-# Resolve links: $0 may be a link
-PRG="$0"
-# Need this for relative symlinks.
-while [ -h "$PRG" ] ; do
-    ls=`ls -ld "$PRG"`
-    link=`expr "$ls" : '.*-> \(.*\)$'`
-    if expr "$link" : '/.*' > /dev/null; then
-        PRG="$link"
-    else
-        PRG=`dirname "$PRG"`"/$link"
-    fi
-done
-SAVED="`pwd`"
-cd "`dirname \"$PRG\"`/" >&-
-APP_HOME="`pwd -P`"
-cd "$SAVED" >&-
-
 CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar
 
 # Determine the Java command to use to start the JVM.
@@ -90,7 +89,7 @@ location of your Java installation."
 fi
 
 # Increase the maximum file descriptors if we can.
-if [ "$cygwin" = "false" -a "$darwin" = "false" ] ; then
+if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then
     MAX_FD_LIMIT=`ulimit -H -n`
     if [ $? -eq 0 ] ; then
         if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then
@@ -114,6 +113,7 @@ fi
 if $cygwin ; then
     APP_HOME=`cygpath --path --mixed "$APP_HOME"`
     CLASSPATH=`cygpath --path --mixed "$CLASSPATH"`
+    JAVACMD=`cygpath --unix "$JAVACMD"`
 
     # We build the pattern for arguments to be converted via cygpath
     ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null`
@@ -154,11 +154,19 @@ if $cygwin ; then
     esac
 fi
 
-# Split up the JVM_OPTS And GRADLE_OPTS values into an array, following the shell quoting and substitution rules
-function splitJvmOpts() {
-    JVM_OPTS=("$@")
+# Escape application args
+save () {
+    for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done
+    echo " "
 }
-eval splitJvmOpts $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS
-JVM_OPTS[${#JVM_OPTS[*]}]="-Dorg.gradle.appname=$APP_BASE_NAME"
+APP_ARGS=$(save "$@")
+
+# Collect all arguments for the java command, following the shell quoting and substitution rules
+eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS"
+
+# by default we should be in the correct project dir, but when run from Finder on Mac, the cwd is wrong
+if [ "$(uname)" = "Darwin" ] && [ "$HOME" = "$PWD" ]; then
+  cd "$(dirname "$0")"
+fi
 
-exec "$JAVACMD" "${JVM_OPTS[@]}" -classpath "$CLASSPATH" org.gradle.wrapper.GradleWrapperMain "$@"
+exec "$JAVACMD" "$@"

data/gradlew.bat

@@ -8,14 +8,14 @@
 @rem Set local scope for the variables with windows NT shell
 if "%OS%"=="Windows_NT" setlocal
 
-@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
-set DEFAULT_JVM_OPTS=
-
 set DIRNAME=%~dp0
 if "%DIRNAME%" == "" set DIRNAME=.
 set APP_BASE_NAME=%~n0
 set APP_HOME=%DIRNAME%
 
+@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
+set DEFAULT_JVM_OPTS=
+
 @rem Find java.exe
 if defined JAVA_HOME goto findJavaFromJavaHome
 
@@ -46,10 +46,9 @@ echo location of your Java installation.
 goto fail
 
 :init
-@rem Get command-line arguments, handling Windowz variants
+@rem Get command-line arguments, handling Windows variants
 
 if not "%OS%" == "Windows_NT" goto win9xME_args
-if "%@eval[2+2]" == "4" goto 4NT_args
 
 :win9xME_args
 @rem Slurp the command line arguments.
@@ -60,11 +59,6 @@ set _SKIP=2
 if "x%~1" == "x" goto execute
 
 set CMD_LINE_ARGS=%*
-goto execute
-
-:4NT_args
-@rem Get arguments from the 4NT Shell from JP Software
-set CMD_LINE_ARGS=%$
 
 :execute
 @rem Setup the command line

data/src/integration-test/java/org/embulk/filter/TestIntegration.java

@@ -111,6 +111,20 @@ public class TestIntegration {
                 getChecksumFromFiles("result_concatenated_bzip2_000.00.csv"));
     }
 
+    @Test
+    public void testNoConcatenatedGzip() throws Exception {
+        assertEquals("Verify input and output contents are identical.",
+                getChecksumFromFiles(SAMPLE_1_SRC_FILES),
+                getChecksumFromFiles("result_no_concatenated_gzip_000.00.csv"));
+    }
+
+    @Test
+    public void testNoConcatenatedBzip2() throws Exception {
+        assertEquals("Verify input and output contents are identical.",
+                getChecksumFromFiles(SAMPLE_1_SRC_FILES),
+                getChecksumFromFiles("result_no_concatenated_bzip2_000.00.csv"));
+    }
+
     private long getChecksumFromFiles(String ... files) throws IOException {
         Checksum cksum = new CRC32();
 

data/src/integration-test/resources/config_concatenated_bzip2.yml

@@ -3,7 +3,6 @@ in:
   path_prefix: ./concatenated.csv.bz2
   decoders:
     - type: commons-compress
-      format: bzip2
   parser:
     charset: UTF-8
     newline: CRLF

data/src/integration-test/resources/config_concatenated_gz.yml

@@ -3,7 +3,6 @@ in:
   path_prefix: ./concatenated.csv.gz
   decoders:
     - type: commons-compress
-      format: gz
   parser:
     charset: UTF-8
     newline: CRLF

data/src/integration-test/resources/config_no_concatenated_bzip2.yml

@@ -0,0 +1,27 @@
+in:
+  type: file
+  path_prefix: ./concatenated.csv.bz2
+  decoders:
+    - type: commons-compress
+      decompress_concatenated: false
+  parser:
+    charset: UTF-8
+    newline: CRLF
+    type: csv
+    delimiter: ','
+    quote: '"'
+    trim_if_not_quoted: false
+    skip_header_lines: 0
+    allow_extra_columns: false
+    allow_optional_columns: false
+    columns:
+    - {name: id, type: long}
+    - {name: comment, type: string}
+out:
+  type: file
+  path_prefix: ./result_no_concatenated_bzip2_
+  file_ext: csv
+  formatter:
+    type: csv
+    quote_policy: MINIMAL
+    newline: LF

data/src/integration-test/resources/config_no_concatenated_gzip.yml

@@ -0,0 +1,27 @@
+in:
+  type: file
+  path_prefix: ./concatenated.csv.gz
+  decoders:
+    - type: commons-compress
+      decompress_concatenated: false
+  parser:
+    charset: UTF-8
+    newline: CRLF
+    type: csv
+    delimiter: ','
+    quote: '"'
+    trim_if_not_quoted: false
+    skip_header_lines: 0
+    allow_extra_columns: false
+    allow_optional_columns: false
+    columns:
+    - {name: id, type: long}
+    - {name: comment, type: string}
+out:
+  type: file
+  path_prefix: ./result_no_concatenated_gzip_
+  file_ext: csv
+  formatter:
+    type: csv
+    quote_policy: MINIMAL
+    newline: LF

data/src/main/java/org/embulk/decoder/ArchiveInputStreamIterator.java

@@ -10,6 +10,7 @@ import org.apache.commons.compress.archivers.ArchiveInputStream;
 class ArchiveInputStreamIterator implements Iterator<InputStream> {
     private ArchiveInputStream ain;
     private ArchiveEntry entry;
+    private String matchRegex = "";
     private boolean endOfArchive = false;
 
     ArchiveInputStreamIterator(ArchiveInputStream ain)
@@ -17,6 +18,11 @@ class ArchiveInputStreamIterator implements Iterator {
         this.ain = ain;
     }
 
+    ArchiveInputStreamIterator(ArchiveInputStream ain, String matchRegex) {
+        this.ain = ain;
+        this.matchRegex = matchRegex;
+    }
+
     @Override
     public boolean hasNext() {
         try {
@@ -60,9 +66,22 @@ class ArchiveInputStreamIterator implements Iterator {
                 return false;
             } else if (entry.isDirectory()) {
                 continue;
+            } else if (!matchName(entry, matchRegex)){
+                continue;
             } else {
                 return true;
             }
         }
     }
+
+    private boolean matchName(ArchiveEntry entry, String regex) {
+        String name = entry.getName();
+        if(regex == null || regex.equals("")){
+            return true;
+        } else if(name == null) {
+            return false;
+        } else {
+            return name.matches(regex);
+        }
+    }
 }

data/src/main/java/org/embulk/decoder/CommonsCompressDecoderPlugin.java

@@ -21,6 +21,14 @@ public class CommonsCompressDecoderPlugin
         @ConfigDefault("\"\"")
         public String getFormat();
 
+        @Config("decompress_concatenated")
+        @ConfigDefault("true")
+        public boolean getDecompressConcatenated();
+
+        @Config("match_name")
+        @ConfigDefault("\"\"")
+        public String getMatchName();
+
         @ConfigInject
         public BufferAllocator getBufferAllocator();
     }

data/src/main/java/org/embulk/decoder/CommonsCompressProvider.java

@@ -26,6 +26,8 @@ class CommonsCompressProvider implements Provider {
     private final boolean formatAutoDetection;
     private Iterator<InputStream> inputStreamIterator;
     private String[] formats;
+    private final boolean decompressConcatenated;
+    private final String matchName;
 
     CommonsCompressProvider(PluginTask task, FileInputInputStream files) {
         this.files = files;
@@ -37,6 +39,9 @@ class CommonsCompressProvider implements Provider {
                 throw new RuntimeException("Failed to get a format.");
             }
         }
+        this.decompressConcatenated = task == null
+            || task.getDecompressConcatenated();
+        this.matchName = (task == null)? "" : task.getMatchName();
     }
 
     @Override
@@ -85,7 +90,9 @@ class CommonsCompressProvider implements Provider {
         in = in.markSupported() ? in : new BufferedInputStream(in);
         try {
             return new ArchiveInputStreamIterator(
-                    createArchiveInputStream(AUTO_DETECT_FORMAT, in));
+                    createArchiveInputStream(AUTO_DETECT_FORMAT, in),
+                    this.matchName
+            );
         } catch (IOException | ArchiveException e) {
             // ArchiveStreamFactory set mark and reset the stream.
             // So, we can use the same stream to check compressor.
@@ -117,7 +124,8 @@ class CommonsCompressProvider implements Provider {
             String format = inputFormats[pos];
             if (CommonsCompressUtil.isArchiveFormat(format)) {
                 return new ArchiveInputStreamIterator(
-                        createArchiveInputStream(format, in));
+                        createArchiveInputStream(format, in),
+                        this.matchName);
             } else if (CommonsCompressUtil.isCompressorFormat(format)) {
                 return createInputStreamIterator(inputFormats, pos + 1,
                         createCompressorInputStream(format, in));
@@ -162,6 +170,7 @@ class CommonsCompressProvider implements Provider {
     CompressorInputStream createCompressorInputStream(String format,
             InputStream in) throws IOException, CompressorException {
         CompressorStreamFactory factory = new CompressorStreamFactory();
+        factory.setDecompressConcatenated(decompressConcatenated);
         if (CommonsCompressUtil.isAutoDetect(format)) {
             in = in.markSupported() ? in : new BufferedInputStream(in);
             try {
@@ -171,12 +180,6 @@ class CommonsCompressProvider implements Provider {
                         "Failed to detect a file format. Please try to set a format explicitly.",
                         e);
             }
-        }
-
-        if (CompressorStreamFactory.GZIP.equalsIgnoreCase(format)) {
-            return new GzipCompressorInputStream(in, true);
-        } else if (CompressorStreamFactory.BZIP2.equalsIgnoreCase(format)) {
-            return new BZip2CompressorInputStream(in, true);
         } else {
             return factory.createCompressorInputStream(format, in);
         }

data/src/test/java/org/embulk/decoder/TestArchiveInputStreamIterator.java

@@ -70,6 +70,21 @@ public class TestArchiveInputStreamIterator {
         assertNull("Verify there is no stream.", it.next());
     }
 
+    @Test
+    public void testHasNextForNameMatch(@Mocked final ArchiveInputStream ain, @Mocked final ArchiveEntry entry) throws Exception {
+        new NonStrictExpectations() {{
+            ain.getNextEntry(); result = entry; result = entry; result = entry; result = null;
+            entry.getName(); result = "first.csv"; result = "second.txt"; result = "third.csv";
+        }};
+        ArchiveInputStreamIterator it = new ArchiveInputStreamIterator(ain, ".*\\.csv");
+        assertTrue("Verify 1st file match", it.hasNext());
+        assertEquals("Verify ArchiveInputStream is return.", (InputStream)ain, it.next());
+        assertTrue("Verify 3rd file match", it.hasNext());
+        assertEquals("Verify ArchiveInputStream is return.", (InputStream)ain, it.next());
+        assertFalse("Veryfy no more entry because second.txt is skipped.", it.hasNext());
+        assertNull("Verify there is no stream.", it.next());
+    }
+
     @Test
     public void testArchiveFile() throws Exception {
         InputStream in = getClass().getResourceAsStream("samples.tar");

data/src/test/java/org/embulk/decoder/TestCommonsCompressDecoderPlugin.java

@@ -59,6 +59,16 @@ public class TestCommonsCompressDecoderPlugin
         Assert.assertEquals("Verify the default config value.", DEFAULT_FORMAT_CONFIG, configDefault.value());
     }
 
+    @Test
+    public void testPluginTaskGetDecompressConcatenated() throws Exception {
+        Method method = CommonsCompressDecoderPlugin.PluginTask.class.getMethod("getDecompressConcatenated");
+        Config config = method.getAnnotation(Config.class);
+        ConfigDefault configDefault = method.getAnnotation(ConfigDefault.class);
+
+        Assert.assertEquals("Verify the config name.", "decompress_concatenated", config.value());
+        Assert.assertEquals("Verify the default config value.", "true", configDefault.value());
+    }
+
     @Test
     public void testTransaction(@Mocked final ConfigSource config, @Mocked final DecoderPlugin.Control control)
     {
@@ -552,9 +562,13 @@ public class TestCommonsCompressDecoderPlugin
 
     private class MockPluginTask implements CommonsCompressDecoderPlugin.PluginTask {
         private final String format;
+        private final boolean decompressConcatenated;
+        private final String matchName;
 
         MockPluginTask(String format) {
             this.format = format;
+            this.decompressConcatenated = true;
+            this.matchName = "";
         }
 
         @Override
@@ -571,6 +585,16 @@ public class TestCommonsCompressDecoderPlugin
             return format;
         }
 
+        @Override
+        public boolean getDecompressConcatenated() {
+            return decompressConcatenated;
+        }
+
+        @Override
+        public String getMatchName() {
+            return matchName;
+        }
+
         @Override
         public BufferAllocator getBufferAllocator() {
             return newBufferAllocator();

data/src/test/java/org/embulk/decoder/TestCommonsCompressProvider.java

@@ -243,6 +243,10 @@ public class TestCommonsCompressProvider {
 
     @Test
     public void testCreateInputStreamConcatenatedGZ() throws Exception {
+        new NonStrictExpectations() {{
+            task.getDecompressConcatenated(); result = true;
+        }};
+
         try (CommonsCompressProvider provider = new CommonsCompressProvider(task, files)) {
             Iterator<InputStream> it = provider.createInputStreamIterator(
                     new String[]{CompressorStreamFactory.GZIP}, 0, getResourceInputStream("concatenated.csv.gz"));
@@ -252,6 +256,10 @@ public class TestCommonsCompressProvider {
 
     @Test
     public void testCreateInputStreamConcatenatedGZip() throws Exception {
+        new NonStrictExpectations() {{
+            task.getDecompressConcatenated(); result = true;
+        }};
+
         try (CommonsCompressProvider provider = new CommonsCompressProvider(task, files)) {
             Iterator<InputStream> it = provider.createInputStreamIterator(
                     CommonsCompressUtil.toFormats("gzip"), 0, getResourceInputStream("concatenated.csv.gz"));
@@ -261,6 +269,10 @@ public class TestCommonsCompressProvider {
 
     @Test
     public void testCreateInputStreamConcatenatedBZip2() throws Exception {
+        new NonStrictExpectations() {{
+            task.getDecompressConcatenated(); result = true;
+        }};
+
         try (CommonsCompressProvider provider = new CommonsCompressProvider(task, files)) {
             Iterator<InputStream> it = provider.createInputStreamIterator(
                     new String[]{CompressorStreamFactory.BZIP2}, 0, getResourceInputStream("concatenated.csv.bz2"));
@@ -270,6 +282,10 @@ public class TestCommonsCompressProvider {
 
     @Test
     public void testCreateInputStreamConcatenatedBZ2() throws Exception {
+        new NonStrictExpectations() {{
+            task.getDecompressConcatenated(); result = true;
+        }};
+
         try (CommonsCompressProvider provider = new CommonsCompressProvider(task, files)) {
             Iterator<InputStream> it = provider.createInputStreamIterator(
                     CommonsCompressUtil.toFormats("bz2"), 0, getResourceInputStream("concatenated.csv.bz2"));

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-decoder-commons-compress
 version: !ruby/object:Gem::Version
-  version: 0.3.2
+  version: 0.5.0
 platform: ruby
 authors:
 - hata
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-03-02 00:00:00.000000000 Z
+date: 2020-06-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -65,6 +65,8 @@ files:
 - src/integration-test/resources/config_concatenated_gz.yml
 - src/integration-test/resources/config_concatenated_gzip.yml
 - src/integration-test/resources/config_gz.yml
+- src/integration-test/resources/config_no_concatenated_bzip2.yml
+- src/integration-test/resources/config_no_concatenated_gzip.yml
 - src/integration-test/resources/config_tar.Z.yml
 - src/integration-test/resources/config_tar.bz2.yml
 - src/integration-test/resources/config_tar.gz.yml
@@ -110,8 +112,8 @@ files:
 - src/test/resources/org/embulk/decoder/samples.tar.xz
 - src/test/resources/org/embulk/decoder/samples.tgz
 - src/test/resources/org/embulk/decoder/samples.zip
-- classpath/commons-compress-1.9.jar
-- classpath/embulk-decoder-commons-compress-0.3.2.jar
+- classpath/commons-compress-1.20.jar
+- classpath/embulk-decoder-commons-compress-0.5.0.jar
 homepage: https://github.com/hata/embulk-decoder-commons-compress
 licenses:
 - MIT