embulk-decoder-commons-compress 0.3.3 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: d01788f5dc971e9ee19b2139c5079fc2d54ebc81
-  data.tar.gz: d7cd120f691677b626b5b758f665e2dafab083ce
+  metadata.gz: 991a486a32c2dd2bfddf31f937d04e612870ddc5
+  data.tar.gz: ae04456e55be753f049dd6343995e22873a1a4e8
 SHA512:
-  metadata.gz: bb6b7d5f8717bc3464ca3e6c5272a6eb7a9e3eee262f2effbfb57e0b8784151b417a9d3e8cabad624080eea8adbabb461f9536b051ddbd676335f783bba5f51a
-  data.tar.gz: 4b1c4d7d7b01ea23c388d858717f8762db2bc2b97651bfb26273e02ec2c307d47faade8143269c4b491edb1adf2acaa9204d86594e1c682cfc0d53dd0d3da7e9
+  metadata.gz: a2a6e171d9d98884df6bb4eddafd2d0ad07de835d66a2756926cd919be8bdb84926ca0982e94aeac5c2279406d4f6423bcfcde45daabf13365f262ca7031830f
+  data.tar.gz: a0031b1f6c5279cdd2c49fb098eef837e8f9aeac3f766fd318d190a87f4a708e7b509c846be0efe0a09eeadde7e627e312f49832124caa019047b94db0476d6b

data/README.md

@@ -18,6 +18,7 @@ This decoder plugin for Embulk supports various archive formats using [Apache Co
   - Auto detect is used when there is no configuration. This can use for a single format. If a file format is solid compression like tar.gz, please set format config explicitly.
   - Some listing formats in [Apache Commons Compress](http://commons.apache.org/proper/commons-compress/) may not work in your environment. I could confirm the following formats work well. Your environment may be able to use other formats listed in the site.
 - **decompress_concatenated**: gzip, bzip2, and xz formats support multiple concatenated streams. The default value of this parameter is true. If you want to disable it, then set to false. See [CompressorStreamFactory.setDecompressConcatenated()](https://commons.apache.org/proper/commons-compress/apidocs/org/apache/commons/compress/compressors/CompressorStreamFactory.html#setDecompressConcatenated(boolean)) in ver.1.9 for more details.
+- **match_name**: Only the files in an archive which match to match_name are processed. match_name is set by regular expression.
 
 ## Formats
 
@@ -62,7 +63,7 @@ in:
       format: tgz
 ```
 
-- Set decompress_concatenated to false if you would like to read only the first concatenated entry.
+- Set decompress_concatenated to false if you would like to read the first concatenated gzip/bzip2 archive only.
 
 ```yaml
 in:
@@ -72,6 +73,16 @@ in:
       decompress_concatenated: false
 ```
 
+- Set match_name to extract only the files whose suffix is '.csv' from an archive.
+
+```yaml
+in:
+  type: any input plugin type
+  decoders:
+    - type: commons-compress
+      match_name_: ".*\\.csv"
+```
+
 
 
 ## Build

data/build.gradle

@@ -18,7 +18,7 @@ configurations {
     provided
 }
 
-version = "0.3.3"
+version = "0.4.0"
 
 dependencies {
     compile  "org.embulk:embulk-core:0.7.0"

data/src/main/java/org/embulk/decoder/ArchiveInputStreamIterator.java

@@ -10,6 +10,7 @@ import org.apache.commons.compress.archivers.ArchiveInputStream;
 class ArchiveInputStreamIterator implements Iterator<InputStream> {
     private ArchiveInputStream ain;
     private ArchiveEntry entry;
+    private String matchRegex = "";
     private boolean endOfArchive = false;
 
     ArchiveInputStreamIterator(ArchiveInputStream ain)
@@ -17,6 +18,11 @@ class ArchiveInputStreamIterator implements Iterator<InputStream> {
         this.ain = ain;
     }
 
+    ArchiveInputStreamIterator(ArchiveInputStream ain, String matchRegex) {
+        this.ain = ain;
+        this.matchRegex = matchRegex;
+    }
+
     @Override
     public boolean hasNext() {
         try {
@@ -60,9 +66,22 @@ class ArchiveInputStreamIterator implements Iterator<InputStream> {
                 return false;
             } else if (entry.isDirectory()) {
                 continue;
+            } else if (!matchName(entry, matchRegex)){
+                continue;
             } else {
                 return true;
             }
         }
     }
+
+    private boolean matchName(ArchiveEntry entry, String regex) {
+        String name = entry.getName();
+        if(regex == null || regex.equals("")){
+            return true;
+        } else if(name == null) {
+            return false;
+        } else {
+            return name.matches(regex);
+        }
+    }
 }

data/src/main/java/org/embulk/decoder/CommonsCompressDecoderPlugin.java

@@ -25,6 +25,10 @@ public class CommonsCompressDecoderPlugin
         @ConfigDefault("true")
         public boolean getDecompressConcatenated();
 
+        @Config("match_name")
+        @ConfigDefault("\"\"")
+        public String getMatchName();
+
         @ConfigInject
         public BufferAllocator getBufferAllocator();
     }

data/src/main/java/org/embulk/decoder/CommonsCompressProvider.java

@@ -27,6 +27,7 @@ class CommonsCompressProvider implements Provider {
     private Iterator<InputStream> inputStreamIterator;
     private String[] formats;
     private final boolean decompressConcatenated;
+    private final String matchName;
 
     CommonsCompressProvider(PluginTask task, FileInputInputStream files) {
         this.files = files;
@@ -40,6 +41,7 @@ class CommonsCompressProvider implements Provider {
         }
         this.decompressConcatenated = task == null
             || task.getDecompressConcatenated();
+        this.matchName = (task == null)? "" : task.getMatchName();
     }
 
     @Override
@@ -88,7 +90,9 @@ class CommonsCompressProvider implements Provider {
         in = in.markSupported() ? in : new BufferedInputStream(in);
         try {
             return new ArchiveInputStreamIterator(
-                    createArchiveInputStream(AUTO_DETECT_FORMAT, in));
+                    createArchiveInputStream(AUTO_DETECT_FORMAT, in),
+                    this.matchName
+            );
         } catch (IOException | ArchiveException e) {
             // ArchiveStreamFactory set mark and reset the stream.
             // So, we can use the same stream to check compressor.

data/src/test/java/org/embulk/decoder/TestArchiveInputStreamIterator.java

@@ -70,6 +70,21 @@ public class TestArchiveInputStreamIterator {
         assertNull("Verify there is no stream.", it.next());
     }
 
+    @Test
+    public void testHasNextForNameMatch(@Mocked final ArchiveInputStream ain, @Mocked final ArchiveEntry entry) throws Exception {
+        new NonStrictExpectations() {{
+            ain.getNextEntry(); result = entry; result = entry; result = entry; result = null;
+            entry.getName(); result = "first.csv"; result = "second.txt"; result = "third.csv";
+        }};
+        ArchiveInputStreamIterator it = new ArchiveInputStreamIterator(ain, ".*\\.csv");
+        assertTrue("Verify 1st file match", it.hasNext());
+        assertEquals("Verify ArchiveInputStream is return.", (InputStream)ain, it.next());
+        assertTrue("Verify 3rd file match", it.hasNext());
+        assertEquals("Verify ArchiveInputStream is return.", (InputStream)ain, it.next());
+        assertFalse("Veryfy no more entry because second.txt is skipped.", it.hasNext());
+        assertNull("Verify there is no stream.", it.next());
+    }
+
     @Test
     public void testArchiveFile() throws Exception {
         InputStream in = getClass().getResourceAsStream("samples.tar");

data/src/test/java/org/embulk/decoder/TestCommonsCompressDecoderPlugin.java

@@ -563,10 +563,12 @@ public class TestCommonsCompressDecoderPlugin
     private class MockPluginTask implements CommonsCompressDecoderPlugin.PluginTask {
         private final String format;
         private final boolean decompressConcatenated;
+        private final String matchName;
 
         MockPluginTask(String format) {
             this.format = format;
             this.decompressConcatenated = true;
+            this.matchName = "";
         }
 
         @Override
@@ -588,6 +590,11 @@ public class TestCommonsCompressDecoderPlugin
             return decompressConcatenated;
         }
 
+        @Override
+        public String getMatchName() {
+            return matchName;
+        }
+
         @Override
         public BufferAllocator getBufferAllocator() {
             return newBufferAllocator();

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-decoder-commons-compress
 version: !ruby/object:Gem::Version
-  version: 0.3.3
+  version: 0.4.0
 platform: ruby
 authors:
 - hata
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-03-02 00:00:00.000000000 Z
+date: 2016-05-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -113,7 +113,7 @@ files:
 - src/test/resources/org/embulk/decoder/samples.tgz
 - src/test/resources/org/embulk/decoder/samples.zip
 - classpath/commons-compress-1.9.jar
-- classpath/embulk-decoder-commons-compress-0.3.3.jar
+- classpath/embulk-decoder-commons-compress-0.4.0.jar
 homepage: https://github.com/hata/embulk-decoder-commons-compress
 licenses:
 - MIT