embulk-encoder-commons-compress 0.1.0 → 0.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 23a1109315cf57fbdf2ffde3d4eec038d0dbea18
-  data.tar.gz: 391ee97d553804487bee7e5421dc1cef952bb99d
+  metadata.gz: a0ad3ce0045781de0d2faac7790c076dbc636187
+  data.tar.gz: 3d0d8425b24195917ec0dc101c553ceaff8982b6
 SHA512:
-  metadata.gz: 789058ac973dfcb777cf0216bacf107dc27a92dcb2016a2fdc190095b5c5295a0e57ba26f6f7f8b83a4e2a96471a2cba479c497830641d91be0a46f2294dcd2c
-  data.tar.gz: 43e265e8be8e80bccabcf032f964f8cbea0fdeefa1c898730e344ff3e0a9931c57f6310c900bd00e919cc67ce2c8d9089045c40bd91d67351709393666d13c26
+  metadata.gz: 497dd5d2c8908446ed70d69e26b407ffe2afa3119916f3419eb1e0fde5e11f10ea7186b30c54898374519580719ca7fbd108f8d2596d8703972ba53cff9a8e1c
+  data.tar.gz: c8ca43ee86d84c6841e6eccff86f66de9ea6b77d11d5bdd9de1e8ac61e5bc11410f8e277397adaabef8b34944f231fd88eae3fa97431e8180a760d94b6f97964

data/.travis.yml

@@ -0,0 +1,3 @@
+language: java
+script: ./gradlew -DenableIntegrationTest=true gem
+

data/README.md

@@ -1,4 +1,5 @@
 # Commons Compress Encoder plugin for Embulk
+[![Build Status](https://travis-ci.org/hata/embulk-encoder-commons-compress.svg)](https://travis-ci.org/hata/embulk-encoder-commons-compress)
 
 This encoder plugin supports to encode a format like bzip2 and gzip.
 
@@ -10,7 +11,7 @@ This encoder plugin supports to encode a format like bzip2 and gzip.
 
 ## Configuration
 
-- **format**: File format like bzip2, gzip (string, required)
+- **format**: File format like bzip2 and gzip (string, required)
 
 ## Example
 
@@ -18,7 +19,7 @@ This encoder plugin supports to encode a format like bzip2 and gzip.
 out:
   type: file
   path_prefix: any path
-  file_ext: .csv.bz2
+  file_ext: csv.bz2
   formatter:
     type: csv
   encoders:
@@ -31,3 +32,9 @@ out:
 ```
 $ ./gradlew gem
 ```
+
+To run integrationTest
+```
+$ ./gradlew -DenableIntegrationTest=true gem
+```
+

data/build.gradle

@@ -5,6 +5,8 @@ plugins {
 }
 import com.github.jrubygradle.JRubyExec
 
+apply from: 'https://raw.githubusercontent.com/hata/gradle-plugins/master/embulk-integration-test.gradle'
+
 sourceCompatibility = '1.7'
 targetCompatibility = '1.7'
 
@@ -16,12 +18,12 @@ configurations {
     provided
 }
 
-version = "0.1.0"
+version = "0.1.1"
 
 dependencies {
-    compile  "org.embulk:embulk-core:0.4.6"
-    compile "org.apache.commons:commons-compress:1.9"
-    provided "org.embulk:embulk-core:0.4.6"
+    compile  "org.embulk:embulk-core:0.8.0"
+    compile "org.apache.commons:commons-compress:1.13"
+    provided "org.embulk:embulk-core:0.8.0"
     // compile "YOUR_JAR_DEPENDENCY_GROUP:YOUR_JAR_DEPENDENCY_MODULE:YOUR_JAR_DEPENDENCY_VERSION"
     testCompile "org.jmockit:jmockit:1.15"
     testCompile "junit:junit:4.+"
@@ -61,3 +63,9 @@ Gem::Specification.new do |spec|
 end
 /$)
 }
+
+//
+// To enable integrationTest, run like this:
+// ./gradlew -DenableIntegrationTest=true gem
+//
+project.tasks.integrationTest.dependsOn(classpath)

data/src/integration-test/java/org/embulk/encoder/TestIntegration.java

@@ -0,0 +1,80 @@
+package org.embulk.encoder;
+
+import static org.junit.Assert.assertEquals;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.util.zip.CRC32;
+import java.util.zip.Checksum;
+
+import org.apache.commons.compress.compressors.CompressorException;
+import org.apache.commons.compress.compressors.CompressorStreamFactory;
+import org.junit.Test;
+
+public class TestIntegration {
+    static final String TEST_DIR = System.getProperty("embulk.integrationtest.dir");
+
+    private static String getTestFile(String name) {
+        return TEST_DIR + File.separator + name;
+    }
+
+    @Test
+    public void testSolidCompressionFormatTarGz() throws Exception {
+        assertEquals("Verify input and output contents are identical.",
+                getChecksumFromFiles("sample_1.csv"),
+                getChecksumFromCompressedFiles("gz", "result_gz_000.00.csv.gz"));
+    }
+
+    @Test
+    public void testSolidCompressionFormatTarZ() throws Exception {
+        assertEquals("Verify input and output contents are identical.",
+                getChecksumFromFiles("sample_1.csv"),
+                getChecksumFromCompressedFiles("bzip2", "result_bz2_000.00.csv.bz2"));
+        
+        
+    }
+
+    private long getChecksumFromFiles(String ... files) throws IOException {
+        Checksum cksum = new CRC32();
+
+        for (String srcFile : files) {
+            try (BufferedReader reader = new BufferedReader(new FileReader(getTestFile(srcFile)))) {
+                getChecksum(cksum, reader);
+            }
+        }
+        
+        return cksum.getValue();
+    }
+
+    private long getChecksumFromCompressedFiles(String compressName, String ... files) throws IOException, CompressorException {
+        Checksum cksum = new CRC32();
+
+        CompressorStreamFactory factory = new CompressorStreamFactory();
+
+        for (String srcFile : files) {
+            try(BufferedReader reader = new BufferedReader(new InputStreamReader(
+                    factory.createCompressorInputStream(compressName, new FileInputStream(getTestFile(srcFile)))))) {
+                getChecksum(cksum, reader);
+            }
+        }
+
+        return cksum.getValue();
+    }
+
+    private long getChecksum(Checksum cksum, BufferedReader reader) throws IOException {
+        String line = reader.readLine();
+        while (line != null) {
+            byte[] lineBuf = line.trim().getBytes();
+            if (lineBuf.length > 0) {
+                // System.out.println("line:" + new String(lineBuf));
+                cksum.update(lineBuf, 0, lineBuf.length);
+            }
+            line = reader.readLine();
+        }
+        return cksum.getValue();
+    }
+}

data/src/integration-test/resources/config_bz2.yml

@@ -0,0 +1,27 @@
+in:
+  type: file
+  path_prefix: ./sample_1.csv
+  parser:
+    charset: UTF-8
+    newline: CRLF
+    type: csv
+    delimiter: ','
+    quote: '"'
+    trim_if_not_quoted: false
+    skip_header_lines: 1
+    allow_extra_columns: false
+    allow_optional_columns: false
+    columns:
+    - {name: id, type: long}
+    - {name: comment, type: string}
+out:
+  type: file
+  path_prefix: ./result_bz2_
+  file_ext: csv.bz2
+  formatter:
+    type: csv
+    quote_policy: MINIMAL
+    newline: LF
+  encoders:
+  - type: commons-compress
+    format: bzip2

data/src/integration-test/resources/config_gzip.yml

@@ -0,0 +1,27 @@
+in:
+  type: file
+  path_prefix: ./sample_1.csv
+  parser:
+    charset: UTF-8
+    newline: CRLF
+    type: csv
+    delimiter: ','
+    quote: '"'
+    trim_if_not_quoted: false
+    skip_header_lines: 1
+    allow_extra_columns: false
+    allow_optional_columns: false
+    columns:
+    - {name: id, type: long}
+    - {name: comment, type: string}
+out:
+  type: file
+  path_prefix: ./result_gz_
+  file_ext: csv.gz
+  formatter:
+    type: csv
+    quote_policy: MINIMAL
+    newline: LF
+  encoders:
+  - type: commons-compress
+    format: gzip

data/src/integration-test/resources/sample_1.csv

@@ -0,0 +1,3 @@
+id,comment
+1,foo
+2,bar

data/src/test/java/org/embulk/encoder/TestCommonsCompressCompressorProvider.java

@@ -0,0 +1,189 @@
+/*
+ * $Id$
+ * 
+ * Copyright 2015 Hiroki Ata
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.embulk.encoder;
+
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+
+import java.io.OutputStream;
+
+import mockit.Mocked;
+import mockit.NonStrictExpectations;
+import mockit.Verifications;
+
+import org.apache.commons.compress.compressors.bzip2.BZip2CompressorOutputStream;
+import org.apache.commons.compress.compressors.deflate.DeflateCompressorOutputStream;
+import org.apache.commons.compress.compressors.gzip.GzipCompressorOutputStream;
+import org.embulk.encoder.CommonsCompressEncoderPlugin.PluginTask;
+import org.embulk.spi.Buffer;
+import org.embulk.spi.BufferAllocator;
+import org.embulk.spi.FileOutput;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+public class TestCommonsCompressCompressorProvider {
+    @Mocked
+    PluginTask task;
+
+    @Mocked
+    FileOutput fileOutput;
+
+    CommonsCompressCompressorProvider provider;
+
+    @Before
+    public void setUp() throws Exception {
+    }
+
+    @After
+    public void tearDown() throws Exception {
+        provider = null;
+    }
+
+    @Test
+    public void testIsCompressorFormat() {
+        assertTrue(CommonsCompressCompressorProvider.isCompressorFormat("bzip2"));
+        assertTrue(CommonsCompressCompressorProvider.isCompressorFormat("gzip"));
+        assertTrue(CommonsCompressCompressorProvider.isCompressorFormat("deflate"));
+        assertFalse(CommonsCompressCompressorProvider.isCompressorFormat("tar"));
+        assertFalse(CommonsCompressCompressorProvider.isCompressorFormat("zip"));
+    }
+
+    @Test
+    public void testCommonsCompressCompressorProvider() {
+        new NonStrictExpectations() {{
+            task.getFormat(); result = "gzip";
+            task.getBufferAllocator(); result = new MockBufferAllocator();
+        }};
+
+        assertNotNull("Check constructor.", new CommonsCompressCompressorProvider(task, fileOutput));
+    }
+
+    @Test
+    public void testOpenNext() throws Exception {
+        new NonStrictExpectations() {{
+            task.getFormat(); result = "gzip";
+            task.getBufferAllocator(); result = new MockBufferAllocator();
+        }};
+
+        provider = new CommonsCompressCompressorProvider(task, fileOutput);
+        OutputStream out = provider.openNext();
+        assertTrue("Verify a stream instance.", out instanceof GzipCompressorOutputStream);
+        provider.close();
+        
+        new Verifications() {{
+            fileOutput.nextFile(); times = 1;
+        }};
+    }
+
+    @Test
+    public void testFinish() throws Exception {
+        new NonStrictExpectations() {{
+            task.getFormat(); result = "gzip";
+            task.getBufferAllocator(); result = new MockBufferAllocator();
+        }};
+
+        provider = new CommonsCompressCompressorProvider(task, fileOutput);
+        provider.finish();
+        provider.close();
+        
+        new Verifications() {{
+            fileOutput.finish(); times = 1;
+        }};
+    }
+
+    @Test
+    public void testClose() throws Exception {
+        new NonStrictExpectations() {{
+            task.getFormat(); result = "gzip";
+            task.getBufferAllocator(); result = new MockBufferAllocator();
+        }};
+
+        provider = new CommonsCompressCompressorProvider(task, fileOutput);
+        provider.close();
+        
+        new Verifications() {{
+            fileOutput.close(); times = 1;
+        }};
+    }
+
+    @Test
+    public void testCreateCompressorOutputStreamGzip() throws Exception {
+        new NonStrictExpectations() {{
+            task.getFormat(); result = "gzip";
+            task.getBufferAllocator(); result = new MockBufferAllocator();
+        }};
+
+        provider = new CommonsCompressCompressorProvider(task, fileOutput);
+        OutputStream out = provider.createCompressorOutputStream();
+        assertTrue("Verify a stream instance.", out instanceof GzipCompressorOutputStream);
+        provider.close();
+        
+        new Verifications() {{
+            fileOutput.close(); times = 1;
+        }};
+    }
+
+    @Test
+    public void testCreateCompressorOutputStreamBzip2() throws Exception {
+        new NonStrictExpectations() {{
+            task.getFormat(); result = "bzip2";
+            task.getBufferAllocator(); result = new MockBufferAllocator();
+        }};
+
+        provider = new CommonsCompressCompressorProvider(task, fileOutput);
+        OutputStream out = provider.createCompressorOutputStream();
+        assertTrue("Verify a stream instance.", out instanceof BZip2CompressorOutputStream);
+        provider.close();
+        
+        new Verifications() {{
+            fileOutput.close(); times = 1;
+        }};
+    }
+
+
+    @Test
+    public void testCreateCompressorOutputStreamDeflate() throws Exception {
+        new NonStrictExpectations() {{
+            task.getFormat(); result = "deflate";
+            task.getBufferAllocator(); result = new MockBufferAllocator();
+        }};
+
+        provider = new CommonsCompressCompressorProvider(task, fileOutput);
+        OutputStream out = provider.createCompressorOutputStream();
+        assertTrue("Verify a stream instance.", out instanceof DeflateCompressorOutputStream);
+        provider.close();
+        
+        new Verifications() {{
+            fileOutput.close(); times = 1;
+        }};
+    }
+
+    private class MockBufferAllocator implements BufferAllocator {
+        @Override
+        public Buffer allocate() {
+            return allocate(8192);
+        }
+
+        @Override
+        public Buffer allocate(int size) {
+            return Buffer.allocate(size);
+        }
+    }
+}

data/src/test/java/org/embulk/encoder/TestCommonsCompressEncoderPlugin.java

@@ -1,5 +1,81 @@
 package org.embulk.encoder;
 
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+import mockit.Mocked;
+import mockit.NonStrictExpectations;
+import mockit.Verifications;
+
+import org.embulk.config.ConfigSource;
+import org.embulk.config.TaskSource;
+import org.embulk.encoder.CommonsCompressEncoderPlugin.PluginTask;
+import org.embulk.spi.EncoderPlugin;
+import org.embulk.spi.FileOutput;
+import org.embulk.spi.util.OutputStreamFileOutput;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
 public class TestCommonsCompressEncoderPlugin
 {
+    @Mocked
+    CommonsCompressEncoderPlugin.PluginTask task;
+
+    @Mocked
+    FileOutput output;
+    
+    @Mocked
+    TaskSource taskSource;
+
+    @Mocked
+    ConfigSource config;
+    
+    @Mocked
+    EncoderPlugin.Control control;
+
+    CommonsCompressEncoderPlugin plugin;
+
+    @Before
+    public void setUp() throws Exception {
+        plugin = new CommonsCompressEncoderPlugin();
+    }
+
+    @After
+    public void tearDown() throws Exception {
+        plugin = null;
+    }
+
+    @Test
+    public void testTransaction() {
+        new NonStrictExpectations() {{
+            config.loadConfig(PluginTask.class); result = task;
+            task.dump(); result = taskSource;
+        }};
+
+        plugin.transaction(config, control);
+        
+        new Verifications() {{
+            control.run(taskSource); times = 1;
+        }};
+    }
+
+    @Test
+    public void testOpen() {
+        new NonStrictExpectations() {{
+            task.getFormat(); result = "gzip";
+            taskSource.loadTask(PluginTask.class); result = task;
+        }};
+
+        assertNotNull(plugin.open(taskSource, output));
+    }
+
+    @Test
+    public void testCreateProvider() {
+        new NonStrictExpectations() {{
+            task.getFormat(); result = "gzip";
+        }};
+
+        OutputStreamFileOutput.Provider provider = plugin.createProvider(task, output);
+        assertTrue("Verify a provider instance.", provider instanceof CommonsCompressCompressorProvider);
+    }
 }

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-encoder-commons-compress
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.1
 platform: ruby
 authors:
 - hata
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-10-16 00:00:00.000000000 Z
+date: 2017-02-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -46,6 +46,7 @@ extensions: []
 extra_rdoc_files: []
 files:
 - .gitignore
+- .travis.yml
 - LICENSE.txt
 - README.md
 - build.gradle
@@ -54,12 +55,17 @@ files:
 - gradlew
 - gradlew.bat
 - lib/embulk/encoder/commons-compress.rb
+- src/integration-test/java/org/embulk/encoder/TestIntegration.java
+- src/integration-test/resources/config_bz2.yml
+- src/integration-test/resources/config_gzip.yml
+- src/integration-test/resources/sample_1.csv
 - src/main/java/org/embulk/encoder/CommonsCompressArchiveProvider.java
 - src/main/java/org/embulk/encoder/CommonsCompressCompressorProvider.java
 - src/main/java/org/embulk/encoder/CommonsCompressEncoderPlugin.java
+- src/test/java/org/embulk/encoder/TestCommonsCompressCompressorProvider.java
 - src/test/java/org/embulk/encoder/TestCommonsCompressEncoderPlugin.java
-- classpath/commons-compress-1.9.jar
-- classpath/embulk-encoder-commons-compress-0.1.0.jar
+- classpath/commons-compress-1.13.jar
+- classpath/embulk-encoder-commons-compress-0.1.1.jar
 homepage: https://github.com/hata/embulk-encoder-commons-compress
 licenses:
 - MIT