embulk-output-gcs_streaming 0.1.0

data/src/test/java/org/embulk/output/gcs_streaming/TestGcsClient.java

@@ -0,0 +1,50 @@
+package org.embulk.output.gcs_streaming;
+
+import org.junit.BeforeClass;
+import org.junit.Test;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assume.assumeNotNull;
+
+import java.util.Optional;
+
+public class TestGcsClient
+{
+    private static String gcpProjectId;
+    private static Optional<String> gcpJsonKeyfile;
+    /*
+     * This test case requires environment variables
+     *   GCP_PROJECT_ID
+     *   GCP_JSON_KEYFILE
+     * And prepare gcloud authentication ADC, following command.
+     *   $ gcloud auth application-default login
+     */
+    @BeforeClass
+    public static void initializeConstant()
+    {
+        gcpProjectId = System.getenv("GCP_PROJECT_ID");
+        gcpJsonKeyfile = Optional.of(System.getenv("GCP_JSON_KEYFILE"));
+        assumeNotNull(gcpJsonKeyfile, gcpProjectId);
+        // skip test cases, if environment variables are not set.
+    }
+
+    @Test
+    public void testGetStorageSuccess() throws RuntimeException
+    {
+        Optional<String> empty = Optional.empty();
+        GcsClient client = new GcsClient(gcpProjectId, empty);
+    }
+
+    @Test
+    public void testGetStorageSuccessFromJsonKeyfile() throws RuntimeException
+    {
+        GcsClient client = new GcsClient(gcpProjectId, gcpJsonKeyfile);
+    }
+
+    @Test(expected = RuntimeException.class)
+    public void testGetStorageFailFromJsonKeyfile() throws RuntimeException
+    {
+        Optional<String> notFoundJsonKeyfile = Optional.of("/path/to/key.json");
+        GcsClient client = new GcsClient(gcpProjectId, notFoundJsonKeyfile);
+        assertEquals(1, 2);
+    }
+}

data/src/test/java/org/embulk/output/gcs_streaming/TestGcsStreamingFileOutputPlugin.java

@@ -0,0 +1,305 @@
+package org.embulk.output.gcs_streaming;
+
+import com.google.cloud.storage.Blob;
+import com.google.cloud.storage.BlobId;
+import com.google.cloud.storage.Storage;
+import com.google.cloud.storage.StorageOptions;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Lists;
+import com.google.common.io.Resources;
+
+import org.embulk.EmbulkTestRuntime;
+import org.embulk.config.ConfigSource;
+import org.embulk.config.TaskReport;
+import org.embulk.config.TaskSource;
+
+import org.embulk.output.gcs_streaming.GcsStreamingFileOutputPlugin.PluginTask;
+
+import org.embulk.spi.Buffer;
+import org.embulk.spi.Exec;
+import org.embulk.spi.FileOutputRunner;
+import org.embulk.spi.OutputPlugin;
+import org.embulk.spi.Schema;
+import org.embulk.spi.TransactionalFileOutput;
+import org.embulk.standards.CsvParserPlugin;
+
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Rule;
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assume.assumeNotNull;
+
+import java.io.BufferedReader;
+import java.io.ByteArrayOutputStream;
+import java.io.FileInputStream;
+import java.io.InputStream;
+import java.io.IOException;
+import java.io.StringReader;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Optional;
+
+public class TestGcsStreamingFileOutputPlugin
+{
+    private static String gcpProjectId;
+    private static String gcpBucket;
+    private static Optional<String> gcpJsonKeyfile;
+    private static String localPathPrefix;
+    private FileOutputRunner runner;
+    /*
+     * This test case requires environment variables
+     *   GCP_PROJECT_ID
+     *   GCP_BUCKET
+     *   GCP_JSON_KEYFILE
+     * And prepare gcloud authentication ADC, following command.
+     *   $ gcloud auth application-default login
+     */
+    @BeforeClass
+    public static void initializeConstant()
+    {
+        gcpProjectId = System.getenv("GCP_PROJECT_ID");
+        gcpJsonKeyfile = Optional.of(System.getenv("GCP_JSON_KEYFILE"));
+        gcpBucket = System.getenv("GCP_BUCKET");
+        localPathPrefix = Resources.getResource("test.000.csv").getPath();
+        assumeNotNull(gcpJsonKeyfile, gcpProjectId, gcpBucket, localPathPrefix);
+        // skip test cases, if environment variables are not set.
+    }
+
+    @Rule
+    public EmbulkTestRuntime runtime = new EmbulkTestRuntime();
+    private GcsStreamingFileOutputPlugin plugin;
+
+    @Before
+    public void createResources()
+    {
+        plugin = new GcsStreamingFileOutputPlugin();
+        runner = new FileOutputRunner(runtime.getInstance(GcsStreamingFileOutputPlugin.class));
+    }
+
+    @Test
+    public void checkDefaultValues()
+    {
+        ConfigSource config = Exec.newConfigSource()
+                .set("in", inputConfig())
+                .set("parser", parserConfig(schemaConfig()))
+                .set("type", "gcs_streaming")
+                .set("project_id", gcpProjectId)
+                .set("bucket", gcpBucket)
+                .set("json_keyfile", gcpJsonKeyfile)
+                .set("path_prefix", "tests/data")
+                .set("file_ext", ".csv")
+                .set("formatter", formatterConfig());
+
+        PluginTask task = config.loadConfig(PluginTask.class);
+
+        assertEquals(gcpProjectId, task.getProjectId());
+        assertEquals(gcpBucket, task.getBucket());
+        assertEquals(gcpJsonKeyfile, task.getJsonKeyfile());
+        assertEquals(".%03d.%02d.", task.getSequenceFormat());
+        assertEquals("application/octet-stream", task.getContentType());
+        assertEquals("tests/data", task.getPathPrefix());
+        assertEquals(".csv", task.getFileNameExtension());
+    }
+
+    @Test
+    public void testTransaction()
+    {
+        ConfigSource config = Exec.newConfigSource()
+                .set("in", inputConfig())
+                .set("parser", parserConfig(schemaConfig()))
+                .set("type", "gcs_streaming")
+                .set("project_id", gcpProjectId)
+                .set("bucket", gcpBucket)
+                .set("path_prefix", "tests/data")
+                .set("file_ext", "csv")
+                .set("formatter", formatterConfig());
+
+        Schema schema = config.getNested("parser").loadConfig(CsvParserPlugin.PluginTask.class).getSchemaConfig().toSchema();
+
+        runner.transaction(config, schema, 0, new Control());
+    }
+
+    @Test
+    public void testTransactionWithJsonKeyfile()
+    {
+        ConfigSource config = Exec.newConfigSource()
+                .set("in", inputConfig())
+                .set("parser", parserConfig(schemaConfig()))
+                .set("type", "gcs_streaming")
+                .set("project_id", gcpProjectId)
+                .set("bucket", gcpBucket)
+                .set("json_keyfile", gcpJsonKeyfile)
+                .set("path_prefix", "tests/data")
+                .set("file_ext", "csv")
+                .set("formatter", formatterConfig());
+
+        Schema schema = config.getNested("parser").loadConfig(CsvParserPlugin.PluginTask.class).getSchemaConfig().toSchema();
+
+        runner.transaction(config, schema, 0, new Control());
+    }
+
+    @Test
+    public void testResume()
+    {
+        // no support resume
+    }
+
+    @Test
+    public void testCleanup()
+    {
+        PluginTask task = config().loadConfig(PluginTask.class);
+        plugin.cleanup(task.dump(), 0, Lists.newArrayList()); // no errors happens
+    }
+
+    @Test
+    public void testGcsFileOutputByOpen() throws Exception
+    {
+        ConfigSource configSource = config();
+        PluginTask task = configSource.loadConfig(PluginTask.class);
+        Schema schema = configSource.getNested("parser").loadConfig(CsvParserPlugin.PluginTask.class).getSchemaConfig().toSchema();
+        runner.transaction(configSource, schema, 0, new Control());
+
+        TransactionalFileOutput output = plugin.open(task.dump(), 0);
+
+        output.nextFile();
+
+        FileInputStream is = new FileInputStream(localPathPrefix);
+        byte[] bytes = convertInputStreamToByte(is);
+        Buffer buffer = Buffer.wrap(bytes);
+        output.add(buffer);
+
+        output.finish();
+        output.commit();
+
+        assertRecords(getFileContentsFromGcs(task));
+    }
+
+    public ConfigSource config()
+    {
+        return Exec.newConfigSource()
+                .set("in", inputConfig())
+                .set("parser", parserConfig(schemaConfig()))
+                .set("type", "gcs_streaming")
+                .set("project_id", gcpProjectId)
+                .set("bucket", gcpBucket)
+                .set("path_prefix", "tests/data")
+                .set("file_ext", "csv")
+                .set("formatter", formatterConfig());
+    }
+
+    private class Control implements OutputPlugin.Control
+    {
+        @Override
+        public List<TaskReport> run(TaskSource taskSource)
+        {
+            return Lists.newArrayList(Exec.newTaskReport());
+        }
+    }
+
+    private ImmutableMap<String, Object> inputConfig()
+    {
+        ImmutableMap.Builder<String, Object> builder = new ImmutableMap.Builder<>();
+        builder.put("type", "file");
+        builder.put("path_prefix", localPathPrefix);
+        builder.put("last_path", "");
+        return builder.build();
+    }
+
+    private ImmutableMap<String, Object> parserConfig(ImmutableList<Object> schemaConfig)
+    {
+        ImmutableMap.Builder<String, Object> builder = new ImmutableMap.Builder<>();
+        builder.put("type", "csv");
+        builder.put("newline", "CRLF");
+        builder.put("delimiter", ",");
+        builder.put("quote", "\"");
+        builder.put("escape", "\"");
+        builder.put("trim_if_not_quoted", false);
+        builder.put("skip_header_lines", 1);
+        builder.put("allow_extra_columns", false);
+        builder.put("allow_optional_columns", false);
+        builder.put("columns", schemaConfig);
+        return builder.build();
+    }
+
+    private ImmutableList<Object> schemaConfig()
+    {
+        ImmutableList.Builder<Object> builder = new ImmutableList.Builder<>();
+        builder.add(ImmutableMap.of("name", "id", "type", "long"));
+        builder.add(ImmutableMap.of("name", "account", "type", "string"));
+        builder.add(ImmutableMap.of("name", "ts", "type", "timestamp", "format", "%Y-%m-%d %H:%M:%S"));
+        builder.add(ImmutableMap.of("name", "dt", "type", "timestamp", "format", "%Y%m%d"));
+        builder.add(ImmutableMap.of("name", "message", "type", "string"));
+        return builder.build();
+    }
+
+    private ImmutableMap<String, Object> formatterConfig()
+    {
+        ImmutableMap.Builder<String, Object> builder = new ImmutableMap.Builder<>();
+        builder.put("type", "csv");
+        builder.put("header_line", true);
+        return builder.build();
+    }
+
+    private void assertRecords(ImmutableList<List<String>> records)
+    {
+        assertEquals(3, records.size());
+        {
+            List<String> record = records.get(1);
+            assertEquals("1", record.get(0));
+            assertEquals("account1", record.get(1));
+            assertEquals("2020-01-01 00:00:00", record.get(2));
+            assertEquals("20200101", record.get(3));
+            assertEquals("init", record.get(4));
+        }
+
+        {
+            List<String> record = records.get(2);
+            assertEquals("2", record.get(0));
+            assertEquals("account2", record.get(1));
+            assertEquals("2020-02-01 12:00:00", record.get(2));
+            assertEquals("20200201", record.get(3));
+            assertEquals("init", record.get(4));
+        }
+    }
+
+    private ImmutableList<List<String>> getFileContentsFromGcs(PluginTask task) throws Exception
+    {
+        Storage storage = StorageOptions.newBuilder()
+                .setProjectId(task.getProjectId())
+                .build()
+                .getService();
+
+        String blobName = task.getPathPrefix() + String.format(task.getSequenceFormat(), 0, 0) + task.getFileNameExtension();
+        BlobId blobId = BlobId.of(task.getBucket(), blobName);
+        Blob blob = storage.get(blobId);
+        byte[] byteContent = blob.getContent(); // one or multiple RPC calls will be issued
+        String strContent = new String(byteContent);
+
+        ImmutableList.Builder<List<String>> builder = new ImmutableList.Builder<>();
+
+        String line;
+        BufferedReader reader = new BufferedReader(new StringReader(strContent));
+        while ((line = reader.readLine()) != null) {
+            List<String> records = Arrays.asList(line.split(",", 0));
+            builder.add(records);
+        }
+        return builder.build();
+    }
+
+    private byte[] convertInputStreamToByte(InputStream is) throws IOException
+    {
+        ByteArrayOutputStream bo = new ByteArrayOutputStream();
+        byte [] buffer = new byte[1024];
+        while (true) {
+            int len = is.read(buffer);
+            if (len < 0) {
+                break;
+            }
+            bo.write(buffer, 0, len);
+        }
+        return bo.toByteArray();
+    }
+}

data/src/test/resources/test.000.csv

@@ -0,0 +1,3 @@
+id,account,ts,dt,message
+1,account1,2020-01-01 00:00:00,20200101,init
+2,account2,2020-02-01 12:00:00,20200201,init

metadata

@@ -0,0 +1,127 @@
+--- !ruby/object:Gem::Specification
+name: embulk-output-gcs_streaming
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- irotoris
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2020-09-16 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  name: bundler
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '12.0'
+  name: rake
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '12.0'
+description: Stores files on Google Cloud Storage with streaming uplaod.
+email:
+- shiroto00@yahoo.co.jp
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- LICENSE.txt
+- README.md
+- build.gradle
+- classpath/animal-sniffer-annotations-1.17.jar
+- classpath/api-common-1.7.0.jar
+- classpath/checker-compat-qual-2.5.2.jar
+- classpath/commons-codec-1.10.jar
+- classpath/commons-logging-1.2.jar
+- classpath/embulk-output-gcs_streaming-0.1.0.jar
+- classpath/error_prone_annotations-2.3.2.jar
+- classpath/failureaccess-1.0.1.jar
+- classpath/gax-1.44.0.jar
+- classpath/gax-httpjson-0.61.0.jar
+- classpath/google-api-client-1.27.0.jar
+- classpath/google-api-services-storage-v1-rev20181109-1.27.0.jar
+- classpath/google-auth-library-credentials-0.15.0.jar
+- classpath/google-auth-library-oauth2-http-0.15.0.jar
+- classpath/google-cloud-core-1.71.0.jar
+- classpath/google-cloud-core-http-1.71.0.jar
+- classpath/google-cloud-storage-1.71.0.jar
+- classpath/google-http-client-1.29.0.jar
+- classpath/google-http-client-apache-2.1.0.jar
+- classpath/google-http-client-appengine-1.29.0.jar
+- classpath/google-http-client-jackson2-1.29.0.jar
+- classpath/google-oauth-client-1.27.0.jar
+- classpath/grpc-context-1.18.0.jar
+- classpath/gson-2.7.jar
+- classpath/guava-27.1-android.jar
+- classpath/httpclient-4.5.5.jar
+- classpath/httpcore-4.4.9.jar
+- classpath/j2objc-annotations-1.1.jar
+- classpath/jackson-core-2.9.6.jar
+- classpath/javax.annotation-api-1.3.2.jar
+- classpath/jsr305-3.0.2.jar
+- classpath/listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar
+- classpath/opencensus-api-0.19.2.jar
+- classpath/opencensus-contrib-http-util-0.19.2.jar
+- classpath/proto-google-common-protos-1.15.0.jar
+- classpath/proto-google-iam-v1-0.12.0.jar
+- classpath/protobuf-java-3.7.0.jar
+- classpath/protobuf-java-util-3.7.0.jar
+- classpath/threetenbp-1.3.3.jar
+- config/checkstyle/checkstyle.xml
+- config/checkstyle/default.xml
+- gradle/wrapper/gradle-wrapper.jar
+- gradle/wrapper/gradle-wrapper.properties
+- gradlew
+- gradlew.bat
+- lib/embulk/output/gcs_streaming.rb
+- src/main/java/org/embulk/output/gcs_streaming/GcsClient.java
+- src/main/java/org/embulk/output/gcs_streaming/GcsStreamingFileOutputPlugin.java
+- src/test/java/org/embulk/output/gcs_streaming/TestGcsClient.java
+- src/test/java/org/embulk/output/gcs_streaming/TestGcsStreamingFileOutputPlugin.java
+- src/test/resources/test.000.csv
+homepage: https://github.com/irotoris/embulk-output-gcs_streaming
+licenses:
+- MIT
+metadata: {}
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project:
+rubygems_version: 2.6.8
+signing_key:
+specification_version: 4
+summary: Gcs Streaming file output plugin for Embulk
+test_files: []