embulk-output-td 0.5.2 → 0.5.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 4fe85fa7501f82e0d7660c88ef9369629fe91827
-  data.tar.gz: 9858b8eea57c1574c346d919dd9a57ba7ea44058
+  metadata.gz: a2b9d332f7623b5bcb26571bed0f2a021303d342
+  data.tar.gz: 5dde90bc97bbd9e19e76b553c81a750f8a4bf753
 SHA512:
-  metadata.gz: 602fa7e84ed8e32444a6de50ceeb8333ba071e7ba546dbb46424a8e7c20ebdc21cd67705031e29fe602db1f5bd9a50b6bc08f682507aedef986b2d6741e6a20e
-  data.tar.gz: 1960a94319e23a719273853bdffcd3151caa3cfe053bcdab45c5f70fe419a0650dcc07cb77a9ff495e9bea8aff7f3bd40e7c538fe3d9260eadffa08f387b2c2c
+  metadata.gz: c8500157a6ca8483718d72148064fab8b8b571d103e270bfd187175607b2dbac0af7196998329bc33c0813e584902158446cb5fad66475c41904ad7a652097c1
+  data.tar.gz: 4a564bbaec11724badc1a48a432578881c4489e44f495fbd68a3a93a957e71b79d3bc0b6870d8345e3ae5db41efc94c52be37e0adda525313d3773caf4685ef5

data/CHANGELOG.md

@@ -1,3 +1,6 @@
+## 0.5.3 - 2019-11-19
+* [new feature] Added additional headers to the Http requests [#94](https://github.com/treasure-data/embulk-output-td/pul/94) addressed by [#93](https://github.com/treasure-data/embulk-output-td/issues/93)
+* Also exposes `port` configuration (by default will use 80 and 443). 
 
 ## 0.5.2 - 2019-10-24
 * [maintenance] Fix columns order bug described in issue [91](https://github.com/treasure-data/embulk-output-td/issues/91) [#92](https://github.com/treasure-data/embulk-output-td/pull/92)

data/README.md

@@ -40,6 +40,8 @@
 - **retry_limit**: indicates how many retries are allowed (int, default: 20)
 - **retry_initial_interval_millis**: the initial intervals (int, default: 1000)
 - **retry_max_interval_millis**: the maximum intervals. The interval doubles every retry until retry_max_interval_millis is reached. (int, default: 90000)
+- **additional_http_headers**: add additional headers to the requests (a key & value map, default: null)
+- **port**: set port for Http requests. By default will connect to port 443 or 80 if `use_ssl: false` (int, optional)
 
 ## Modes
 * **append**:
@@ -88,6 +90,22 @@ out:
   mode: append
 ```
 
+### Additional Http headers
+```yaml
+out:
+  type: td
+  apikey: <your apikey>
+  endpoint: api.treasuredata.com
+  database: my_db
+  table: my_table
+  time_column: created_at
+  auto_create_table: true
+  mode: append
+  additional_http_headers:
+    Content_Type: 'application/json'
+    foo: bar
+```
+
 
 
 ## Build

data/build.gradle

@@ -19,7 +19,7 @@ configurations {
 }
 
 group = "org.embulk.output.td"
-version = "0.5.2"
+version = "0.5.3"
 
 compileJava.options.encoding = 'UTF-8' // source encoding
 sourceCompatibility = 1.8
@@ -38,6 +38,7 @@ dependencies {
     testCompile "org.bigtesting:fixd:1.0.0"
     testCompile  "org.embulk:embulk-core:0.8.+:tests"
     testCompile  "org.mockito:mockito-core:1.9.5"
+    testCompile "com.github.tomakehurst:wiremock-jre8:2.25.1"
 }
 
 javadoc {

data/embulk-output-td.gemspec

@@ -1,7 +1,7 @@
 
 Gem::Specification.new do |spec|
   spec.name          = "embulk-output-td"
-  spec.version       = "0.5.2"
+  spec.version       = "0.5.3"
   spec.authors       = ["Muga Nishizawa"]
   spec.summary       = %[TreasureData output plugin for Embulk]
   spec.description   = %[TreasureData output plugin is an Embulk plugin that loads records to TreasureData read by any input plugins. Search the input plugins by 'embulk-output' keyword.]

data/src/main/java/org/embulk/output/td/TdOutputPlugin.java

@@ -1,29 +1,14 @@
 package org.embulk.output.td;
 
-import java.io.IOException;
-import java.io.InputStream;
-import java.util.LinkedHashMap;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.HashMap;
-import java.nio.charset.StandardCharsets;
-import java.util.Properties;
-import java.util.UUID;
-import java.util.regex.Pattern;
-import java.util.zip.GZIPInputStream;
-
-import javax.validation.constraints.Min;
-import javax.validation.constraints.Max;
-
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonValue;
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Function;
 import com.google.common.base.Optional;
-import com.google.common.base.Predicates;
 import com.google.common.base.Throwables;
+import com.google.common.collect.ArrayListMultimap;
 import com.google.common.collect.Lists;
-import com.fasterxml.jackson.annotation.JsonCreator;
-import com.fasterxml.jackson.annotation.JsonValue;
+import com.google.common.collect.Multimap;
 import com.treasuredata.client.ProxyConfig;
 import com.treasuredata.client.TDClient;
 import com.treasuredata.client.TDClientBuilder;
@@ -35,22 +20,22 @@ import com.treasuredata.client.model.TDColumn;
 import com.treasuredata.client.model.TDColumnType;
 import com.treasuredata.client.model.TDTable;
 import org.embulk.EmbulkVersion;
-import org.embulk.config.TaskReport;
 import org.embulk.config.Config;
 import org.embulk.config.ConfigDefault;
 import org.embulk.config.ConfigDiff;
-import org.embulk.config.ConfigSource;
 import org.embulk.config.ConfigException;
+import org.embulk.config.ConfigSource;
 import org.embulk.config.Task;
+import org.embulk.config.TaskReport;
 import org.embulk.config.TaskSource;
 import org.embulk.output.td.writer.FieldWriterSet;
+import org.embulk.spi.Column;
+import org.embulk.spi.ColumnVisitor;
 import org.embulk.spi.DataException;
 import org.embulk.spi.Exec;
-import org.embulk.spi.ColumnVisitor;
 import org.embulk.spi.ExecSession;
 import org.embulk.spi.OutputPlugin;
 import org.embulk.spi.Schema;
-import org.embulk.spi.Column;
 import org.embulk.spi.TransactionalPageOutput;
 import org.embulk.spi.time.Timestamp;
 import org.embulk.spi.time.TimestampFormatter;
@@ -60,6 +45,21 @@ import org.msgpack.core.MessageUnpacker;
 import org.msgpack.value.Value;
 import org.slf4j.Logger;
 
+import javax.validation.constraints.Max;
+import javax.validation.constraints.Min;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+import java.util.UUID;
+import java.util.regex.Pattern;
+import java.util.zip.GZIPInputStream;
+
 import static com.google.common.base.Optional.fromNullable;
 import static java.lang.Integer.parseInt;
 
@@ -70,74 +70,74 @@ public class TdOutputPlugin
             extends Task, TimestampFormatter.Task
     {
         @Config("apikey")
-        public String getApiKey();
+        String getApiKey();
 
         @Config("endpoint")
         @ConfigDefault("\"api.treasuredata.com\"")
-        public String getEndpoint();
+        String getEndpoint();
 
         @Config("use_ssl")
         @ConfigDefault("true")
-        public boolean getUseSsl();
+        boolean getUseSsl();
 
         @Config("http_proxy")
         @ConfigDefault("null")
-        public Optional<HttpProxyTask> getHttpProxy();
+        Optional<HttpProxyTask> getHttpProxy();
 
         //  TODO connect_timeout, read_timeout, send_timeout
 
         @Config("mode")
         @ConfigDefault("\"append\"")
-        public Mode getMode();
+        Mode getMode();
 
         @Config("auto_create_table")
         @ConfigDefault("true")
-        public boolean getAutoCreateTable();
+        boolean getAutoCreateTable();
 
         @Config("database")
-        public String getDatabase();
+        String getDatabase();
 
         @Config("table")
-        public String getTable();
+        String getTable();
 
-        public void setLoadTargetTableName(String name);
-        public String getLoadTargetTableName();
+        void setLoadTargetTableName(String name);
+        String getLoadTargetTableName();
 
         @Config("session")
         @ConfigDefault("null")
-        public Optional<String> getSession();
+        Optional<String> getSession();
 
         @Config("default_timestamp_type_convert_to")
         @ConfigDefault("\"string\"")
-        public ConvertTimestampType getConvertTimestampType();
+        ConvertTimestampType getConvertTimestampType();
 
         @Config("time_column")
         @ConfigDefault("null")
-        public Optional<String> getTimeColumn();
+        Optional<String> getTimeColumn();
 
         @Config("time_value")
         @ConfigDefault("null")
-        public Optional<TimeValueConfig> getTimeValue(); // TODO allow timestamp format such as {from: "2015-01-01 00:00:00 UTC", to: "2015-01-02 00:00:00 UTC"} as well as unixtime integer
-        public void setTimeValue(Optional<TimeValueConfig> timeValue);
+        Optional<TimeValueConfig> getTimeValue(); // TODO allow timestamp format such as {from: "2015-01-01 00:00:00 UTC", to: "2015-01-02 00:00:00 UTC"} as well as unixtime integer
+        void setTimeValue(Optional<TimeValueConfig> timeValue);
 
         @Config("unix_timestamp_unit")
         @ConfigDefault("\"sec\"")
-        public UnixTimestampUnit getUnixTimestampUnit();
+        UnixTimestampUnit getUnixTimestampUnit();
 
         @Config("tmpdir")
         @ConfigDefault("null")
-        public Optional<String> getTempDir();
-        public void setTempDir(Optional<String> dir);
+        Optional<String> getTempDir();
+        void setTempDir(Optional<String> dir);
 
         @Config("upload_concurrency")
         @ConfigDefault("2")
         @Min(1)
         @Max(8)
-        public int getUploadConcurrency();
+        int getUploadConcurrency();
 
         @Config("file_split_size")
         @ConfigDefault("16384") // default 16MB (unit: kb)
-        public long getFileSplitSize();
+        long getFileSplitSize();
 
         @Override
         @Config("default_timestamp_format")
@@ -153,11 +153,11 @@ public class TdOutputPlugin
         //   * can parse SQL timestamp with microseconds like '2015-02-03 04:05:06.789012'
         //   * can parse SQL timestamp with milliseconds like '2015-02-03 04:05:06.789'
         @ConfigDefault("\"%Y-%m-%d %H:%M:%S.%3N\"")
-        public String getDefaultTimestampFormat();
+        String getDefaultTimestampFormat();
 
         @Config("column_options")
         @ConfigDefault("{}")
-        public Map<String, TimestampColumnOption> getColumnOptions();
+        Map<String, TimestampColumnOption> getColumnOptions();
 
         @Config("stop_on_invalid_record")
         @ConfigDefault("false")
@@ -182,13 +182,22 @@ public class TdOutputPlugin
 
         @Config("pool_name")
         @ConfigDefault("null")
-        public Optional<String> getPoolName();
+        Optional<String> getPoolName();
 
-        public boolean getDoUpload();
-        public void setDoUpload(boolean doUpload);
+        @Config("additional_http_headers")
+        @ConfigDefault("null")
+        Optional<Map<String, String>> getAdditionalHttpHeaders();
+
+        @Config("port")
+        @ConfigDefault("null")
+        Optional<Integer> getPort();
+        void setPort(Optional<Integer> port);
 
-        public String getSessionName();
-        public void setSessionName(String session);
+        boolean getDoUpload();
+        void setDoUpload(boolean doUpload);
+
+        String getSessionName();
+        void setSessionName(String session);
     }
 
     public interface TimestampColumnOption
@@ -234,22 +243,22 @@ public class TdOutputPlugin
             extends Task
     {
         @Config("host")
-        public String getHost();
+        String getHost();
 
         @Config("port")
-        public int getPort();
+        int getPort();
 
         @Config("use_ssl")
         @ConfigDefault("false")
-        public boolean getUseSsl();
+        boolean getUseSsl();
 
         @Config("user")
         @ConfigDefault("null")
-        public Optional<String> getUser();
+        Optional<String> getUser();
 
         @Config("password")
         @ConfigDefault("null")
-        public Optional<String> getPassword();
+        Optional<String> getPassword();
     }
 
     public static enum ConvertTimestampType
@@ -476,6 +485,14 @@ public class TdOutputPlugin
         builder.setRetryInitialIntervalMillis(task.getRetryInitialIntervalMillis());
         builder.setRetryMaxIntervalMillis(task.getRetryMaxIntervalMillis());
 
+        if (task.getPort().isPresent()) {
+            builder.setPort(task.getPort().get());
+        }
+
+        if (task.getAdditionalHttpHeaders().isPresent()) {
+            builder.setHeaders(buildMultiMapHeaders(task.getAdditionalHttpHeaders().get()));
+        }
+
         Optional<ProxyConfig> proxyConfig = newProxyConfig(task.getHttpProxy());
         if (proxyConfig.isPresent()) {
             builder.setProxy(proxyConfig.get());
@@ -484,6 +501,15 @@ public class TdOutputPlugin
         return builder.build();
     }
 
+    private Multimap<String, String> buildMultiMapHeaders(Map<String, String> headers)
+    {
+        Multimap<String, String> multimap = ArrayListMultimap.create();
+        for (Map.Entry<String, String> entry : headers.entrySet()) {
+            multimap.put(entry.getKey(), entry.getValue());
+        }
+        return multimap;
+    }
+
     @VisibleForTesting
     Optional<ProxyConfig> newProxyConfig(Optional<HttpProxyTask> task)
     {
@@ -776,7 +802,7 @@ public class TdOutputPlugin
         try {
             return client.showTable(databaseName, tableName);
         }
-        catch(TDClientHttpNotFoundException e) {
+        catch (TDClientHttpNotFoundException e) {
             return null;
         }
     }

data/src/test/java/org/embulk/output/td/TestTdOutputPlugin.java

@@ -1,5 +1,6 @@
 package org.embulk.output.td;
 
+import com.github.tomakehurst.wiremock.junit.WireMockRule;
 import com.google.common.base.Optional;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.Lists;
@@ -13,14 +14,16 @@ import com.treasuredata.client.model.TDColumn;
 import com.treasuredata.client.model.TDColumnType;
 import com.treasuredata.client.model.TDTable;
 import com.treasuredata.client.model.TDTableType;
+import org.apache.http.Header;
+import org.apache.http.message.BasicHeader;
 import org.embulk.EmbulkTestRuntime;
-import org.embulk.config.TaskReport;
 import org.embulk.config.ConfigDiff;
 import org.embulk.config.ConfigException;
 import org.embulk.config.ConfigSource;
+import org.embulk.config.TaskReport;
 import org.embulk.config.TaskSource;
-import org.embulk.output.td.TdOutputPlugin.PluginTask;
 import org.embulk.output.td.TdOutputPlugin.HttpProxyTask;
+import org.embulk.output.td.TdOutputPlugin.PluginTask;
 import org.embulk.output.td.TdOutputPlugin.TimestampColumnOption;
 import org.embulk.output.td.TdOutputPlugin.UnixTimestampUnit;
 import org.embulk.output.td.writer.FieldWriterSet;
@@ -39,10 +42,18 @@ import org.junit.Test;
 import org.mockito.ArgumentCaptor;
 import org.slf4j.Logger;
 
-import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
-
+import java.util.Map;
+
+import static com.github.tomakehurst.wiremock.client.WireMock.aResponse;
+import static com.github.tomakehurst.wiremock.client.WireMock.equalTo;
+import static com.github.tomakehurst.wiremock.client.WireMock.get;
+import static com.github.tomakehurst.wiremock.client.WireMock.getRequestedFor;
+import static com.github.tomakehurst.wiremock.client.WireMock.stubFor;
+import static com.github.tomakehurst.wiremock.client.WireMock.urlMatching;
+import static com.github.tomakehurst.wiremock.client.WireMock.verify;
+import static com.github.tomakehurst.wiremock.core.WireMockConfiguration.options;
 import static com.treasuredata.client.model.TDBulkImportSession.ImportStatus.COMMITTED;
 import static com.treasuredata.client.model.TDBulkImportSession.ImportStatus.COMMITTING;
 import static com.treasuredata.client.model.TDBulkImportSession.ImportStatus.PERFORMING;
@@ -56,7 +67,6 @@ import static org.junit.Assert.fail;
 import static org.mockito.Matchers.any;
 import static org.mockito.Matchers.anyInt;
 import static org.mockito.Matchers.anyString;
-import static org.mockito.Mockito.doAnswer;
 import static org.mockito.Mockito.doNothing;
 import static org.mockito.Mockito.doReturn;
 import static org.mockito.Mockito.doThrow;
@@ -68,6 +78,11 @@ public class TestTdOutputPlugin
     @Rule
     public EmbulkTestRuntime runtime = new EmbulkTestRuntime();
 
+    private final int wireMockPort = 10888;
+
+    @Rule
+    public WireMockRule wireMockRule = new WireMockRule(options().port(wireMockPort), false);
+
     private ConfigSource config; // not mock
     private TdOutputPlugin plugin; // mock
 
@@ -567,7 +582,7 @@ public class TestTdOutputPlugin
         ArgumentCaptor<List<TDColumn>> schemaCaptor = ArgumentCaptor.forClass((Class) List.class);
         doNothing().when(client).appendTableSchema(anyString(), anyString(), schemaCaptor.capture());
 
-        plugin.updateSchema(client, schema,task);
+        plugin.updateSchema(client, schema, task);
 
         List<Column> inputCols = schema.getColumns();
         List<TDColumn> uploadedCols = schemaCaptor.getValue();
@@ -577,6 +592,51 @@ public class TestTdOutputPlugin
         assertEquals(inputCols.get(2).getName(), uploadedCols.get(2).getName());
     }
 
+    @Test
+    public void testTDClientSendsExtraHeader()
+    {
+        final String urlRegx = "/v3/table/show/.*";
+        final Header header1 = new BasicHeader("h_name", "h_value");
+        final Header header2 = new BasicHeader("ABC", "XYZ");
+        final Map<String, String> headers = ImmutableMap.of(
+                                                header1.getName(), header1.getValue(),
+                                                header2.getName(), header2.getValue());
+
+        PluginTask task = config()
+                .set("endpoint", "localhost")
+                .set("port", wireMockPort)
+                .set("use_ssl", "false") // ssl disabled for wiremock
+                .set("additional_http_headers", headers)
+                .loadConfig(PluginTask.class);
+
+        stubFor(get(urlMatching(urlRegx))
+                .willReturn(aResponse()
+                        .withBody("{\n" +
+                                "  \"id\": 46732,\n" +
+                                "  \"name\": \"sample_csv11\",\n" +
+                                "  \"estimated_storage_size\": 0,\n" +
+                                "  \"counter_updated_at\": null,\n" +
+                                "  \"last_log_timestamp\": null,\n" +
+                                "  \"delete_protected\": false,\n" +
+                                "  \"created_at\": \"2019-10-23 07:49:02 UTC\",\n" +
+                                "  \"updated_at\": \"2019-10-23 07:51:13 UTC\",\n" +
+                                "  \"type\": \"log\",\n" +
+                                "  \"include_v\": true,\n" +
+                                "  \"count\": 15,\n" +
+                                "  \"schema\": \"[]\",\n" +
+                                "  \"expire_days\": null\n" +
+                                "}")));
+
+        TDClient client = plugin.newTDClient(task);
+
+        // issue an API request
+        plugin.validateTableExists(client, "tmp_db", "tmp_table");
+
+        verify(getRequestedFor(urlMatching(urlRegx))
+                .withHeader(header1.getName(), equalTo(header1.getValue()))
+                .withHeader(header2.getName(), equalTo(header2.getValue())));
+    }
+
     public static ConfigSource config()
     {
         return Exec.newConfigSource()

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-output-td
 version: !ruby/object:Gem::Version
-  version: 0.5.2
+  version: 0.5.3
 platform: ruby
 authors:
 - Muga Nishizawa
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2019-10-24 00:00:00.000000000 Z
+date: 2020-01-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -96,7 +96,7 @@ files:
 - classpath/hamcrest-core-1.1.jar
 - classpath/jackson-datatype-json-org-2.8.1.jar
 - classpath/td-client-0.8.4.jar
-- classpath/embulk-output-td-0.5.2.jar
+- classpath/embulk-output-td-0.5.3.jar
 - classpath/okio-1.13.0.jar
 homepage: https://github.com/treasure-data/embulk-output-td
 licenses: