embulk-output-td 0.1.0 → 0.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 58e133b33691a9a323d67be520799e68a8ec7e83
-  data.tar.gz: bd184003c46843eba457d648e3b557d8d2566373
+  metadata.gz: 0e96fa2666efc130272325571d56cfd9e77b5086
+  data.tar.gz: b4ed6f4e05cf72ab403427a452b87a6b0da1d575
 SHA512:
-  metadata.gz: b6e9317a6a47f6de4388f8d78f64d180396a82444cdb3405abcb723caaf4f0c7fab7c2e33993a9f8bca13eedd2fe514f6311869cd907f0eca0bea9cdf5d63cb5
-  data.tar.gz: 5daaad4fd6463747b9d5f83d2caff817187187ce7397067bc3627bdacad6e7d39fbcb8a340c430d3399cc8637814a4871d8354af3d6353896e4012adb6e4bd89
+  metadata.gz: 25a373bb171c0280913f452e5a220b2ab12e350a4ddd4e476606e29541c79a12dc16ebd172243ae48b5962ff7a7e531c74ce4347b94705a85c3768c77529a3ff
+  data.tar.gz: b5910c054b1e9bdc74f2fc269789901bc8652bdb9a56d3b92686e38ebc5d4a1d27a4e86d47c3f0bb560e99c27c3d9bbbfdf216054b378368c3efd8f3c4d44fa9

data/CHANGELOG.md

@@ -0,0 +1,9 @@
+## 0.1.1 - 2015-07-14
+
+* [maintenance] Make part name unique and idempotent [#9](https://github.com/treasure-data/embulk-output-td/pull/9)
+* [maintenance] Delete temp files after uploading [#7](https://github.com/treasure-data/embulk-output-td/pull/7)
+* [new feature] Add unix_timestamp_unit option [#6](https://github.com/treasure-data/embulk-output-td/pull/6)
+
+## 0.1.0 - 2015-06-23
+
+The first release!!

data/README.md

@@ -19,6 +19,7 @@ TODO: Write short description here
 - **table**: table name (string, required)
 - **session**: bulk_import session name (string, optional)
 - **time_column**: user-defined time column (string, optional)
+- **unix_timestamp_unit**: if type of "time" or **time_column** is long, it's considered unix timestamp. This option specify its unit in sec, milli, micro or nano (enum, default: `sec`)
 - **tmpdir**: temporal directory
 - **upload_concurrency**: upload concurrency (int, default=2). max concurrency is 8.
 - **file_split_size**: split size (long, default=16384 (16MB)).

data/build.gradle

@@ -13,7 +13,7 @@ configurations {
     provided
 }
 
-version = "0.1.0"
+version = "0.1.1"
 
 compileJava.options.encoding = 'UTF-8' // source encoding
 sourceCompatibility = 1.7

data/embulk-output-td.gemspec

@@ -1,7 +1,7 @@
 
 Gem::Specification.new do |spec|
   spec.name          = "embulk-output-td"
-  spec.version       = "0.1.0"
+  spec.version       = "0.1.1"
   spec.authors       = ["Muga Nishizawa"]
   spec.summary       = %[TreasureData output plugin for Embulk]
   spec.description   = %[TreasureData output plugin is an Embulk plugin that loads records to TreasureData read by any input plugins. Search the input plugins by 'embulk-output' keyword.]

data/src/main/java/com/treasuredata/api/TdApiClient.java

@@ -171,6 +171,7 @@ public class TdApiClient
         return session;
     }
 
+    @Deprecated
     public void uploadBulkImport(String sessionName, File path)
             throws IOException
     {
@@ -181,6 +182,15 @@ public class TdApiClient
         ContentResponse response = executeExchange(request);
     }
 
+    public void uploadBulkImportPart(String sessionName, String uniquePartName, File path)
+            throws IOException
+    {
+        Request request = prepareExchange(HttpMethod.PUT,
+                buildUrl("/v3/bulk_import/upload_part", sessionName, uniquePartName));
+        request.file(path.toPath());
+        ContentResponse response = executeExchange(request);
+    }
+
     public void freezeBulkImportSession(String sessionName)
     {
         Request request = prepareExchange(HttpMethod.POST,

data/src/main/java/org/embulk/output/MsgpackGZFileBuilder.java

@@ -1,5 +1,6 @@
 package org.embulk.output;
 
+import org.embulk.spi.Exec;
 import org.msgpack.MessagePack;
 import org.msgpack.packer.Packer;
 
@@ -83,6 +84,11 @@ public class MsgpackGZFileBuilder
         return file;
     }
 
+    public boolean delete()
+    {
+        return file.delete();
+    }
+
     public void finish()
             throws IOException
     {

data/src/main/java/org/embulk/output/RecordWriter.java

@@ -1,6 +1,7 @@
 package org.embulk.output;
 
 import com.google.common.base.Optional;
+import com.google.common.base.Stopwatch;
 import com.google.common.base.Throwables;
 import com.treasuredata.api.TdApiClient;
 import org.embulk.config.CommitReport;
@@ -26,11 +27,15 @@ import org.msgpack.MessagePack;
 import org.slf4j.Logger;
 
 import java.io.File;
+import java.io.Closeable;
 import java.io.IOException;
+import java.util.Locale;
 import java.text.NumberFormat;
 import java.util.concurrent.Callable;
+import java.util.concurrent.TimeUnit;
 
 import static com.google.common.base.Preconditions.checkNotNull;
+import org.embulk.output.TdOutputPlugin.UnixTimestampUnit;
 
 public class RecordWriter
         implements TransactionalPageOutput
@@ -38,12 +43,13 @@ public class RecordWriter
     private final Logger log;
     private final TdApiClient client;
     private final String sessionName;
+    private final int taskIndex;
 
     private final MessagePack msgpack;
     private final FieldWriterSet fieldWriters;
     private final File tempDir;
 
-    private int seqid = 0;
+    private int partSeqId = 0;
     private PageReader pageReader;
     private MsgpackGZFileBuilder builder;
 
@@ -51,11 +57,12 @@ public class RecordWriter
     private final int uploadConcurrency;
     private final long fileSplitSize; // unit: kb
 
-    public RecordWriter(PluginTask task, TdApiClient client, FieldWriterSet fieldWriters)
+    public RecordWriter(PluginTask task, int taskIndex, TdApiClient client, FieldWriterSet fieldWriters)
     {
         this.log = Exec.getLogger(getClass());
         this.client = checkNotNull(client);
         this.sessionName = task.getSessionName();
+        this.taskIndex = taskIndex;
 
         this.msgpack = new MessagePack();
         this.fieldWriters = fieldWriters;
@@ -80,7 +87,7 @@ public class RecordWriter
     private void prepareNextBuilder()
             throws IOException
     {
-        String prefix = String.format("%s-%d-", sessionName, seqid);
+        String prefix = String.format("%s-", sessionName);
         File tempFile = File.createTempFile(prefix, ".msgpack.gz", tempDir);
         this.builder = new MsgpackGZFileBuilder(msgpack, tempFile);
     }
@@ -140,6 +147,7 @@ public class RecordWriter
 
                 if (builder.getWrittenSize() > fileSplitSize) {
                     flush();
+                    prepareNextBuilder();
                 }
             }
 
@@ -150,31 +158,48 @@ public class RecordWriter
 
     public void flush() throws IOException
     {
-        builder.finish();
-
         if (builder.getRecordCount() > 0) {
+            builder.finish();
+
             log.info("{uploading: {rows: {}, size: {} bytes (compressed)}}",
                     builder.getRecordCount(),
                     NumberFormat.getNumberInstance().format(builder.getWrittenSize()));
-            upload(builder);
+            upload(builder, String.format(Locale.ENGLISH, "task-%d_%d", taskIndex, partSeqId));
+            partSeqId++;
             builder = null;
         }
-
-        prepareNextBuilder();
     }
 
-    private void upload(final MsgpackGZFileBuilder builder)
+    private void upload(final MsgpackGZFileBuilder builder, final String uniquePartName)
             throws IOException
     {
         executor.joinPartial(uploadConcurrency - 1);
         executor.submit(new Callable<Void>() {
             @Override
-            public Void call() throws Exception {
-                client.uploadBulkImport(sessionName, builder.getFile());
+            public Void call() throws Exception
+            {
+                File file = builder.getFile();
+
+                log.debug("{uploading: {file: {}}}", file.getAbsolutePath());
+                Stopwatch stopwatch = Stopwatch.createStarted();
+
+                client.uploadBulkImportPart(sessionName, uniquePartName, builder.getFile());
+
+                stopwatch.stop();
+                stopwatch.elapsed(TimeUnit.MILLISECONDS);
+                log.debug("{uploaded: {file: {}, time: {}}}", file.getAbsolutePath(), stopwatch);
                 return null;
             }
-        }, builder);
-        seqid++;
+        },
+        new Closeable() {
+            public void close() throws IOException
+            {
+                builder.close();
+                if (!builder.delete()) {
+                    log.warn("Failed to delete local temporary file {}. Ignoring.", builder.getFile());
+                }
+            }
+        });
     }
 
     @Override
@@ -199,6 +224,7 @@ public class RecordWriter
             } finally {
                 if (builder != null) {
                     builder.close();
+                    builder.delete();
                     builder = null;
                 }
 
@@ -285,7 +311,10 @@ public class RecordWriter
                     case PRIMARY_KEY:
                         log.info("Using {}:{} column as the data partitioning key", columnName, columnType);
                         if (columnType instanceof LongType) {
-                            writer = new LongFieldWriter(columnName);
+                            if (task.getUnixTimestampUnit() != UnixTimestampUnit.SEC) {
+                                log.warn("time column is converted from {} to seconds", task.getUnixTimestampUnit());
+                            }
+                            writer = new UnixTimestampLongFieldWriter(columnName, task.getUnixTimestampUnit().getFractionUnit());
                             hasPkWriter = true;
                         } else if (columnType instanceof TimestampType) {
                             writer = new TimestampStringFieldWriter(task.getJRuby(), columnName);
@@ -344,13 +373,14 @@ public class RecordWriter
                 String columnName = schema.getColumnName(duplicatePrimaryKeySourceIndex);
                 Type columnType = schema.getColumnType(duplicatePrimaryKeySourceIndex);
 
-                log.info("Duplicating {}:{} column to 'time' column for the data partitioning",
-                        columnName, columnType);
-
                 FieldWriter writer;
                 if (columnType instanceof LongType) {
-                    writer = new LongFieldDuplicator(columnName, "time");
+                    log.info("Duplicating {}:{} column (unix timestamp {}) to 'time' column as seconds for the data partitioning",
+                            columnName, columnType, task.getUnixTimestampUnit());
+                    writer = new UnixTimestampFieldDuplicator(columnName, "time", task.getUnixTimestampUnit().getFractionUnit());
                 } else if (columnType instanceof TimestampType) {
+                    log.info("Duplicating {}:{} column to 'time' column as seconds for the data partitioning",
+                            columnName, columnType);
                     writer = new TimestampFieldLongDuplicator(task.getJRuby(), columnName, "time");
                 } else {
                     throw new ConfigException(String.format("Type of '%s' column must be long or timestamp but got %s",
@@ -473,6 +503,25 @@ public class RecordWriter
         }
     }
 
+    static class UnixTimestampLongFieldWriter
+            extends FieldWriter
+    {
+        private final int fractionUnit;
+
+        UnixTimestampLongFieldWriter(String keyName, int fractionUnit)
+        {
+            super(keyName);
+            this.fractionUnit = fractionUnit;
+        }
+
+        @Override
+        public void writeValue(MsgpackGZFileBuilder builder, PageReader reader, Column column)
+                throws IOException
+        {
+            builder.writeLong(reader.getLong(column) / fractionUnit);
+        }
+    }
+
     static class StringFieldWriter
             extends FieldWriter
     {
@@ -525,15 +574,15 @@ public class RecordWriter
         }
     }
 
-    static class LongFieldDuplicator
+    static class UnixTimestampFieldDuplicator
             extends LongFieldWriter
     {
-        private final LongFieldWriter timeFieldWriter;
+        private final UnixTimestampLongFieldWriter timeFieldWriter;
 
-        public LongFieldDuplicator(String keyName, String duplicateKeyName)
+        public UnixTimestampFieldDuplicator(String keyName, String duplicateKeyName, int fractionUnit)
         {
             super(keyName);
-            timeFieldWriter = new LongFieldWriter(duplicateKeyName);
+            timeFieldWriter = new UnixTimestampLongFieldWriter(duplicateKeyName, fractionUnit);
         }
 
         @Override

data/src/main/java/org/embulk/output/TdOutputPlugin.java

@@ -7,6 +7,8 @@ import javax.validation.constraints.Max;
 
 import com.google.common.base.Optional;
 import com.google.common.base.Throwables;
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonValue;
 import com.treasuredata.api.TdApiClient;
 import com.treasuredata.api.TdApiClientConfig;
 import com.treasuredata.api.TdApiClientConfig.HttpProxyConfig;
@@ -78,6 +80,10 @@ public class TdOutputPlugin
         @ConfigDefault("null")
         public Optional<String> getTimeColumn();
 
+        @Config("unix_timestamp_unit")
+        @ConfigDefault("\"sec\"")
+        public UnixTimestampUnit getUnixTimestampUnit();
+
         @Config("tmpdir")
         @ConfigDefault("\"/tmp\"")
         public String getTempDir();
@@ -116,6 +122,47 @@ public class TdOutputPlugin
         public boolean getUseSsl();
     }
 
+    public static enum UnixTimestampUnit
+    {
+        SEC(1),
+        MILLI(1000),
+        MICRO(1000000),
+        NANO(1000000000);
+
+        private final int unit;
+
+        private UnixTimestampUnit(int unit)
+        {
+            this.unit = unit;
+        }
+
+        public int getFractionUnit()
+        {
+            return unit;
+        }
+
+        @JsonCreator
+        public static UnixTimestampUnit of(String s)
+        {
+            switch (s) {
+            case "sec": return SEC;
+            case "milli": return MILLI;
+            case "micro": return MICRO;
+            case "nano": return NANO;
+            default:
+                throw new ConfigException(
+                        String.format("Unknown unix_timestamp_unit '%s'. Supported units are sec, milli, micro, and nano"));
+            }
+        }
+
+        @JsonValue
+        @Override
+        public String toString()
+        {
+            return name().toLowerCase();
+        }
+    }
+
     private final Logger log;
 
     public TdOutputPlugin()
@@ -367,14 +414,14 @@ public class TdOutputPlugin
     }
 
     @Override
-    public TransactionalPageOutput open(TaskSource taskSource, Schema schema, int processorIndex)
+    public TransactionalPageOutput open(TaskSource taskSource, Schema schema, int taskIndex)
     {
         final PluginTask task = taskSource.loadTask(PluginTask.class);
 
         RecordWriter closeLater = null;
         try {
             FieldWriterSet fieldWriters = new FieldWriterSet(log, task, schema);
-            RecordWriter recordWriter = closeLater = new RecordWriter(task, newTdApiClient(task), fieldWriters);
+            RecordWriter recordWriter = closeLater = new RecordWriter(task, taskIndex, newTdApiClient(task), fieldWriters);
             recordWriter.open(schema);
             closeLater = null;
             return recordWriter;

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-output-td
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.1
 platform: ruby
 authors:
 - Muga Nishizawa
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-06-22 00:00:00.000000000 Z
+date: 2015-07-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -46,6 +46,7 @@ extensions: []
 extra_rdoc_files: []
 files:
 - .gitignore
+- CHANGELOG.md
 - README.md
 - build.gradle
 - embulk-output-td.gemspec
@@ -84,7 +85,7 @@ files:
 - src/main/java/org/embulk/output/RecordWriter.java
 - src/main/java/org/embulk/output/TdOutputPlugin.java
 - src/test/java/org/embulk/output/TestTdOutputPlugin.java
-- classpath/embulk-output-td-0.1.0.jar
+- classpath/embulk-output-td-0.1.1.jar
 - classpath/javassist-3.18.1-GA.jar
 - classpath/jetty-client-9.2.2.v20140723.jar
 - classpath/jetty-http-9.2.2.v20140723.jar