embulk-filter-speedometer 0.3.3 → 0.3.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 4439cf02600ec5d3100d5e55278db0f621ea0379
-  data.tar.gz: 50d8c16f4517f224d220a5dfea9e00e7f18ede42
+  metadata.gz: 5548ab739e2722a078ef056885ea9e579e8630fb
+  data.tar.gz: 6f7317591c00c5d0e05bd30d91fc9063b112fa87
 SHA512:
-  metadata.gz: 961d57c198263a6a5288e9b5d88ae88a41db7207782aabe9f6a8a237f4ae92029f82c58ce4e07cbe6beddc0e136ede9559971aeeaac2f19430950f5801773cae
-  data.tar.gz: e7afe0a2ca10f05912444ad06619b3b88b61f91ae1f4b0669097c0c478d116d316cdb00cf74a141334d2918399776ee1f9335dbddbc4aaafab91258b320b5464
+  metadata.gz: 301275d05d35c6ac17651ba81a548d4c73ac1a675bcd9793383da4832437b3edb869c820f96a81f810ad8c14a33c0bbbfdd292b1e3f3822a64fac1bcac99ee9f
+  data.tar.gz: ddf85d31b06c47f9880ce29dcf60f5223a1d37df6fe70322f886f968c007e2f91c2bcf0f9fc7749e0ecf67571b473c45c05f7d27673974a55ff6818da7b52aa6

data/README.md

@@ -16,6 +16,7 @@ This plugin works with embulk:0.8.0 or later versions.
 - **delimiter**: Delimiter text to calculate delimiter length. (string, optional, default: ",")
 - **record_padding_size**: Additional byte size for each record like a return code length. (integer, optional, default: 1)
 - **column_options**: A map whose keys are name of columns like csv formatter plugin (hash, optional, default: {})
+- **label**: Add this text to log output as label value. (string, optional, default: null)
 
 ## Example of Configuration
 
@@ -66,6 +67,19 @@ filters:
 ```
 
 
+- Set label text to distinguish log file. The following example can see label text like **{speedometer: {label: foo, active: ...**
+
+```yaml
+filters:
+  - type: speedometer
+    speed_limit: 250000
+    column_options:
+      time: {format: '%Y-%m-%d %H:%M:%S'}
+      purchase: {format: '%Y%m%d'}
+    label: 'foo'
+```
+
+
 ## Sample Log Message
 
 ```

data/build.gradle

@@ -18,7 +18,7 @@ configurations {
     provided
 }
 
-version = "0.3.3"
+version = "0.3.4"
 
 dependencies {
     compile  "org.embulk:embulk-core:0.8.0+"

data/src/integration-test/java/org/embulk/filter/TestSingleRun.java

@@ -5,19 +5,17 @@ import static org.junit.Assert.assertTrue;
 
 import java.io.BufferedReader;
 import java.io.File;
-import java.io.FilenameFilter;
 import java.io.FileInputStream;
 import java.io.FileReader;
-import java.io.InputStreamReader;
+import java.io.FilenameFilter;
 import java.io.IOException;
-import java.io.InputStream;
-import java.util.regex.Pattern;
-import java.util.zip.GZIPInputStream;
+import java.io.InputStreamReader;
 import java.util.ArrayList;
-import java.util.TreeSet;
-import java.util.List;
 import java.util.Collections;
-import java.util.Set;
+import java.util.List;
+import java.util.TreeSet;
+import java.util.regex.Pattern;
+import java.util.zip.GZIPInputStream;
 
 import org.junit.Test;
 
@@ -27,6 +25,9 @@ public class TestSingleRun {
     // e.g. {speedometer: {active: 4, total: 13.5mb, sec: 1:51, speed: 121kb/s, records: 269,748, record-speed: 2,435/s}}
     static final Pattern logLinePattern = Pattern.compile("\\{speedometer: \\{active: [^,]+, total: [^,]+, sec: [^,]+, speed: [^,]+, records: \\S+, record-speed: [^\\}]+\\}\\}");
 
+    // Add label to the log.
+    static final Pattern logLabelLinePattern = Pattern.compile("\\{speedometer: \\{label: [^,]+, active: [^,]+, total: [^,]+, sec: [^,]+, speed: [^,]+, records: \\S+, record-speed: [^\\}]+\\}\\}");
+
     private static String getTestFile(String name) {
         return TEST_DIR + File.separator + name;
     }
@@ -46,6 +47,11 @@ public class TestSingleRun {
         validateJsonResultFiles("ref_json_result_01.csv.gz", "result_json_");
     }
 
+    @Test
+    public void testValidateLabelOutputFile() throws Exception {
+        validateResultFiles("label_01.csv.gz", "result_label_");
+    }
+
     @Test
     public void testSpeedometerMinLog() throws Exception {
         validateSpeedometerLog("config_min.yml.run.log");
@@ -61,12 +67,21 @@ public class TestSingleRun {
         validateSpeedometerLog("config_json.yml.run.log");
     }
 
+    @Test
+    public void testSpeedometerLabelLog() throws Exception {
+        validateSpeedometerLog("config_label.yml.run.log", logLabelLinePattern);
+    }
+
     private void validateSpeedometerLog(String logFile) throws Exception {
+        validateSpeedometerLog(logFile, logLinePattern);
+    }
+
+    private void validateSpeedometerLog(String logFile, Pattern pattern) throws Exception {
         boolean found = false;
         try (BufferedReader r = new BufferedReader(new FileReader(getTestFile(logFile)))) {
             String line = r.readLine();
             while (line != null) {
-                if (logLinePattern.matcher(line).find()) {
+                if (pattern.matcher(line).find()) {
                     found = true;
                     break;
                 }

data/src/integration-test/resources/config_label.yml

@@ -0,0 +1,43 @@
+in:
+  type: file
+  path_prefix: ./min_
+  decoders:
+  - {type: gzip}
+  parser:
+    charset: UTF-8
+    newline: CRLF
+    type: csv
+    delimiter: ','
+    quote: '"'
+    trim_if_not_quoted: false
+    skip_header_lines: 1
+    allow_extra_columns: false
+    allow_optional_columns: false
+    columns:
+    - {name: id, type: long}
+    - {name: account, type: long}
+    - {name: time, type: timestamp, format: '%Y-%m-%d %H:%M:%S'}
+    - {name: purchase, type: timestamp, format: '%Y%m%d'}
+    - {name: enabled, type: boolean}
+    - {name: comment, type: string}
+filters:
+  - type: speedometer
+    speed_limit: 250000
+    column_options:
+      time: {format: '%Y-%m-%d %H:%M:%S'}
+      purchase: {format: '%Y%m%d'}
+    label: 'foo'
+out:
+  type: file
+  path_prefix: ./result_label_
+  file_ext: csv
+  formatter:
+    type: csv
+    quote_policy: MINIMAL
+    default_timezone: UTC
+    newline: LF
+    column_options:
+      time: {format: '%Y-%m-%d %H:%M:%S'}
+      purchase: {format: '%Y%m%d'}
+
+

data/src/main/java/org/embulk/filter/SpeedometerFilterPlugin.java

@@ -24,6 +24,8 @@ import org.embulk.spi.time.TimestampFormatter;
 import org.embulk.spi.util.Timestamps;
 import org.msgpack.value.Value;
 
+import com.google.common.base.Optional;
+
 public class SpeedometerFilterPlugin
         implements FilterPlugin
 {
@@ -58,7 +60,11 @@ public class SpeedometerFilterPlugin
 
         @Config("column_options")
         @ConfigDefault("{}")
-        Map<String, TimestampColumnOption> getColumnOptions();
+        public Map<String, TimestampColumnOption> getColumnOptions();
+
+        @Config("label")
+        @ConfigDefault("null")
+        public Optional<String> getLabel();
 
         @ConfigInject
         public BufferAllocator getBufferAllocator();
@@ -97,7 +103,7 @@ public class SpeedometerFilterPlugin
         private final PageBuilder pageBuilder;
 
         SpeedControlPageOutput(PluginTask task, Schema schema, PageOutput pageOutput) {
-            this.controller = new SpeedometerSpeedController(task, SpeedometerSpeedAggregator.getInstance());
+            this.controller = new SpeedometerSpeedController(task, SpeedometerSpeedAggregator.getInstance(task));
             this.schema = schema;
             this.allocator = task.getBufferAllocator();
             this.delimiterLength = task.getDelimiter().length();

data/src/main/java/org/embulk/filter/SpeedometerSpeedAggregator.java

@@ -5,13 +5,13 @@ import java.util.List;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicLong;
 
+import org.embulk.filter.SpeedometerFilterPlugin.PluginTask;
 import org.embulk.spi.Exec;
 import org.slf4j.Logger;
 
 class SpeedometerSpeedAggregator {
-    private static class SpeedometerSpeedAggregatorHolder {
-        private static final SpeedometerSpeedAggregator INSTANCE = new SpeedometerSpeedAggregator();
-    }
+    private static final Object INSTANCE_LOCK = new Object();
+    private static SpeedometerSpeedAggregator INSTANCE;
 
     private final long INITAL_START_TIME = 0;
 
@@ -20,15 +20,26 @@ class SpeedometerSpeedAggregator {
     private final AtomicLong globalTotalBytes = new AtomicLong(0);
     private final AtomicLong globalTotalRecords = new AtomicLong(0);
     private final AtomicLong previousLogReportTimeMillisec = new AtomicLong(INITAL_START_TIME);
+    private final String logFormat;
 
     // TODO: We can use google's library.
     private final List<SpeedometerSpeedController> controllerList = new ArrayList<>();
 
-    public static SpeedometerSpeedAggregator getInstance() {
-        return SpeedometerSpeedAggregatorHolder.INSTANCE;
+    public static SpeedometerSpeedAggregator getInstance(PluginTask task) {
+        synchronized (INSTANCE_LOCK) {
+            if (INSTANCE == null) {
+                INSTANCE = new SpeedometerSpeedAggregator(task != null && task.getLabel().isPresent() ? task.getLabel().get() : null);
+            }
+            return INSTANCE;
+        }
+    }
+
+    SpeedometerSpeedAggregator() {
+        this(null);
     }
 
-    public SpeedometerSpeedAggregator() {
+    SpeedometerSpeedAggregator(String label) {
+        logFormat = initLogFormat(label);
         showLogMessage(activeControllerCount.get(), 0, 0, 0, 0, 0);
     }
 
@@ -106,6 +117,10 @@ class SpeedometerSpeedAggregator {
         return copyList;
     }
 
+    String getLogFormat() {
+        return logFormat;
+    }
+
     private void renewPeriods() {
         for (SpeedometerSpeedController controller : getControllerList()) {
             controller.renewPeriod();
@@ -142,7 +157,7 @@ class SpeedometerSpeedAggregator {
     private void showLogMessage(int activeThreads, long totalBytes, long timeMilliSec, long bytesPerSec, long totalRecords, long recordsPerSec) {
         Logger logger = getLogger();
         if (logger != null) {
-            logger.info(String.format("{speedometer: {active: %d, total: %s, sec: %s, speed: %s/s, records: %s, record-speed: %s/s}}",
+            logger.info(String.format(logFormat,
                     activeThreads,
                     SpeedometerUtil.toByteText(totalBytes),
                     SpeedometerUtil.toTimeText(timeMilliSec),
@@ -151,4 +166,14 @@ class SpeedometerSpeedAggregator {
                     SpeedometerUtil.toDecimalText(recordsPerSec)));
         }
     }
+
+    private String initLogFormat(String label) {
+        StringBuilder builder = new StringBuilder();
+        builder.append("{speedometer: {");
+        if (label != null && label.length() > 0) {
+            builder.append("label: ").append(label).append(", ");
+        }
+        builder.append("active: %d, total: %s, sec: %s, speed: %s/s, records: %s, record-speed: %s/s}}");
+        return builder.toString();
+    }
 }

data/src/test/java/org/embulk/filter/TestSpeedometerSpeedAggregator.java

@@ -4,21 +4,24 @@ import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.assertTrue;
-import mockit.Mocked;
-import mockit.NonStrictExpectations;
-import mockit.Verifications;
 
+import org.embulk.filter.SpeedometerFilterPlugin.PluginTask;
 import org.embulk.spi.Exec;
 import org.junit.Test;
 import org.slf4j.Logger;
 
+import mockit.Mocked;
+import mockit.NonStrictExpectations;
+import mockit.Verifications;
+
 public class TestSpeedometerSpeedAggregator {
     @Mocked SpeedometerSpeedController controller;
     @Mocked Exec exec;
+    @Mocked PluginTask task;
 
     @Test
     public void testGetInstance() {
-        assertNotNull("Verify there is a singleton.", SpeedometerSpeedAggregator.getInstance());
+        assertNotNull("Verify there is a singleton.", SpeedometerSpeedAggregator.getInstance(task));
     }
 
     @Test
@@ -26,6 +29,15 @@ public class TestSpeedometerSpeedAggregator {
         SpeedometerSpeedAggregator aggregator = new SpeedometerSpeedAggregator();
         assertEquals("Verify the default global start time is zero.", 0, aggregator.getGlobalStartTime());
         assertEquals("Verify the default active count is zero.", 0, aggregator.getActiveControllerCount());
+        assertEquals("{speedometer: {active: %d, total: %s, sec: %s, speed: %s/s, records: %s, record-speed: %s/s}}", aggregator.getLogFormat());
+    }
+
+    @Test
+    public void testSpeedometerSpeedAggregatorWithLabel() {
+        SpeedometerSpeedAggregator aggregator = new SpeedometerSpeedAggregator("foo");
+        assertEquals("Verify the default global start time is zero.", 0, aggregator.getGlobalStartTime());
+        assertEquals("Verify the default active count is zero.", 0, aggregator.getActiveControllerCount());
+        assertEquals("{speedometer: {label: foo, active: %d, total: %s, sec: %s, speed: %s/s, records: %s, record-speed: %s/s}}", aggregator.getLogFormat());
     }
 
     @Test

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-filter-speedometer
 version: !ruby/object:Gem::Version
-  version: 0.3.3
+  version: 0.3.4
 platform: ruby
 authors:
 - hata
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2017-02-17 00:00:00.000000000 Z
+date: 2017-03-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -59,8 +59,10 @@ files:
 - src/integration-test/resources/big_01.csv.gz
 - src/integration-test/resources/config_big.yml
 - src/integration-test/resources/config_json.yml
+- src/integration-test/resources/config_label.yml
 - src/integration-test/resources/config_min.yml
 - src/integration-test/resources/json_01.json.gz
+- src/integration-test/resources/label_01.csv.gz
 - src/integration-test/resources/min_01.csv.gz
 - src/integration-test/resources/ref_json_result_01.csv.gz
 - src/main/java/org/embulk/filter/SpeedometerFilterPlugin.java
@@ -71,7 +73,7 @@ files:
 - src/test/java/org/embulk/filter/TestSpeedometerSpeedAggregator.java
 - src/test/java/org/embulk/filter/TestSpeedometerSpeedController.java
 - src/test/java/org/embulk/filter/TestSpeedometerUtil.java
-- classpath/embulk-filter-speedometer-0.3.3.jar
+- classpath/embulk-filter-speedometer-0.3.4.jar
 homepage: https://github.com/hata/embulk-filter-speedometer
 licenses:
 - MIT