embulk-filter-speedometer 0.3.2 → 0.3.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 958b4d942aed7b7887b9e1ef551eff708fc6c772
-  data.tar.gz: 494333e5bdf90fe38a4b4cb2bfe51d3fd4243ae7
+  metadata.gz: ff41aeb838dae6df3aaf4b6d5847e83c36ea9fce
+  data.tar.gz: 0537d47fbc97b2d9b2a30d3fea19e22c4cabb4d7
 SHA512:
-  metadata.gz: a079f858ae03565450f89bfcb549da32cdc4ee0decbb132ae711a9a48e99b8a084f92ad22f02843d57324879ba17863ebb5db643acd681d626f610ebc9385716
-  data.tar.gz: 61c318a6d621d39f54a4493e7113d94dd273c7bb417a01bcd91e91945cdad10f54fddbdbe40fafaac1c3aece860fe6ecd7c7fdbe084fcf5be8dd335815dd6e7c
+  metadata.gz: ab2110df906ba8bdaf82e9d24dc798fa47aa8aa4fee70f8bcbb7237459f9b4fa0d4f416adcd53fc43613639a83a442dde10e9c0e44599cd02d6024644ac56d98
+  data.tar.gz: cc962bc8123ad29385835348b333b33559bd96f9c8b212d021c740766177edff79df20d19bb1280933e37fe51d4a70e8f759029f46b9e5d0f7a4c632c6b3080f

data/README.md

@@ -3,7 +3,7 @@
 [![Build Status](https://travis-ci.org/hata/embulk-filter-speedometer.svg?branch=master)](https://travis-ci.org/hata/embulk-filter-speedometer.svg?branch=master)
 
 Write log message of processed bytes and throughput periodically.
-This plugin works with embulk:0.6.17 or later versions because of removing deprecated APIs.
+This plugin works with embulk:0.8.0 or later versions.
 
 ## Overview
 
@@ -16,6 +16,7 @@ This plugin works with embulk:0.6.17 or later versions because of removing depre
 - **delimiter**: Delimiter text to calculate delimiter length. (string, optional, default: ",")
 - **record_padding_size**: Additional byte size for each record like a return code length. (integer, optional, default: 1)
 - **column_options**: A map whose keys are name of columns like csv formatter plugin (hash, optional, default: {})
+- **label**: Add this text to log output as label value. (string, optional, default: null)
 
 ## Example of Configuration
 
@@ -66,6 +67,19 @@ filters:
 ```
 
 
+- Set label text to distinguish log file. The following example can see label text like **{speedometer: {label: foo, active: ...**
+
+```yaml
+filters:
+  - type: speedometer
+    speed_limit: 250000
+    column_options:
+      time: {format: '%Y-%m-%d %H:%M:%S'}
+      purchase: {format: '%Y%m%d'}
+    label: 'foo'
+```
+
+
 ## Sample Log Message
 
 ```

data/build.gradle

@@ -18,11 +18,12 @@ configurations {
     provided
 }
 
-version = "0.3.2"
+version = "0.3.5"
 
 dependencies {
-    compile  "org.embulk:embulk-core:0.6.17+"
-    provided "org.embulk:embulk-core:0.6.17+"
+    compile  "org.embulk:embulk-core:0.8.0+"
+    compile  "org.msgpack:msgpack-core:0.7.1"
+    provided "org.embulk:embulk-core:0.8.0+"
     // compile "YOUR_JAR_DEPENDENCY_GROUP:YOUR_JAR_DEPENDENCY_MODULE:YOUR_JAR_DEPENDENCY_VERSION"
     testCompile "org.jmockit:jmockit:1.15+"
     testCompile "junit:junit:4.+"

data/src/integration-test/java/org/embulk/filter/TestSingleRun.java

@@ -5,16 +5,17 @@ import static org.junit.Assert.assertTrue;
 
 import java.io.BufferedReader;
 import java.io.File;
-import java.io.FilenameFilter;
 import java.io.FileInputStream;
 import java.io.FileReader;
-import java.io.InputStreamReader;
+import java.io.FilenameFilter;
 import java.io.IOException;
-import java.io.InputStream;
-import java.util.regex.Pattern;
-import java.util.zip.GZIPInputStream;
+import java.io.InputStreamReader;
 import java.util.ArrayList;
 import java.util.Collections;
+import java.util.List;
+import java.util.TreeSet;
+import java.util.regex.Pattern;
+import java.util.zip.GZIPInputStream;
 
 import org.junit.Test;
 
@@ -24,6 +25,9 @@ public class TestSingleRun {
     // e.g. {speedometer: {active: 4, total: 13.5mb, sec: 1:51, speed: 121kb/s, records: 269,748, record-speed: 2,435/s}}
     static final Pattern logLinePattern = Pattern.compile("\\{speedometer: \\{active: [^,]+, total: [^,]+, sec: [^,]+, speed: [^,]+, records: \\S+, record-speed: [^\\}]+\\}\\}");
 
+    // Add label to the log.
+    static final Pattern logLabelLinePattern = Pattern.compile("\\{speedometer: \\{label: [^,]+, active: [^,]+, total: [^,]+, sec: [^,]+, speed: [^,]+, records: \\S+, record-speed: [^\\}]+\\}\\}");
+
     private static String getTestFile(String name) {
         return TEST_DIR + File.separator + name;
     }
@@ -38,6 +42,16 @@ public class TestSingleRun {
         validateResultFiles("big_01.csv.gz", "result_big_");
     }
 
+    @Test
+    public void testValidateJsonOutputFile() throws Exception {
+        validateJsonResultFiles("ref_json_result_01.csv.gz", "result_json_");
+    }
+
+    @Test
+    public void testValidateLabelOutputFile() throws Exception {
+        validateResultFiles("label_01.csv.gz", "result_label_");
+    }
+
     @Test
     public void testSpeedometerMinLog() throws Exception {
         validateSpeedometerLog("config_min.yml.run.log");
@@ -48,12 +62,26 @@ public class TestSingleRun {
         validateSpeedometerLog("config_big.yml.run.log");
     }
 
+    @Test
+    public void testSpeedometerJsonLog() throws Exception {
+        validateSpeedometerLog("config_json.yml.run.log");
+    }
+
+    @Test
+    public void testSpeedometerLabelLog() throws Exception {
+        validateSpeedometerLog("config_label.yml.run.log", logLabelLinePattern);
+    }
+
     private void validateSpeedometerLog(String logFile) throws Exception {
+        validateSpeedometerLog(logFile, logLinePattern);
+    }
+
+    private void validateSpeedometerLog(String logFile, Pattern pattern) throws Exception {
         boolean found = false;
         try (BufferedReader r = new BufferedReader(new FileReader(getTestFile(logFile)))) {
             String line = r.readLine();
             while (line != null) {
-                if (logLinePattern.matcher(line).find()) {
+                if (pattern.matcher(line).find()) {
                     found = true;
                     break;
                 }
@@ -64,19 +92,44 @@ public class TestSingleRun {
     }
 
     private void validateResultFiles(String gzipSrcFile, final String prefix) throws Exception {
-        ArrayList inList = new ArrayList();
-        ArrayList outList = new ArrayList();
+        ArrayList<String> inList = new ArrayList();
+        ArrayList<String> outList = new ArrayList();
+
+        readToListFromGzipFile(gzipSrcFile, inList);
+        readToListFromPrefixMatching(prefix, outList);
 
+        Collections.sort(inList);
+        Collections.sort(outList);
+
+        assertEquals("Verify input and output lines are identical. in:" +
+            inList.size() + ", out:" + outList.size(), inList.toString(), outList.toString());
+    }
+
+
+    private void validateJsonResultFiles(String gzipSrcFile, final String prefix) throws Exception {
+        ArrayList<String> inList = new ArrayList();
+        ArrayList<String> outList = new ArrayList();
+
+        readToListFromGzipFile(gzipSrcFile, inList);
+        readToListFromPrefixMatching(prefix, outList);
+
+        assertEquals("Verify input and output lines are identical. in:" +
+            inList.size() + ", out:" + outList.size(), readToSet(inList), readToSet(outList));
+    }
+
+    private void readToListFromGzipFile(String gzipSrcFile, List<String> lineList) throws IOException {
         try (BufferedReader reader = new BufferedReader(new InputStreamReader(
             new GZIPInputStream(new FileInputStream(getTestFile(gzipSrcFile)))))) {
             String line = reader.readLine(); // Discard a header line
             line = reader.readLine();
             while (line != null) {
-                inList.add(line);
+                lineList.add(line);
                 line = reader.readLine();
             }
         }
+    }
 
+    private void readToListFromPrefixMatching(final String prefix, List<String> lineList) throws IOException {
         // In travis env, there are many cpus and it may be different from
         // my local environment. From this, list all files using File.list method.
         String[] resultFiles = new File(TEST_DIR).list(new FilenameFilter() {
@@ -90,16 +143,32 @@ public class TestSingleRun {
                 String line = reader.readLine(); // Discard a header line
                 line = reader.readLine();
                 while (line != null) {
-                    outList.add(line);
+                    lineList.add(line);
                     line = reader.readLine();
                 }
             }
         }
+    }
 
-        Collections.sort(inList);
-        Collections.sort(outList);
+    private TreeSet<String> readToSet(List<String> lineList) throws Exception {
+        TreeSet<String> set = new TreeSet<>();
+        for (String line : lineList) {
+            line = stripQuote(line);
+            if (line.startsWith("{") && line.endsWith("}")) {
+                ArrayList<String> fields = new ArrayList<>();
+                for (String field : line.substring(1, line.length() - 1).split(",")) {
+                    fields.add(field);
+                }
+                Collections.sort(fields);
+                set.add(fields.toString());
+            } else {
+                throw new Exception("Unexpected lines." + lineList);
+            }
+        }
+        return set;
+    }
 
-        assertEquals("Verify input and output lines are identical. in:" +
-            inList.size() + ", out:" + outList.size(), inList.toString(), outList.toString());
+    private String stripQuote(String line) {
+        return line.startsWith("'") && line.endsWith("'") ? line.substring(1, line.length() -1) : line;
     }
 }

data/src/integration-test/resources/config_json.yml

@@ -0,0 +1,20 @@
+in:
+  type: file
+  path_prefix: ./json_
+  decoders:
+  - {type: gzip}
+  parser:
+    type: json
+filters:
+  - type: speedometer
+    speed_limit: 250000
+out:
+  type: file
+  path_prefix: ./result_json_
+  file_ext: csv
+  formatter:
+    type: csv
+    quote_policy: MINIMAL
+    default_timezone: UTC
+    newline: LF
+    quote: "'"

data/src/integration-test/resources/config_label.yml

@@ -0,0 +1,43 @@
+in:
+  type: file
+  path_prefix: ./min_
+  decoders:
+  - {type: gzip}
+  parser:
+    charset: UTF-8
+    newline: CRLF
+    type: csv
+    delimiter: ','
+    quote: '"'
+    trim_if_not_quoted: false
+    skip_header_lines: 1
+    allow_extra_columns: false
+    allow_optional_columns: false
+    columns:
+    - {name: id, type: long}
+    - {name: account, type: long}
+    - {name: time, type: timestamp, format: '%Y-%m-%d %H:%M:%S'}
+    - {name: purchase, type: timestamp, format: '%Y%m%d'}
+    - {name: enabled, type: boolean}
+    - {name: comment, type: string}
+filters:
+  - type: speedometer
+    speed_limit: 250000
+    column_options:
+      time: {format: '%Y-%m-%d %H:%M:%S'}
+      purchase: {format: '%Y%m%d'}
+    label: 'foo'
+out:
+  type: file
+  path_prefix: ./result_label_
+  file_ext: csv
+  formatter:
+    type: csv
+    quote_policy: MINIMAL
+    default_timezone: UTC
+    newline: LF
+    column_options:
+      time: {format: '%Y-%m-%d %H:%M:%S'}
+      purchase: {format: '%Y%m%d'}
+
+

data/src/main/java/org/embulk/filter/SpeedometerFilterPlugin.java

@@ -22,6 +22,9 @@ import org.embulk.spi.Schema;
 import org.embulk.spi.time.Timestamp;
 import org.embulk.spi.time.TimestampFormatter;
 import org.embulk.spi.util.Timestamps;
+import org.msgpack.value.Value;
+
+import com.google.common.base.Optional;
 
 public class SpeedometerFilterPlugin
         implements FilterPlugin
@@ -57,7 +60,11 @@ public class SpeedometerFilterPlugin
 
         @Config("column_options")
         @ConfigDefault("{}")
-        Map<String, TimestampColumnOption> getColumnOptions();
+        public Map<String, TimestampColumnOption> getColumnOptions();
+
+        @Config("label")
+        @ConfigDefault("null")
+        public Optional<String> getLabel();
 
         @ConfigInject
         public BufferAllocator getBufferAllocator();
@@ -96,7 +103,7 @@ public class SpeedometerFilterPlugin
         private final PageBuilder pageBuilder;
 
         SpeedControlPageOutput(PluginTask task, Schema schema, PageOutput pageOutput) {
-            this.controller = new SpeedometerSpeedController(task, SpeedometerSpeedAggregator.getInstance());
+            this.controller = new SpeedometerSpeedController(task, SpeedometerSpeedAggregator.getInstance(task));
             this.schema = schema;
             this.allocator = task.getBufferAllocator();
             this.delimiterLength = task.getDelimiter().length();
@@ -104,6 +111,7 @@ public class SpeedometerFilterPlugin
             this.pageReader = new PageReader(schema);
             this.timestampFormatters = Timestamps.newTimestampColumnFormatters(task, schema, task.getColumnOptions());
             this.pageBuilder = new PageBuilder(allocator, schema, pageOutput);
+            this.controller.start(System.currentTimeMillis());
         }
 
         @Override
@@ -120,14 +128,13 @@ public class SpeedometerFilterPlugin
 
         @Override
         public void finish() {
+            controller.stop();
             pageBuilder.finish();
         }
 
         @Override
         public void close() {
-            controller.stop();
             pageBuilder.close();
-            pageReader.close();
         }
 
         class ColumnVisitorImpl implements ColumnVisitor {
@@ -188,6 +195,16 @@ public class SpeedometerFilterPlugin
                 }
             }
 
+            @Override
+            public void jsonColumn(Column column) {
+                if (pageReader.isNull(column)) {
+                    speedMonitor(column);
+                    pageBuilder.setNull(column);
+                } else {
+                    pageBuilder.setJson(column, speedMonitor(column, pageReader.getJson(column)));
+                }
+            }
+
             private void speedMonitorStartRecord() {
                 startRecordTime = System.currentTimeMillis();
             }
@@ -232,6 +249,14 @@ public class SpeedometerFilterPlugin
                 return t;
             }
 
+            private Value speedMonitor(Column column, Value v) {
+                speedMonitorForDelimiter(column);
+                // NOTE: This may not be good for performance. But, I have no other idea.
+                String s = v.toJson();
+                controller.checkSpeedLimit(startRecordTime, s != null ? s.length() : 0);
+                return v;
+            }
+
             private void speedMonitorForDelimiter(Column column) {
                 if (column.getIndex() > 0) {
                     controller.checkSpeedLimit(startRecordTime, delimiterLength);

data/src/main/java/org/embulk/filter/SpeedometerSpeedAggregator.java

@@ -5,13 +5,13 @@ import java.util.List;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicLong;
 
+import org.embulk.filter.SpeedometerFilterPlugin.PluginTask;
 import org.embulk.spi.Exec;
 import org.slf4j.Logger;
 
 class SpeedometerSpeedAggregator {
-    private static class SpeedometerSpeedAggregatorHolder {
-        private static final SpeedometerSpeedAggregator INSTANCE = new SpeedometerSpeedAggregator();
-    }
+    private static final Object INSTANCE_LOCK = new Object();
+    private static SpeedometerSpeedAggregator INSTANCE;
 
     private final long INITAL_START_TIME = 0;
 
@@ -20,15 +20,26 @@ class SpeedometerSpeedAggregator {
     private final AtomicLong globalTotalBytes = new AtomicLong(0);
     private final AtomicLong globalTotalRecords = new AtomicLong(0);
     private final AtomicLong previousLogReportTimeMillisec = new AtomicLong(INITAL_START_TIME);
+    private final String logFormat;
 
     // TODO: We can use google's library.
     private final List<SpeedometerSpeedController> controllerList = new ArrayList<>();
 
-    public static SpeedometerSpeedAggregator getInstance() {
-        return SpeedometerSpeedAggregatorHolder.INSTANCE;
+    public static SpeedometerSpeedAggregator getInstance(PluginTask task) {
+        synchronized (INSTANCE_LOCK) {
+            if (INSTANCE == null) {
+                INSTANCE = new SpeedometerSpeedAggregator(task != null && task.getLabel().isPresent() ? task.getLabel().get() : null);
+            }
+            return INSTANCE;
+        }
+    }
+
+    SpeedometerSpeedAggregator() {
+        this(null);
     }
 
-    public SpeedometerSpeedAggregator() {
+    SpeedometerSpeedAggregator(String label) {
+        logFormat = initLogFormat(label);
         showLogMessage(activeControllerCount.get(), 0, 0, 0, 0, 0);
     }
 
@@ -106,6 +117,10 @@ class SpeedometerSpeedAggregator {
         return copyList;
     }
 
+    String getLogFormat() {
+        return logFormat;
+    }
+
     private void renewPeriods() {
         for (SpeedometerSpeedController controller : getControllerList()) {
             controller.renewPeriod();
@@ -142,7 +157,7 @@ class SpeedometerSpeedAggregator {
     private void showLogMessage(int activeThreads, long totalBytes, long timeMilliSec, long bytesPerSec, long totalRecords, long recordsPerSec) {
         Logger logger = getLogger();
         if (logger != null) {
-            logger.info(String.format("{speedometer: {active: %d, total: %s, sec: %s, speed: %s/s, records: %s, record-speed: %s/s}}",
+            logger.info(String.format(logFormat,
                     activeThreads,
                     SpeedometerUtil.toByteText(totalBytes),
                     SpeedometerUtil.toTimeText(timeMilliSec),
@@ -151,4 +166,14 @@ class SpeedometerSpeedAggregator {
                     SpeedometerUtil.toDecimalText(recordsPerSec)));
         }
     }
+
+    private String initLogFormat(String label) {
+        StringBuilder builder = new StringBuilder();
+        builder.append("{speedometer: {");
+        if (label != null && label.length() > 0) {
+            builder.append("label: ").append(label).append(", ");
+        }
+        builder.append("active: %d, total: %s, sec: %s, speed: %s/s, records: %s, record-speed: %s/s}}");
+        return builder.toString();
+    }
 }

data/src/main/java/org/embulk/filter/SpeedometerSpeedController.java

@@ -25,6 +25,13 @@ class SpeedometerSpeedController {
         this.aggregator = aggregator;
     }
 
+    public void start(long nowTime) {
+        if (startTime == 0) {
+            startTime = nowTime;
+            aggregator.startController(this, startTime);
+        }
+    }
+
     public void stop() {
         startNewPeriod(0);
         aggregator.stopController(this);

data/src/test/java/org/embulk/filter/TestSpeedometerFilterPlugin.java

@@ -108,7 +108,6 @@ public class TestSpeedometerFilterPlugin
         new Verifications() {{
             taskSource.loadTask(PluginTask.class); times = 1;
             builder.close(); times = 1;
-            reader.close(); times = 1;
         }};
     }
 }

data/src/test/java/org/embulk/filter/TestSpeedometerSpeedAggregator.java

@@ -4,21 +4,24 @@ import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.assertTrue;
-import mockit.Mocked;
-import mockit.NonStrictExpectations;
-import mockit.Verifications;
 
+import org.embulk.filter.SpeedometerFilterPlugin.PluginTask;
 import org.embulk.spi.Exec;
 import org.junit.Test;
 import org.slf4j.Logger;
 
+import mockit.Mocked;
+import mockit.NonStrictExpectations;
+import mockit.Verifications;
+
 public class TestSpeedometerSpeedAggregator {
     @Mocked SpeedometerSpeedController controller;
     @Mocked Exec exec;
+    @Mocked PluginTask task;
 
     @Test
     public void testGetInstance() {
-        assertNotNull("Verify there is a singleton.", SpeedometerSpeedAggregator.getInstance());
+        assertNotNull("Verify there is a singleton.", SpeedometerSpeedAggregator.getInstance(task));
     }
 
     @Test
@@ -26,6 +29,15 @@ public class TestSpeedometerSpeedAggregator {
         SpeedometerSpeedAggregator aggregator = new SpeedometerSpeedAggregator();
         assertEquals("Verify the default global start time is zero.", 0, aggregator.getGlobalStartTime());
         assertEquals("Verify the default active count is zero.", 0, aggregator.getActiveControllerCount());
+        assertEquals("{speedometer: {active: %d, total: %s, sec: %s, speed: %s/s, records: %s, record-speed: %s/s}}", aggregator.getLogFormat());
+    }
+
+    @Test
+    public void testSpeedometerSpeedAggregatorWithLabel() {
+        SpeedometerSpeedAggregator aggregator = new SpeedometerSpeedAggregator("foo");
+        assertEquals("Verify the default global start time is zero.", 0, aggregator.getGlobalStartTime());
+        assertEquals("Verify the default active count is zero.", 0, aggregator.getActiveControllerCount());
+        assertEquals("{speedometer: {label: foo, active: %d, total: %s, sec: %s, speed: %s/s, records: %s, record-speed: %s/s}}", aggregator.getLogFormat());
     }
 
     @Test

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-filter-speedometer
 version: !ruby/object:Gem::Version
-  version: 0.3.2
+  version: 0.3.5
 platform: ruby
 authors:
 - hata
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-04-23 00:00:00.000000000 Z
+date: 2022-03-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -58,8 +58,13 @@ files:
 - src/integration-test/java/org/embulk/filter/TestSingleRun.java
 - src/integration-test/resources/big_01.csv.gz
 - src/integration-test/resources/config_big.yml
+- src/integration-test/resources/config_json.yml
+- src/integration-test/resources/config_label.yml
 - src/integration-test/resources/config_min.yml
+- src/integration-test/resources/json_01.json.gz
+- src/integration-test/resources/label_01.csv.gz
 - src/integration-test/resources/min_01.csv.gz
+- src/integration-test/resources/ref_json_result_01.csv.gz
 - src/main/java/org/embulk/filter/SpeedometerFilterPlugin.java
 - src/main/java/org/embulk/filter/SpeedometerSpeedAggregator.java
 - src/main/java/org/embulk/filter/SpeedometerSpeedController.java
@@ -68,7 +73,7 @@ files:
 - src/test/java/org/embulk/filter/TestSpeedometerSpeedAggregator.java
 - src/test/java/org/embulk/filter/TestSpeedometerSpeedController.java
 - src/test/java/org/embulk/filter/TestSpeedometerUtil.java
-- classpath/embulk-filter-speedometer-0.3.2.jar
+- classpath/embulk-filter-speedometer-0.3.5.jar
 homepage: https://github.com/hata/embulk-filter-speedometer
 licenses:
 - MIT