embulk-filter-speedometer 0.3.2 → 0.3.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 958b4d942aed7b7887b9e1ef551eff708fc6c772
-  data.tar.gz: 494333e5bdf90fe38a4b4cb2bfe51d3fd4243ae7
+  metadata.gz: 4439cf02600ec5d3100d5e55278db0f621ea0379
+  data.tar.gz: 50d8c16f4517f224d220a5dfea9e00e7f18ede42
 SHA512:
-  metadata.gz: a079f858ae03565450f89bfcb549da32cdc4ee0decbb132ae711a9a48e99b8a084f92ad22f02843d57324879ba17863ebb5db643acd681d626f610ebc9385716
-  data.tar.gz: 61c318a6d621d39f54a4493e7113d94dd273c7bb417a01bcd91e91945cdad10f54fddbdbe40fafaac1c3aece860fe6ecd7c7fdbe084fcf5be8dd335815dd6e7c
+  metadata.gz: 961d57c198263a6a5288e9b5d88ae88a41db7207782aabe9f6a8a237f4ae92029f82c58ce4e07cbe6beddc0e136ede9559971aeeaac2f19430950f5801773cae
+  data.tar.gz: e7afe0a2ca10f05912444ad06619b3b88b61f91ae1f4b0669097c0c478d116d316cdb00cf74a141334d2918399776ee1f9335dbddbc4aaafab91258b320b5464

data/README.md

@@ -3,7 +3,7 @@
 [![Build Status](https://travis-ci.org/hata/embulk-filter-speedometer.svg?branch=master)](https://travis-ci.org/hata/embulk-filter-speedometer.svg?branch=master)
 
 Write log message of processed bytes and throughput periodically.
-This plugin works with embulk:0.6.17 or later versions because of removing deprecated APIs.
+This plugin works with embulk:0.8.0 or later versions.
 
 ## Overview
 

data/build.gradle

@@ -18,11 +18,12 @@ configurations {
     provided
 }
 
-version = "0.3.2"
+version = "0.3.3"
 
 dependencies {
-    compile  "org.embulk:embulk-core:0.6.17+"
-    provided "org.embulk:embulk-core:0.6.17+"
+    compile  "org.embulk:embulk-core:0.8.0+"
+    compile  "org.msgpack:msgpack-core:0.7.1"
+    provided "org.embulk:embulk-core:0.8.0+"
     // compile "YOUR_JAR_DEPENDENCY_GROUP:YOUR_JAR_DEPENDENCY_MODULE:YOUR_JAR_DEPENDENCY_VERSION"
     testCompile "org.jmockit:jmockit:1.15+"
     testCompile "junit:junit:4.+"

data/src/integration-test/java/org/embulk/filter/TestSingleRun.java

@@ -14,7 +14,10 @@ import java.io.InputStream;
 import java.util.regex.Pattern;
 import java.util.zip.GZIPInputStream;
 import java.util.ArrayList;
+import java.util.TreeSet;
+import java.util.List;
 import java.util.Collections;
+import java.util.Set;
 
 import org.junit.Test;
 
@@ -38,6 +41,11 @@ public class TestSingleRun {
         validateResultFiles("big_01.csv.gz", "result_big_");
     }
 
+    @Test
+    public void testValidateJsonOutputFile() throws Exception {
+        validateJsonResultFiles("ref_json_result_01.csv.gz", "result_json_");
+    }
+
     @Test
     public void testSpeedometerMinLog() throws Exception {
         validateSpeedometerLog("config_min.yml.run.log");
@@ -48,6 +56,11 @@ public class TestSingleRun {
         validateSpeedometerLog("config_big.yml.run.log");
     }
 
+    @Test
+    public void testSpeedometerJsonLog() throws Exception {
+        validateSpeedometerLog("config_json.yml.run.log");
+    }
+
     private void validateSpeedometerLog(String logFile) throws Exception {
         boolean found = false;
         try (BufferedReader r = new BufferedReader(new FileReader(getTestFile(logFile)))) {
@@ -64,19 +77,44 @@ public class TestSingleRun {
     }
 
     private void validateResultFiles(String gzipSrcFile, final String prefix) throws Exception {
-        ArrayList inList = new ArrayList();
-        ArrayList outList = new ArrayList();
+        ArrayList<String> inList = new ArrayList();
+        ArrayList<String> outList = new ArrayList();
+
+        readToListFromGzipFile(gzipSrcFile, inList);
+        readToListFromPrefixMatching(prefix, outList);
+
+        Collections.sort(inList);
+        Collections.sort(outList);
+
+        assertEquals("Verify input and output lines are identical. in:" +
+            inList.size() + ", out:" + outList.size(), inList.toString(), outList.toString());
+    }
+
 
+    private void validateJsonResultFiles(String gzipSrcFile, final String prefix) throws Exception {
+        ArrayList<String> inList = new ArrayList();
+        ArrayList<String> outList = new ArrayList();
+
+        readToListFromGzipFile(gzipSrcFile, inList);
+        readToListFromPrefixMatching(prefix, outList);
+
+        assertEquals("Verify input and output lines are identical. in:" +
+            inList.size() + ", out:" + outList.size(), readToSet(inList), readToSet(outList));
+    }
+
+    private void readToListFromGzipFile(String gzipSrcFile, List<String> lineList) throws IOException {
         try (BufferedReader reader = new BufferedReader(new InputStreamReader(
             new GZIPInputStream(new FileInputStream(getTestFile(gzipSrcFile)))))) {
             String line = reader.readLine(); // Discard a header line
             line = reader.readLine();
             while (line != null) {
-                inList.add(line);
+                lineList.add(line);
                 line = reader.readLine();
             }
         }
+    }
 
+    private void readToListFromPrefixMatching(final String prefix, List<String> lineList) throws IOException {
         // In travis env, there are many cpus and it may be different from
         // my local environment. From this, list all files using File.list method.
         String[] resultFiles = new File(TEST_DIR).list(new FilenameFilter() {
@@ -90,16 +128,32 @@ public class TestSingleRun {
                 String line = reader.readLine(); // Discard a header line
                 line = reader.readLine();
                 while (line != null) {
-                    outList.add(line);
+                    lineList.add(line);
                     line = reader.readLine();
                 }
             }
         }
+    }
 
-        Collections.sort(inList);
-        Collections.sort(outList);
+    private TreeSet<String> readToSet(List<String> lineList) throws Exception {
+        TreeSet<String> set = new TreeSet<>();
+        for (String line : lineList) {
+            line = stripQuote(line);
+            if (line.startsWith("{") && line.endsWith("}")) {
+                ArrayList<String> fields = new ArrayList<>();
+                for (String field : line.substring(1, line.length() - 1).split(",")) {
+                    fields.add(field);
+                }
+                Collections.sort(fields);
+                set.add(fields.toString());
+            } else {
+                throw new Exception("Unexpected lines." + lineList);
+            }
+        }
+        return set;
+    }
 
-        assertEquals("Verify input and output lines are identical. in:" +
-            inList.size() + ", out:" + outList.size(), inList.toString(), outList.toString());
+    private String stripQuote(String line) {
+        return line.startsWith("'") && line.endsWith("'") ? line.substring(1, line.length() -1) : line;
     }
 }

data/src/integration-test/resources/config_json.yml

@@ -0,0 +1,20 @@
+in:
+  type: file
+  path_prefix: ./json_
+  decoders:
+  - {type: gzip}
+  parser:
+    type: json
+filters:
+  - type: speedometer
+    speed_limit: 250000
+out:
+  type: file
+  path_prefix: ./result_json_
+  file_ext: csv
+  formatter:
+    type: csv
+    quote_policy: MINIMAL
+    default_timezone: UTC
+    newline: LF
+    quote: "'"

data/src/main/java/org/embulk/filter/SpeedometerFilterPlugin.java

@@ -22,6 +22,7 @@ import org.embulk.spi.Schema;
 import org.embulk.spi.time.Timestamp;
 import org.embulk.spi.time.TimestampFormatter;
 import org.embulk.spi.util.Timestamps;
+import org.msgpack.value.Value;
 
 public class SpeedometerFilterPlugin
         implements FilterPlugin
@@ -188,6 +189,16 @@ public class SpeedometerFilterPlugin
                 }
             }
 
+            @Override
+            public void jsonColumn(Column column) {
+                if (pageReader.isNull(column)) {
+                    speedMonitor(column);
+                    pageBuilder.setNull(column);
+                } else {
+                    pageBuilder.setJson(column, speedMonitor(column, pageReader.getJson(column)));
+                }
+            }
+
             private void speedMonitorStartRecord() {
                 startRecordTime = System.currentTimeMillis();
             }
@@ -232,6 +243,14 @@ public class SpeedometerFilterPlugin
                 return t;
             }
 
+            private Value speedMonitor(Column column, Value v) {
+                speedMonitorForDelimiter(column);
+                // NOTE: This may not be good for performance. But, I have no other idea.
+                String s = v.toJson();
+                controller.checkSpeedLimit(startRecordTime, s != null ? s.length() : 0);
+                return v;
+            }
+
             private void speedMonitorForDelimiter(Column column) {
                 if (column.getIndex() > 0) {
                     controller.checkSpeedLimit(startRecordTime, delimiterLength);

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-filter-speedometer
 version: !ruby/object:Gem::Version
-  version: 0.3.2
+  version: 0.3.3
 platform: ruby
 authors:
 - hata
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-04-23 00:00:00.000000000 Z
+date: 2017-02-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -58,8 +58,11 @@ files:
 - src/integration-test/java/org/embulk/filter/TestSingleRun.java
 - src/integration-test/resources/big_01.csv.gz
 - src/integration-test/resources/config_big.yml
+- src/integration-test/resources/config_json.yml
 - src/integration-test/resources/config_min.yml
+- src/integration-test/resources/json_01.json.gz
 - src/integration-test/resources/min_01.csv.gz
+- src/integration-test/resources/ref_json_result_01.csv.gz
 - src/main/java/org/embulk/filter/SpeedometerFilterPlugin.java
 - src/main/java/org/embulk/filter/SpeedometerSpeedAggregator.java
 - src/main/java/org/embulk/filter/SpeedometerSpeedController.java
@@ -68,7 +71,7 @@ files:
 - src/test/java/org/embulk/filter/TestSpeedometerSpeedAggregator.java
 - src/test/java/org/embulk/filter/TestSpeedometerSpeedController.java
 - src/test/java/org/embulk/filter/TestSpeedometerUtil.java
-- classpath/embulk-filter-speedometer-0.3.2.jar
+- classpath/embulk-filter-speedometer-0.3.3.jar
 homepage: https://github.com/hata/embulk-filter-speedometer
 licenses:
 - MIT