embulk-filter-speedometer 0.3.0 → 0.3.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 412ffe6e6afcc9ad7adcc2c3bf115ff69b46e068
-  data.tar.gz: 26edc64ab161aed9f413778eefb4de6e2e61fca2
+  metadata.gz: 8b34d1dc9dc74d1b823018a0b69e4f31465df617
+  data.tar.gz: efb4575ac811267c0bf70101252293e6ce061cb5
 SHA512:
-  metadata.gz: 49cc5903582acc3e75f80e6f4a1eeddcb6b43b9751dadbd3fa51c8367c9c47adbdc56483ad0cf0b514d3446e6984a3abe299a73d28b140197f42d8d758f8ebad
-  data.tar.gz: a4a89b2f63c83e0e2c222e775058aa9176f92ad7487936aa2fbf4752e74c324e1e9603a75c212df057d2f6da0fc13f73c22fdae6d1020e588f038906548436e8
+  metadata.gz: 27b149f6be1961a37ef6120aa892b605dc1a99790bab45f40c03b7ca43bdb4acda4790b0a2eb218f3ab523ed0e32cd3584c4918690268c2d570b41dad1f26224
+  data.tar.gz: 65c229df6124ddba77c84b65be36cbf1b67cc701f17e333f8c03180cdf762bbf349fe4bf41e6a1fa734812a618529ff38dd16d069c5ec6257b04818715f505d2

data/.travis.yml

@@ -0,0 +1,3 @@
+language: java
+script: ./gradlew -DenableIntegrationTest=true gem
+

data/README.md

@@ -1,5 +1,7 @@
 # Speedometer Filter Plugin for Embulk
 
+[![Build Status](https://travis-ci.org/hata/embulk-filter-speedometer.svg?branch=master)](https://travis-ci.org/hata/embulk-filter-speedometer.svg?branch=master)
+
 Write log message of processed bytes and throughput periodically.
 This plugin works with embulk:0.6.17 or later versions because of removing deprecated APIs.
 
@@ -85,6 +87,13 @@ filters:
 $ ./gradlew gem
 ```
 
+Build with integrationTest
+
+```
+$ ./gradlew -DenableIntegrationTest=true gem
+```
+
+
 ## Note
 
 The shown data is caled based on text data size while using this filter plugin. So, the data is not the same data as read bytes and write bytes n input and output plugins. And this plugin has a little overhead to measure the bytes.

data/build.gradle

@@ -5,6 +5,8 @@ plugins {
 }
 import com.github.jrubygradle.JRubyExec
 
+apply from: 'https://raw.githubusercontent.com/hata/gradle-plugins/master/embulk-integration-test.gradle'
+
 sourceCompatibility = '1.7'
 targetCompatibility = '1.7'
 
@@ -16,7 +18,7 @@ configurations {
     provided
 }
 
-version = "0.3.0"
+version = "0.3.1"
 
 dependencies {
     compile  "org.embulk:embulk-core:0.6.17+"
@@ -70,3 +72,5 @@ Gem::Specification.new do |spec|
 end
 /$)
 }
+
+project.tasks.integrationTest.dependsOn(classpath)

data/src/integration-test/java/org/embulk/filter/TestSingleRun.java

@@ -0,0 +1,95 @@
+package org.embulk.filter;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileReader;
+import java.io.InputStreamReader;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.regex.Pattern;
+import java.util.zip.GZIPInputStream;
+import java.util.ArrayList;
+import java.util.Collections;
+
+import org.junit.Test;
+
+public class TestSingleRun {
+    static final String TEST_DIR = System.getProperty("embulk.integrationtest.dir");
+
+    // e.g. {speedometer: {active: 4, total: 13.5mb, sec: 1:51, speed: 121kb/s, records: 269,748, record-speed: 2,435/s}}
+    static final Pattern logLinePattern = Pattern.compile("\\{speedometer: \\{active: [^,]+, total: [^,]+, sec: [^,]+, speed: [^,]+, records: \\S+, record-speed: [^\\}]+\\}\\}");
+
+    private static String getTestFile(String name) {
+        return TEST_DIR + File.separator + name;
+    }
+
+    @Test
+    public void testValidateMinOutputFile() throws Exception {
+        validateResultFiles("min_01.csv.gz", "result_min_000.00.csv", "result_min_001.00.csv");
+    }
+
+    @Test
+    public void testValidateBigOutputFile() throws Exception {
+        validateResultFiles("big_01.csv.gz", "result_big_000.00.csv", "result_big_001.00.csv");
+    }
+
+    @Test
+    public void testSpeedometerMinLog() throws Exception {
+        validateSpeedometerLog("config_min.yml.run.log");
+    }
+
+    @Test
+    public void testSpeedometerBigLog() throws Exception {
+        validateSpeedometerLog("config_big.yml.run.log");
+    }
+
+    private void validateSpeedometerLog(String logFile) throws Exception {
+        boolean found = false;
+        try (BufferedReader r = new BufferedReader(new FileReader(getTestFile(logFile)))) {
+            String line = r.readLine();
+            while (line != null) {
+                if (logLinePattern.matcher(line).find()) {
+                    found = true;
+                    break;
+                }
+                line = r.readLine();
+            }
+        }
+        assertTrue("Verify there are speedometer log lines.", found);
+    }
+
+    private void validateResultFiles(String gzipSrcFile, String... resultFiles) throws Exception {
+        ArrayList inList = new ArrayList();
+        ArrayList outList = new ArrayList();
+
+        try (BufferedReader reader = new BufferedReader(new InputStreamReader(
+            new GZIPInputStream(new FileInputStream(getTestFile(gzipSrcFile)))))) {
+            String line = reader.readLine(); // Discard a header line
+            line = reader.readLine();
+            while (line != null) {
+                inList.add(line);
+                line = reader.readLine();
+            }
+        }
+
+        for (String resultFile : resultFiles) {
+            try (BufferedReader reader = new BufferedReader(new FileReader(getTestFile(resultFile)))) {
+                String line = reader.readLine(); // Discard a header line
+                line = reader.readLine();
+                while (line != null) {
+                    outList.add(line);
+                    line = reader.readLine();
+                }
+            }
+        }
+
+        Collections.sort(inList);
+        Collections.sort(outList);
+
+        assertEquals("Verify input and output lines are identical.", inList, outList);
+    }
+}

data/src/integration-test/resources/config_big.yml

@@ -0,0 +1,45 @@
+in:
+  type: file
+  path_prefix: ./big_
+  decoders:
+  - {type: gzip}
+  parser:
+    charset: UTF-8
+    newline: CRLF
+    type: csv
+    delimiter: ','
+    quote: '"'
+    trim_if_not_quoted: false
+    skip_header_lines: 1
+    allow_extra_columns: false
+    allow_optional_columns: false
+    columns:
+    - {name: id, type: long}
+    - {name: account, type: long}
+    - {name: time, type: timestamp, format: '%Y-%m-%d %H:%M:%S'}
+    - {name: purchase, type: timestamp, format: '%Y%m%d'}
+    - {name: enabled, type: boolean}
+    - {name: comment, type: string}
+filters:
+  - type: speedometer
+    log_interval_seconds: 5
+    delimiter: ", "
+    record_padding_size: 2
+    speed_limit: 250000
+    column_options:
+      time: {format: '%Y-%m-%d %H:%M:%S'}
+      purchase: {format: '%Y%m%d'}
+out:
+  type: file
+  path_prefix: ./result_big_
+  file_ext: csv
+  formatter:
+    type: csv
+    quote_policy: MINIMAL
+    default_timezone: UTC
+    newline: LF
+    column_options:
+      time: {format: '%Y-%m-%d %H:%M:%S'}
+      purchase: {format: '%Y%m%d'}
+
+

data/src/integration-test/resources/config_min.yml

@@ -0,0 +1,42 @@
+in:
+  type: file
+  path_prefix: ./min_
+  decoders:
+  - {type: gzip}
+  parser:
+    charset: UTF-8
+    newline: CRLF
+    type: csv
+    delimiter: ','
+    quote: '"'
+    trim_if_not_quoted: false
+    skip_header_lines: 1
+    allow_extra_columns: false
+    allow_optional_columns: false
+    columns:
+    - {name: id, type: long}
+    - {name: account, type: long}
+    - {name: time, type: timestamp, format: '%Y-%m-%d %H:%M:%S'}
+    - {name: purchase, type: timestamp, format: '%Y%m%d'}
+    - {name: enabled, type: boolean}
+    - {name: comment, type: string}
+filters:
+  - type: speedometer
+    speed_limit: 250000
+    column_options:
+      time: {format: '%Y-%m-%d %H:%M:%S'}
+      purchase: {format: '%Y%m%d'}
+out:
+  type: file
+  path_prefix: ./result_min_
+  file_ext: csv
+  formatter:
+    type: csv
+    quote_policy: MINIMAL
+    default_timezone: UTC
+    newline: LF
+    column_options:
+      time: {format: '%Y-%m-%d %H:%M:%S'}
+      purchase: {format: '%Y%m%d'}
+
+

data/src/main/java/org/embulk/filter/SpeedometerFilterPlugin.java

@@ -123,7 +123,6 @@ public class SpeedometerFilterPlugin
         @Override
         public void finish() {
             pageBuilder.finish();
-            pageOutput.finish();
         }
 
         @Override

data/src/test/java/org/embulk/filter/TestSpeedometerFilterPlugin.java

@@ -91,7 +91,6 @@ public class TestSpeedometerFilterPlugin
         new Verifications() {{
             taskSource.loadTask(PluginTask.class); times = 1;
             builder.finish(); times = 1;
-            inPageOutput.finish(); times = 1;
         }};
     }
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-filter-speedometer
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.3.1
 platform: ruby
 authors:
 - hata
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-11-27 00:00:00.000000000 Z
+date: 2016-04-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -46,6 +46,7 @@ extensions: []
 extra_rdoc_files: []
 files:
 - .gitignore
+- .travis.yml
 - LICENSE.txt
 - README.md
 - build.gradle
@@ -54,6 +55,11 @@ files:
 - gradlew
 - gradlew.bat
 - lib/embulk/filter/speedometer.rb
+- src/integration-test/java/org/embulk/filter/TestSingleRun.java
+- src/integration-test/resources/big_01.csv.gz
+- src/integration-test/resources/config_big.yml
+- src/integration-test/resources/config_min.yml
+- src/integration-test/resources/min_01.csv.gz
 - src/main/java/org/embulk/filter/SpeedometerFilterPlugin.java
 - src/main/java/org/embulk/filter/SpeedometerSpeedAggregator.java
 - src/main/java/org/embulk/filter/SpeedometerSpeedController.java
@@ -62,7 +68,7 @@ files:
 - src/test/java/org/embulk/filter/TestSpeedometerSpeedAggregator.java
 - src/test/java/org/embulk/filter/TestSpeedometerSpeedController.java
 - src/test/java/org/embulk/filter/TestSpeedometerUtil.java
-- classpath/embulk-filter-speedometer-0.3.0.jar
+- classpath/embulk-filter-speedometer-0.3.1.jar
 homepage: https://github.com/hata/embulk-filter-speedometer
 licenses:
 - MIT