embulk-filter-speedometer 0.3.0 → 0.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.travis.yml +3 -0
- data/README.md +9 -0
- data/build.gradle +5 -1
- data/src/integration-test/java/org/embulk/filter/TestSingleRun.java +95 -0
- data/src/integration-test/resources/big_01.csv.gz +0 -0
- data/src/integration-test/resources/config_big.yml +45 -0
- data/src/integration-test/resources/config_min.yml +42 -0
- data/src/integration-test/resources/min_01.csv.gz +0 -0
- data/src/main/java/org/embulk/filter/SpeedometerFilterPlugin.java +0 -1
- data/src/test/java/org/embulk/filter/TestSpeedometerFilterPlugin.java +0 -1
- metadata +9 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 8b34d1dc9dc74d1b823018a0b69e4f31465df617
|
4
|
+
data.tar.gz: efb4575ac811267c0bf70101252293e6ce061cb5
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 27b149f6be1961a37ef6120aa892b605dc1a99790bab45f40c03b7ca43bdb4acda4790b0a2eb218f3ab523ed0e32cd3584c4918690268c2d570b41dad1f26224
|
7
|
+
data.tar.gz: 65c229df6124ddba77c84b65be36cbf1b67cc701f17e333f8c03180cdf762bbf349fe4bf41e6a1fa734812a618529ff38dd16d069c5ec6257b04818715f505d2
|
data/.travis.yml
ADDED
data/README.md
CHANGED
@@ -1,5 +1,7 @@
|
|
1
1
|
# Speedometer Filter Plugin for Embulk
|
2
2
|
|
3
|
+
[](https://travis-ci.org/hata/embulk-filter-speedometer.svg?branch=master)
|
4
|
+
|
3
5
|
Write log message of processed bytes and throughput periodically.
|
4
6
|
This plugin works with embulk:0.6.17 or later versions because of removing deprecated APIs.
|
5
7
|
|
@@ -85,6 +87,13 @@ filters:
|
|
85
87
|
$ ./gradlew gem
|
86
88
|
```
|
87
89
|
|
90
|
+
Build with integrationTest
|
91
|
+
|
92
|
+
```
|
93
|
+
$ ./gradlew -DenableIntegrationTest=true gem
|
94
|
+
```
|
95
|
+
|
96
|
+
|
88
97
|
## Note
|
89
98
|
|
90
99
|
The shown data is caled based on text data size while using this filter plugin. So, the data is not the same data as read bytes and write bytes n input and output plugins. And this plugin has a little overhead to measure the bytes.
|
data/build.gradle
CHANGED
@@ -5,6 +5,8 @@ plugins {
|
|
5
5
|
}
|
6
6
|
import com.github.jrubygradle.JRubyExec
|
7
7
|
|
8
|
+
apply from: 'https://raw.githubusercontent.com/hata/gradle-plugins/master/embulk-integration-test.gradle'
|
9
|
+
|
8
10
|
sourceCompatibility = '1.7'
|
9
11
|
targetCompatibility = '1.7'
|
10
12
|
|
@@ -16,7 +18,7 @@ configurations {
|
|
16
18
|
provided
|
17
19
|
}
|
18
20
|
|
19
|
-
version = "0.3.
|
21
|
+
version = "0.3.1"
|
20
22
|
|
21
23
|
dependencies {
|
22
24
|
compile "org.embulk:embulk-core:0.6.17+"
|
@@ -70,3 +72,5 @@ Gem::Specification.new do |spec|
|
|
70
72
|
end
|
71
73
|
/$)
|
72
74
|
}
|
75
|
+
|
76
|
+
project.tasks.integrationTest.dependsOn(classpath)
|
@@ -0,0 +1,95 @@
|
|
1
|
+
package org.embulk.filter;
|
2
|
+
|
3
|
+
import static org.junit.Assert.assertEquals;
|
4
|
+
import static org.junit.Assert.assertTrue;
|
5
|
+
|
6
|
+
import java.io.BufferedReader;
|
7
|
+
import java.io.File;
|
8
|
+
import java.io.FileInputStream;
|
9
|
+
import java.io.FileReader;
|
10
|
+
import java.io.InputStreamReader;
|
11
|
+
import java.io.IOException;
|
12
|
+
import java.io.InputStream;
|
13
|
+
import java.util.regex.Pattern;
|
14
|
+
import java.util.zip.GZIPInputStream;
|
15
|
+
import java.util.ArrayList;
|
16
|
+
import java.util.Collections;
|
17
|
+
|
18
|
+
import org.junit.Test;
|
19
|
+
|
20
|
+
public class TestSingleRun {
|
21
|
+
static final String TEST_DIR = System.getProperty("embulk.integrationtest.dir");
|
22
|
+
|
23
|
+
// e.g. {speedometer: {active: 4, total: 13.5mb, sec: 1:51, speed: 121kb/s, records: 269,748, record-speed: 2,435/s}}
|
24
|
+
static final Pattern logLinePattern = Pattern.compile("\\{speedometer: \\{active: [^,]+, total: [^,]+, sec: [^,]+, speed: [^,]+, records: \\S+, record-speed: [^\\}]+\\}\\}");
|
25
|
+
|
26
|
+
private static String getTestFile(String name) {
|
27
|
+
return TEST_DIR + File.separator + name;
|
28
|
+
}
|
29
|
+
|
30
|
+
@Test
|
31
|
+
public void testValidateMinOutputFile() throws Exception {
|
32
|
+
validateResultFiles("min_01.csv.gz", "result_min_000.00.csv", "result_min_001.00.csv");
|
33
|
+
}
|
34
|
+
|
35
|
+
@Test
|
36
|
+
public void testValidateBigOutputFile() throws Exception {
|
37
|
+
validateResultFiles("big_01.csv.gz", "result_big_000.00.csv", "result_big_001.00.csv");
|
38
|
+
}
|
39
|
+
|
40
|
+
@Test
|
41
|
+
public void testSpeedometerMinLog() throws Exception {
|
42
|
+
validateSpeedometerLog("config_min.yml.run.log");
|
43
|
+
}
|
44
|
+
|
45
|
+
@Test
|
46
|
+
public void testSpeedometerBigLog() throws Exception {
|
47
|
+
validateSpeedometerLog("config_big.yml.run.log");
|
48
|
+
}
|
49
|
+
|
50
|
+
private void validateSpeedometerLog(String logFile) throws Exception {
|
51
|
+
boolean found = false;
|
52
|
+
try (BufferedReader r = new BufferedReader(new FileReader(getTestFile(logFile)))) {
|
53
|
+
String line = r.readLine();
|
54
|
+
while (line != null) {
|
55
|
+
if (logLinePattern.matcher(line).find()) {
|
56
|
+
found = true;
|
57
|
+
break;
|
58
|
+
}
|
59
|
+
line = r.readLine();
|
60
|
+
}
|
61
|
+
}
|
62
|
+
assertTrue("Verify there are speedometer log lines.", found);
|
63
|
+
}
|
64
|
+
|
65
|
+
private void validateResultFiles(String gzipSrcFile, String... resultFiles) throws Exception {
|
66
|
+
ArrayList inList = new ArrayList();
|
67
|
+
ArrayList outList = new ArrayList();
|
68
|
+
|
69
|
+
try (BufferedReader reader = new BufferedReader(new InputStreamReader(
|
70
|
+
new GZIPInputStream(new FileInputStream(getTestFile(gzipSrcFile)))))) {
|
71
|
+
String line = reader.readLine(); // Discard a header line
|
72
|
+
line = reader.readLine();
|
73
|
+
while (line != null) {
|
74
|
+
inList.add(line);
|
75
|
+
line = reader.readLine();
|
76
|
+
}
|
77
|
+
}
|
78
|
+
|
79
|
+
for (String resultFile : resultFiles) {
|
80
|
+
try (BufferedReader reader = new BufferedReader(new FileReader(getTestFile(resultFile)))) {
|
81
|
+
String line = reader.readLine(); // Discard a header line
|
82
|
+
line = reader.readLine();
|
83
|
+
while (line != null) {
|
84
|
+
outList.add(line);
|
85
|
+
line = reader.readLine();
|
86
|
+
}
|
87
|
+
}
|
88
|
+
}
|
89
|
+
|
90
|
+
Collections.sort(inList);
|
91
|
+
Collections.sort(outList);
|
92
|
+
|
93
|
+
assertEquals("Verify input and output lines are identical.", inList, outList);
|
94
|
+
}
|
95
|
+
}
|
Binary file
|
@@ -0,0 +1,45 @@
|
|
1
|
+
in:
|
2
|
+
type: file
|
3
|
+
path_prefix: ./big_
|
4
|
+
decoders:
|
5
|
+
- {type: gzip}
|
6
|
+
parser:
|
7
|
+
charset: UTF-8
|
8
|
+
newline: CRLF
|
9
|
+
type: csv
|
10
|
+
delimiter: ','
|
11
|
+
quote: '"'
|
12
|
+
trim_if_not_quoted: false
|
13
|
+
skip_header_lines: 1
|
14
|
+
allow_extra_columns: false
|
15
|
+
allow_optional_columns: false
|
16
|
+
columns:
|
17
|
+
- {name: id, type: long}
|
18
|
+
- {name: account, type: long}
|
19
|
+
- {name: time, type: timestamp, format: '%Y-%m-%d %H:%M:%S'}
|
20
|
+
- {name: purchase, type: timestamp, format: '%Y%m%d'}
|
21
|
+
- {name: enabled, type: boolean}
|
22
|
+
- {name: comment, type: string}
|
23
|
+
filters:
|
24
|
+
- type: speedometer
|
25
|
+
log_interval_seconds: 5
|
26
|
+
delimiter: ", "
|
27
|
+
record_padding_size: 2
|
28
|
+
speed_limit: 250000
|
29
|
+
column_options:
|
30
|
+
time: {format: '%Y-%m-%d %H:%M:%S'}
|
31
|
+
purchase: {format: '%Y%m%d'}
|
32
|
+
out:
|
33
|
+
type: file
|
34
|
+
path_prefix: ./result_big_
|
35
|
+
file_ext: csv
|
36
|
+
formatter:
|
37
|
+
type: csv
|
38
|
+
quote_policy: MINIMAL
|
39
|
+
default_timezone: UTC
|
40
|
+
newline: LF
|
41
|
+
column_options:
|
42
|
+
time: {format: '%Y-%m-%d %H:%M:%S'}
|
43
|
+
purchase: {format: '%Y%m%d'}
|
44
|
+
|
45
|
+
|
@@ -0,0 +1,42 @@
|
|
1
|
+
in:
|
2
|
+
type: file
|
3
|
+
path_prefix: ./min_
|
4
|
+
decoders:
|
5
|
+
- {type: gzip}
|
6
|
+
parser:
|
7
|
+
charset: UTF-8
|
8
|
+
newline: CRLF
|
9
|
+
type: csv
|
10
|
+
delimiter: ','
|
11
|
+
quote: '"'
|
12
|
+
trim_if_not_quoted: false
|
13
|
+
skip_header_lines: 1
|
14
|
+
allow_extra_columns: false
|
15
|
+
allow_optional_columns: false
|
16
|
+
columns:
|
17
|
+
- {name: id, type: long}
|
18
|
+
- {name: account, type: long}
|
19
|
+
- {name: time, type: timestamp, format: '%Y-%m-%d %H:%M:%S'}
|
20
|
+
- {name: purchase, type: timestamp, format: '%Y%m%d'}
|
21
|
+
- {name: enabled, type: boolean}
|
22
|
+
- {name: comment, type: string}
|
23
|
+
filters:
|
24
|
+
- type: speedometer
|
25
|
+
speed_limit: 250000
|
26
|
+
column_options:
|
27
|
+
time: {format: '%Y-%m-%d %H:%M:%S'}
|
28
|
+
purchase: {format: '%Y%m%d'}
|
29
|
+
out:
|
30
|
+
type: file
|
31
|
+
path_prefix: ./result_min_
|
32
|
+
file_ext: csv
|
33
|
+
formatter:
|
34
|
+
type: csv
|
35
|
+
quote_policy: MINIMAL
|
36
|
+
default_timezone: UTC
|
37
|
+
newline: LF
|
38
|
+
column_options:
|
39
|
+
time: {format: '%Y-%m-%d %H:%M:%S'}
|
40
|
+
purchase: {format: '%Y%m%d'}
|
41
|
+
|
42
|
+
|
Binary file
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: embulk-filter-speedometer
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.3.
|
4
|
+
version: 0.3.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- hata
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2016-04-23 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
requirement: !ruby/object:Gem::Requirement
|
@@ -46,6 +46,7 @@ extensions: []
|
|
46
46
|
extra_rdoc_files: []
|
47
47
|
files:
|
48
48
|
- .gitignore
|
49
|
+
- .travis.yml
|
49
50
|
- LICENSE.txt
|
50
51
|
- README.md
|
51
52
|
- build.gradle
|
@@ -54,6 +55,11 @@ files:
|
|
54
55
|
- gradlew
|
55
56
|
- gradlew.bat
|
56
57
|
- lib/embulk/filter/speedometer.rb
|
58
|
+
- src/integration-test/java/org/embulk/filter/TestSingleRun.java
|
59
|
+
- src/integration-test/resources/big_01.csv.gz
|
60
|
+
- src/integration-test/resources/config_big.yml
|
61
|
+
- src/integration-test/resources/config_min.yml
|
62
|
+
- src/integration-test/resources/min_01.csv.gz
|
57
63
|
- src/main/java/org/embulk/filter/SpeedometerFilterPlugin.java
|
58
64
|
- src/main/java/org/embulk/filter/SpeedometerSpeedAggregator.java
|
59
65
|
- src/main/java/org/embulk/filter/SpeedometerSpeedController.java
|
@@ -62,7 +68,7 @@ files:
|
|
62
68
|
- src/test/java/org/embulk/filter/TestSpeedometerSpeedAggregator.java
|
63
69
|
- src/test/java/org/embulk/filter/TestSpeedometerSpeedController.java
|
64
70
|
- src/test/java/org/embulk/filter/TestSpeedometerUtil.java
|
65
|
-
- classpath/embulk-filter-speedometer-0.3.
|
71
|
+
- classpath/embulk-filter-speedometer-0.3.1.jar
|
66
72
|
homepage: https://github.com/hata/embulk-filter-speedometer
|
67
73
|
licenses:
|
68
74
|
- MIT
|