embulk-input-randomj 0.3.0 → 0.4.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.travis.yml +2 -0
- data/README.md +7 -0
- data/build.gradle +2 -1
- data/gradle/wrapper/gradle-wrapper.jar +0 -0
- data/gradle/wrapper/gradle-wrapper.properties +2 -2
- data/gradlew +11 -8
- data/src/main/java/org/embulk/input/randomj/RandomjColumnVisitor.java +68 -39
- data/src/main/java/org/embulk/input/randomj/RandomjInputPlugin.java +2 -7
- metadata +3 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 1165a1e2c182b649c7a2fbf73de530de55a5497f
|
4
|
+
data.tar.gz: ab71604a4f4af3d7765433b7e380e5f509f7b7e2
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 9fc2bea3f88fcc01b2a2e33061d0a28d2588345d8b293ca567cb7aaadcd1daf3975c70f24328e2facbb2e4dd511e06dec5f854520885e77984bfa7a03a508866
|
7
|
+
data.tar.gz: a6d3961d0f3fcd6ebd3f02545f6272a05805680de3923cb2ff1a3dc8db588219e429549a7279b4cca14d719feeed126046f465d7ab7ede26a92f123fe58ea787
|
data/.travis.yml
CHANGED
data/README.md
CHANGED
data/build.gradle
CHANGED
@@ -4,6 +4,7 @@ plugins {
|
|
4
4
|
id "java"
|
5
5
|
id "checkstyle"
|
6
6
|
id "findbugs"
|
7
|
+
id "org.sonarqube" version "2.5"
|
7
8
|
}
|
8
9
|
import com.github.jrubygradle.JRubyExec
|
9
10
|
repositories {
|
@@ -14,7 +15,7 @@ configurations {
|
|
14
15
|
provided
|
15
16
|
}
|
16
17
|
|
17
|
-
version = "0.
|
18
|
+
version = "0.4.0"
|
18
19
|
|
19
20
|
sourceCompatibility = 1.8
|
20
21
|
targetCompatibility = 1.8
|
Binary file
|
@@ -1,6 +1,6 @@
|
|
1
|
-
#Sun
|
1
|
+
#Sun Oct 08 14:24:25 JST 2017
|
2
2
|
distributionBase=GRADLE_USER_HOME
|
3
3
|
distributionPath=wrapper/dists
|
4
4
|
zipStoreBase=GRADLE_USER_HOME
|
5
5
|
zipStorePath=wrapper/dists
|
6
|
-
distributionUrl=https\://services.gradle.org/distributions/gradle-
|
6
|
+
distributionUrl=https\://services.gradle.org/distributions/gradle-4.2.1-bin.zip
|
data/gradlew
CHANGED
@@ -1,4 +1,4 @@
|
|
1
|
-
#!/usr/bin/env
|
1
|
+
#!/usr/bin/env sh
|
2
2
|
|
3
3
|
##############################################################################
|
4
4
|
##
|
@@ -154,16 +154,19 @@ if $cygwin ; then
|
|
154
154
|
esac
|
155
155
|
fi
|
156
156
|
|
157
|
-
#
|
158
|
-
|
159
|
-
|
157
|
+
# Escape application args
|
158
|
+
save ( ) {
|
159
|
+
for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done
|
160
|
+
echo " "
|
160
161
|
}
|
161
|
-
|
162
|
-
|
162
|
+
APP_ARGS=$(save "$@")
|
163
|
+
|
164
|
+
# Collect all arguments for the java command, following the shell quoting and substitution rules
|
165
|
+
eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS"
|
163
166
|
|
164
167
|
# by default we should be in the correct project dir, but when run from Finder on Mac, the cwd is wrong
|
165
|
-
if [
|
168
|
+
if [ "$(uname)" = "Darwin" ] && [ "$HOME" = "$PWD" ]; then
|
166
169
|
cd "$(dirname "$0")"
|
167
170
|
fi
|
168
171
|
|
169
|
-
exec "$JAVACMD" "
|
172
|
+
exec "$JAVACMD" "$@"
|
@@ -10,7 +10,6 @@ import org.embulk.spi.time.Timestamp;
|
|
10
10
|
|
11
11
|
import java.time.LocalDateTime;
|
12
12
|
import java.time.ZoneId;
|
13
|
-
import java.util.HashMap;
|
14
13
|
import java.util.Map;
|
15
14
|
import java.util.Random;
|
16
15
|
|
@@ -21,14 +20,14 @@ public class RandomjColumnVisitor
|
|
21
20
|
private final PluginTask task;
|
22
21
|
private final Integer row;
|
23
22
|
private final Random rnd;
|
24
|
-
private final
|
23
|
+
private final Map<Column, Map<String, Integer>> columnOptions;
|
25
24
|
private final RandomStringGenerator generator = new RandomStringGenerator.Builder()
|
26
25
|
.withinRange('0', 'z')
|
27
26
|
.filteredBy(CharacterPredicates.LETTERS, CharacterPredicates.DIGITS)
|
28
27
|
.build();
|
29
28
|
private final ZoneId zoneId = ZoneId.systemDefault();
|
30
29
|
|
31
|
-
public RandomjColumnVisitor(PageBuilder pageBuilder, PluginTask task, Integer row,
|
30
|
+
public RandomjColumnVisitor(PageBuilder pageBuilder, PluginTask task, Integer row, Map<Column, Map<String, Integer>> columnOptions)
|
32
31
|
{
|
33
32
|
this.task = task;
|
34
33
|
this.pageBuilder = pageBuilder;
|
@@ -40,11 +39,17 @@ public class RandomjColumnVisitor
|
|
40
39
|
@Override
|
41
40
|
public void booleanColumn(Column column)
|
42
41
|
{
|
43
|
-
|
44
|
-
|
42
|
+
Integer nrate = columnOptions.get(column).get("null_rate");
|
43
|
+
if (Math.random() < (double) nrate / 10000) {
|
44
|
+
pageBuilder.setNull(column);
|
45
45
|
}
|
46
46
|
else {
|
47
|
-
|
47
|
+
if (Math.random() < 0.5) {
|
48
|
+
pageBuilder.setBoolean(column, false);
|
49
|
+
}
|
50
|
+
else {
|
51
|
+
pageBuilder.setBoolean(column, true);
|
52
|
+
}
|
48
53
|
}
|
49
54
|
}
|
50
55
|
|
@@ -56,66 +61,90 @@ public class RandomjColumnVisitor
|
|
56
61
|
pageBuilder.setLong(column, row);
|
57
62
|
}
|
58
63
|
else {
|
59
|
-
Integer
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
|
64
|
-
|
64
|
+
Integer nrate = columnOptions.get(column).get("null_rate");
|
65
|
+
if (Math.random() < (double) nrate / 10000) {
|
66
|
+
pageBuilder.setNull(column);
|
67
|
+
}
|
68
|
+
else {
|
69
|
+
Integer max = columnOptions.get(column).get("max_value");
|
70
|
+
Integer min = columnOptions.get(column).get("min_value");
|
71
|
+
if (max != null) {
|
72
|
+
if (min != null) {
|
73
|
+
Integer s = min + rnd.nextInt((max - min));
|
74
|
+
pageBuilder.setLong(column, s);
|
75
|
+
}
|
76
|
+
else {
|
77
|
+
pageBuilder.setLong(column, rnd.nextInt(max));
|
78
|
+
}
|
65
79
|
}
|
66
80
|
else {
|
67
|
-
pageBuilder.setLong(column, rnd.nextInt(
|
81
|
+
pageBuilder.setLong(column, rnd.nextInt(10000));
|
68
82
|
}
|
69
83
|
}
|
70
|
-
else {
|
71
|
-
pageBuilder.setLong(column, rnd.nextInt(10000));
|
72
|
-
}
|
73
84
|
}
|
74
85
|
}
|
75
86
|
|
76
87
|
@Override
|
77
88
|
public void doubleColumn(Column column)
|
78
89
|
{
|
79
|
-
Integer
|
80
|
-
|
81
|
-
|
82
|
-
|
83
|
-
|
84
|
-
|
90
|
+
Integer nrate = columnOptions.get(column).get("null_rate");
|
91
|
+
if (Math.random() < (double) nrate / 10000) {
|
92
|
+
pageBuilder.setNull(column);
|
93
|
+
}
|
94
|
+
else {
|
95
|
+
Integer max = columnOptions.get(column).get("max_value");
|
96
|
+
Integer min = columnOptions.get(column).get("min_value");
|
97
|
+
if (max != null) {
|
98
|
+
if (min != null) {
|
99
|
+
Double d = min + rnd.nextInt((max - min) - 1) + rnd.nextDouble();
|
100
|
+
pageBuilder.setDouble(column, d);
|
101
|
+
}
|
102
|
+
else {
|
103
|
+
Double d = rnd.nextInt(max - 1) + rnd.nextDouble();
|
104
|
+
pageBuilder.setDouble(column, d);
|
105
|
+
}
|
85
106
|
}
|
86
107
|
else {
|
87
|
-
|
88
|
-
pageBuilder.setDouble(column, d);
|
108
|
+
pageBuilder.setDouble(column, rnd.nextDouble() * 10000);
|
89
109
|
}
|
90
110
|
}
|
91
|
-
else {
|
92
|
-
pageBuilder.setDouble(column, rnd.nextDouble() * 10000);
|
93
|
-
}
|
94
111
|
}
|
95
112
|
|
96
113
|
@Override
|
97
114
|
public void stringColumn(Column column)
|
98
115
|
{
|
99
|
-
|
100
|
-
if (
|
101
|
-
pageBuilder.
|
116
|
+
Integer nrate = columnOptions.get(column).get("null_rate");
|
117
|
+
if (Math.random() < (double) nrate / 10000) {
|
118
|
+
pageBuilder.setNull(column);
|
102
119
|
}
|
103
120
|
else {
|
104
|
-
|
121
|
+
final Integer length = columnOptions.get(column).getOrDefault("length", 0);
|
122
|
+
if (length == 0) {
|
123
|
+
pageBuilder.setString(column, generator.generate(32));
|
124
|
+
}
|
125
|
+
else {
|
126
|
+
pageBuilder.setString(column, generator.generate(length));
|
127
|
+
}
|
105
128
|
}
|
106
129
|
}
|
107
130
|
|
108
131
|
@Override
|
109
132
|
public void timestampColumn(Column column)
|
110
133
|
{
|
111
|
-
|
112
|
-
|
113
|
-
|
114
|
-
|
115
|
-
|
116
|
-
|
117
|
-
|
118
|
-
|
134
|
+
Integer nrate = columnOptions.get(column).get("null_rate");
|
135
|
+
if (Math.random() < (double) nrate / 10000) {
|
136
|
+
pageBuilder.setNull(column);
|
137
|
+
}
|
138
|
+
else {
|
139
|
+
final double randd = Math.random();
|
140
|
+
LocalDateTime randomDate = LocalDateTime.now()
|
141
|
+
.plusDays((long) (randd * 100))
|
142
|
+
.plusSeconds((long) (randd * 1000000));
|
143
|
+
Timestamp timestamp = Timestamp.ofEpochSecond(
|
144
|
+
randomDate.atZone(zoneId).toEpochSecond()
|
145
|
+
);
|
146
|
+
pageBuilder.setTimestamp(column, timestamp);
|
147
|
+
}
|
119
148
|
}
|
120
149
|
|
121
150
|
@Override
|
@@ -1,7 +1,5 @@
|
|
1
1
|
package org.embulk.input.randomj;
|
2
2
|
|
3
|
-
import org.apache.commons.text.CharacterPredicates;
|
4
|
-
import org.apache.commons.text.RandomStringGenerator;
|
5
3
|
import org.embulk.config.Config;
|
6
4
|
import org.embulk.config.ConfigDefault;
|
7
5
|
import org.embulk.config.ConfigDiff;
|
@@ -26,11 +24,6 @@ import java.util.stream.IntStream;
|
|
26
24
|
public class RandomjInputPlugin
|
27
25
|
implements InputPlugin
|
28
26
|
{
|
29
|
-
private final RandomStringGenerator generator = new RandomStringGenerator.Builder()
|
30
|
-
.withinRange('0', 'z')
|
31
|
-
.filteredBy(CharacterPredicates.LETTERS, CharacterPredicates.DIGITS)
|
32
|
-
.build();
|
33
|
-
|
34
27
|
public interface PluginTask
|
35
28
|
extends Task
|
36
29
|
{
|
@@ -77,6 +70,7 @@ public class RandomjInputPlugin
|
|
77
70
|
Schema schema, int taskCount,
|
78
71
|
List<TaskReport> successTaskReports)
|
79
72
|
{
|
73
|
+
// throw new UnsupportedOperationException("randomj input plugin does not support cleanup");
|
80
74
|
}
|
81
75
|
|
82
76
|
@Override
|
@@ -115,6 +109,7 @@ public class RandomjInputPlugin
|
|
115
109
|
HashMap<String, Integer> miniMap = new HashMap<>();
|
116
110
|
ColumnConfig c = schemaConfig.getColumn(column.getIndex());
|
117
111
|
miniMap.put("length", c.getOption().get(Integer.class, "length", 0));
|
112
|
+
miniMap.put("null_rate", c.getOption().get(Integer.class, "null_rate", 0));
|
118
113
|
miniMap.put("max_value", c.getOption().get(Integer.class, "max_value", null));
|
119
114
|
miniMap.put("min_value", c.getOption().get(Integer.class, "min_value", null));
|
120
115
|
lengthMap.put(column, miniMap);
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: embulk-input-randomj
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.4.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- yuokada
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2017-
|
11
|
+
date: 2017-12-02 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
requirement: !ruby/object:Gem::Requirement
|
@@ -62,7 +62,7 @@ files:
|
|
62
62
|
- src/test/java/org/embulk/input/randomj/TestRandomjInputPlugin.java
|
63
63
|
- classpath/commons-lang3-3.5.jar
|
64
64
|
- classpath/commons-text-1.1.jar
|
65
|
-
- classpath/embulk-input-randomj-0.
|
65
|
+
- classpath/embulk-input-randomj-0.4.0.jar
|
66
66
|
homepage: https://github.com/yuokada/embulk-input-randomj
|
67
67
|
licenses:
|
68
68
|
- MIT
|