embulk-input-randomj 0.2.0 → 0.3.0
Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 664734a687060fbda08c7e14593684f63a501767
|
4
|
+
data.tar.gz: 3e3aa292e1eb2916cf8b516c3bc486f194e67417
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: c07094226ddafc4ce96a2f8738a8a73c86a9e143d3de802b048ee80990459f6b9fff0db75adc0b1d15a5127e91a61241f1556db7540bf1d1f6f0e5ccdc055180
|
7
|
+
data.tar.gz: 6a97183220e98b63c92c2cccb7b54d8ae6456b796670abeda1d10b1bc3a6effbdde9f6182980dbb1ff13289802741f89ee9a60f8b7b01723d18d7cfc278f9bee
|
data/README.md
CHANGED
@@ -45,8 +45,24 @@ in:
|
|
45
45
|
- {name: purchase, type: timestamp, format: '%Y/%m/%d'}
|
46
46
|
```
|
47
47
|
|
48
|
+
Add `length`, `max_value`, `min_value` option (from 0.3.0)
|
49
|
+
```yaml
|
50
|
+
in:
|
51
|
+
type: randomj
|
52
|
+
rows: 16
|
53
|
+
threads: 1
|
54
|
+
primary_key: myid
|
55
|
+
schema:
|
56
|
+
- {name: myid, type: long}
|
57
|
+
- {name: named, type: string, length: 12}
|
58
|
+
- {name: price, type: long, max_value: 1080, min_value: 100}
|
59
|
+
- {name: purchase, type: timestamp, format: '%Y/%m/%d'}
|
60
|
+
```
|
61
|
+
|
48
62
|
## Usage
|
49
63
|
|
64
|
+
### Example1
|
65
|
+
|
50
66
|
```shell
|
51
67
|
|
52
68
|
% cat example/config.yml
|
@@ -96,6 +112,61 @@ in:
|
|
96
112
|
|
97
113
|
```
|
98
114
|
|
115
|
+
### Example2
|
116
|
+
|
117
|
+
- `named_s` return string with length 8
|
118
|
+
- `score` return value between `100~255`
|
119
|
+
- `rate` return value between `-100~100`
|
120
|
+
|
121
|
+
```shell
|
122
|
+
|
123
|
+
% cat example/config.yml
|
124
|
+
in:
|
125
|
+
type: randomj
|
126
|
+
rows: 16
|
127
|
+
threads: 1
|
128
|
+
# default_timezone: Asia/Tokyo
|
129
|
+
primary_key: myid
|
130
|
+
schema:
|
131
|
+
- {name: myid, type: long}
|
132
|
+
- {name: named, type: string}
|
133
|
+
- {name: named_s, type: string, length: 8}
|
134
|
+
- {name: x_flag, type: boolean}
|
135
|
+
- {name: rate, type: double, max_value: 100, min_value: -100}
|
136
|
+
- {name: score, type: long, max_value: 255, min_value: 100}
|
137
|
+
- {name: time, type: timestamp, format: '%Y-%m-%d %H:%M:%S'}
|
138
|
+
- {name: purchase, type: timestamp, format: '%Y/%m/%d'}
|
139
|
+
|
140
|
+
out:
|
141
|
+
type: stdout
|
142
|
+
|
143
|
+
|
144
|
+
% embulk run -I lib example/config.yml
|
145
|
+
2017-09-10 04:45:04.894 +0900: Embulk v0.8.32
|
146
|
+
2017-09-10 04:45:10.212 +0900 [INFO] (0001:transaction): Loaded plugin embulk/input/randomj from a load path
|
147
|
+
2017-09-10 04:45:10.246 +0900 [INFO] (0001:transaction): Using local thread executor with max_threads=8 / output tasks 4 = input tasks 1 * 4
|
148
|
+
2017-09-10 04:45:10.263 +0900 [INFO] (0001:transaction): {done: 0 / 1, running: 0}
|
149
|
+
1,BOcbVJX5bWL5wRBJc532trxvwhQpmg3d,yHwXATfG,true,-79.62544211154894,129,2017-12-05 22:31:35,2017/12/26
|
150
|
+
2,N2gljQxd4yDBzJjK9iSRUdROtaZGUEl7,zSrEMjzC,false,-11.47506884041689,194,2017-09-17 15:56:18,2017/12/06
|
151
|
+
3,PJvKkf0wwpGqGMlc7OjUhjZNi0pTEZIU,q6TgdoaZ,false,85.17356188437738,137,2017-10-07 17:28:43,2017/10/22
|
152
|
+
4,DA6wWE4p3zIPDK0Mp81bWczewNSMY2sq,KeobJmS1,false,79.95787440150436,221,2017-09-28 19:35:17,2017/11/20
|
153
|
+
5,8DNF4TzhVDLCFey2x1eCHryf4GdvHlyW,D2jddtEN,true,19.801687906161735,182,2017-11-24 18:43:38,2017/12/29
|
154
|
+
6,veyIxBc9u0FMwsGksMfLhvBMuIF2D7XO,6Mtz4MN9,true,26.922649237294582,176,2017-09-23 07:43:40,2017/11/18
|
155
|
+
7,HHCTLuaxAJIRHHG7cB2u9Ake9p9OSIcy,UHHKp5xX,true,9.960707451320626,108,2017-09-14 08:11:49,2017/11/05
|
156
|
+
8,HcQhHMQ4sYiXTBpvNiTqDGskuTeVEC6r,d0VSR8K8,false,-62.405292711551624,118,2017-11-11 08:06:20,2017/10/20
|
157
|
+
9,si5BWUPEEvVHvveeqSxG6ypc7pSsKtC7,bW5p9boG,false,-76.91915279000274,192,2017-09-28 19:46:53,2017/11/04
|
158
|
+
10,xnfU0aJgigJG9rPan2rwoffhN9pzLQCy,R8MV0Jpa,true,-79.40738909989871,104,2017-11-19 02:50:07,2017/09/11
|
159
|
+
11,KiRzQqfE6wRw3WjMPAmedqtHyG3MttGU,SowzDTSb,true,77.22509797548325,163,2017-12-23 18:16:30,2017/12/27
|
160
|
+
12,pQLz3fMIkN6UANwSbzJ5vhBWzF2FI7uo,uPGyHyuW,true,71.19680005107371,180,2017-11-23 16:31:30,2017/11/14
|
161
|
+
13,aFOc2qCAu5oYbxTCGkMNcZob6Tl3wl3Y,apFu34Ps,false,82.8406608691031,226,2017-10-03 06:09:25,2017/10/06
|
162
|
+
14,Kz3JGL23k7f8SR17xQBw063ApuGdeWIP,r0c0KnUC,true,-26.484829732050134,113,2017-10-01 02:40:37,2017/11/26
|
163
|
+
15,p5vGY02BzrHqk345JyAhFU7xVsA2jEZD,nhzsefns,false,-79.0184308849151,119,2017-12-15 22:59:28,2017/11/25
|
164
|
+
16,1jyxot60lCrRFMUfjyHcZ07dq05eu76a,WewnLZfw,false,-55.315211168770816,141,2017-12-11 10:36:46,2017/12/05
|
165
|
+
2017-09-10 04:45:10.344 +0900 [INFO] (0001:transaction): {done: 1 / 1, running: 0}
|
166
|
+
2017-09-10 04:45:10.351 +0900 [INFO] (main): Committed.
|
167
|
+
2017-09-10 04:45:10.351 +0900 [INFO] (main): Next config diff: {"in":{},"out":{}}
|
168
|
+
|
169
|
+
```
|
99
170
|
|
100
171
|
## Build
|
101
172
|
|
data/build.gradle
CHANGED
data/example/config.yml
CHANGED
@@ -7,9 +7,10 @@ in:
|
|
7
7
|
schema:
|
8
8
|
- {name: myid, type: long}
|
9
9
|
- {name: named, type: string}
|
10
|
+
- {name: named_s, type: string, length: 8}
|
10
11
|
- {name: x_flag, type: boolean}
|
11
|
-
- {name:
|
12
|
-
- {name: score, type: long}
|
12
|
+
- {name: rate, type: double, max_value: 100, min_value: -100}
|
13
|
+
- {name: score, type: long, max_value: 255, min_value: 100}
|
13
14
|
- {name: time, type: timestamp, format: '%Y-%m-%d %H:%M:%S'}
|
14
15
|
- {name: purchase, type: timestamp, format: '%Y/%m/%d'}
|
15
16
|
|
data/src/main/java/org/embulk/input/randomj/{RandomColumnVisitor.java → RandomjColumnVisitor.java}
RENAMED
@@ -10,26 +10,30 @@ import org.embulk.spi.time.Timestamp;
|
|
10
10
|
|
11
11
|
import java.time.LocalDateTime;
|
12
12
|
import java.time.ZoneId;
|
13
|
+
import java.util.HashMap;
|
14
|
+
import java.util.Map;
|
13
15
|
import java.util.Random;
|
14
16
|
|
15
|
-
public class
|
17
|
+
public class RandomjColumnVisitor
|
16
18
|
implements ColumnVisitor
|
17
19
|
{
|
18
20
|
private final PageBuilder pageBuilder;
|
19
21
|
private final PluginTask task;
|
20
22
|
private final Integer row;
|
21
23
|
private final Random rnd;
|
24
|
+
private final HashMap<Column, Map<String, Integer>> columnOptions;
|
22
25
|
private final RandomStringGenerator generator = new RandomStringGenerator.Builder()
|
23
26
|
.withinRange('0', 'z')
|
24
27
|
.filteredBy(CharacterPredicates.LETTERS, CharacterPredicates.DIGITS)
|
25
28
|
.build();
|
26
29
|
private final ZoneId zoneId = ZoneId.systemDefault();
|
27
30
|
|
28
|
-
public
|
31
|
+
public RandomjColumnVisitor(PageBuilder pageBuilder, PluginTask task, Integer row, HashMap<Column, Map<String, Integer>> columnOptions)
|
29
32
|
{
|
30
33
|
this.task = task;
|
31
34
|
this.pageBuilder = pageBuilder;
|
32
35
|
this.row = row;
|
36
|
+
this.columnOptions = columnOptions;
|
33
37
|
this.rnd = new Random();
|
34
38
|
}
|
35
39
|
|
@@ -52,20 +56,53 @@ public class RandomColumnVisitor
|
|
52
56
|
pageBuilder.setLong(column, row);
|
53
57
|
}
|
54
58
|
else {
|
55
|
-
|
59
|
+
Integer max = columnOptions.get(column).get("max_value");
|
60
|
+
Integer min = columnOptions.get(column).get("min_value");
|
61
|
+
if (max != null) {
|
62
|
+
if (min != null) {
|
63
|
+
Integer s = min + rnd.nextInt((max - min));
|
64
|
+
pageBuilder.setLong(column, s);
|
65
|
+
}
|
66
|
+
else {
|
67
|
+
pageBuilder.setLong(column, rnd.nextInt(max));
|
68
|
+
}
|
69
|
+
}
|
70
|
+
else {
|
71
|
+
pageBuilder.setLong(column, rnd.nextInt(10000));
|
72
|
+
}
|
56
73
|
}
|
57
74
|
}
|
58
75
|
|
59
76
|
@Override
|
60
77
|
public void doubleColumn(Column column)
|
61
78
|
{
|
62
|
-
|
79
|
+
Integer max = columnOptions.get(column).get("max_value");
|
80
|
+
Integer min = columnOptions.get(column).get("min_value");
|
81
|
+
if (max != null) {
|
82
|
+
if (min != null) {
|
83
|
+
Double d = min + rnd.nextInt((max - min) - 1) + rnd.nextDouble();
|
84
|
+
pageBuilder.setDouble(column, d);
|
85
|
+
}
|
86
|
+
else {
|
87
|
+
Double d = rnd.nextInt(max - 1) + rnd.nextDouble();
|
88
|
+
pageBuilder.setDouble(column, d);
|
89
|
+
}
|
90
|
+
}
|
91
|
+
else {
|
92
|
+
pageBuilder.setDouble(column, rnd.nextDouble() * 10000);
|
93
|
+
}
|
63
94
|
}
|
64
95
|
|
65
96
|
@Override
|
66
97
|
public void stringColumn(Column column)
|
67
98
|
{
|
68
|
-
|
99
|
+
final Integer length = columnOptions.get(column).getOrDefault("length", 0);
|
100
|
+
if (length == 0) {
|
101
|
+
pageBuilder.setString(column, generator.generate(32));
|
102
|
+
}
|
103
|
+
else {
|
104
|
+
pageBuilder.setString(column, generator.generate(length));
|
105
|
+
}
|
69
106
|
}
|
70
107
|
|
71
108
|
@Override
|
@@ -9,6 +9,8 @@ import org.embulk.config.ConfigSource;
|
|
9
9
|
import org.embulk.config.Task;
|
10
10
|
import org.embulk.config.TaskReport;
|
11
11
|
import org.embulk.config.TaskSource;
|
12
|
+
import org.embulk.spi.Column;
|
13
|
+
import org.embulk.spi.ColumnConfig;
|
12
14
|
import org.embulk.spi.Exec;
|
13
15
|
import org.embulk.spi.InputPlugin;
|
14
16
|
import org.embulk.spi.PageBuilder;
|
@@ -16,7 +18,9 @@ import org.embulk.spi.PageOutput;
|
|
16
18
|
import org.embulk.spi.Schema;
|
17
19
|
import org.embulk.spi.SchemaConfig;
|
18
20
|
|
21
|
+
import java.util.HashMap;
|
19
22
|
import java.util.List;
|
23
|
+
import java.util.Map;
|
20
24
|
import java.util.stream.IntStream;
|
21
25
|
|
22
26
|
public class RandomjInputPlugin
|
@@ -82,13 +86,14 @@ public class RandomjInputPlugin
|
|
82
86
|
{
|
83
87
|
PluginTask task = taskSource.loadTask(PluginTask.class);
|
84
88
|
Integer rows = task.getRows();
|
89
|
+
final HashMap<Column, Map<String, Integer>> columnOptions = getColumnOptions(task);
|
85
90
|
try (PageBuilder pagebuilder =
|
86
91
|
new PageBuilder(Exec.getBufferAllocator(), schema, output)) {
|
87
92
|
IntStream.rangeClosed(
|
88
93
|
taskIndex * rows + 1,
|
89
94
|
taskIndex * rows + rows
|
90
95
|
).boxed().forEach(rowNumber -> {
|
91
|
-
|
96
|
+
RandomjColumnVisitor visitor = new RandomjColumnVisitor(pagebuilder, task, rowNumber, columnOptions);
|
92
97
|
schema.visitColumns(visitor);
|
93
98
|
pagebuilder.addRecord();
|
94
99
|
});
|
@@ -101,6 +106,22 @@ public class RandomjInputPlugin
|
|
101
106
|
return taskReport;
|
102
107
|
}
|
103
108
|
|
109
|
+
HashMap<Column, Map<String, Integer>> getColumnOptions(PluginTask task)
|
110
|
+
{
|
111
|
+
SchemaConfig schemaConfig = task.getSchema();
|
112
|
+
Schema schema = schemaConfig.toSchema();
|
113
|
+
HashMap<Column, Map<String, Integer>> lengthMap = new HashMap<>();
|
114
|
+
for (Column column : schema.getColumns()) {
|
115
|
+
HashMap<String, Integer> miniMap = new HashMap<>();
|
116
|
+
ColumnConfig c = schemaConfig.getColumn(column.getIndex());
|
117
|
+
miniMap.put("length", c.getOption().get(Integer.class, "length", 0));
|
118
|
+
miniMap.put("max_value", c.getOption().get(Integer.class, "max_value", null));
|
119
|
+
miniMap.put("min_value", c.getOption().get(Integer.class, "min_value", null));
|
120
|
+
lengthMap.put(column, miniMap);
|
121
|
+
}
|
122
|
+
return lengthMap;
|
123
|
+
}
|
124
|
+
|
104
125
|
@Override
|
105
126
|
public ConfigDiff guess(ConfigSource config)
|
106
127
|
{
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: embulk-input-randomj
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.3.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- yuokada
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2017-09-
|
11
|
+
date: 2017-09-10 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
requirement: !ruby/object:Gem::Requirement
|
@@ -57,12 +57,12 @@ files:
|
|
57
57
|
- gradle/wrapper/gradle-wrapper.properties
|
58
58
|
- gradlew
|
59
59
|
- lib/embulk/input/randomj.rb
|
60
|
-
- src/main/java/org/embulk/input/randomj/
|
60
|
+
- src/main/java/org/embulk/input/randomj/RandomjColumnVisitor.java
|
61
61
|
- src/main/java/org/embulk/input/randomj/RandomjInputPlugin.java
|
62
62
|
- src/test/java/org/embulk/input/randomj/TestRandomjInputPlugin.java
|
63
63
|
- classpath/commons-lang3-3.5.jar
|
64
64
|
- classpath/commons-text-1.1.jar
|
65
|
-
- classpath/embulk-input-randomj-0.
|
65
|
+
- classpath/embulk-input-randomj-0.3.0.jar
|
66
66
|
homepage: https://github.com/yuokada/embulk-input-randomj
|
67
67
|
licenses:
|
68
68
|
- MIT
|