embulk-input-randomj 0.2.0 → 0.3.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (7) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +71 -0
  3. data/build.gradle +1 -1
  4. data/example/config.yml +3 -2
  5. data/src/main/java/org/embulk/input/randomj/{RandomColumnVisitor.java → RandomjColumnVisitor.java} +42 -5
  6. data/src/main/java/org/embulk/input/randomj/RandomjInputPlugin.java +22 -1
  7. metadata +4 -4
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: ab0f7f5168b89c9ae46492aca972aebfda891f20
4
- data.tar.gz: d3ea8441e1dd85765cf8f5ca940a67d6c4b45f9d
3
+ metadata.gz: 664734a687060fbda08c7e14593684f63a501767
4
+ data.tar.gz: 3e3aa292e1eb2916cf8b516c3bc486f194e67417
5
5
  SHA512:
6
- metadata.gz: 094c98bdd73e06a67981422f0a1b4974e95b921056902b8eb3f1438d5171a0e926a8ee9d1d483103861a10b2902d72cd11390e96f9bc5dca7f9f5490d01a36bb
7
- data.tar.gz: 2c8ed6182a6891aa39a72894abcdc983756cd8cc411159fa8e730d0628425bd0492a4c1e73de62f1d1f0cc21deb97747ada43c409c3f9d4b50ae9c28ed7bac5b
6
+ metadata.gz: c07094226ddafc4ce96a2f8738a8a73c86a9e143d3de802b048ee80990459f6b9fff0db75adc0b1d15a5127e91a61241f1556db7540bf1d1f6f0e5ccdc055180
7
+ data.tar.gz: 6a97183220e98b63c92c2cccb7b54d8ae6456b796670abeda1d10b1bc3a6effbdde9f6182980dbb1ff13289802741f89ee9a60f8b7b01723d18d7cfc278f9bee
data/README.md CHANGED
@@ -45,8 +45,24 @@ in:
45
45
  - {name: purchase, type: timestamp, format: '%Y/%m/%d'}
46
46
  ```
47
47
 
48
+ Add `length`, `max_value`, `min_value` option (from 0.3.0)
49
+ ```yaml
50
+ in:
51
+ type: randomj
52
+ rows: 16
53
+ threads: 1
54
+ primary_key: myid
55
+ schema:
56
+ - {name: myid, type: long}
57
+ - {name: named, type: string, length: 12}
58
+ - {name: price, type: long, max_value: 1080, min_value: 100}
59
+ - {name: purchase, type: timestamp, format: '%Y/%m/%d'}
60
+ ```
61
+
48
62
  ## Usage
49
63
 
64
+ ### Example1
65
+
50
66
  ```shell
51
67
 
52
68
  % cat example/config.yml
@@ -96,6 +112,61 @@ in:
96
112
 
97
113
  ```
98
114
 
115
+ ### Example2
116
+
117
+ - `named_s` return string with length 8
118
+ - `score` return value between `100~255`
119
+ - `rate` return value between `-100~100`
120
+
121
+ ```shell
122
+
123
+ % cat example/config.yml
124
+ in:
125
+ type: randomj
126
+ rows: 16
127
+ threads: 1
128
+ # default_timezone: Asia/Tokyo
129
+ primary_key: myid
130
+ schema:
131
+ - {name: myid, type: long}
132
+ - {name: named, type: string}
133
+ - {name: named_s, type: string, length: 8}
134
+ - {name: x_flag, type: boolean}
135
+ - {name: rate, type: double, max_value: 100, min_value: -100}
136
+ - {name: score, type: long, max_value: 255, min_value: 100}
137
+ - {name: time, type: timestamp, format: '%Y-%m-%d %H:%M:%S'}
138
+ - {name: purchase, type: timestamp, format: '%Y/%m/%d'}
139
+
140
+ out:
141
+ type: stdout
142
+
143
+
144
+ % embulk run -I lib example/config.yml
145
+ 2017-09-10 04:45:04.894 +0900: Embulk v0.8.32
146
+ 2017-09-10 04:45:10.212 +0900 [INFO] (0001:transaction): Loaded plugin embulk/input/randomj from a load path
147
+ 2017-09-10 04:45:10.246 +0900 [INFO] (0001:transaction): Using local thread executor with max_threads=8 / output tasks 4 = input tasks 1 * 4
148
+ 2017-09-10 04:45:10.263 +0900 [INFO] (0001:transaction): {done: 0 / 1, running: 0}
149
+ 1,BOcbVJX5bWL5wRBJc532trxvwhQpmg3d,yHwXATfG,true,-79.62544211154894,129,2017-12-05 22:31:35,2017/12/26
150
+ 2,N2gljQxd4yDBzJjK9iSRUdROtaZGUEl7,zSrEMjzC,false,-11.47506884041689,194,2017-09-17 15:56:18,2017/12/06
151
+ 3,PJvKkf0wwpGqGMlc7OjUhjZNi0pTEZIU,q6TgdoaZ,false,85.17356188437738,137,2017-10-07 17:28:43,2017/10/22
152
+ 4,DA6wWE4p3zIPDK0Mp81bWczewNSMY2sq,KeobJmS1,false,79.95787440150436,221,2017-09-28 19:35:17,2017/11/20
153
+ 5,8DNF4TzhVDLCFey2x1eCHryf4GdvHlyW,D2jddtEN,true,19.801687906161735,182,2017-11-24 18:43:38,2017/12/29
154
+ 6,veyIxBc9u0FMwsGksMfLhvBMuIF2D7XO,6Mtz4MN9,true,26.922649237294582,176,2017-09-23 07:43:40,2017/11/18
155
+ 7,HHCTLuaxAJIRHHG7cB2u9Ake9p9OSIcy,UHHKp5xX,true,9.960707451320626,108,2017-09-14 08:11:49,2017/11/05
156
+ 8,HcQhHMQ4sYiXTBpvNiTqDGskuTeVEC6r,d0VSR8K8,false,-62.405292711551624,118,2017-11-11 08:06:20,2017/10/20
157
+ 9,si5BWUPEEvVHvveeqSxG6ypc7pSsKtC7,bW5p9boG,false,-76.91915279000274,192,2017-09-28 19:46:53,2017/11/04
158
+ 10,xnfU0aJgigJG9rPan2rwoffhN9pzLQCy,R8MV0Jpa,true,-79.40738909989871,104,2017-11-19 02:50:07,2017/09/11
159
+ 11,KiRzQqfE6wRw3WjMPAmedqtHyG3MttGU,SowzDTSb,true,77.22509797548325,163,2017-12-23 18:16:30,2017/12/27
160
+ 12,pQLz3fMIkN6UANwSbzJ5vhBWzF2FI7uo,uPGyHyuW,true,71.19680005107371,180,2017-11-23 16:31:30,2017/11/14
161
+ 13,aFOc2qCAu5oYbxTCGkMNcZob6Tl3wl3Y,apFu34Ps,false,82.8406608691031,226,2017-10-03 06:09:25,2017/10/06
162
+ 14,Kz3JGL23k7f8SR17xQBw063ApuGdeWIP,r0c0KnUC,true,-26.484829732050134,113,2017-10-01 02:40:37,2017/11/26
163
+ 15,p5vGY02BzrHqk345JyAhFU7xVsA2jEZD,nhzsefns,false,-79.0184308849151,119,2017-12-15 22:59:28,2017/11/25
164
+ 16,1jyxot60lCrRFMUfjyHcZ07dq05eu76a,WewnLZfw,false,-55.315211168770816,141,2017-12-11 10:36:46,2017/12/05
165
+ 2017-09-10 04:45:10.344 +0900 [INFO] (0001:transaction): {done: 1 / 1, running: 0}
166
+ 2017-09-10 04:45:10.351 +0900 [INFO] (main): Committed.
167
+ 2017-09-10 04:45:10.351 +0900 [INFO] (main): Next config diff: {"in":{},"out":{}}
168
+
169
+ ```
99
170
 
100
171
  ## Build
101
172
 
data/build.gradle CHANGED
@@ -14,7 +14,7 @@ configurations {
14
14
  provided
15
15
  }
16
16
 
17
- version = "0.2.0"
17
+ version = "0.3.0"
18
18
 
19
19
  sourceCompatibility = 1.8
20
20
  targetCompatibility = 1.8
data/example/config.yml CHANGED
@@ -7,9 +7,10 @@ in:
7
7
  schema:
8
8
  - {name: myid, type: long}
9
9
  - {name: named, type: string}
10
+ - {name: named_s, type: string, length: 8}
10
11
  - {name: x_flag, type: boolean}
11
- - {name: pit_rate, type: double}
12
- - {name: score, type: long}
12
+ - {name: rate, type: double, max_value: 100, min_value: -100}
13
+ - {name: score, type: long, max_value: 255, min_value: 100}
13
14
  - {name: time, type: timestamp, format: '%Y-%m-%d %H:%M:%S'}
14
15
  - {name: purchase, type: timestamp, format: '%Y/%m/%d'}
15
16
 
data/src/main/java/org/embulk/input/randomj/{RandomColumnVisitor.java → RandomjColumnVisitor.java} RENAMED
@@ -10,26 +10,30 @@ import org.embulk.spi.time.Timestamp;
10
10
 
11
11
  import java.time.LocalDateTime;
12
12
  import java.time.ZoneId;
13
+ import java.util.HashMap;
14
+ import java.util.Map;
13
15
  import java.util.Random;
14
16
 
15
- public class RandomColumnVisitor
17
+ public class RandomjColumnVisitor
16
18
  implements ColumnVisitor
17
19
  {
18
20
  private final PageBuilder pageBuilder;
19
21
  private final PluginTask task;
20
22
  private final Integer row;
21
23
  private final Random rnd;
24
+ private final HashMap<Column, Map<String, Integer>> columnOptions;
22
25
  private final RandomStringGenerator generator = new RandomStringGenerator.Builder()
23
26
  .withinRange('0', 'z')
24
27
  .filteredBy(CharacterPredicates.LETTERS, CharacterPredicates.DIGITS)
25
28
  .build();
26
29
  private final ZoneId zoneId = ZoneId.systemDefault();
27
30
 
28
- public RandomColumnVisitor(PageBuilder pageBuilder, PluginTask task, Integer row)
31
+ public RandomjColumnVisitor(PageBuilder pageBuilder, PluginTask task, Integer row, HashMap<Column, Map<String, Integer>> columnOptions)
29
32
  {
30
33
  this.task = task;
31
34
  this.pageBuilder = pageBuilder;
32
35
  this.row = row;
36
+ this.columnOptions = columnOptions;
33
37
  this.rnd = new Random();
34
38
  }
35
39
 
@@ -52,20 +56,53 @@ public class RandomColumnVisitor
52
56
  pageBuilder.setLong(column, row);
53
57
  }
54
58
  else {
55
- pageBuilder.setLong(column, rnd.nextInt(10000));
59
+ Integer max = columnOptions.get(column).get("max_value");
60
+ Integer min = columnOptions.get(column).get("min_value");
61
+ if (max != null) {
62
+ if (min != null) {
63
+ Integer s = min + rnd.nextInt((max - min));
64
+ pageBuilder.setLong(column, s);
65
+ }
66
+ else {
67
+ pageBuilder.setLong(column, rnd.nextInt(max));
68
+ }
69
+ }
70
+ else {
71
+ pageBuilder.setLong(column, rnd.nextInt(10000));
72
+ }
56
73
  }
57
74
  }
58
75
 
59
76
  @Override
60
77
  public void doubleColumn(Column column)
61
78
  {
62
- pageBuilder.setDouble(column, rnd.nextDouble() * 10000);
79
+ Integer max = columnOptions.get(column).get("max_value");
80
+ Integer min = columnOptions.get(column).get("min_value");
81
+ if (max != null) {
82
+ if (min != null) {
83
+ Double d = min + rnd.nextInt((max - min) - 1) + rnd.nextDouble();
84
+ pageBuilder.setDouble(column, d);
85
+ }
86
+ else {
87
+ Double d = rnd.nextInt(max - 1) + rnd.nextDouble();
88
+ pageBuilder.setDouble(column, d);
89
+ }
90
+ }
91
+ else {
92
+ pageBuilder.setDouble(column, rnd.nextDouble() * 10000);
93
+ }
63
94
  }
64
95
 
65
96
  @Override
66
97
  public void stringColumn(Column column)
67
98
  {
68
- pageBuilder.setString(column, generator.generate(32));
99
+ final Integer length = columnOptions.get(column).getOrDefault("length", 0);
100
+ if (length == 0) {
101
+ pageBuilder.setString(column, generator.generate(32));
102
+ }
103
+ else {
104
+ pageBuilder.setString(column, generator.generate(length));
105
+ }
69
106
  }
70
107
 
71
108
  @Override
data/src/main/java/org/embulk/input/randomj/RandomjInputPlugin.java CHANGED
@@ -9,6 +9,8 @@ import org.embulk.config.ConfigSource;
9
9
  import org.embulk.config.Task;
10
10
  import org.embulk.config.TaskReport;
11
11
  import org.embulk.config.TaskSource;
12
+ import org.embulk.spi.Column;
13
+ import org.embulk.spi.ColumnConfig;
12
14
  import org.embulk.spi.Exec;
13
15
  import org.embulk.spi.InputPlugin;
14
16
  import org.embulk.spi.PageBuilder;
@@ -16,7 +18,9 @@ import org.embulk.spi.PageOutput;
16
18
  import org.embulk.spi.Schema;
17
19
  import org.embulk.spi.SchemaConfig;
18
20
 
21
+ import java.util.HashMap;
19
22
  import java.util.List;
23
+ import java.util.Map;
20
24
  import java.util.stream.IntStream;
21
25
 
22
26
  public class RandomjInputPlugin
@@ -82,13 +86,14 @@ public class RandomjInputPlugin
82
86
  {
83
87
  PluginTask task = taskSource.loadTask(PluginTask.class);
84
88
  Integer rows = task.getRows();
89
+ final HashMap<Column, Map<String, Integer>> columnOptions = getColumnOptions(task);
85
90
  try (PageBuilder pagebuilder =
86
91
  new PageBuilder(Exec.getBufferAllocator(), schema, output)) {
87
92
  IntStream.rangeClosed(
88
93
  taskIndex * rows + 1,
89
94
  taskIndex * rows + rows
90
95
  ).boxed().forEach(rowNumber -> {
91
- RandomColumnVisitor visitor = new RandomColumnVisitor(pagebuilder, task, rowNumber);
96
+ RandomjColumnVisitor visitor = new RandomjColumnVisitor(pagebuilder, task, rowNumber, columnOptions);
92
97
  schema.visitColumns(visitor);
93
98
  pagebuilder.addRecord();
94
99
  });
@@ -101,6 +106,22 @@ public class RandomjInputPlugin
101
106
  return taskReport;
102
107
  }
103
108
 
109
+ HashMap<Column, Map<String, Integer>> getColumnOptions(PluginTask task)
110
+ {
111
+ SchemaConfig schemaConfig = task.getSchema();
112
+ Schema schema = schemaConfig.toSchema();
113
+ HashMap<Column, Map<String, Integer>> lengthMap = new HashMap<>();
114
+ for (Column column : schema.getColumns()) {
115
+ HashMap<String, Integer> miniMap = new HashMap<>();
116
+ ColumnConfig c = schemaConfig.getColumn(column.getIndex());
117
+ miniMap.put("length", c.getOption().get(Integer.class, "length", 0));
118
+ miniMap.put("max_value", c.getOption().get(Integer.class, "max_value", null));
119
+ miniMap.put("min_value", c.getOption().get(Integer.class, "min_value", null));
120
+ lengthMap.put(column, miniMap);
121
+ }
122
+ return lengthMap;
123
+ }
124
+
104
125
  @Override
105
126
  public ConfigDiff guess(ConfigSource config)
106
127
  {
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: embulk-input-randomj
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.2.0
4
+ version: 0.3.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - yuokada
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2017-09-05 00:00:00.000000000 Z
11
+ date: 2017-09-10 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  requirement: !ruby/object:Gem::Requirement
@@ -57,12 +57,12 @@ files:
57
57
  - gradle/wrapper/gradle-wrapper.properties
58
58
  - gradlew
59
59
  - lib/embulk/input/randomj.rb
60
- - src/main/java/org/embulk/input/randomj/RandomColumnVisitor.java
60
+ - src/main/java/org/embulk/input/randomj/RandomjColumnVisitor.java
61
61
  - src/main/java/org/embulk/input/randomj/RandomjInputPlugin.java
62
62
  - src/test/java/org/embulk/input/randomj/TestRandomjInputPlugin.java
63
63
  - classpath/commons-lang3-3.5.jar
64
64
  - classpath/commons-text-1.1.jar
65
- - classpath/embulk-input-randomj-0.2.0.jar
65
+ - classpath/embulk-input-randomj-0.3.0.jar
66
66
  homepage: https://github.com/yuokada/embulk-input-randomj
67
67
  licenses:
68
68
  - MIT