embulk-input-randomj 0.4.0 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 1165a1e2c182b649c7a2fbf73de530de55a5497f
-  data.tar.gz: ab71604a4f4af3d7765433b7e380e5f509f7b7e2
+  metadata.gz: 82a2c62219bbf24fdfc2cf2bb24f8f0b2210b829
+  data.tar.gz: bd0cad5900b36c009f6e92fa86357b5edd97c741
 SHA512:
-  metadata.gz: 9fc2bea3f88fcc01b2a2e33061d0a28d2588345d8b293ca567cb7aaadcd1daf3975c70f24328e2facbb2e4dd511e06dec5f854520885e77984bfa7a03a508866
-  data.tar.gz: a6d3961d0f3fcd6ebd3f02545f6272a05805680de3923cb2ff1a3dc8db588219e429549a7279b4cca14d719feeed126046f465d7ab7ede26a92f123fe58ea787
+  metadata.gz: ff0b599c31cf975d09f28ca2e3328abc5fcbee510832d551d0ab5b0e0dc7b63005f269478a9ff3bccaddaf68ba4435964a9f126ed736d62d86eed7843f66f2cd
+  data.tar.gz: 3a6ec59cbc4d120e15b5012c26633bf4494e47870b58cc67be520851c5695807bb1553012b82748f97ed7ca6f16f0dfa556f9b690cbeef6d3d2484a0115cc998

data/README.md

@@ -14,7 +14,7 @@ Original: [kumagi/embulk\-input\-random](https://github.com/kumagi/embulk-input-
 * **Cleanup supported**: no
 * **Guess supported**: no
 
-## Install 
+## Install
 
 ``` shell
 % embulk gem install embulk-input-randomj
@@ -59,6 +59,25 @@ in:
       - {name: purchase, type: timestamp, format: '%Y/%m/%d'}
 ```
 
+Add `null_rate` option (from 0.4.0)
+
+This configuration is that inserted `null` into `price` filed with a probability `8` of 10000.
+
+```yaml
+in:
+    type: randomj
+    rows: 16
+    threads: 1
+    primary_key: myid
+    schema:
+      - {name: myid,     type: long}
+      - {name: named,    type: string, length: 12}
+      - {name: price,    type: long, max_value: 1080, min_value: 100, null_rate: 8}
+      - {name: purchase, type: timestamp, format: '%Y/%m/%d'}
+```
+
+
+
 ## Usage
 
 ### Example1
@@ -80,7 +99,7 @@ in:
       - {name: score,    type: long}
       - {name: time,     type: timestamp, format: '%Y-%m-%d %H:%M:%S'}
       - {name: purchase, type: timestamp, format: '%Y/%m/%d'}
-  
+
   out:
     type: stdout
 
@@ -116,7 +135,7 @@ in:
 
 - `named_s` return string with length 8
 - `score` return value between `100~255`
-- `rate` return value between `-100~100` 
+- `rate` return value between `-100~100`
 
 ```shell
 
@@ -176,7 +195,7 @@ $ ./gradlew gem  # -t to watch change of files and rebuild continuously
 ```
 
 ```
-$ ./gradlew build && ./gradlew classpath 
+$ ./gradlew build && ./gradlew classpath
 $ embulk run -I lib config/example.yml     
 ```
 

data/build.gradle

@@ -15,7 +15,7 @@ configurations {
     provided
 }
 
-version = "0.4.0"
+version = "0.5.0"
 
 sourceCompatibility = 1.8
 targetCompatibility = 1.8

data/example/config.yml

@@ -13,6 +13,8 @@ in:
     - {name: score,    type: long, max_value: 255, min_value: 100}
     - {name: time,     type: timestamp, format: '%Y-%m-%d %H:%M:%S'}
     - {name: purchase, type: timestamp, format: '%Y/%m/%d'}
+    - {name: json_obj, type: json, schema: '[{"name": "obj1", "type": "object", "items": [{"name": "key1", "type": "string"}, {"name": "key2", "type": "number"} ] }]' }
+    - {name: json_key, type: json, schema: '[{"name": "baz", "type": "array", "items": {"type": "string", "size": 1}}]' }
 
 out:
   type: stdout

data/src/main/java/org/embulk/input/randomj/JsonColumnVisitor.java

@@ -0,0 +1,142 @@
+package org.embulk.input.randomj;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import org.apache.commons.text.CharacterPredicates;
+import org.apache.commons.text.RandomStringGenerator;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+
+public class JsonColumnVisitor
+{
+
+    private final Map<String, Object> map;
+    private final Random rnd = new Random();
+    private final RandomStringGenerator generator = new RandomStringGenerator.Builder()
+            .withinRange('0', 'z')
+            .filteredBy(CharacterPredicates.LETTERS, CharacterPredicates.DIGITS)
+            .build();
+
+    private static final String ITEMS = "items";
+
+    public JsonColumnVisitor(Map<String, Object> gmap)
+    {
+        map = gmap;
+    }
+
+    public void booleanNode(JsonNode node)
+    {
+        String key = node.get("name").asText();
+        if (Math.random() < 0.5) {
+            map.put(key, true);
+        }
+        else {
+            map.put(key, false);
+        }
+    }
+
+    public void doubleNode(JsonNode node)
+    {
+        String key = node.get("name").asText();
+        map.put(key, rnd.nextDouble() * 10000);
+    }
+
+    public void integerNode(JsonNode node)
+    {
+        String key = node.get("name").asText();
+        map.put(key, rnd.nextInt(10000));
+    }
+
+    public void stringNode(JsonNode node)
+    {
+        String key = node.get("name").asText();
+        map.put(key, generator.generate(8));
+    }
+
+    public void arrayNode(JsonNode node) // NOSONAR
+    {
+        String key = node.get("name").asText();
+        String dataType = node.get(ITEMS).get("type").asText();
+        int arraySize = node.get(ITEMS).get("size").asInt(1);
+        SupportedJsonObject jtype = SupportedJsonObject.valueOf(dataType.toUpperCase());
+        switch (jtype) {
+            case BOOLEAN: { // NOSONAR
+                List<Boolean> m = new ArrayList<>();
+                for (int i = 0; i < arraySize; i++) {
+                    if (Math.random() < 0.5) {
+                        m.add(true);
+                    }
+                    else {
+                        m.add(false);
+                    }
+                }
+                map.put(key, m);
+                break;
+            }
+            case INTEGER: { // NOSONAR
+                ArrayList<Integer> m = new ArrayList<>();
+                for (int i = 0; i < arraySize; i++) {
+                    m.add(rnd.nextInt(100));
+                }
+                map.put(key, m);
+                break;
+            }
+            case NUMBER: { // NOSONAR
+                ArrayList<Number> m = new ArrayList<>();
+                for (int i = 0; i < arraySize; i++) {
+                    m.add(rnd.nextDouble() * 100);
+                }
+                map.put(key, m);
+                break;
+            }
+            case STRING: { // NOSONAR
+                int length = 8;
+                ArrayList<String> m = new ArrayList<>();
+                for (int i = 0; i < arraySize; i++) {
+                    m.add(generator.generate(length));
+                }
+                map.put(key, m);
+                break;
+            }
+            default:
+                throw new UnsupportedOperationException("randomj input plugin does not support json-array-data type");
+        }
+    }
+
+    public void objectNode(JsonNode node)
+    {
+        Map<String, Object> objectMap = new HashMap<>();
+
+        for (JsonNode jsonNode : node.findValues(ITEMS).listIterator().next()) {
+            String nestKey = jsonNode.get("name").asText();
+            SupportedJsonObject jtype = SupportedJsonObject.valueOf(jsonNode.get("type").asText().toUpperCase());
+
+            switch (jtype) {
+                case BOOLEAN:
+                    if (Math.random() < 0.5) {
+                        objectMap.put(nestKey, true);
+                    }
+                    else {
+                        objectMap.put(nestKey, false);
+                    }
+                    break;
+                case NUMBER:
+                    objectMap.put(nestKey, rnd.nextDouble() * 100);
+                    break;
+                case INTEGER:
+                    objectMap.put(nestKey, rnd.nextInt(10000));
+                    break;
+                case STRING:
+                    int length = 8;
+                    objectMap.put(nestKey, generator.generate(length));
+                    break;
+                default:
+                    throw new UnsupportedOperationException("randomj input plugin does not support json-data type");
+            }
+        }
+        map.put(node.get("name").asText(), objectMap);
+    }
+}

data/src/main/java/org/embulk/input/randomj/PluginTask.java

@@ -0,0 +1,26 @@
+package org.embulk.input.randomj;
+
+import org.embulk.config.Config;
+import org.embulk.config.ConfigDefault;
+import org.embulk.config.Task;
+import org.embulk.spi.SchemaConfig;
+
+public interface PluginTask
+        extends Task
+{
+    // configuration row (required integer)
+    @Config("rows")
+    int getRows();
+
+    // ref: https://github.com/embulk/embulk-input-jdbc/blob/master/embulk-input-mysql/src/main/java/org/embulk/input/MySQLInputPlugin.java#L33-L35
+    @Config("threads")
+    @ConfigDefault("1")
+    Integer getThreads();
+
+    @Config("primary_key")
+    @ConfigDefault("")
+    String getPrimaryKey();
+
+    @Config("schema")
+    SchemaConfig getSchema();
+}

data/src/main/java/org/embulk/input/randomj/RandomjColumnVisitor.java

@@ -1,15 +1,20 @@
 package org.embulk.input.randomj;
 
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
 import org.apache.commons.text.CharacterPredicates;
 import org.apache.commons.text.RandomStringGenerator;
-import org.embulk.input.randomj.RandomjInputPlugin.PluginTask;
 import org.embulk.spi.Column;
 import org.embulk.spi.ColumnVisitor;
 import org.embulk.spi.PageBuilder;
+import org.embulk.spi.json.JsonParser;
 import org.embulk.spi.time.Timestamp;
 
 import java.time.LocalDateTime;
 import java.time.ZoneId;
+import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 import java.util.Random;
 
@@ -21,25 +26,33 @@ public class RandomjColumnVisitor
     private final Integer row;
     private final Random rnd;
     private final Map<Column, Map<String, Integer>> columnOptions;
+    private final Map<Column, List<JsonNode>> schemaOptions;
     private final RandomStringGenerator generator = new RandomStringGenerator.Builder()
             .withinRange('0', 'z')
             .filteredBy(CharacterPredicates.LETTERS, CharacterPredicates.DIGITS)
             .build();
     private final ZoneId zoneId = ZoneId.systemDefault();
+    private final JsonParser jsonParser = new JsonParser();
+    private final ObjectMapper mapper = new ObjectMapper();
 
-    public RandomjColumnVisitor(PageBuilder pageBuilder, PluginTask task, Integer row, Map<Column, Map<String, Integer>> columnOptions)
+    private static final String NULL_RATE = "null_rate";
+
+    public RandomjColumnVisitor(PageBuilder pageBuilder, PluginTask task, Integer row,
+            Map<Column, Map<String, Integer>> columnOptions,
+            Map<Column, List<JsonNode>> schemaOptions)
     {
         this.task = task;
         this.pageBuilder = pageBuilder;
         this.row = row;
         this.columnOptions = columnOptions;
         this.rnd = new Random();
+        this.schemaOptions = schemaOptions;
     }
 
     @Override
     public void booleanColumn(Column column)
     {
-        Integer nrate = columnOptions.get(column).get("null_rate");
+        Integer nrate = columnOptions.get(column).get(NULL_RATE);
         if (Math.random() < (double) nrate / 10000) {
             pageBuilder.setNull(column);
         }
@@ -61,7 +74,7 @@ public class RandomjColumnVisitor
             pageBuilder.setLong(column, row);
         }
         else {
-            Integer nrate = columnOptions.get(column).get("null_rate");
+            Integer nrate = columnOptions.get(column).get(NULL_RATE);
             if (Math.random() < (double) nrate / 10000) {
                 pageBuilder.setNull(column);
             }
@@ -87,7 +100,7 @@ public class RandomjColumnVisitor
     @Override
     public void doubleColumn(Column column)
     {
-        Integer nrate = columnOptions.get(column).get("null_rate");
+        Integer nrate = columnOptions.get(column).get(NULL_RATE);
         if (Math.random() < (double) nrate / 10000) {
             pageBuilder.setNull(column);
         }
@@ -113,7 +126,7 @@ public class RandomjColumnVisitor
     @Override
     public void stringColumn(Column column)
     {
-        Integer nrate = columnOptions.get(column).get("null_rate");
+        Integer nrate = columnOptions.get(column).get(NULL_RATE);
         if (Math.random() < (double) nrate / 10000) {
             pageBuilder.setNull(column);
         }
@@ -131,7 +144,7 @@ public class RandomjColumnVisitor
     @Override
     public void timestampColumn(Column column)
     {
-        Integer nrate = columnOptions.get(column).get("null_rate");
+        Integer nrate = columnOptions.get(column).get(NULL_RATE);
         if (Math.random() < (double) nrate / 10000) {
             pageBuilder.setNull(column);
         }
@@ -150,6 +163,45 @@ public class RandomjColumnVisitor
     @Override
     public void jsonColumn(Column column)
     {
-        throw new UnsupportedOperationException("orc output plugin does not support json type");
+        Map<String, Object> map = new HashMap<>();
+        JsonColumnVisitor visitor = new JsonColumnVisitor(map);
+
+        List<JsonNode> nodes = schemaOptions.get(column);
+        for (JsonNode node : nodes) {
+            visit(node, visitor);
+        }
+
+        try {
+            pageBuilder.setJson(column, jsonParser.parse(mapper.writeValueAsString(map)));
+        }
+        catch (JsonProcessingException e) {
+            e.printStackTrace(); // NOSONAR
+        }
+    }
+
+    private void visit(JsonNode node, JsonColumnVisitor visitor)
+    {
+        SupportedJsonObject object = SupportedJsonObject.valueOf(node.get("type").asText().toUpperCase());
+        if (object.equals(SupportedJsonObject.BOOLEAN)) {
+            visitor.booleanNode(node);
+        }
+        else if (object.equals(SupportedJsonObject.NUMBER)) {
+            visitor.doubleNode(node);
+        }
+        else if (object.equals(SupportedJsonObject.INTEGER)) {
+            visitor.integerNode(node);
+        }
+        else if (object.equals(SupportedJsonObject.STRING)) {
+            visitor.stringNode(node);
+        }
+        else if (object.equals(SupportedJsonObject.ARRAY)) {
+            visitor.arrayNode(node);
+        }
+        else if (object.equals(SupportedJsonObject.OBJECT)) {
+            visitor.objectNode(node);
+        }
+        else {
+            throw new UnsupportedOperationException("randomj input plugin does not support json-data type");
+        }
     }
 }

data/src/main/java/org/embulk/input/randomj/RandomjInputPlugin.java

@@ -1,10 +1,10 @@
 package org.embulk.input.randomj;
 
-import org.embulk.config.Config;
-import org.embulk.config.ConfigDefault;
+import com.fasterxml.jackson.core.type.TypeReference;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
 import org.embulk.config.ConfigDiff;
 import org.embulk.config.ConfigSource;
-import org.embulk.config.Task;
 import org.embulk.config.TaskReport;
 import org.embulk.config.TaskSource;
 import org.embulk.spi.Column;
@@ -16,6 +16,7 @@ import org.embulk.spi.PageOutput;
 import org.embulk.spi.Schema;
 import org.embulk.spi.SchemaConfig;
 
+import java.io.IOException;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -24,26 +25,6 @@ import java.util.stream.IntStream;
 public class RandomjInputPlugin
         implements InputPlugin
 {
-    public interface PluginTask
-            extends Task
-    {
-        // configuration row (required integer)
-        @Config("rows")
-        int getRows();
-
-        // ref: https://github.com/embulk/embulk-input-jdbc/blob/master/embulk-input-mysql/src/main/java/org/embulk/input/MySQLInputPlugin.java#L33-L35
-        @Config("threads")
-        @ConfigDefault("1")
-        Integer getThreads();
-
-        @Config("primary_key")
-        @ConfigDefault("")
-        String getPrimaryKey();
-
-        @Config("schema")
-        SchemaConfig getSchema();
-    }
-
     @Override
     public ConfigDiff transaction(ConfigSource config,
             InputPlugin.Control control)
@@ -70,7 +51,7 @@ public class RandomjInputPlugin
             Schema schema, int taskCount,
             List<TaskReport> successTaskReports)
     {
-        // throw new UnsupportedOperationException("randomj input plugin does not support cleanup");
+        // throw new UnsupportedOperationException("randomj input plugin does not support cleanup"); // NOSONAR
     }
 
     @Override
@@ -81,13 +62,14 @@ public class RandomjInputPlugin
         PluginTask task = taskSource.loadTask(PluginTask.class);
         Integer rows = task.getRows();
         final HashMap<Column, Map<String, Integer>> columnOptions = getColumnOptions(task);
+        final HashMap<Column, List<JsonNode>> columnSchemas = getColumnSchemas(task);
         try (PageBuilder pagebuilder =
                 new PageBuilder(Exec.getBufferAllocator(), schema, output)) {
             IntStream.rangeClosed(
                     taskIndex * rows + 1,
                     taskIndex * rows + rows
             ).boxed().forEach(rowNumber -> {
-                RandomjColumnVisitor visitor = new RandomjColumnVisitor(pagebuilder, task, rowNumber, columnOptions);
+                RandomjColumnVisitor visitor = new RandomjColumnVisitor(pagebuilder, task, rowNumber, columnOptions, columnSchemas);
                 schema.visitColumns(visitor);
                 pagebuilder.addRecord();
             });
@@ -117,6 +99,28 @@ public class RandomjInputPlugin
         return lengthMap;
     }
 
+    HashMap<Column, List<JsonNode>> getColumnSchemas(PluginTask task)
+    {
+        SchemaConfig schemaConfig = task.getSchema();
+        Schema schema = schemaConfig.toSchema();
+        HashMap<Column, List<JsonNode>> schemaMap = new HashMap<>();
+        for (Column column : schema.getColumns()) {
+            String schemaString = schemaConfig
+                    .getColumn(column.getIndex())
+                    .getOption().get(String.class, "schema", "");
+            if (!schemaString.isEmpty()) {
+                try {
+                    List<JsonNode> jsonNodes = new ObjectMapper().readValue(schemaString, new TypeReference<List<JsonNode>>() {});
+                    schemaMap.put(column, jsonNodes);
+                }
+                catch (IOException e) {
+                    e.printStackTrace(); // NOSONAR
+                }
+            }
+        }
+        return schemaMap;
+    }
+
     @Override
     public ConfigDiff guess(ConfigSource config)
     {

data/src/main/java/org/embulk/input/randomj/SupportedJsonObject.java

@@ -0,0 +1,18 @@
+package org.embulk.input.randomj;
+
+public enum SupportedJsonObject
+{
+    OBJECT("object"),
+    STRING("string"),
+    BOOLEAN("boolean"),
+    INTEGER("integer"),
+    NUMBER("number"),
+    ARRAY("array");
+
+    private final String typeName;
+
+    private SupportedJsonObject(final String typeName)
+    {
+        this.typeName = typeName;
+    }
+}

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-input-randomj
 version: !ruby/object:Gem::Version
-  version: 0.4.0
+  version: 0.5.0
 platform: ruby
 authors:
 - yuokada
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2017-12-02 00:00:00.000000000 Z
+date: 2018-02-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -57,12 +57,15 @@ files:
 - gradle/wrapper/gradle-wrapper.properties
 - gradlew
 - lib/embulk/input/randomj.rb
+- src/main/java/org/embulk/input/randomj/JsonColumnVisitor.java
+- src/main/java/org/embulk/input/randomj/PluginTask.java
 - src/main/java/org/embulk/input/randomj/RandomjColumnVisitor.java
 - src/main/java/org/embulk/input/randomj/RandomjInputPlugin.java
+- src/main/java/org/embulk/input/randomj/SupportedJsonObject.java
 - src/test/java/org/embulk/input/randomj/TestRandomjInputPlugin.java
 - classpath/commons-lang3-3.5.jar
 - classpath/commons-text-1.1.jar
-- classpath/embulk-input-randomj-0.4.0.jar
+- classpath/embulk-input-randomj-0.5.0.jar
 homepage: https://github.com/yuokada/embulk-input-randomj
 licenses:
 - MIT