embulk-input-randomj 0.3.0 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 664734a687060fbda08c7e14593684f63a501767
-  data.tar.gz: 3e3aa292e1eb2916cf8b516c3bc486f194e67417
+  metadata.gz: 1165a1e2c182b649c7a2fbf73de530de55a5497f
+  data.tar.gz: ab71604a4f4af3d7765433b7e380e5f509f7b7e2
 SHA512:
-  metadata.gz: c07094226ddafc4ce96a2f8738a8a73c86a9e143d3de802b048ee80990459f6b9fff0db75adc0b1d15a5127e91a61241f1556db7540bf1d1f6f0e5ccdc055180
-  data.tar.gz: 6a97183220e98b63c92c2cccb7b54d8ae6456b796670abeda1d10b1bc3a6effbdde9f6182980dbb1ff13289802741f89ee9a60f8b7b01723d18d7cfc278f9bee
+  metadata.gz: 9fc2bea3f88fcc01b2a2e33061d0a28d2588345d8b293ca567cb7aaadcd1daf3975c70f24328e2facbb2e4dd511e06dec5f854520885e77984bfa7a03a508866
+  data.tar.gz: a6d3961d0f3fcd6ebd3f02545f6272a05805680de3923cb2ff1a3dc8db588219e429549a7279b4cca14d719feeed126046f465d7ab7ede26a92f123fe58ea787

data/.travis.yml

@@ -10,3 +10,5 @@ sudo: false
 script:
   - ./gradlew --info checkstyle
   - ./gradlew --info check
+
+after_success: ./gradlew sonarqube -Dsonar.organization=yuokada-github -Dsonar.host.url=https://sonarcloud.io -Dsonar.login=${SONAR_TOKEN}

data/README.md

@@ -179,3 +179,10 @@ $ ./gradlew gem  # -t to watch change of files and rebuild continuously
 $ ./gradlew build && ./gradlew classpath 
 $ embulk run -I lib config/example.yml     
 ```
+
+
+## ChangeLog
+
+### v0.4
+
+- Support null_rate parameter

data/build.gradle

@@ -4,6 +4,7 @@ plugins {
     id "java"
     id "checkstyle"
     id "findbugs"
+    id "org.sonarqube" version "2.5"
 }
 import com.github.jrubygradle.JRubyExec
 repositories {
@@ -14,7 +15,7 @@ configurations {
     provided
 }
 
-version = "0.3.0"
+version = "0.4.0"
 
 sourceCompatibility = 1.8
 targetCompatibility = 1.8

data/gradle/wrapper/gradle-wrapper.properties

@@ -1,6 +1,6 @@
-#Sun Jan 08 00:35:58 PST 2017
+#Sun Oct 08 14:24:25 JST 2017
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-3.2.1-bin.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-4.2.1-bin.zip

data/gradlew

@@ -1,4 +1,4 @@
-#!/usr/bin/env bash
+#!/usr/bin/env sh
 
 ##############################################################################
 ##
@@ -154,16 +154,19 @@ if $cygwin ; then
     esac
 fi
 
-# Split up the JVM_OPTS And GRADLE_OPTS values into an array, following the shell quoting and substitution rules
-function splitJvmOpts() {
-    JVM_OPTS=("$@")
+# Escape application args
+save ( ) {
+    for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done
+    echo " "
 }
-eval splitJvmOpts $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS
-JVM_OPTS[${#JVM_OPTS[*]}]="-Dorg.gradle.appname=$APP_BASE_NAME"
+APP_ARGS=$(save "$@")
+
+# Collect all arguments for the java command, following the shell quoting and substitution rules
+eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS"
 
 # by default we should be in the correct project dir, but when run from Finder on Mac, the cwd is wrong
-if [[ "$(uname)" == "Darwin" ]] && [[ "$HOME" == "$PWD" ]]; then
+if [ "$(uname)" = "Darwin" ] && [ "$HOME" = "$PWD" ]; then
   cd "$(dirname "$0")"
 fi
 
-exec "$JAVACMD" "${JVM_OPTS[@]}" -classpath "$CLASSPATH" org.gradle.wrapper.GradleWrapperMain "$@"
+exec "$JAVACMD" "$@"

data/src/main/java/org/embulk/input/randomj/RandomjColumnVisitor.java

@@ -10,7 +10,6 @@ import org.embulk.spi.time.Timestamp;
 
 import java.time.LocalDateTime;
 import java.time.ZoneId;
-import java.util.HashMap;
 import java.util.Map;
 import java.util.Random;
 
@@ -21,14 +20,14 @@ public class RandomjColumnVisitor
     private final PluginTask task;
     private final Integer row;
     private final Random rnd;
-    private final HashMap<Column, Map<String, Integer>> columnOptions;
+    private final Map<Column, Map<String, Integer>> columnOptions;
     private final RandomStringGenerator generator = new RandomStringGenerator.Builder()
             .withinRange('0', 'z')
             .filteredBy(CharacterPredicates.LETTERS, CharacterPredicates.DIGITS)
             .build();
     private final ZoneId zoneId = ZoneId.systemDefault();
 
-    public RandomjColumnVisitor(PageBuilder pageBuilder, PluginTask task, Integer row, HashMap<Column, Map<String, Integer>> columnOptions)
+    public RandomjColumnVisitor(PageBuilder pageBuilder, PluginTask task, Integer row, Map<Column, Map<String, Integer>> columnOptions)
     {
         this.task = task;
         this.pageBuilder = pageBuilder;
@@ -40,11 +39,17 @@ public class RandomjColumnVisitor
     @Override
     public void booleanColumn(Column column)
     {
-        if (Math.random() < 0.5) {
-            pageBuilder.setBoolean(column, false);
+        Integer nrate = columnOptions.get(column).get("null_rate");
+        if (Math.random() < (double) nrate / 10000) {
+            pageBuilder.setNull(column);
         }
         else {
-            pageBuilder.setBoolean(column, true);
+            if (Math.random() < 0.5) {
+                pageBuilder.setBoolean(column, false);
+            }
+            else {
+                pageBuilder.setBoolean(column, true);
+            }
         }
     }
 
@@ -56,66 +61,90 @@ public class RandomjColumnVisitor
             pageBuilder.setLong(column, row);
         }
         else {
-            Integer max = columnOptions.get(column).get("max_value");
-            Integer min = columnOptions.get(column).get("min_value");
-            if (max != null) {
-                if (min != null) {
-                    Integer s = min + rnd.nextInt((max - min));
-                    pageBuilder.setLong(column, s);
+            Integer nrate = columnOptions.get(column).get("null_rate");
+            if (Math.random() < (double) nrate / 10000) {
+                pageBuilder.setNull(column);
+            }
+            else {
+                Integer max = columnOptions.get(column).get("max_value");
+                Integer min = columnOptions.get(column).get("min_value");
+                if (max != null) {
+                    if (min != null) {
+                        Integer s = min + rnd.nextInt((max - min));
+                        pageBuilder.setLong(column, s);
+                    }
+                    else {
+                        pageBuilder.setLong(column, rnd.nextInt(max));
+                    }
                 }
                 else {
-                    pageBuilder.setLong(column, rnd.nextInt(max));
+                    pageBuilder.setLong(column, rnd.nextInt(10000));
                 }
             }
-            else {
-                pageBuilder.setLong(column, rnd.nextInt(10000));
-            }
         }
     }
 
     @Override
     public void doubleColumn(Column column)
     {
-        Integer max = columnOptions.get(column).get("max_value");
-        Integer min = columnOptions.get(column).get("min_value");
-        if (max != null) {
-            if (min != null) {
-                Double d = min + rnd.nextInt((max - min) - 1) + rnd.nextDouble();
-                pageBuilder.setDouble(column, d);
+        Integer nrate = columnOptions.get(column).get("null_rate");
+        if (Math.random() < (double) nrate / 10000) {
+            pageBuilder.setNull(column);
+        }
+        else {
+            Integer max = columnOptions.get(column).get("max_value");
+            Integer min = columnOptions.get(column).get("min_value");
+            if (max != null) {
+                if (min != null) {
+                    Double d = min + rnd.nextInt((max - min) - 1) + rnd.nextDouble();
+                    pageBuilder.setDouble(column, d);
+                }
+                else {
+                    Double d = rnd.nextInt(max - 1) + rnd.nextDouble();
+                    pageBuilder.setDouble(column, d);
+                }
             }
             else {
-                Double d = rnd.nextInt(max - 1) + rnd.nextDouble();
-                pageBuilder.setDouble(column, d);
+                pageBuilder.setDouble(column, rnd.nextDouble() * 10000);
             }
         }
-        else {
-            pageBuilder.setDouble(column, rnd.nextDouble() * 10000);
-        }
     }
 
     @Override
     public void stringColumn(Column column)
     {
-        final Integer length = columnOptions.get(column).getOrDefault("length", 0);
-        if (length == 0) {
-            pageBuilder.setString(column, generator.generate(32));
+        Integer nrate = columnOptions.get(column).get("null_rate");
+        if (Math.random() < (double) nrate / 10000) {
+            pageBuilder.setNull(column);
         }
         else {
-            pageBuilder.setString(column, generator.generate(length));
+            final Integer length = columnOptions.get(column).getOrDefault("length", 0);
+            if (length == 0) {
+                pageBuilder.setString(column, generator.generate(32));
+            }
+            else {
+                pageBuilder.setString(column, generator.generate(length));
+            }
         }
     }
 
     @Override
     public void timestampColumn(Column column)
     {
-        final double randd = Math.random();
-        LocalDateTime randomDate = LocalDateTime.now()
-                .plusDays((long) (randd * 100))
-                .plusSeconds((long) (randd * 1000000));
-        Timestamp timestamp = Timestamp.ofEpochSecond(
-                randomDate.atZone(zoneId).toEpochSecond()
-        );
-        pageBuilder.setTimestamp(column, timestamp);
+        Integer nrate = columnOptions.get(column).get("null_rate");
+        if (Math.random() < (double) nrate / 10000) {
+            pageBuilder.setNull(column);
+        }
+        else {
+            final double randd = Math.random();
+            LocalDateTime randomDate = LocalDateTime.now()
+                    .plusDays((long) (randd * 100))
+                    .plusSeconds((long) (randd * 1000000));
+            Timestamp timestamp = Timestamp.ofEpochSecond(
+                    randomDate.atZone(zoneId).toEpochSecond()
+            );
+            pageBuilder.setTimestamp(column, timestamp);
+        }
     }
 
     @Override

data/src/main/java/org/embulk/input/randomj/RandomjInputPlugin.java

@@ -1,7 +1,5 @@
 package org.embulk.input.randomj;
 
-import org.apache.commons.text.CharacterPredicates;
-import org.apache.commons.text.RandomStringGenerator;
 import org.embulk.config.Config;
 import org.embulk.config.ConfigDefault;
 import org.embulk.config.ConfigDiff;
@@ -26,11 +24,6 @@ import java.util.stream.IntStream;
 public class RandomjInputPlugin
         implements InputPlugin
 {
-    private final RandomStringGenerator generator = new RandomStringGenerator.Builder()
-            .withinRange('0', 'z')
-            .filteredBy(CharacterPredicates.LETTERS, CharacterPredicates.DIGITS)
-            .build();
-
     public interface PluginTask
             extends Task
     {
@@ -77,6 +70,7 @@ public class RandomjInputPlugin
             Schema schema, int taskCount,
             List<TaskReport> successTaskReports)
     {
+        // throw new UnsupportedOperationException("randomj input plugin does not support cleanup");
     }
 
     @Override
@@ -115,6 +109,7 @@ public class RandomjInputPlugin
             HashMap<String, Integer> miniMap = new HashMap<>();
             ColumnConfig c = schemaConfig.getColumn(column.getIndex());
             miniMap.put("length", c.getOption().get(Integer.class, "length", 0));
+            miniMap.put("null_rate", c.getOption().get(Integer.class, "null_rate", 0));
             miniMap.put("max_value", c.getOption().get(Integer.class, "max_value", null));
             miniMap.put("min_value", c.getOption().get(Integer.class, "min_value", null));
             lengthMap.put(column, miniMap);

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-input-randomj
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.4.0
 platform: ruby
 authors:
 - yuokada
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2017-09-10 00:00:00.000000000 Z
+date: 2017-12-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -62,7 +62,7 @@ files:
 - src/test/java/org/embulk/input/randomj/TestRandomjInputPlugin.java
 - classpath/commons-lang3-3.5.jar
 - classpath/commons-text-1.1.jar
-- classpath/embulk-input-randomj-0.3.0.jar
+- classpath/embulk-input-randomj-0.4.0.jar
 homepage: https://github.com/yuokada/embulk-input-randomj
 licenses:
 - MIT