embulk-input-randomj 0.5.0 → 0.5.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 82a2c62219bbf24fdfc2cf2bb24f8f0b2210b829
-  data.tar.gz: bd0cad5900b36c009f6e92fa86357b5edd97c741
+  metadata.gz: 312c2e6df7d5ebe9d43775931f076d6d63502615
+  data.tar.gz: 68749dee15b28f53a187bac33dd8c5fa63380641
 SHA512:
-  metadata.gz: ff0b599c31cf975d09f28ca2e3328abc5fcbee510832d551d0ab5b0e0dc7b63005f269478a9ff3bccaddaf68ba4435964a9f126ed736d62d86eed7843f66f2cd
-  data.tar.gz: 3a6ec59cbc4d120e15b5012c26633bf4494e47870b58cc67be520851c5695807bb1553012b82748f97ed7ca6f16f0dfa556f9b690cbeef6d3d2484a0115cc998
+  metadata.gz: 8671f2b06c0eb6e36fb01fe31e96b8b9a8f6078100e3c749d5c7e284e923ecce8d5431a173bf271b00992def93a469d9f283377f410507906d04fc7b1d1f65d1
+  data.tar.gz: 10ae95c775793701dd4e88eb6f73e4173f9bfb16f3afb19a8dc321ef6f2fd9fa0010c04fafd889e5ecc74c5d03ec68aaafcf57dafb147e2afbf831b7157d86cb

data/.travis.yml

@@ -11,4 +11,4 @@ script:
   - ./gradlew --info checkstyle
   - ./gradlew --info check
 
-after_success: ./gradlew sonarqube -Dsonar.organization=yuokada-github -Dsonar.host.url=https://sonarcloud.io -Dsonar.login=${SONAR_TOKEN}
+# after_success: ./gradlew sonarqube -Dsonar.organization=yuokada-github -Dsonar.host.url=https://sonarcloud.io -Dsonar.login=${SONAR_TOKEN}

data/README.md

@@ -45,23 +45,14 @@ in:
       - {name: purchase, type: timestamp, format: '%Y/%m/%d'}
 ```
 
-Add `length`, `max_value`, `min_value` option (from 0.3.0)
-```yaml
-in:
-    type: randomj
-    rows: 16
-    threads: 1
-    primary_key: myid
-    schema:
-      - {name: myid,     type: long}
-      - {name: named,    type: string, length: 12}
-      - {name: price,    type: long, max_value: 1080, min_value: 100}
-      - {name: purchase, type: timestamp, format: '%Y/%m/%d'}
-```
+- Add `length`, `max_value`, `min_value` option (from 0.3.0)
+- Add `null_rate` option (from 0.4.0)  
+  This configuration is that inserted `null` into `price` filed with a probability `8` of 10000.
+- Support json type (from 0.5.0)
+- Support `start_date` & `end_date` key in **Timestamp** field.
 
-Add `null_rate` option (from 0.4.0)
-
-This configuration is that inserted `null` into `price` filed with a probability `8` of 10000.
+    - Ex1. `{name: created_at, type: timestamp, format: '%Y-%m-%d %H:%M:%S', start_date: 20180331, end_date: 20180430}`
+    - Ex2. `{name: created_at, type: timestamp, format: '%Y-%m-%d %H:%M:%S', start_date: 20180331}`
 
 ```yaml
 in:
@@ -74,10 +65,10 @@ in:
       - {name: named,    type: string, length: 12}
       - {name: price,    type: long, max_value: 1080, min_value: 100, null_rate: 8}
       - {name: purchase, type: timestamp, format: '%Y/%m/%d'}
+      - {name: json_key, type: json, schema: '[{"name": "baz", "type": "array", "items": {"type": "string", "size": 1}}]' }
 ```
 
 
-
 ## Usage
 
 ### Example1
@@ -202,6 +193,14 @@ $ embulk run -I lib config/example.yml
 
 ## ChangeLog
 
+### 0.5.1
+
+- Support start_date & end_date key with Timestamp field.
+
+### 0.5.0
+
+- Support `json` datatype
+
 ### v0.4
 
 - Support null_rate parameter

data/build.gradle

@@ -15,7 +15,7 @@ configurations {
     provided
 }
 
-version = "0.5.0"
+version = "0.5.1"
 
 sourceCompatibility = 1.8
 targetCompatibility = 1.8

data/example/README.md

@@ -0,0 +1,9 @@
+# HELP
+
+## How to run with "ROWS" Environment
+
+You can change the number of output-records by using the environment variable "ROWS".
+
+```
+% ROWS=160 embulk run etc/config.yml.liquid
+```

data/example/config.yml

@@ -11,7 +11,7 @@ in:
     - {name: x_flag,   type: boolean}
     - {name: rate,     type: double, max_value: 100, min_value: -100}
     - {name: score,    type: long, max_value: 255, min_value: 100}
-    - {name: time,     type: timestamp, format: '%Y-%m-%d %H:%M:%S'}
+    - {name: time,     type: timestamp, format: '%Y-%m-%d %H:%M:%S', start_date: 20171221, end_date: 20171231}
     - {name: purchase, type: timestamp, format: '%Y/%m/%d'}
     - {name: json_obj, type: json, schema: '[{"name": "obj1", "type": "object", "items": [{"name": "key1", "type": "string"}, {"name": "key2", "type": "number"} ] }]' }
     - {name: json_key, type: json, schema: '[{"name": "baz", "type": "array", "items": {"type": "string", "size": 1}}]' }

data/example/config.yml.liquid

@@ -0,0 +1,18 @@
+in:
+  type: randomj
+  rows: {{ env.ROWS }}
+  threads: 1
+  # default_timezone: Asia/Tokyo
+  primary_key: myid
+  schema:
+    - {name: myid,     type: long}
+    - {name: named,    type: string}
+    - {name: named_s,  type: string, length: 8}
+    - {name: x_flag,   type: boolean}
+    - {name: rate,     type: double, max_value: 100, min_value: -100}
+    - {name: score,    type: long, max_value: 255, min_value: 100}
+    - {name: time,     type: timestamp, format: '%Y-%m-%d %H:%M:%S', start_date: 20180331}
+    - {name: purchase, type: timestamp, format: '%Y/%m/%d'}
+
+out:
+  type: stdout

data/src/main/java/org/embulk/input/randomj/JsonColumnVisitor.java

@@ -12,7 +12,6 @@ import java.util.Random;
 
 public class JsonColumnVisitor
 {
-
     private final Map<String, Object> map;
     private final Random rnd = new Random();
     private final RandomStringGenerator generator = new RandomStringGenerator.Builder()

data/src/main/java/org/embulk/input/randomj/RandomjColumnVisitor.java

@@ -3,6 +3,8 @@ package org.embulk.input.randomj;
 import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.common.cache.Cache;
+import com.google.common.cache.CacheBuilder;
 import org.apache.commons.text.CharacterPredicates;
 import org.apache.commons.text.RandomStringGenerator;
 import org.embulk.spi.Column;
@@ -11,8 +13,13 @@ import org.embulk.spi.PageBuilder;
 import org.embulk.spi.json.JsonParser;
 import org.embulk.spi.time.Timestamp;
 
+import java.time.Duration;
+import java.time.LocalDate;
 import java.time.LocalDateTime;
 import java.time.ZoneId;
+import java.time.ZonedDateTime;
+import java.time.format.DateTimeFormatter;
+import java.time.format.ResolverStyle;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -35,6 +42,19 @@ public class RandomjColumnVisitor
     private final JsonParser jsonParser = new JsonParser();
     private final ObjectMapper mapper = new ObjectMapper();
 
+    private static final DateTimeFormatter formatter = DateTimeFormatter
+            .ofPattern("yyyyMMdd")
+            .withResolverStyle(ResolverStyle.LENIENT);
+    private static final long cacheSize = 64;
+    private static final Cache<String, ZonedDateTime> zonedDateTimeCache = CacheBuilder
+            .newBuilder()
+            .maximumSize(cacheSize)
+            .build();
+    private static final Cache<String, Long> durationCache = CacheBuilder
+            .newBuilder()
+            .maximumSize(cacheSize)
+            .build();
+
     private static final String NULL_RATE = "null_rate";
 
     public RandomjColumnVisitor(PageBuilder pageBuilder, PluginTask task, Integer row,
@@ -149,15 +169,56 @@ public class RandomjColumnVisitor
             pageBuilder.setNull(column);
         }
         else {
-            final double randd = Math.random();
-            LocalDateTime randomDate = LocalDateTime.now()
-                    .plusDays((long) (randd * 100))
-                    .plusSeconds((long) (randd * 1000000));
-            Timestamp timestamp = Timestamp.ofEpochSecond(
-                    randomDate.atZone(zoneId).toEpochSecond()
-            );
-            pageBuilder.setTimestamp(column, timestamp);
+            ZonedDateTime start = getZonedDatetime(column, "start_date");
+            ZonedDateTime end = getZonedDatetime(column, "end_date");
+            long duration = getDuration(column, start, end);
+            if (duration != 0) {
+                int plus = rnd.nextInt((int) duration);
+                Timestamp timestamp = Timestamp.ofEpochSecond(
+                        start.plusSeconds(plus).toEpochSecond()
+                );
+                pageBuilder.setTimestamp(column, timestamp);
+            }
+            else {
+                final double randd = Math.random();
+                LocalDateTime randomDate = LocalDateTime.now()
+                        .plusDays((long) (randd * 100))
+                        .plusSeconds((long) (randd * 1000000));
+                Timestamp timestamp = Timestamp.ofEpochSecond(
+                        randomDate.atZone(zoneId).toEpochSecond()
+                );
+                pageBuilder.setTimestamp(column, timestamp);
+            }
+        }
+    }
+
+    private ZonedDateTime getZonedDatetime(Column column, String dateString)
+    {
+        String cacheKey = String.format("%s::%s", column.getName(), dateString);
+        ZonedDateTime start = zonedDateTimeCache.getIfPresent(cacheKey);
+        if (start == null) {
+            Integer startDate = columnOptions.get(column).getOrDefault(dateString, null);
+            if (startDate == null) {
+                start = LocalDate.now().atStartOfDay(zoneId);
+            }
+            else {
+                start = LocalDate.parse(startDate.toString(), formatter)
+                        .atStartOfDay(zoneId);
+            }
+            zonedDateTimeCache.put(cacheKey, start);
+        }
+
+        return start;
+    }
+
+    private long getDuration(Column column, ZonedDateTime start, ZonedDateTime end)
+    {
+        Long duration = durationCache.getIfPresent(column.getName());
+        if (duration == null) {
+            duration = Duration.between(start, end).getSeconds();
+            durationCache.put(column.getName(), duration);
         }
+        return duration;
     }
 
     @Override
@@ -181,7 +242,8 @@ public class RandomjColumnVisitor
 
     private void visit(JsonNode node, JsonColumnVisitor visitor)
     {
-        SupportedJsonObject object = SupportedJsonObject.valueOf(node.get("type").asText().toUpperCase());
+        SupportedJsonObject object = SupportedJsonObject
+                .valueOf(node.get("type").asText().toUpperCase());
         if (object.equals(SupportedJsonObject.BOOLEAN)) {
             visitor.booleanNode(node);
         }
@@ -201,7 +263,8 @@ public class RandomjColumnVisitor
             visitor.objectNode(node);
         }
         else {
-            throw new UnsupportedOperationException("randomj input plugin does not support json-data type");
+            throw new UnsupportedOperationException(
+                    "randomj input plugin does not support json-data type");
         }
     }
 }

data/src/main/java/org/embulk/input/randomj/RandomjInputPlugin.java

@@ -94,6 +94,9 @@ public class RandomjInputPlugin
             miniMap.put("null_rate", c.getOption().get(Integer.class, "null_rate", 0));
             miniMap.put("max_value", c.getOption().get(Integer.class, "max_value", null));
             miniMap.put("min_value", c.getOption().get(Integer.class, "min_value", null));
+            // For datetime
+            miniMap.put("start_date", c.getOption().get(Integer.class, "start_date", null));
+            miniMap.put("end_date", c.getOption().get(Integer.class, "end_date", null));
             lengthMap.put(column, miniMap);
         }
         return lengthMap;

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-input-randomj
 version: !ruby/object:Gem::Version
-  version: 0.5.0
+  version: 0.5.1
 platform: ruby
 authors:
 - yuokada
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2018-02-25 00:00:00.000000000 Z
+date: 2018-05-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -52,7 +52,9 @@ files:
 - build.gradle
 - config/checkstyle/checkstyle.xml
 - config/checkstyle/default.xml
+- example/README.md
 - example/config.yml
+- example/config.yml.liquid
 - gradle/wrapper/gradle-wrapper.jar
 - gradle/wrapper/gradle-wrapper.properties
 - gradlew
@@ -65,7 +67,7 @@ files:
 - src/test/java/org/embulk/input/randomj/TestRandomjInputPlugin.java
 - classpath/commons-lang3-3.5.jar
 - classpath/commons-text-1.1.jar
-- classpath/embulk-input-randomj-0.5.0.jar
+- classpath/embulk-input-randomj-0.5.1.jar
 homepage: https://github.com/yuokada/embulk-input-randomj
 licenses:
 - MIT