embulk-filter-add_time 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 8d4e2b549cb21e8c80956270dffd3a7d3fde0a96
+  data.tar.gz: da794c2b1309cf71015a035876eb142e0056dfdb
+SHA512:
+  metadata.gz: 14cff8010192c946d9349fe1d22198784205f4ddbc8d4537b2b9ceb98269553abeeff028fea808300a7c13489ddded45966ea39e81add2835cf36b474bbe9e6e
+  data.tar.gz: 66d5eada349e6502a170d05236d4f92223083b131a424bb3b946d856b5462523b19d923df03c2b73e60646e57995f0da3aa2977e4d268d108d364d6934feb0d7

data/.gitignore

@@ -0,0 +1,15 @@
+*~
+/pkg/
+/tmp/
+*.gemspec
+.gradle/
+/.gradle
+/classpath/
+build/
+*.iml
+.idea
+/.settings/
+/.metadata/
+.classpath
+.project
+/pkg/

data/.travis.yml

@@ -0,0 +1,7 @@
+language: java
+jdk: oraclejdk7
+
+script: ./gradlew clean test
+
+after_success:
+- ./gradlew check jacocoRootReport

data/CHANGELOG.md

@@ -0,0 +1,3 @@
+## 0.1.0 - 2016-01-28
+
+The first release!!

data/COPYING

@@ -0,0 +1,14 @@
+Copyright (C) 2016 Muga Nishizawa
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+

data/README.md

@@ -0,0 +1,212 @@
+# Add time filter plugin for Embulk
+
+The `embulk-filter-add_time` plugin allows users to add a new time-based column to the existing schema either by copying the value from another existing column in the schema or by specifying the value. The latter use case is particularly useful to tag all events bulk loaded with one Embulk run with a specific timestamp or time range.
+
+## Overview
+
+* **Plugin type**: filter
+
+## Configuration
+
+### to_column configuration
+
+This configuration is **required**.
+
+The `to_column` configuration specifies the name, type, and optional format of a new column to be added to the Embulk schema.
+
+This configuration specifies the name, type, and format for the added time column as a set of key-value pairs:
+- **name**<br/>
+a string specifying the name of the new column (**required**).
+- **type**<br/>
+a string specifying the data type of the column: `long` or `timestamp` are the supported values for this parameter since they need to express a valid timestamp (**required**).
+- **unix_timestamp_unit**<br/>
+a string specifying the unit for the unix timestamp - it is required if the column type is `long`. `sec`, `milli` (for milliseconds), `micro` (for microseconds), or `nano` (for nanoseconds) are the supported values for this option (default is `sec`).
+
+For example, the `to_column` configuration can be used to add a `'time'` column to the schema:
+
+```yaml
+filters:
+- type: add_time
+  to_column:
+    name: time
+    type: timestamp
+...
+```
+
+Note that if a column with the **same name** as `name` already exists in the schema, the existing column name is changed by appending '\_' to its name. For example, if `'created_at'` is specified as `to_column` but the column name already exists, the existing column name is changed to `'created_at_'`.
+
+When `long` type is specified for the `type` parameter, `unix_timestamp_unit` is required to convert from the unit timestamp value to `long` and tell the plugin how to exactly parse the timestamp number; `sec` for seconds, `milli` for milliseconds, `micro` for microseconds, or `nano` for nanoseconds are the supported values for this option:
+
+```yaml
+filters:
+- type: add_time
+  to_column:
+    name: time
+    type: long
+    unix_timestamp_unit: sec
+...
+```
+
+### from_value configuration
+
+This configuration is mutually exclusive with `from_column` below. One amongst `from_column` and `from_value` configurations is required.
+
+The `from_value` configuration specifies a specific value or range to be used as value of the column added by the `to_column` configuration. This configuration supports different modes and the behavior will vary accordingly. Refer to the examples below.
+
+This configuration specifies the mode, the timestamp_format, and one amongst value, from and to, or nothing depending on the mode of choice as a set of key-value pairs.
+These parameters are required:
+- **mode**<br/>
+a string specifying the mode. There are 3 valid modes (default is `fixed_time`):
+  * `fixed_time`<br/>
+  In this mode, all values of the added `to_column` column are set with the fixed value specified by the `value` parameter, which becomes a required parameter, see below;
+  * `incremental_time`<br/>
+  In this mode, all values of the added `to_column` column are set to a value that increments by 1 second for each record, starting at `from` timestamp and up to `to` timestamp, after which it wrap around and starts from `from` again and so on - hence the additional `from` and `to` parameters are required in this mode.
+  * `upload_time`<br/>
+  In this mode, all values of the added `to_column` column are set with the fixed value corresponding to the time the Embulk upload was started. This mode does not require additional parameters.
+- **timestamp_format**<br/>
+a string specifying how to parse the string value provided as either `value` or `from`/`to` parameters depending on the `mode` in use (default is `"%Y-%m-%d %H:%M:%S %z"`).<br/>
+It follow the [Ruby's `strptime` format](http://ruby-doc.org/stdlib-2.0.0/libdoc/date/rdoc/DateTime.html#method-c-strptime). Note that at the moment also Unix timestamps and epoch times need to be specified as string, therefore the `timestamp_format` parameter needs to specify how to parse it.
+- **unix_timestamp_unit**<br/>
+a string specifying the expected unit of the unix timestamp for the long value provides as either `value` or `from`/`to` parameters `sec`, `milli` (for milliseconds), `micro` (for microseconds), or `nano` (for nanoseconds) are the supported values for this option (default is `sec`).
+
+These parameters are mutually exclusive and the usage depend on the `mode`. The type of them is string or long. The format of the string value used here needs to match the rule provided in the `timestamp_format` parameter. The long value is converted to the unix timestamp with the unit provided in the `unix_timestamp_unit` parameter:
+- **value**<br/>
+a string or long values specifying a fixed value for the added time column. This options is required if the mode is `fixed_time`.
+- **from** and **to**<br/>
+two strings or long values specifying the value for the beginning and end of the range of time for `mode: incremental_time`. The format needs to be consistent between these two parameters.
+
+Example: `mode: fixed_time`
+```yaml
+filters:
+- type: add_time
+  to_column:
+    name: time
+    type: timestamp
+  from_value:
+    mode: fixed_time
+    value: "2016-01-01 00:00:00 UTC"
+    timestamp_format: "%Y-%m-%d %H:%M:%S %z"
+```
+
+The mode `fixed_time` is default and can be omitted:
+
+```yaml
+filters:
+- type: add_time
+  to_column:
+    name: time
+    type: timestamp
+  from_value:
+    value: "2016-01-01 00:00:00 UTC"
+```
+```yaml
+filters:
+- type: add_time
+  to_column:
+    name: time
+    type: timestamp
+  from_value:
+    value: 1453939479
+```
+
+where in this example the format of the value is also corresponding to the default, therefore the `timestamp_format` option is also omitted.
+
+Example: `mode: incremental_time`
+```yaml
+filters:
+- type: add_time
+  to_column:
+    name: time
+    type: timestamp
+  from_value:
+    mode: incremental_time
+    from: "2016-01-01 00:00:00 UTC"
+    to: "2016-01-01 01:00:00 UTC"
+```
+
+Example: `mode: upload_time`
+```yaml
+filters:
+- type: add_time
+  to_column:
+    name: time
+    type: timestamp
+  from_value:
+    mode: upload_time
+```
+
+where neither of the additional parameters `value`, `to`, and `from` is specified. Embulk will associate the fixed bulk upload start time to every record.
+
+### from_column configuration
+
+This configuration is mutually exclusive with `from_value` above. One amongst `from_column` and `from_value` configurations is required.
+
+The `from_column` configuration specifies the name of one of the columns found in the Embulk schema and the format to be used to parse and feed values in the column added by the `to_column` configuration. This configuration makes a copy of the values from the column specified by `name`, instead of renaming the source column itself.
+The parameters for this configuration are expressed as a set of key-value pairs.
+- **name**<br/>
+a string specifying the name of the source column from the Embulk schema. The column type must be one of `long`, `timestamp` or `string` (**required**).
+- **unix_timestamp_unit**<br/>
+a string specifying the expected unit of the unix timestamp for the values of the source column: it is **required** only if the type of the source column is `long` (see above for supported types). The supported values are `sec` for seconds, `milli` for milliseconds, `micro` for microseconds, and `nano` for nanoseconds (default is `sec`).
+- **timestamp_format**<br/>
+a string specifying the expected format of the values in the source column: it is **required** only if the type of the column is `string` (see above for supported types) (default is `"%Y-%m-%d %H:%M:%S %z"`). It follow the [Ruby's `strptime` format](http://ruby-doc.org/stdlib-2.0.0/libdoc/date/rdoc/DateTime.html#method-c-strptime).
+
+Note that if neither `timestamp_format` or `unix_timestamp_unit` parameters are specified, the column type is expected to be `timestamp`.
+
+Example: `created_at` column with `timestamp` data type
+```yaml
+filters:
+- type: add_time
+  to_column:
+    name: time
+    type: timestamp
+  from_column:
+    name: created_at
+```
+
+In this example the expected type for the source column `created_at` is `timestamp`. If the `created_at` column type is string, the `timestamp_format` parameter is required to provide Embulk with a way to parse the values and convert them to a `timestamp`.
+
+```yaml
+filters:
+- type: add_time
+  to_column:
+    name: time
+    type: timestamp
+  from_column:
+    name: created_at
+    timestamp_format: "%Y-%m-%d %H:%M:%S"
+    timezone: UTC
+```
+
+If the type of the source column is long, the `unix_timestamp_unit` parameter is required. The additional parameter provides Embulk the information to properly parse the `long` values of the source column and convert them to type specified in the `to_column` configuration.
+
+```yaml
+filters:
+- type: add_time
+  to_column:
+    name: time
+    type: long
+    unix_timestamp_unit: sec
+  from_column:
+    name: created_at
+    unixtime_unit: milli
+```
+
+## Install
+
+```
+$ embulk gem install embulk-filter-add_time
+```
+
+## Build
+
+### Build by Gradle
+```
+$ git clone https://github.com/treasure-data/embulk-filter-add_time.git
+$ cd embulk-filter-add_time
+$ ./gradlew gem classpath
+```
+
+### Run on Embulk
+```
+$ bin/embulk run -I embulk-filter-add_time/lib/ config.yml
+```

data/build.gradle

@@ -0,0 +1,82 @@
+plugins {
+    id "com.jfrog.bintray" version "1.1"
+    id "com.github.jruby-gradle.base" version "0.1.5"
+    id "java"
+}
+
+apply from: 'gradle/check.gradle'
+
+import com.github.jrubygradle.JRubyExec
+repositories {
+    mavenCentral()
+    jcenter()
+}
+configurations {
+    provided
+}
+
+version = "0.1.0"
+
+compileJava.options.encoding = 'UTF-8' // source encoding
+sourceCompatibility = 1.7
+targetCompatibility = 1.7
+
+dependencies {
+    compile  "org.embulk:embulk-core:0.7.10"
+    provided "org.embulk:embulk-core:0.7.10"
+
+    testCompile "junit:junit:4.+"
+    testCompile "org.embulk:embulk-standards:0.7.10"
+    testCompile "org.embulk:embulk-core:0.7.10:tests"
+    testCompile "org.mockito:mockito-core:1.9.5"
+}
+
+task classpath(type: Copy, dependsOn: ["jar"]) {
+    doFirst { file("classpath").deleteDir() }
+    from (configurations.runtime - configurations.provided + files(jar.archivePath))
+    into "classpath"
+}
+clean { delete "classpath" }
+
+task gem(type: JRubyExec, dependsOn: ["gemspec", "classpath"]) {
+    jrubyArgs "-rrubygems/gem_runner", "-eGem::GemRunner.new.run(ARGV)", "build"
+    script "${project.name}.gemspec"
+    doLast { ant.move(file: "${project.name}-${project.version}.gem", todir: "pkg") }
+}
+
+task gemPush(type: JRubyExec, dependsOn: ["gem"]) {
+    jrubyArgs "-rrubygems/gem_runner", "-eGem::GemRunner.new.run(ARGV)", "push"
+    script "pkg/${project.name}-${project.version}.gem"
+}
+
+task "package"(dependsOn: ["gemspec", "classpath"]) << {
+    println "> Build succeeded."
+    println "> You can run embulk with '-L ${file(".").absolutePath}' argument."
+}
+
+task gemspec {
+    ext.gemspecFile = file("${project.name}.gemspec")
+    inputs.file "build.gradle"
+    outputs.file gemspecFile
+    doLast { gemspecFile.write($/
+Gem::Specification.new do |spec|
+  spec.name          = "${project.name}"
+  spec.version       = "${project.version}"
+  spec.authors       = ["Muga Nishizawa"]
+  spec.summary       = %[Add time filter plugin for Embulk]
+  spec.description   = %[Add time column to the schema]
+  spec.email         = ["muga.nishizawa@gmail.com"]
+  spec.licenses      = ["Apache 2.0"]
+  spec.homepage      = "https://github.com/treasure-data/embulk-filter-add_time"
+
+  spec.files         = `git ls-files`.split("\n") + Dir["classpath/*.jar"]
+  spec.test_files    = spec.files.grep(%r"^(test|spec)/")
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency 'bundler', ['~> 1.0']
+  spec.add_development_dependency 'rake', ['>= 10.0']
+end
+/$)
+    }
+}
+clean { delete "${project.name}.gemspec" }

data/gradle/check.gradle

@@ -0,0 +1,34 @@
+// Checkstyle
+// @see https://docs.gradle.org/2.5/userguide/checkstyle_plugin.html
+apply plugin: 'checkstyle'
+checkstyle {
+    ignoreFailures = true
+    // @see https://github.com/facebook/presto/blob/master/src/checkstyle/checks.xml
+    //configFile = rootProject.file('./checkstyle.xml') // default {project.projectDir}/config/checkstyle/checkstyle.xml
+}
+
+// FindBugs
+// @see https://docs.gradle.org/2.5/userguide/findbugs_plugin.html
+apply plugin: 'findbugs'
+findbugs {
+    ignoreFailures = true
+}
+
+// PMD
+// @see https://docs.gradle.org/2.5/userguide/pmd_plugin.html
+apply plugin: 'pmd'
+tasks.withType(Pmd) {
+    ignoreFailures = true
+    reports.html.enabled true
+}
+
+// JaCoCo
+// @see https://docs.gradle.org/2.5/userguide/jacoco_plugin.html
+apply plugin: 'jacoco'
+jacocoTestReport { // will use td-client v0.6.x
+    afterEvaluate {
+        classDirectories = files(classDirectories.files.collect {
+            fileTree(dir: it, exclude: 'com/treasuredata/api/**')
+        })
+    }
+}

data/gradle/wrapper/gradle-wrapper.properties

@@ -0,0 +1,6 @@
+#Tue Aug 11 00:26:20 PDT 2015
+distributionBase=GRADLE_USER_HOME
+distributionPath=wrapper/dists
+zipStoreBase=GRADLE_USER_HOME
+zipStorePath=wrapper/dists
+distributionUrl=https\://services.gradle.org/distributions/gradle-2.6-bin.zip

data/gradlew

@@ -0,0 +1,164 @@
+#!/usr/bin/env bash
+
+##############################################################################
+##
+##  Gradle start up script for UN*X
+##
+##############################################################################
+
+# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
+DEFAULT_JVM_OPTS=""
+
+APP_NAME="Gradle"
+APP_BASE_NAME=`basename "$0"`
+
+# Use the maximum available, or set MAX_FD != -1 to use that value.
+MAX_FD="maximum"
+
+warn ( ) {
+    echo "$*"
+}
+
+die ( ) {
+    echo
+    echo "$*"
+    echo
+    exit 1
+}
+
+# OS specific support (must be 'true' or 'false').
+cygwin=false
+msys=false
+darwin=false
+case "`uname`" in
+  CYGWIN* )
+    cygwin=true
+    ;;
+  Darwin* )
+    darwin=true
+    ;;
+  MINGW* )
+    msys=true
+    ;;
+esac
+
+# For Cygwin, ensure paths are in UNIX format before anything is touched.
+if $cygwin ; then
+    [ -n "$JAVA_HOME" ] && JAVA_HOME=`cygpath --unix "$JAVA_HOME"`
+fi
+
+# Attempt to set APP_HOME
+# Resolve links: $0 may be a link
+PRG="$0"
+# Need this for relative symlinks.
+while [ -h "$PRG" ] ; do
+    ls=`ls -ld "$PRG"`
+    link=`expr "$ls" : '.*-> \(.*\)$'`
+    if expr "$link" : '/.*' > /dev/null; then
+        PRG="$link"
+    else
+        PRG=`dirname "$PRG"`"/$link"
+    fi
+done
+SAVED="`pwd`"
+cd "`dirname \"$PRG\"`/" >&-
+APP_HOME="`pwd -P`"
+cd "$SAVED" >&-
+
+CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar
+
+# Determine the Java command to use to start the JVM.
+if [ -n "$JAVA_HOME" ] ; then
+    if [ -x "$JAVA_HOME/jre/sh/java" ] ; then
+        # IBM's JDK on AIX uses strange locations for the executables
+        JAVACMD="$JAVA_HOME/jre/sh/java"
+    else
+        JAVACMD="$JAVA_HOME/bin/java"
+    fi
+    if [ ! -x "$JAVACMD" ] ; then
+        die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME
+
+Please set the JAVA_HOME variable in your environment to match the
+location of your Java installation."
+    fi
+else
+    JAVACMD="java"
+    which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
+
+Please set the JAVA_HOME variable in your environment to match the
+location of your Java installation."
+fi
+
+# Increase the maximum file descriptors if we can.
+if [ "$cygwin" = "false" -a "$darwin" = "false" ] ; then
+    MAX_FD_LIMIT=`ulimit -H -n`
+    if [ $? -eq 0 ] ; then
+        if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then
+            MAX_FD="$MAX_FD_LIMIT"
+        fi
+        ulimit -n $MAX_FD
+        if [ $? -ne 0 ] ; then
+            warn "Could not set maximum file descriptor limit: $MAX_FD"
+        fi
+    else
+        warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT"
+    fi
+fi
+
+# For Darwin, add options to specify how the application appears in the dock
+if $darwin; then
+    GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\""
+fi
+
+# For Cygwin, switch paths to Windows format before running java
+if $cygwin ; then
+    APP_HOME=`cygpath --path --mixed "$APP_HOME"`
+    CLASSPATH=`cygpath --path --mixed "$CLASSPATH"`
+
+    # We build the pattern for arguments to be converted via cygpath
+    ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null`
+    SEP=""
+    for dir in $ROOTDIRSRAW ; do
+        ROOTDIRS="$ROOTDIRS$SEP$dir"
+        SEP="|"
+    done
+    OURCYGPATTERN="(^($ROOTDIRS))"
+    # Add a user-defined pattern to the cygpath arguments
+    if [ "$GRADLE_CYGPATTERN" != "" ] ; then
+        OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)"
+    fi
+    # Now convert the arguments - kludge to limit ourselves to /bin/sh
+    i=0
+    for arg in "$@" ; do
+        CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -`
+        CHECK2=`echo "$arg"|egrep -c "^-"`                                 ### Determine if an option
+
+        if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then                    ### Added a condition
+            eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"`
+        else
+            eval `echo args$i`="\"$arg\""
+        fi
+        i=$((i+1))
+    done
+    case $i in
+        (0) set -- ;;
+        (1) set -- "$args0" ;;
+        (2) set -- "$args0" "$args1" ;;
+        (3) set -- "$args0" "$args1" "$args2" ;;
+        (4) set -- "$args0" "$args1" "$args2" "$args3" ;;
+        (5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;;
+        (6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;;
+        (7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;;
+        (8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;;
+        (9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;;
+    esac
+fi
+
+# Split up the JVM_OPTS And GRADLE_OPTS values into an array, following the shell quoting and substitution rules
+function splitJvmOpts() {
+    JVM_OPTS=("$@")
+}
+eval splitJvmOpts $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS
+JVM_OPTS[${#JVM_OPTS[*]}]="-Dorg.gradle.appname=$APP_BASE_NAME"
+
+exec "$JAVACMD" "${JVM_OPTS[@]}" -classpath "$CLASSPATH" org.gradle.wrapper.GradleWrapperMain "$@"