embulk-filter-hash 0.1.2 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 55e32c59aa1c3fc35b3ea853b56be1ade5eb7544
-  data.tar.gz: 543dbf94ee96d0b2d8c5df445c02e1c38c5df69c
+  metadata.gz: 5ded1ac8ae36443e9654c2192980f2032b692e11
+  data.tar.gz: 379c360184e10e28c39eeaf1e7eccd25f582200f
 SHA512:
-  metadata.gz: 00fed17b6821661a3d3770020e4f70c5d5340542d040bd2f18f46befe1d045a86af0f2942f1abec5477e5e48ca5e39215b8df5a8aabcebeffaaa6be1dbd5de41
-  data.tar.gz: 6863c2ac3fbf8111efc9945f4b6963cdefe1eec6e8dc2a759620e17913e2561af5366fb6ecd56e564346f63f59c0bf3a913aaf4f58f62a9ef62add4052a1b2d3
+  metadata.gz: 72cfce290a43b1ff0df9f036482647fea65c589fc4ef2db8abfdb4f625c95902efb6216140b83f01546251ad2191406962ad04ea12ad1a3c22bfb20457ebe6f0
+  data.tar.gz: c333b812cc4807dd514682f8930a07400fc4d23851ee4010eaf6274021a83ce408c6db7d8b15cf819c7d9d1654508a668aaac3e54dbf551eaba799aad7bf2030

data/.travis.yml

@@ -0,0 +1,9 @@
+sudo: required
+
+language: java
+
+jdk:
+  - oraclejdk8
+
+script:
+  - ./gradlew --info check

data/README.md

@@ -1,3 +1,5 @@
+[![Build Status](https://travis-ci.org/kamatama41/embulk-filter-hash.svg?branch=master)](https://travis-ci.org/kamatama41/embulk-filter-hash)
+
 # Hash filter plugin for Embulk
 
 Embulk filter plugin to convert a input to hash.

data/build.gradle

@@ -7,18 +7,20 @@ import com.github.jrubygradle.JRubyExec
 repositories {
     mavenCentral()
     jcenter()
+    maven { url "http://kamatama41.github.com/embulk-test-helpers/repository" }
 }
 configurations {
     provided
 }
 
-version = "0.1.2"
+version = "0.2.0"
 
 dependencies {
-    compile  "org.embulk:embulk-core:0.8.8"
-    provided "org.embulk:embulk-core:0.8.8"
-    // compile "YOUR_JAR_DEPENDENCY_GROUP:YOUR_JAR_DEPENDENCY_MODULE:YOUR_JAR_DEPENDENCY_VERSION"
-    testCompile "junit:junit:4.+"
+    compile  "org.embulk:embulk-core:0.8.16"
+    provided "org.embulk:embulk-core:0.8.16"
+    testCompile "com.kamatama41:embulk-test-helpers:0.1.1"
+    // Uncomment when using local embulk-test-helpers (and settings.gradle as well)
+    //testCompile project(':embulk-test-helpers')
 }
 
 task classpath(type: Copy, dependsOn: ["jar"]) {

data/gradle/wrapper/gradle-wrapper.properties

@@ -1,6 +1,6 @@
-#Tue Aug 11 00:26:20 PDT 2015
+#Sun Jan 29 22:41:06 JST 2017
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-2.6-bin.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-3.3-bin.zip

data/gradlew

@@ -1,4 +1,4 @@
-#!/usr/bin/env bash
+#!/usr/bin/env sh
 
 ##############################################################################
 ##
@@ -6,12 +6,30 @@
 ##
 ##############################################################################
 
-# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
-DEFAULT_JVM_OPTS=""
+# Attempt to set APP_HOME
+# Resolve links: $0 may be a link
+PRG="$0"
+# Need this for relative symlinks.
+while [ -h "$PRG" ] ; do
+    ls=`ls -ld "$PRG"`
+    link=`expr "$ls" : '.*-> \(.*\)$'`
+    if expr "$link" : '/.*' > /dev/null; then
+        PRG="$link"
+    else
+        PRG=`dirname "$PRG"`"/$link"
+    fi
+done
+SAVED="`pwd`"
+cd "`dirname \"$PRG\"`/" >/dev/null
+APP_HOME="`pwd -P`"
+cd "$SAVED" >/dev/null
 
 APP_NAME="Gradle"
 APP_BASE_NAME=`basename "$0"`
 
+# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
+DEFAULT_JVM_OPTS=""
+
 # Use the maximum available, or set MAX_FD != -1 to use that value.
 MAX_FD="maximum"
 
@@ -30,6 +48,7 @@ die ( ) {
 cygwin=false
 msys=false
 darwin=false
+nonstop=false
 case "`uname`" in
   CYGWIN* )
     cygwin=true
@@ -40,31 +59,11 @@ case "`uname`" in
   MINGW* )
     msys=true
     ;;
+  NONSTOP* )
+    nonstop=true
+    ;;
 esac
 
-# For Cygwin, ensure paths are in UNIX format before anything is touched.
-if $cygwin ; then
-    [ -n "$JAVA_HOME" ] && JAVA_HOME=`cygpath --unix "$JAVA_HOME"`
-fi
-
-# Attempt to set APP_HOME
-# Resolve links: $0 may be a link
-PRG="$0"
-# Need this for relative symlinks.
-while [ -h "$PRG" ] ; do
-    ls=`ls -ld "$PRG"`
-    link=`expr "$ls" : '.*-> \(.*\)$'`
-    if expr "$link" : '/.*' > /dev/null; then
-        PRG="$link"
-    else
-        PRG=`dirname "$PRG"`"/$link"
-    fi
-done
-SAVED="`pwd`"
-cd "`dirname \"$PRG\"`/" >&-
-APP_HOME="`pwd -P`"
-cd "$SAVED" >&-
-
 CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar
 
 # Determine the Java command to use to start the JVM.
@@ -90,7 +89,7 @@ location of your Java installation."
 fi
 
 # Increase the maximum file descriptors if we can.
-if [ "$cygwin" = "false" -a "$darwin" = "false" ] ; then
+if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then
     MAX_FD_LIMIT=`ulimit -H -n`
     if [ $? -eq 0 ] ; then
         if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then
@@ -114,6 +113,7 @@ fi
 if $cygwin ; then
     APP_HOME=`cygpath --path --mixed "$APP_HOME"`
     CLASSPATH=`cygpath --path --mixed "$CLASSPATH"`
+    JAVACMD=`cygpath --unix "$JAVACMD"`
 
     # We build the pattern for arguments to be converted via cygpath
     ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null`
@@ -154,11 +154,19 @@ if $cygwin ; then
     esac
 fi
 
-# Split up the JVM_OPTS And GRADLE_OPTS values into an array, following the shell quoting and substitution rules
-function splitJvmOpts() {
-    JVM_OPTS=("$@")
+# Escape application args
+save ( ) {
+    for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done
+    echo " "
 }
-eval splitJvmOpts $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS
-JVM_OPTS[${#JVM_OPTS[*]}]="-Dorg.gradle.appname=$APP_BASE_NAME"
+APP_ARGS=$(save "$@")
+
+# Collect all arguments for the java command, following the shell quoting and substitution rules
+eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS"
+
+# by default we should be in the correct project dir, but when run from Finder on Mac, the cwd is wrong
+if [ "$(uname)" = "Darwin" ] && [ "$HOME" = "$PWD" ]; then
+  cd "$(dirname "$0")"
+fi
 
-exec "$JAVACMD" "${JVM_OPTS[@]}" -classpath "$CLASSPATH" org.gradle.wrapper.GradleWrapperMain "$@"
+exec "$JAVACMD" "$@"

data/gradlew.bat

@@ -8,14 +8,14 @@
 @rem Set local scope for the variables with windows NT shell
 if "%OS%"=="Windows_NT" setlocal
 
-@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
-set DEFAULT_JVM_OPTS=
-
 set DIRNAME=%~dp0
 if "%DIRNAME%" == "" set DIRNAME=.
 set APP_BASE_NAME=%~n0
 set APP_HOME=%DIRNAME%
 
+@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
+set DEFAULT_JVM_OPTS=
+
 @rem Find java.exe
 if defined JAVA_HOME goto findJavaFromJavaHome
 
@@ -46,10 +46,9 @@ echo location of your Java installation.
 goto fail
 
 :init
-@rem Get command-line arguments, handling Windowz variants
+@rem Get command-line arguments, handling Windows variants
 
 if not "%OS%" == "Windows_NT" goto win9xME_args
-if "%@eval[2+2]" == "4" goto 4NT_args
 
 :win9xME_args
 @rem Slurp the command line arguments.
@@ -60,11 +59,6 @@ set _SKIP=2
 if "x%~1" == "x" goto execute
 
 set CMD_LINE_ARGS=%*
-goto execute
-
-:4NT_args
-@rem Get arguments from the 4NT Shell from JP Software
-set CMD_LINE_ARGS=%$
 
 :execute
 @rem Setup the command line

data/settings.gradle

@@ -0,0 +1,3 @@
+// Uncomment when using local embulk-test-helpers (and build.gradle as well)
+//include ':embulk-test-helpers'
+//project(':embulk-test-helpers').projectDir = new File(settingsDir, '../embulk-test-helpers')

data/src/test/java/org/embulk/filter/hash/TestHashFilterPlugin.java

@@ -1,27 +1,90 @@
 package org.embulk.filter.hash;
 
-//import com.fasterxml.jackson.databind.ObjectMapper;
-//import com.google.inject.AbstractModule;
-//import com.google.inject.Guice;
-//import org.embulk.EmbulkEmbed;
-//import org.embulk.config.ConfigLoader;
-//import org.embulk.config.ModelManager;
-//import org.junit.Test;
-//
-//import java.io.File;
-//import java.io.IOException;
-
-public class TestHashFilterPlugin
-{
-//    @Test
-//    public void test() throws IOException {
-//        EmbulkEmbed embed = new EmbulkEmbed.Bootstrap().initialize();
-//        ConfigLoader loader = new ConfigLoader(new ModelManager(Guice.createInjector(new AbstractModule() {
-//            @Override
-//            protected void configure() {}
-//        }),
-//                new ObjectMapper()
-//        ));
-//        embed.run(loader.fromYamlFile(new File("./test.yml")));
-//    }
+import org.embulk.config.ConfigSource;
+import org.embulk.spi.FilterPlugin;
+import org.embulk.test.ExtendedTestingEmbulk;
+import org.junit.Rule;
+import org.junit.Test;
+
+import java.util.Arrays;
+import java.util.Collections;
+
+import static org.embulk.spi.type.Types.STRING;
+import static org.embulk.test.TestOutputPlugin.assertRecords;
+import static org.embulk.test.TestOutputPlugin.assertSchema;
+import static org.embulk.test.Utils.column;
+import static org.embulk.test.Utils.record;
+
+public class TestHashFilterPlugin {
+    @Rule
+    public ExtendedTestingEmbulk embulk = (ExtendedTestingEmbulk) ExtendedTestingEmbulk
+            .builder()
+            .registerPlugin(FilterPlugin.class, "hash", HashFilterPlugin.class)
+            .build();
+
+    @Test
+    public void specifiedColumnIsHashedAndRenamed() {
+        final String inConfigPath = "yaml/input_basic.yml";
+
+        ConfigSource config = embulk.newConfig()
+                .set("type", "hash")
+                .set("columns", Collections.singletonList(
+                        config().set("name", "age").set("algorithm", "MD5").set("new_name", "hashed_age")
+                        )
+                );
+
+        embulk.runFilter(config, inConfigPath);
+
+        assertSchema(
+                column("username", STRING),
+                column("hashed_age", STRING)
+        );
+
+        assertRecords(
+                record("user1", "98f13708210194c475687be6106a3b84")
+        );
+    }
+
+    @Test
+    public void allColumnTypesAreHashed() {
+        final String inConfigPath = "yaml/input_column_types.yml";
+
+        ConfigSource config = embulk.newConfig()
+                .set("type", "hash")
+                .set("columns", Arrays.asList(
+                        config().set("name", "username"),
+                        config().set("name", "age"),
+                        config().set("name", "weight"),
+                        config().set("name", "active"),
+                        config().set("name", "created_at"),
+                        config().set("name", "options")
+                        )
+                );
+
+        embulk.runFilter(config, inConfigPath);
+
+        assertSchema(
+                column("username", STRING),
+                column("age", STRING),
+                column("weight", STRING),
+                column("active", STRING),
+                column("created_at", STRING),
+                column("options", STRING)
+        );
+
+        assertRecords(
+                record(
+                        "0a041b9462caa4a31bac3567e0b6e6fd9100787db2ab433d96f6d178cabfce90",
+                        "6f4b6612125fb3a0daecd2799dfd6c9c299424fd920f9b308110a2c1fbd8f443",
+                        "70822ecbef5bee37d162492107a3127fc0a4de0564f34ce92713a7baaeb582b0",
+                        "b5bea41b6c623f7c09f1bf24dcae58ebab3c0cdd90ad966bc43a45b44867e12b",
+                        "9673fe7b67d880e2c9071428c63f6e1bea9dde98283297277a20b92ea0acdc72",
+                        "3ff0e331ca59a2a1194bac0e36359ed4540a97383e1cdf6eb95c7de9309143fc"
+                )
+        );
+    }
+
+    private ConfigSource config() {
+        return embulk.newConfig();
+    }
 }

data/src/test/resources/yaml/input_basic.yml

@@ -0,0 +1,8 @@
+type: test
+data:
+- user1,20
+parser:
+  type: csv
+  columns:
+  - {name: username, type: string}
+  - {name: age, type: long}

data/src/test/resources/yaml/input_column_types.yml

@@ -0,0 +1,13 @@
+type: test
+data:
+- >-
+ user1,21,65.4,true,2016-03-27 01:23:45,{"a":{"b":"c"}}"
+parser:
+  type: csv
+  columns:
+  - {name: username,   type: string}
+  - {name: age,        type: long}
+  - {name: weight,     type: double}
+  - {name: active,     type: boolean}
+  - {name: created_at, type: timestamp, format: '%Y-%m-%d %H:%M:%S'}
+  - {name: options,    type: json}

metadata

@@ -1,43 +1,43 @@
 --- !ruby/object:Gem::Specification
 name: embulk-filter-hash
 version: !ruby/object:Gem::Version
-  version: 0.1.2
+  version: 0.2.0
 platform: ruby
 authors:
 - Shinichi Ishimura
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-03-24 00:00:00.000000000 Z
+date: 2017-01-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: bundler
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ~>
-      - !ruby/object:Gem::Version
-        version: '1.0'
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
         version: '1.0'
+  name: bundler
   prerelease: false
   type: :development
-- !ruby/object:Gem::Dependency
-  name: rake
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ~>
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '1.0'
+- !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
         version: '10.0'
+  name: rake
   prerelease: false
   type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '10.0'
 description: Embulk filter plugin to convert a input to hash.
 email:
 - shiketaudonko41@gmail.com
@@ -46,6 +46,7 @@ extensions: []
 extra_rdoc_files: []
 files:
 - .gitignore
+- .travis.yml
 - LICENSE.txt
 - README.md
 - build.gradle
@@ -54,10 +55,12 @@ files:
 - gradlew
 - gradlew.bat
 - lib/embulk/filter/hash.rb
-- sample.csv
+- settings.gradle
 - src/main/java/org/embulk/filter/hash/HashFilterPlugin.java
 - src/test/java/org/embulk/filter/hash/TestHashFilterPlugin.java
-- classpath/embulk-filter-hash-0.1.2.jar
+- src/test/resources/yaml/input_basic.yml
+- src/test/resources/yaml/input_column_types.yml
+- classpath/embulk-filter-hash-0.2.0.jar
 homepage: https://github.com/kamatama41/embulk-filter-hash
 licenses:
 - MIT

data/sample.csv

@@ -1,2 +0,0 @@
-id,name,json
-1,あ,{"a":{"b":"c"}}