embulk-filter-hash 0.4.0 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 681859abe9dc6462e1ca8bfc9d32da28b11be9d4
-  data.tar.gz: 364021a86b7841541741d68ad0dfa2c4a42e196a
+  metadata.gz: df57d10cff5411c1b2d482c912fd2b2d352919ba
+  data.tar.gz: ef0fe81747abaa517010653c6d509c8c1b2af846
 SHA512:
-  metadata.gz: f0cb9f3e5b8fd629e6ab69940a605ff5c2631494d3b4f63968f602f66225c5596f3016c30bbf108017c83491b1421a28babb21509657efce9fe5696c522a1c09
-  data.tar.gz: 3229e2f0fdff0a6bd9617c8a977d2473c64a3d4e11f341d48ef41a8c69a13d6ae25bfe46b09cd1244675527517281dff74e7b03dc6132685e6d167380581188a
+  metadata.gz: 315b799b9893dbae9c60abd9614d7efdc2978e67a4e3c3969d5b98efd441166cfbfdb655017d8aa648166d9fd2b794f041378653e30b8bfd3642d9a9ebac5c4c
+  data.tar.gz: dfda22d8e8403757d2e7865f243b92aae836b73a6c62fb3df2642d9eceaaded297a56bbd78b829f8ae103e60207c3c3d14463e7a57e2d42832437c8e843ca080

data/.gitignore

@@ -10,3 +10,4 @@ build/
 /.metadata/
 .classpath
 .project
+*.iml

data/README.md

@@ -12,8 +12,9 @@ Embulk filter plugin to convert an input to a hash value.
 
 - **columns**: Columns to hash (array, required)
   - **name**: Name of input column (string, required)
-  - **algorithm**: A hash algorithm. [See also](#hash_algorithm) (string, default:`"SHA-256"`)
-  - **new_name**: New column name if you want to rename (string, default: `null`)
+  - **algorithm**: Hash algorithm. [See also](#hash_algorithm) (string, default:`"SHA-256"`)
+  - **secret_key**: Secret key for HMAC hashing. (string, required when specifying HMAC algorithm)
+  - **new_name**: New column name if you want to rename the column (string, default: `null`)
 
 ## Example
 
@@ -23,22 +24,22 @@ filters:
     columns:
     - { name: username }
     - { name: email, algorithm: SHA-512, new_name: hashed_email }
+    - { name: phone_number, algorithm: HmacSHA256, secret_key: passw0rd }
 ```
 
 ## Hash Algorithm
 <a name ="hash_algorithm">
 
-This plugin uses [MessageDigest](https://docs.oracle.com/javase/7/docs/api/java/security/MessageDigest.html) for hashing.
-Every implementation of the Java platform supports the following MessageDigest algorithms:  
-- MD5
-- SHA-1
-- SHA-256
-
+You can choose either of [MessageDigest](https://docs.oracle.com/javase/8/docs/api/java/security/MessageDigest.html) algorithm or [HMAC](https://docs.oracle.com/javase/8/docs/api/javax/crypto/Mac.html) algorithm.
 If you want to know all algorithms that your platform supports, run the following snippet.
+
 ```java
 for (String algorithm : java.security.Security.getAlgorithms("MessageDigest")) {
     System.out.println(algorithm);
 }
+for (String algorithm : java.security.Security.getAlgorithms("Mac")) {
+    System.out.println(algorithm);
+}
 ```
 
 ## Build

data/build.gradle

@@ -1,24 +1,21 @@
 buildscript {
-    ext.kotlinVersion = '1.2.31'
     repositories {
-        mavenCentral()
         jcenter()
         maven { url 'http://kamatama41.github.com/maven-repository/repository' }
     }
     dependencies {
-        classpath "org.jetbrains.kotlin:kotlin-gradle-plugin:$kotlinVersion"
-        classpath "com.github.kamatama41:gradle-embulk-plugin:0.1.4"
+        classpath "com.github.kamatama41:gradle-embulk-plugin:0.3.0"
         classpath "net.researchgate:gradle-release:2.5.0"
     }
 }
 
-apply plugin: "kotlin"
+plugins {
+    id "org.jetbrains.kotlin.jvm" version "1.2.31"
+}
 apply plugin: "com.github.kamatama41.embulk"
 apply plugin: 'net.researchgate.release'
 
 repositories {
-    mavenCentral()
-    jcenter()
     maven { url 'http://kamatama41.github.com/maven-repository/repository' }
 }
 
@@ -26,12 +23,12 @@ sourceCompatibility = 1.8
 targetCompatibility = 1.8
 
 dependencies {
-    compile "org.jetbrains.kotlin:kotlin-stdlib:$kotlinVersion"
-    testCompile "com.github.kamatama41:embulk-test-helpers:0.4.0"
+    compile "org.jetbrains.kotlin:kotlin-stdlib"
+    testCompile "com.github.kamatama41:embulk-test-helpers:0.5.0"
 }
 
 embulk {
-    version = "0.9.7"
+    version = "0.9.9"
     category = "filter"
     name = "hash"
     authors = ["Shinichi Ishimura"]

data/circle.yml

@@ -1,23 +1,44 @@
-machine:
-  java:
-    version: oraclejdk8
+version: 2
+jobs:
+  build:
+    docker:
+      - image: openjdk:8
+    working_directory: ~/embulk-filter-hash
+    steps:
+      - checkout
 
-dependencies:
-  pre:
-    - git config --global user.email "shiketaudonko41@gmail.com"
-    - git config --global user.name "kamatama41"
-  override:
-    - ./gradlew dependencies
+      - restore_cache:
+          keys:
+           - v1-gradle-{{ checksum "build.gradle" }}
+           - v1-gradle-
+      - run: ./gradlew checkstyle
+      - run: ./gradlew check --info
+      - run:
+          name: Save test results
+          command: |
+            mkdir -p ~/junit/
+            find . -type f -regex ".*/build/test-results/.*xml" -exec cp {} ~/junit/ \;
+          when: always
+      - store_test_results:
+          path: ~/junit
+      - store_artifacts:
+          path: ~/junit
 
-test:
-  override:
-    - ./gradlew check --stacktrace
+      - save_cache:
+          paths:
+            - "~/.gradle"
+            - "~/.m2"
+          key: v1-gradle-{{ checksum "build.gradle" }}
 
-deployment:
-  release:
-    branch: release
-    commands:
-      - curl -f -u $RUBYGEMS_USER:$RUBYGEMS_PASSWORD https://rubygems.org/api/v1/api_key.yaml > ~/.gem/credentials; chmod 0600 ~/.gem/credentials
-      - git checkout master
-      - git reset --hard origin/master
-      - ./gradlew release -Prelease.useAutomaticVersion=true
+      - deploy:
+          name: Push Gem to RubyGems.org and bump up
+          command: |
+            if [ "${CIRCLE_BRANCH}" == "release" ]; then
+              mkdir -p ~/.gem
+              curl -f -u $RUBYGEMS_USER:$RUBYGEMS_PASSWORD https://rubygems.org/api/v1/api_key.yaml > ~/.gem/credentials; chmod 0600 ~/.gem/credentials
+              git config --global user.email "shiketaudonko41@gmail.com"
+              git config --global user.name "kamatama41"
+              git checkout master
+              git reset --hard origin/master
+              ./gradlew release -Prelease.useAutomaticVersion=true
+            fi

data/gradle.properties

@@ -1 +1 @@
-version=0.4.0
+version=0.5.0

data/gradle/wrapper/gradle-wrapper.properties

@@ -1,6 +1,6 @@
-#Sun Jan 29 22:41:06 JST 2017
+#Sat Apr 28 09:20:37 JST 2018
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-3.3-bin.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-4.5.1-all.zip

data/src/main/kotlin/org/embulk/filter/hash/HashFilterPlugin.kt

@@ -1,11 +1,7 @@
 package org.embulk.filter.hash
 
 import com.google.common.base.Optional
-import org.embulk.config.Config
-import org.embulk.config.ConfigDefault
-import org.embulk.config.ConfigSource
-import org.embulk.config.Task
-import org.embulk.config.TaskSource
+import org.embulk.config.*
 import org.embulk.spi.Column
 import org.embulk.spi.DataException
 import org.embulk.spi.Exec
@@ -17,6 +13,9 @@ import org.embulk.spi.PageReader
 import org.embulk.spi.Schema
 import org.embulk.spi.type.Types
 import java.security.MessageDigest
+import java.util.Locale
+import javax.crypto.Mac
+import javax.crypto.spec.SecretKeySpec
 
 class HashFilterPlugin : FilterPlugin {
     interface PluginTask : Task {
@@ -32,6 +31,10 @@ class HashFilterPlugin : FilterPlugin {
         @get:ConfigDefault("\"SHA-256\"")
         val algorithm: Optional<String>
 
+        @get:Config("secret_key")
+        @get:ConfigDefault("null")
+        val secretKey: Optional<String>
+
         @get:Config("new_name")
         @get:ConfigDefault("null")
         val newName: Optional<String>
@@ -45,6 +48,8 @@ class HashFilterPlugin : FilterPlugin {
         inputSchema.columns.forEach { column ->
             val hashColumn = hashColumnMap[column.name]
             if (hashColumn != null) {
+                // Check algorithm is valid
+                getAlgorithmType(hashColumn.algorithm.get()).validate(hashColumn)
                 builder.add(hashColumn.newName.or(column.name), Types.STRING)
             } else {
                 builder.add(column.name, column.type)
@@ -114,7 +119,7 @@ class HashFilterPlugin : FilterPlugin {
                 hashColumnMap[inputColumn.name]?.let { hashColumn ->
                     // Write hashed value if it's hash column.
                     val outputColumn = outputColumnMap[hashColumn.newName.or(inputColumn.name)]
-                    val hashedValue = generateHash(inputValue.toString(), hashColumn.algorithm.get())
+                    val hashedValue = generateHash(inputValue.toString(), hashColumn)
                     builder.setString(outputColumn, hashedValue)
                 } ?: run {
                     // Write the original data
@@ -122,10 +127,8 @@ class HashFilterPlugin : FilterPlugin {
                 }
             }
 
-            private fun generateHash(value: String, algorithm: String): String {
-                val md = MessageDigest.getInstance(algorithm)
-                md.update(value.toByteArray())
-                return md.digest().joinToString("") { "%02x".format(it) }
+            private fun generateHash(value: String, config: HashColumn): String {
+                return getAlgorithmType(config.algorithm.get()).generateHash(value, config)
             }
 
             override fun finish() {
@@ -145,4 +148,54 @@ class HashFilterPlugin : FilterPlugin {
     private fun convertColumnListToMap(columns: List<Column>?): Map<String, Column> {
         return columns!!.associate { Pair(it.name, it) }
     }
+
+    private fun getAlgorithmType(algorithm: String): AlgorithmType {
+        return when {
+            MD_ALGORITHMS.contains(algorithm.toUpperCase(Locale.ENGLISH)) -> {
+                AlgorithmType.MESSAGE_DIGEST
+            }
+            MAC_ALGORITHMS.contains(algorithm.toUpperCase(Locale.ENGLISH)) -> {
+                AlgorithmType.MAC
+            }
+            else -> throw ConfigException("No such algorithm: $algorithm")
+        }
+    }
+
+    enum class AlgorithmType {
+        MESSAGE_DIGEST {
+            override fun validate(config: HashColumn) {}
+
+            override fun generateHash(value: String, config: HashColumn): String {
+                val algorithm = config.algorithm.get()
+                return MessageDigest.getInstance(algorithm).run {
+                    update(value.toByteArray())
+                    digest().joinToString("") { "%02x".format(it) }
+                }
+            }
+        },
+        MAC {
+            override fun validate(config: HashColumn) {
+                if (!config.secretKey.isPresent) {
+                    throw ConfigException("Secret key must not be null.")
+                }
+            }
+
+            override fun generateHash(value: String, config: HashColumn): String {
+                val secretKey = config.secretKey.get()
+                val algorithm = config.algorithm.get()
+                return Mac.getInstance(algorithm).run {
+                    init(SecretKeySpec(secretKey.toByteArray(), algorithm))
+                    doFinal(value.toByteArray()).joinToString("") { "%02x".format(it) }
+                }
+            }
+        };
+
+        abstract fun validate(config: HashColumn)
+        abstract fun generateHash(value: String, config: HashColumn): String
+    }
+
+    companion object {
+        val MD_ALGORITHMS = java.security.Security.getAlgorithms("MessageDigest") ?: emptySet<String>()
+        val MAC_ALGORITHMS = java.security.Security.getAlgorithms("Mac") ?: emptySet<String>()
+    }
 }

data/src/test/kotlin/org/embulk/filter/hash/TestHashFilterPlugin.kt

@@ -1,5 +1,7 @@
 package org.embulk.filter.hash
 
+import org.embulk.config.ConfigException
+import org.embulk.exec.PartialExecutionException
 import org.embulk.test.EmbulkPluginTest
 import org.junit.Test
 
@@ -9,6 +11,10 @@ import org.embulk.test.TestOutputPlugin.Matcher.assertSchema
 import org.embulk.test.record
 import org.embulk.test.registerPlugins
 import org.embulk.test.set
+import org.hamcrest.Matchers.`is`
+import org.hamcrest.Matchers.instanceOf
+import org.junit.Assert.assertThat
+import org.junit.Assert.fail
 import org.junit.Before
 
 class TestHashFilterPlugin : EmbulkPluginTest() {
@@ -16,7 +22,8 @@ class TestHashFilterPlugin : EmbulkPluginTest() {
         builder.registerPlugins(HashFilterPlugin::class)
     }
 
-    @Test fun specifiedColumnIsHashedAndRenamed() {
+    @Test
+    fun specifiedColumnIsHashedAndRenamed() {
         val config = config().set(
                 "type" to "hash",
                 "columns" to listOf(config().set(
@@ -37,7 +44,8 @@ class TestHashFilterPlugin : EmbulkPluginTest() {
         )
     }
 
-    @Test fun allColumnTypesAreHashed() {
+    @Test
+    fun allColumnTypesAreHashed() {
         val config = config().set(
                 "type" to "hash",
                 "columns" to listOf(
@@ -71,7 +79,65 @@ class TestHashFilterPlugin : EmbulkPluginTest() {
         )
     }
 
-    @Test fun columnIsNull() {
+    @Test
+    fun specifiedColumnIsHashedByMac() {
+        val config = config().set(
+                "type" to "hash",
+                "columns" to listOf(config().set(
+                        "name" to "age",
+                        "algorithm" to "HmacSHA256",
+                        "secret_key" to "passw0rd",
+                        "new_name" to "hashed_age"
+                )))
+
+        runFilter(config, inConfigPath = "yaml/input_basic.yml")
+
+        assertSchema(
+                "username" to STRING,
+                "hashed_age" to STRING
+        )
+
+        assertRecords(
+                record("user1", "5f9959eac71ad30782ebf4d3c98d12a4c33eadee156a6c5d3881204030811989")
+        )
+    }
+
+    @Test
+    fun exceptionThrownWithInvalidAlgorithm() {
+        try {
+            val config = config().set(
+                    "type" to "hash",
+                    "columns" to listOf(config().set(
+                            "name" to "age",
+                            "algorithm" to "Foo"
+                    )))
+            runFilter(config, inConfigPath = "yaml/input_basic.yml")
+            fail("No exception")
+        } catch (e: PartialExecutionException) {
+            assertThat(e.cause, instanceOf(ConfigException::class.java))
+            assertThat(e.cause?.message, `is`("No such algorithm: Foo"))
+        }
+    }
+
+    @Test
+    fun exceptionThrownWithMacAndNoSecretKey() {
+        try {
+            val config = config().set(
+                    "type" to "hash",
+                    "columns" to listOf(config().set(
+                            "name" to "age",
+                            "algorithm" to "HmacSHA256"
+                    )))
+            runFilter(config, inConfigPath = "yaml/input_basic.yml")
+            fail("No exception")
+        } catch (e: PartialExecutionException) {
+            assertThat(e.cause, instanceOf(ConfigException::class.java))
+            assertThat(e.cause?.message, `is`("Secret key must not be null."))
+        }
+    }
+
+    @Test
+    fun columnIsNull() {
         val config = config().set(
                 "type" to "hash",
                 "columns" to listOf(

data/src/test/resources/yaml/input_basic.yml

@@ -1,8 +1,6 @@
-type: test
-data:
-- user1,20
-parser:
-  type: csv
-  columns:
-  - {name: username, type: string}
-  - {name: age, type: long}
+type: config
+columns:
+- {name: username, type: string}
+- {name: age, type: long}
+values:
+- - [user1,20]

data/src/test/resources/yaml/input_column_types.yml

@@ -1,13 +1,10 @@
-type: test
-data:
-- >-
- user1,21,65.4,true,2016-03-27 01:23:45,{"a":{"b":"c"}}"
-parser:
-  type: csv
-  columns:
-  - {name: username,   type: string}
-  - {name: age,        type: long}
-  - {name: weight,     type: double}
-  - {name: active,     type: boolean}
-  - {name: created_at, type: timestamp, format: '%Y-%m-%d %H:%M:%S'}
-  - {name: options,    type: json}
+type: config
+columns:
+- {name: username,   type: string}
+- {name: age,        type: long}
+- {name: weight,     type: double}
+- {name: active,     type: boolean}
+- {name: created_at, type: timestamp, format: '%Y-%m-%d %H:%M:%S'}
+- {name: options,    type: json}
+values:
+- - [user1,21,65.4,true,"2016-03-27 01:23:45",{"a":{"b":"c"}}]

data/src/test/resources/yaml/input_null_column.yml

@@ -1,9 +1,6 @@
-type: test
-data:
-- null,20
-parser:
-  type: csv
-  null_string: 'null'
-  columns:
-  - {name: username, type: string}
-  - {name: age, type: long}
+type: config
+columns:
+- {name: username, type: string}
+- {name: age, type: long}
+values:
+- - [null,20]

metadata

@@ -1,19 +1,19 @@
 --- !ruby/object:Gem::Specification
 name: embulk-filter-hash
 version: !ruby/object:Gem::Version
-  version: 0.4.0
+  version: 0.5.0
 platform: ruby
 authors:
 - Shinichi Ishimura
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2018-04-19 00:00:00.000000000 Z
+date: 2018-11-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.0'
   name: bundler
@@ -21,13 +21,13 @@ dependencies:
   type: :development
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.0'
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '10.0'
   name: rake
@@ -35,7 +35,7 @@ dependencies:
   type: :development
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '10.0'
 description: Embulk filter plugin to convert an input to a hash value.
@@ -45,11 +45,14 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- .gitignore
+- ".gitignore"
 - LICENSE.txt
 - README.md
 - build.gradle
 - circle.yml
+- classpath/annotations-13.0.jar
+- classpath/embulk-filter-hash-0.5.0.jar
+- classpath/kotlin-stdlib-1.2.31.jar
 - gradle.properties
 - gradle/wrapper/gradle-wrapper.jar
 - gradle/wrapper/gradle-wrapper.properties
@@ -62,9 +65,6 @@ files:
 - src/test/resources/yaml/input_basic.yml
 - src/test/resources/yaml/input_column_types.yml
 - src/test/resources/yaml/input_null_column.yml
-- classpath/kotlin-stdlib-1.2.31.jar
-- classpath/annotations-13.0.jar
-- classpath/embulk-filter-hash-0.4.0.jar
 homepage: https://github.com/kamatama41/embulk-filter-hash
 licenses:
 - MIT
@@ -75,17 +75,17 @@ require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project:
-rubygems_version: 2.1.9
+rubygems_version: 2.6.8
 signing_key:
 specification_version: 4
 summary: Hash filter plugin for Embulk