embulk-filter-hash 0.3.0 → 0.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.travis.yml +1 -0
- data/build.gradle +15 -68
- data/config/checkstyle/checkstyle.xml +128 -0
- data/config/checkstyle/default.xml +108 -0
- data/src/main/kotlin/org/embulk/filter/hash/HashFilterPlugin.kt +21 -23
- data/src/main/kotlin/org/embulk/filter/hash/extensions.kt +8 -0
- data/src/test/kotlin/org/embulk/filter/hash/TestHashFilterPlugin.kt +40 -48
- metadata +9 -7
- data/settings.gradle +0 -3
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA1:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 2e03d2e87c786822c72bd8f9b5b700dcf748e511
|
|
4
|
+
data.tar.gz: e5dcc797763d6af0d199199398e8ca35bb48c712
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 1cfcb1495dd8c882490a466435f787c165e619724344eaaf56b6f26d55f8129815597d9ce9689368e0a530f084dc18f13bd9abc016ca7c0534a604c2bd5c552a
|
|
7
|
+
data.tar.gz: c967108d0b47aca9a55f5f8146c930ca4c4f7cd1b76c2db7e05dcdad16ca9a95772bbc4ab027da15cddcc54faccfd85d59f546b5db725bc54498bdcedb36cc2f
|
data/.travis.yml
CHANGED
data/build.gradle
CHANGED
|
@@ -1,90 +1,37 @@
|
|
|
1
1
|
buildscript {
|
|
2
|
-
ext.
|
|
2
|
+
ext.kotlinVersion = '1.1.1'
|
|
3
3
|
repositories {
|
|
4
4
|
mavenCentral()
|
|
5
|
+
jcenter()
|
|
6
|
+
maven { url 'http://kamatama41.github.com/maven-repository/repository' }
|
|
5
7
|
}
|
|
6
8
|
dependencies {
|
|
7
|
-
classpath "org.jetbrains.kotlin:kotlin-gradle-plugin:$
|
|
9
|
+
classpath "org.jetbrains.kotlin:kotlin-gradle-plugin:$kotlinVersion"
|
|
10
|
+
classpath "com.github.kamatama41:gradle-embulk-plugin:0.1.2"
|
|
8
11
|
}
|
|
9
12
|
}
|
|
10
13
|
|
|
11
|
-
plugins {
|
|
12
|
-
id "com.jfrog.bintray" version "1.1"
|
|
13
|
-
id "com.github.jruby-gradle.base" version "0.1.5"
|
|
14
|
-
id "java"
|
|
15
|
-
}
|
|
16
14
|
apply plugin: "kotlin"
|
|
15
|
+
apply plugin: "com.github.kamatama41.embulk"
|
|
17
16
|
|
|
18
|
-
import com.github.jrubygradle.JRubyExec
|
|
19
17
|
repositories {
|
|
20
18
|
mavenCentral()
|
|
21
19
|
jcenter()
|
|
22
|
-
maven { url
|
|
23
|
-
}
|
|
24
|
-
configurations {
|
|
25
|
-
provided
|
|
20
|
+
maven { url 'http://kamatama41.github.com/maven-repository/repository' }
|
|
26
21
|
}
|
|
27
22
|
|
|
28
|
-
version = "0.3.
|
|
23
|
+
version = "0.3.1"
|
|
29
24
|
sourceCompatibility = 1.7
|
|
30
25
|
targetCompatibility = 1.7
|
|
31
26
|
|
|
32
27
|
dependencies {
|
|
33
|
-
compile
|
|
34
|
-
|
|
35
|
-
compile "org.jetbrains.kotlin:kotlin-stdlib:$kotlin_version"
|
|
36
|
-
testCompile "com.kamatama41:embulk-test-helpers:0.1.3"
|
|
37
|
-
// Uncomment when using local embulk-test-helpers (and settings.gradle as well)
|
|
38
|
-
//testCompile project(':embulk-test-helpers')
|
|
39
|
-
}
|
|
40
|
-
|
|
41
|
-
task classpath(type: Copy, dependsOn: ["jar"]) {
|
|
42
|
-
doFirst { file("classpath").deleteDir() }
|
|
43
|
-
from (configurations.runtime - configurations.provided + files(jar.archivePath))
|
|
44
|
-
into "classpath"
|
|
28
|
+
compile "org.jetbrains.kotlin:kotlin-stdlib:$kotlinVersion"
|
|
29
|
+
testCompile "com.github.kamatama41:embulk-test-helpers:0.3.2"
|
|
45
30
|
}
|
|
46
|
-
clean { delete "classpath" }
|
|
47
31
|
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
task gemPush(type: JRubyExec, dependsOn: ["gem"]) {
|
|
55
|
-
jrubyArgs "-rrubygems/gem_runner", "-eGem::GemRunner.new.run(ARGV)", "push"
|
|
56
|
-
script "pkg/${project.name}-${project.version}.gem"
|
|
57
|
-
}
|
|
58
|
-
|
|
59
|
-
task "package"(dependsOn: ["gemspec", "classpath"]) << {
|
|
60
|
-
println "> Build succeeded."
|
|
61
|
-
println "> You can run embulk with '-L ${file(".").absolutePath}' argument."
|
|
62
|
-
}
|
|
63
|
-
|
|
64
|
-
task gemspec {
|
|
65
|
-
ext.gemspecFile = file("${project.name}.gemspec")
|
|
66
|
-
inputs.file "build.gradle"
|
|
67
|
-
outputs.file gemspecFile
|
|
68
|
-
doLast { gemspecFile.write($/
|
|
69
|
-
Gem::Specification.new do |spec|
|
|
70
|
-
spec.name = "${project.name}"
|
|
71
|
-
spec.version = "${project.version}"
|
|
72
|
-
spec.authors = ["Shinichi Ishimura"]
|
|
73
|
-
spec.summary = %[Hash filter plugin for Embulk]
|
|
74
|
-
spec.description = %[Embulk filter plugin to convert a input to hash.]
|
|
75
|
-
spec.email = ["shiketaudonko41@gmail.com"]
|
|
76
|
-
spec.licenses = ["MIT"]
|
|
77
|
-
spec.homepage = "https://github.com/kamatama41/embulk-filter-hash"
|
|
78
|
-
|
|
79
|
-
spec.files = `git ls-files`.split("\n") + Dir["classpath/*.jar"]
|
|
80
|
-
spec.test_files = spec.files.grep(%r"^(test|spec)/")
|
|
81
|
-
spec.require_paths = ["lib"]
|
|
82
|
-
|
|
83
|
-
#spec.add_dependency 'YOUR_GEM_DEPENDENCY', ['~> YOUR_GEM_DEPENDENCY_VERSION']
|
|
84
|
-
spec.add_development_dependency 'bundler', ['~> 1.0']
|
|
85
|
-
spec.add_development_dependency 'rake', ['>= 10.0']
|
|
86
|
-
end
|
|
87
|
-
/$)
|
|
88
|
-
}
|
|
32
|
+
embulk {
|
|
33
|
+
version = "0.8.18"
|
|
34
|
+
category = "filter"
|
|
35
|
+
name = "hash"
|
|
36
|
+
homepage = "https://github.com/kamatama41/embulk-filter-hash"
|
|
89
37
|
}
|
|
90
|
-
clean { delete "${project.name}.gemspec" }
|
|
@@ -0,0 +1,128 @@
|
|
|
1
|
+
<?xml version="1.0" encoding="UTF-8"?>
|
|
2
|
+
<!DOCTYPE module PUBLIC
|
|
3
|
+
"-//Puppy Crawl//DTD Check Configuration 1.3//EN"
|
|
4
|
+
"http://www.puppycrawl.com/dtds/configuration_1_3.dtd">
|
|
5
|
+
<module name="Checker">
|
|
6
|
+
<!-- https://github.com/facebook/presto/blob/master/src/checkstyle/checks.xml -->
|
|
7
|
+
<module name="FileTabCharacter"/>
|
|
8
|
+
<module name="NewlineAtEndOfFile">
|
|
9
|
+
<property name="lineSeparator" value="lf"/>
|
|
10
|
+
</module>
|
|
11
|
+
<module name="RegexpMultiline">
|
|
12
|
+
<property name="format" value="\r"/>
|
|
13
|
+
<property name="message" value="Line contains carriage return"/>
|
|
14
|
+
</module>
|
|
15
|
+
<module name="RegexpMultiline">
|
|
16
|
+
<property name="format" value=" \n"/>
|
|
17
|
+
<property name="message" value="Line has trailing whitespace"/>
|
|
18
|
+
</module>
|
|
19
|
+
<module name="RegexpMultiline">
|
|
20
|
+
<property name="format" value="\{\n\n"/>
|
|
21
|
+
<property name="message" value="Blank line after opening brace"/>
|
|
22
|
+
</module>
|
|
23
|
+
<module name="RegexpMultiline">
|
|
24
|
+
<property name="format" value="\n\n\s*\}"/>
|
|
25
|
+
<property name="message" value="Blank line before closing brace"/>
|
|
26
|
+
</module>
|
|
27
|
+
<module name="RegexpMultiline">
|
|
28
|
+
<property name="format" value="\n\n\n"/>
|
|
29
|
+
<property name="message" value="Multiple consecutive blank lines"/>
|
|
30
|
+
</module>
|
|
31
|
+
<module name="RegexpMultiline">
|
|
32
|
+
<property name="format" value="\n\n\Z"/>
|
|
33
|
+
<property name="message" value="Blank line before end of file"/>
|
|
34
|
+
</module>
|
|
35
|
+
<module name="RegexpMultiline">
|
|
36
|
+
<property name="format" value="Preconditions\.checkNotNull"/>
|
|
37
|
+
<property name="message" value="Use of checkNotNull"/>
|
|
38
|
+
</module>
|
|
39
|
+
|
|
40
|
+
<module name="TreeWalker">
|
|
41
|
+
<module name="EmptyBlock">
|
|
42
|
+
<property name="option" value="text"/>
|
|
43
|
+
<property name="tokens" value="
|
|
44
|
+
LITERAL_DO, LITERAL_ELSE, LITERAL_FINALLY, LITERAL_IF,
|
|
45
|
+
LITERAL_FOR, LITERAL_TRY, LITERAL_WHILE, INSTANCE_INIT, STATIC_INIT"/>
|
|
46
|
+
</module>
|
|
47
|
+
<module name="EmptyStatement"/>
|
|
48
|
+
<module name="EmptyForInitializerPad"/>
|
|
49
|
+
<module name="EmptyForIteratorPad">
|
|
50
|
+
<property name="option" value="space"/>
|
|
51
|
+
</module>
|
|
52
|
+
<module name="MethodParamPad">
|
|
53
|
+
<property name="allowLineBreaks" value="true"/>
|
|
54
|
+
<property name="option" value="nospace"/>
|
|
55
|
+
</module>
|
|
56
|
+
<module name="ParenPad"/>
|
|
57
|
+
<module name="TypecastParenPad"/>
|
|
58
|
+
<module name="NeedBraces"/>
|
|
59
|
+
<module name="LeftCurly">
|
|
60
|
+
<property name="option" value="nl"/>
|
|
61
|
+
<property name="tokens" value="CLASS_DEF, CTOR_DEF, INTERFACE_DEF, METHOD_DEF"/>
|
|
62
|
+
</module>
|
|
63
|
+
<module name="LeftCurly">
|
|
64
|
+
<property name="option" value="eol"/>
|
|
65
|
+
<property name="tokens" value="
|
|
66
|
+
LITERAL_CATCH, LITERAL_DO, LITERAL_ELSE, LITERAL_FINALLY, LITERAL_FOR,
|
|
67
|
+
LITERAL_IF, LITERAL_SWITCH, LITERAL_SYNCHRONIZED, LITERAL_TRY, LITERAL_WHILE"/>
|
|
68
|
+
</module>
|
|
69
|
+
<module name="RightCurly">
|
|
70
|
+
<property name="option" value="alone"/>
|
|
71
|
+
</module>
|
|
72
|
+
<module name="GenericWhitespace"/>
|
|
73
|
+
<module name="WhitespaceAfter"/>
|
|
74
|
+
<module name="NoWhitespaceBefore"/>
|
|
75
|
+
|
|
76
|
+
<module name="UpperEll"/>
|
|
77
|
+
<module name="DefaultComesLast"/>
|
|
78
|
+
<module name="ArrayTypeStyle"/>
|
|
79
|
+
<module name="MultipleVariableDeclarations"/>
|
|
80
|
+
<module name="ModifierOrder"/>
|
|
81
|
+
<module name="OneStatementPerLine"/>
|
|
82
|
+
<module name="StringLiteralEquality"/>
|
|
83
|
+
<module name="MutableException"/>
|
|
84
|
+
<module name="EqualsHashCode"/>
|
|
85
|
+
<module name="InnerAssignment"/>
|
|
86
|
+
<module name="InterfaceIsType"/>
|
|
87
|
+
<module name="HideUtilityClassConstructor"/>
|
|
88
|
+
|
|
89
|
+
<module name="MemberName"/>
|
|
90
|
+
<module name="LocalVariableName"/>
|
|
91
|
+
<module name="LocalFinalVariableName"/>
|
|
92
|
+
<module name="TypeName"/>
|
|
93
|
+
<module name="PackageName"/>
|
|
94
|
+
<module name="ParameterName"/>
|
|
95
|
+
<module name="StaticVariableName"/>
|
|
96
|
+
<module name="ClassTypeParameterName">
|
|
97
|
+
<property name="format" value="^[A-Z][0-9]?$"/>
|
|
98
|
+
</module>
|
|
99
|
+
<module name="MethodTypeParameterName">
|
|
100
|
+
<property name="format" value="^[A-Z][0-9]?$"/>
|
|
101
|
+
</module>
|
|
102
|
+
|
|
103
|
+
<module name="AvoidStarImport"/>
|
|
104
|
+
<module name="RedundantImport"/>
|
|
105
|
+
<module name="UnusedImports"/>
|
|
106
|
+
<module name="ImportOrder">
|
|
107
|
+
<property name="groups" value="*,javax,java"/>
|
|
108
|
+
<property name="separated" value="true"/>
|
|
109
|
+
<property name="option" value="bottom"/>
|
|
110
|
+
<property name="sortStaticImportsAlphabetically" value="true"/>
|
|
111
|
+
</module>
|
|
112
|
+
|
|
113
|
+
<module name="WhitespaceAround">
|
|
114
|
+
<property name="allowEmptyConstructors" value="true"/>
|
|
115
|
+
<property name="allowEmptyMethods" value="true"/>
|
|
116
|
+
<property name="ignoreEnhancedForColon" value="false"/>
|
|
117
|
+
<property name="tokens" value="
|
|
118
|
+
ASSIGN, BAND, BAND_ASSIGN, BOR, BOR_ASSIGN, BSR, BSR_ASSIGN,
|
|
119
|
+
BXOR, BXOR_ASSIGN, COLON, DIV, DIV_ASSIGN, EQUAL, GE, GT, LAND, LE,
|
|
120
|
+
LITERAL_ASSERT, LITERAL_CATCH, LITERAL_DO, LITERAL_ELSE,
|
|
121
|
+
LITERAL_FINALLY, LITERAL_FOR, LITERAL_IF, LITERAL_RETURN,
|
|
122
|
+
LITERAL_SYNCHRONIZED, LITERAL_TRY, LITERAL_WHILE,
|
|
123
|
+
LOR, LT, MINUS, MINUS_ASSIGN, MOD, MOD_ASSIGN, NOT_EQUAL,
|
|
124
|
+
PLUS, PLUS_ASSIGN, QUESTION, SL, SLIST, SL_ASSIGN, SR, SR_ASSIGN,
|
|
125
|
+
STAR, STAR_ASSIGN, TYPE_EXTENSION_AND"/>
|
|
126
|
+
</module>
|
|
127
|
+
</module>
|
|
128
|
+
</module>
|
|
@@ -0,0 +1,108 @@
|
|
|
1
|
+
<?xml version="1.0" encoding="UTF-8"?>
|
|
2
|
+
<!DOCTYPE module PUBLIC
|
|
3
|
+
"-//Puppy Crawl//DTD Check Configuration 1.3//EN"
|
|
4
|
+
"http://www.puppycrawl.com/dtds/configuration_1_3.dtd">
|
|
5
|
+
<!--
|
|
6
|
+
This is a subset of ./checkstyle.xml which allows some loose styles
|
|
7
|
+
-->
|
|
8
|
+
<module name="Checker">
|
|
9
|
+
<module name="FileTabCharacter"/>
|
|
10
|
+
<module name="NewlineAtEndOfFile">
|
|
11
|
+
<property name="lineSeparator" value="lf"/>
|
|
12
|
+
</module>
|
|
13
|
+
<module name="RegexpMultiline">
|
|
14
|
+
<property name="format" value="\r"/>
|
|
15
|
+
<property name="message" value="Line contains carriage return"/>
|
|
16
|
+
</module>
|
|
17
|
+
<module name="RegexpMultiline">
|
|
18
|
+
<property name="format" value=" \n"/>
|
|
19
|
+
<property name="message" value="Line has trailing whitespace"/>
|
|
20
|
+
</module>
|
|
21
|
+
<module name="RegexpMultiline">
|
|
22
|
+
<property name="format" value="\n\n\n"/>
|
|
23
|
+
<property name="message" value="Multiple consecutive blank lines"/>
|
|
24
|
+
</module>
|
|
25
|
+
<module name="RegexpMultiline">
|
|
26
|
+
<property name="format" value="\n\n\Z"/>
|
|
27
|
+
<property name="message" value="Blank line before end of file"/>
|
|
28
|
+
</module>
|
|
29
|
+
|
|
30
|
+
<module name="TreeWalker">
|
|
31
|
+
<module name="EmptyBlock">
|
|
32
|
+
<property name="option" value="text"/>
|
|
33
|
+
<property name="tokens" value="
|
|
34
|
+
LITERAL_DO, LITERAL_ELSE, LITERAL_FINALLY, LITERAL_IF,
|
|
35
|
+
LITERAL_FOR, LITERAL_TRY, LITERAL_WHILE, INSTANCE_INIT, STATIC_INIT"/>
|
|
36
|
+
</module>
|
|
37
|
+
<module name="EmptyStatement"/>
|
|
38
|
+
<module name="EmptyForInitializerPad"/>
|
|
39
|
+
<module name="EmptyForIteratorPad">
|
|
40
|
+
<property name="option" value="space"/>
|
|
41
|
+
</module>
|
|
42
|
+
<module name="MethodParamPad">
|
|
43
|
+
<property name="allowLineBreaks" value="true"/>
|
|
44
|
+
<property name="option" value="nospace"/>
|
|
45
|
+
</module>
|
|
46
|
+
<module name="ParenPad"/>
|
|
47
|
+
<module name="TypecastParenPad"/>
|
|
48
|
+
<module name="NeedBraces"/>
|
|
49
|
+
<module name="LeftCurly">
|
|
50
|
+
<property name="option" value="nl"/>
|
|
51
|
+
<property name="tokens" value="CLASS_DEF, CTOR_DEF, INTERFACE_DEF, METHOD_DEF"/>
|
|
52
|
+
</module>
|
|
53
|
+
<module name="LeftCurly">
|
|
54
|
+
<property name="option" value="eol"/>
|
|
55
|
+
<property name="tokens" value="
|
|
56
|
+
LITERAL_CATCH, LITERAL_DO, LITERAL_ELSE, LITERAL_FINALLY, LITERAL_FOR,
|
|
57
|
+
LITERAL_IF, LITERAL_SWITCH, LITERAL_SYNCHRONIZED, LITERAL_TRY, LITERAL_WHILE"/>
|
|
58
|
+
</module>
|
|
59
|
+
<module name="RightCurly">
|
|
60
|
+
<property name="option" value="alone"/>
|
|
61
|
+
</module>
|
|
62
|
+
<module name="GenericWhitespace"/>
|
|
63
|
+
<module name="WhitespaceAfter"/>
|
|
64
|
+
<module name="NoWhitespaceBefore"/>
|
|
65
|
+
|
|
66
|
+
<module name="UpperEll"/>
|
|
67
|
+
<module name="DefaultComesLast"/>
|
|
68
|
+
<module name="ArrayTypeStyle"/>
|
|
69
|
+
<module name="MultipleVariableDeclarations"/>
|
|
70
|
+
<module name="ModifierOrder"/>
|
|
71
|
+
<module name="OneStatementPerLine"/>
|
|
72
|
+
<module name="StringLiteralEquality"/>
|
|
73
|
+
<module name="MutableException"/>
|
|
74
|
+
<module name="EqualsHashCode"/>
|
|
75
|
+
<module name="InnerAssignment"/>
|
|
76
|
+
<module name="InterfaceIsType"/>
|
|
77
|
+
<module name="HideUtilityClassConstructor"/>
|
|
78
|
+
|
|
79
|
+
<module name="MemberName"/>
|
|
80
|
+
<module name="LocalVariableName"/>
|
|
81
|
+
<module name="LocalFinalVariableName"/>
|
|
82
|
+
<module name="TypeName"/>
|
|
83
|
+
<module name="PackageName"/>
|
|
84
|
+
<module name="ParameterName"/>
|
|
85
|
+
<module name="StaticVariableName"/>
|
|
86
|
+
<module name="ClassTypeParameterName">
|
|
87
|
+
<property name="format" value="^[A-Z][0-9]?$"/>
|
|
88
|
+
</module>
|
|
89
|
+
<module name="MethodTypeParameterName">
|
|
90
|
+
<property name="format" value="^[A-Z][0-9]?$"/>
|
|
91
|
+
</module>
|
|
92
|
+
|
|
93
|
+
<module name="WhitespaceAround">
|
|
94
|
+
<property name="allowEmptyConstructors" value="true"/>
|
|
95
|
+
<property name="allowEmptyMethods" value="true"/>
|
|
96
|
+
<property name="ignoreEnhancedForColon" value="false"/>
|
|
97
|
+
<property name="tokens" value="
|
|
98
|
+
ASSIGN, BAND, BAND_ASSIGN, BOR, BOR_ASSIGN, BSR, BSR_ASSIGN,
|
|
99
|
+
BXOR, BXOR_ASSIGN, COLON, DIV, DIV_ASSIGN, EQUAL, GE, GT, LAND, LE,
|
|
100
|
+
LITERAL_ASSERT, LITERAL_CATCH, LITERAL_DO, LITERAL_ELSE,
|
|
101
|
+
LITERAL_FINALLY, LITERAL_FOR, LITERAL_IF, LITERAL_RETURN,
|
|
102
|
+
LITERAL_SYNCHRONIZED, LITERAL_TRY, LITERAL_WHILE,
|
|
103
|
+
LOR, LT, MINUS, MINUS_ASSIGN, MOD, MOD_ASSIGN, NOT_EQUAL,
|
|
104
|
+
PLUS, PLUS_ASSIGN, QUESTION, SL, SLIST, SL_ASSIGN, SR, SR_ASSIGN,
|
|
105
|
+
STAR, STAR_ASSIGN, TYPE_EXTENSION_AND"/>
|
|
106
|
+
</module>
|
|
107
|
+
</module>
|
|
108
|
+
</module>
|
|
@@ -19,35 +19,33 @@ import org.embulk.spi.type.Types
|
|
|
19
19
|
import java.security.MessageDigest
|
|
20
20
|
|
|
21
21
|
class HashFilterPlugin : FilterPlugin {
|
|
22
|
-
|
|
23
22
|
interface PluginTask : Task {
|
|
24
|
-
@Config("columns")
|
|
25
|
-
|
|
23
|
+
@get:Config("columns")
|
|
24
|
+
val columns: List<HashColumn>
|
|
26
25
|
}
|
|
27
26
|
|
|
28
27
|
interface HashColumn : Task {
|
|
29
|
-
@Config("name")
|
|
30
|
-
|
|
28
|
+
@get:Config("name")
|
|
29
|
+
val name: String
|
|
31
30
|
|
|
32
|
-
@Config("algorithm")
|
|
33
|
-
@ConfigDefault("\"SHA-256\"")
|
|
34
|
-
|
|
31
|
+
@get:Config("algorithm")
|
|
32
|
+
@get:ConfigDefault("\"SHA-256\"")
|
|
33
|
+
val algorithm: Optional<String>
|
|
35
34
|
|
|
36
|
-
@Config("new_name")
|
|
37
|
-
@ConfigDefault("null")
|
|
38
|
-
|
|
35
|
+
@get:Config("new_name")
|
|
36
|
+
@get:ConfigDefault("null")
|
|
37
|
+
val newName: Optional<String>
|
|
39
38
|
}
|
|
40
39
|
|
|
41
40
|
override fun transaction(config: ConfigSource, inputSchema: Schema, control: FilterPlugin.Control) {
|
|
42
|
-
|
|
43
|
-
val
|
|
44
|
-
val hashColumnMap = convertHashColumnListToMap(task.getColumns())
|
|
41
|
+
val task: PluginTask = config.loadConfig()
|
|
42
|
+
val hashColumnMap = convertHashColumnListToMap(task.columns)
|
|
45
43
|
|
|
46
44
|
val builder = Schema.builder()
|
|
47
45
|
inputSchema.columns.forEach { column ->
|
|
48
46
|
val hashColumn = hashColumnMap[column.name]
|
|
49
47
|
if (hashColumn != null) {
|
|
50
|
-
builder.add(hashColumn.
|
|
48
|
+
builder.add(hashColumn.newName.or(column.name), Types.STRING)
|
|
51
49
|
} else {
|
|
52
50
|
builder.add(column.name, column.type)
|
|
53
51
|
}
|
|
@@ -57,9 +55,8 @@ class HashFilterPlugin : FilterPlugin {
|
|
|
57
55
|
|
|
58
56
|
override fun open(taskSource: TaskSource, inputSchema: Schema,
|
|
59
57
|
outputSchema: Schema, output: PageOutput): PageOutput {
|
|
60
|
-
|
|
61
|
-
val
|
|
62
|
-
val hashColumnMap = convertHashColumnListToMap(task.getColumns())
|
|
58
|
+
val task: PluginTask = taskSource.loadTask()
|
|
59
|
+
val hashColumnMap = convertHashColumnListToMap(task.columns)
|
|
63
60
|
val outputColumnMap = convertColumnListToMap(outputSchema.columns)
|
|
64
61
|
|
|
65
62
|
return object : PageOutput {
|
|
@@ -82,7 +79,7 @@ class HashFilterPlugin : FilterPlugin {
|
|
|
82
79
|
}
|
|
83
80
|
|
|
84
81
|
// Write the original data
|
|
85
|
-
val inputValue
|
|
82
|
+
val inputValue: Any = when (inputColumn.type) {
|
|
86
83
|
Types.STRING -> {
|
|
87
84
|
reader.getString(inputColumn).apply { builder.setString(inputColumn, this) }
|
|
88
85
|
}
|
|
@@ -100,15 +97,16 @@ class HashFilterPlugin : FilterPlugin {
|
|
|
100
97
|
}
|
|
101
98
|
Types.JSON -> {
|
|
102
99
|
reader.getJson(inputColumn).apply { builder.setJson(inputColumn, this) }
|
|
103
|
-
}
|
|
100
|
+
}
|
|
101
|
+
else -> {
|
|
104
102
|
throw DataException("Unexpected type:" + inputColumn.type)
|
|
105
103
|
}
|
|
106
104
|
}
|
|
107
105
|
|
|
108
106
|
// Overwrite the column if it's hash column.
|
|
109
107
|
hashColumnMap[inputColumn.name]?.let { hashColumn ->
|
|
110
|
-
val outputColumn = outputColumnMap[hashColumn.
|
|
111
|
-
val hashedValue = generateHash(inputValue.toString(), hashColumn.
|
|
108
|
+
val outputColumn = outputColumnMap[hashColumn.newName.or(inputColumn.name)]
|
|
109
|
+
val hashedValue = generateHash(inputValue.toString(), hashColumn.algorithm.get())
|
|
112
110
|
builder.setString(outputColumn, hashedValue)
|
|
113
111
|
}
|
|
114
112
|
}
|
|
@@ -131,7 +129,7 @@ class HashFilterPlugin : FilterPlugin {
|
|
|
131
129
|
}
|
|
132
130
|
|
|
133
131
|
private fun convertHashColumnListToMap(hashColumns: List<HashColumn>?): Map<String, HashColumn> {
|
|
134
|
-
return hashColumns!!.associate { Pair(it.
|
|
132
|
+
return hashColumns!!.associate { Pair(it.name, it) }
|
|
135
133
|
}
|
|
136
134
|
|
|
137
135
|
private fun convertColumnListToMap(columns: List<Column>?): Map<String, Column> {
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
package org.embulk.filter.hash
|
|
2
|
+
|
|
3
|
+
import org.embulk.config.ConfigSource
|
|
4
|
+
import org.embulk.config.TaskSource
|
|
5
|
+
|
|
6
|
+
inline fun <reified T : Any> ConfigSource.loadConfig() = loadConfig(T::class.java)!!
|
|
7
|
+
|
|
8
|
+
inline fun <reified T : Any> TaskSource.loadTask() = loadTask(T::class.java)!!
|
|
@@ -1,38 +1,35 @@
|
|
|
1
1
|
package org.embulk.filter.hash
|
|
2
2
|
|
|
3
|
-
import org.embulk.spi.FilterPlugin
|
|
4
3
|
import org.embulk.test.EmbulkPluginTest
|
|
5
|
-
import org.embulk.test.TestingEmbulk
|
|
6
4
|
import org.junit.Test
|
|
7
5
|
|
|
8
6
|
import org.embulk.spi.type.Types.STRING
|
|
9
|
-
import org.embulk.test.TestOutputPlugin.assertRecords
|
|
10
|
-
import org.embulk.test.TestOutputPlugin.assertSchema
|
|
11
|
-
import org.embulk.test.
|
|
12
|
-
import org.embulk.test.
|
|
7
|
+
import org.embulk.test.TestOutputPlugin.Matcher.assertRecords
|
|
8
|
+
import org.embulk.test.TestOutputPlugin.Matcher.assertSchema
|
|
9
|
+
import org.embulk.test.record
|
|
10
|
+
import org.embulk.test.registerPlugins
|
|
11
|
+
import org.embulk.test.set
|
|
12
|
+
import org.junit.Before
|
|
13
13
|
|
|
14
14
|
class TestHashFilterPlugin : EmbulkPluginTest() {
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
builder.registerPlugin(FilterPlugin::class.java, "hash", HashFilterPlugin::class.java)
|
|
15
|
+
@Before fun setup() {
|
|
16
|
+
builder.registerPlugins(HashFilterPlugin::class)
|
|
18
17
|
}
|
|
19
18
|
|
|
20
19
|
@Test fun specifiedColumnIsHashedAndRenamed() {
|
|
21
|
-
val
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
.set("new_name", "hashed_age")
|
|
29
|
-
))
|
|
20
|
+
val config = config().set(
|
|
21
|
+
"type" to "hash",
|
|
22
|
+
"columns" to listOf(config().set(
|
|
23
|
+
"name" to "age",
|
|
24
|
+
"algorithm" to "MD5",
|
|
25
|
+
"new_name" to "hashed_age"
|
|
26
|
+
)))
|
|
30
27
|
|
|
31
|
-
runFilter(config, inConfigPath)
|
|
28
|
+
runFilter(config, inConfigPath = "yaml/input_basic.yml")
|
|
32
29
|
|
|
33
30
|
assertSchema(
|
|
34
|
-
|
|
35
|
-
|
|
31
|
+
"username" to STRING,
|
|
32
|
+
"hashed_age" to STRING
|
|
36
33
|
)
|
|
37
34
|
|
|
38
35
|
assertRecords(
|
|
@@ -41,28 +38,25 @@ class TestHashFilterPlugin : EmbulkPluginTest() {
|
|
|
41
38
|
}
|
|
42
39
|
|
|
43
40
|
@Test fun allColumnTypesAreHashed() {
|
|
44
|
-
val
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
newConfig().set("name", "created_at"),
|
|
54
|
-
newConfig().set("name", "options")
|
|
41
|
+
val config = config().set(
|
|
42
|
+
"type" to "hash",
|
|
43
|
+
"columns" to listOf(
|
|
44
|
+
config().set("name" to "username"),
|
|
45
|
+
config().set("name" to "age"),
|
|
46
|
+
config().set("name" to "weight"),
|
|
47
|
+
config().set("name" to "active"),
|
|
48
|
+
config().set("name" to "created_at"),
|
|
49
|
+
config().set("name" to "options")
|
|
55
50
|
))
|
|
56
|
-
|
|
57
|
-
runFilter(config, inConfigPath)
|
|
51
|
+
runFilter(config, inConfigPath = "yaml/input_column_types.yml")
|
|
58
52
|
|
|
59
53
|
assertSchema(
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
54
|
+
"username" to STRING,
|
|
55
|
+
"age" to STRING,
|
|
56
|
+
"weight" to STRING,
|
|
57
|
+
"active" to STRING,
|
|
58
|
+
"created_at" to STRING,
|
|
59
|
+
"options" to STRING
|
|
66
60
|
)
|
|
67
61
|
|
|
68
62
|
assertRecords(
|
|
@@ -78,16 +72,14 @@ class TestHashFilterPlugin : EmbulkPluginTest() {
|
|
|
78
72
|
}
|
|
79
73
|
|
|
80
74
|
@Test fun columnIsNull() {
|
|
81
|
-
val
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
newConfig().set("name", "username"),
|
|
87
|
-
newConfig().set("name", "age")
|
|
75
|
+
val config = config().set(
|
|
76
|
+
"type" to "hash",
|
|
77
|
+
"columns" to listOf(
|
|
78
|
+
config().set("name" to "username"),
|
|
79
|
+
config().set("name" to "age")
|
|
88
80
|
))
|
|
89
81
|
|
|
90
|
-
runFilter(config, inConfigPath)
|
|
82
|
+
runFilter(config, inConfigPath = "yaml/input_null_column.yml")
|
|
91
83
|
|
|
92
84
|
assertRecords(
|
|
93
85
|
record(null, "f5ca38f748a1d6eaf726b8a42fb575c3c71f1864a8143301782de13da2d9202b")
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: embulk-filter-hash
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.3.
|
|
4
|
+
version: 0.3.1
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Shinichi Ishimura
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: bin
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2017-
|
|
11
|
+
date: 2017-03-18 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
requirement: !ruby/object:Gem::Requirement
|
|
@@ -38,7 +38,7 @@ dependencies:
|
|
|
38
38
|
- - '>='
|
|
39
39
|
- !ruby/object:Gem::Version
|
|
40
40
|
version: '10.0'
|
|
41
|
-
description:
|
|
41
|
+
description: hash
|
|
42
42
|
email:
|
|
43
43
|
- shiketaudonko41@gmail.com
|
|
44
44
|
executables: []
|
|
@@ -50,20 +50,22 @@ files:
|
|
|
50
50
|
- LICENSE.txt
|
|
51
51
|
- README.md
|
|
52
52
|
- build.gradle
|
|
53
|
+
- config/checkstyle/checkstyle.xml
|
|
54
|
+
- config/checkstyle/default.xml
|
|
53
55
|
- gradle/wrapper/gradle-wrapper.jar
|
|
54
56
|
- gradle/wrapper/gradle-wrapper.properties
|
|
55
57
|
- gradlew
|
|
56
58
|
- gradlew.bat
|
|
57
59
|
- lib/embulk/filter/hash.rb
|
|
58
|
-
- settings.gradle
|
|
59
60
|
- src/main/kotlin/org/embulk/filter/hash/HashFilterPlugin.kt
|
|
61
|
+
- src/main/kotlin/org/embulk/filter/hash/extensions.kt
|
|
60
62
|
- src/test/kotlin/org/embulk/filter/hash/TestHashFilterPlugin.kt
|
|
61
63
|
- src/test/resources/yaml/input_basic.yml
|
|
62
64
|
- src/test/resources/yaml/input_column_types.yml
|
|
63
65
|
- src/test/resources/yaml/input_null_column.yml
|
|
64
|
-
- classpath/
|
|
65
|
-
- classpath/
|
|
66
|
-
- classpath/kotlin-stdlib-1.
|
|
66
|
+
- classpath/annotations-13.0.jar
|
|
67
|
+
- classpath/embulk-filter-hash-0.3.1.jar
|
|
68
|
+
- classpath/kotlin-stdlib-1.1.1.jar
|
|
67
69
|
homepage: https://github.com/kamatama41/embulk-filter-hash
|
|
68
70
|
licenses:
|
|
69
71
|
- MIT
|
data/settings.gradle
DELETED