embulk-filter-hash 0.3.0 → 0.3.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.travis.yml +1 -0
- data/build.gradle +15 -68
- data/config/checkstyle/checkstyle.xml +128 -0
- data/config/checkstyle/default.xml +108 -0
- data/src/main/kotlin/org/embulk/filter/hash/HashFilterPlugin.kt +21 -23
- data/src/main/kotlin/org/embulk/filter/hash/extensions.kt +8 -0
- data/src/test/kotlin/org/embulk/filter/hash/TestHashFilterPlugin.kt +40 -48
- metadata +9 -7
- data/settings.gradle +0 -3
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA1:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 2e03d2e87c786822c72bd8f9b5b700dcf748e511
|
|
4
|
+
data.tar.gz: e5dcc797763d6af0d199199398e8ca35bb48c712
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 1cfcb1495dd8c882490a466435f787c165e619724344eaaf56b6f26d55f8129815597d9ce9689368e0a530f084dc18f13bd9abc016ca7c0534a604c2bd5c552a
|
|
7
|
+
data.tar.gz: c967108d0b47aca9a55f5f8146c930ca4c4f7cd1b76c2db7e05dcdad16ca9a95772bbc4ab027da15cddcc54faccfd85d59f546b5db725bc54498bdcedb36cc2f
|
data/.travis.yml
CHANGED
data/build.gradle
CHANGED
|
@@ -1,90 +1,37 @@
|
|
|
1
1
|
buildscript {
|
|
2
|
-
ext.
|
|
2
|
+
ext.kotlinVersion = '1.1.1'
|
|
3
3
|
repositories {
|
|
4
4
|
mavenCentral()
|
|
5
|
+
jcenter()
|
|
6
|
+
maven { url 'http://kamatama41.github.com/maven-repository/repository' }
|
|
5
7
|
}
|
|
6
8
|
dependencies {
|
|
7
|
-
classpath "org.jetbrains.kotlin:kotlin-gradle-plugin:$
|
|
9
|
+
classpath "org.jetbrains.kotlin:kotlin-gradle-plugin:$kotlinVersion"
|
|
10
|
+
classpath "com.github.kamatama41:gradle-embulk-plugin:0.1.2"
|
|
8
11
|
}
|
|
9
12
|
}
|
|
10
13
|
|
|
11
|
-
plugins {
|
|
12
|
-
id "com.jfrog.bintray" version "1.1"
|
|
13
|
-
id "com.github.jruby-gradle.base" version "0.1.5"
|
|
14
|
-
id "java"
|
|
15
|
-
}
|
|
16
14
|
apply plugin: "kotlin"
|
|
15
|
+
apply plugin: "com.github.kamatama41.embulk"
|
|
17
16
|
|
|
18
|
-
import com.github.jrubygradle.JRubyExec
|
|
19
17
|
repositories {
|
|
20
18
|
mavenCentral()
|
|
21
19
|
jcenter()
|
|
22
|
-
maven { url
|
|
23
|
-
}
|
|
24
|
-
configurations {
|
|
25
|
-
provided
|
|
20
|
+
maven { url 'http://kamatama41.github.com/maven-repository/repository' }
|
|
26
21
|
}
|
|
27
22
|
|
|
28
|
-
version = "0.3.
|
|
23
|
+
version = "0.3.1"
|
|
29
24
|
sourceCompatibility = 1.7
|
|
30
25
|
targetCompatibility = 1.7
|
|
31
26
|
|
|
32
27
|
dependencies {
|
|
33
|
-
compile
|
|
34
|
-
|
|
35
|
-
compile "org.jetbrains.kotlin:kotlin-stdlib:$kotlin_version"
|
|
36
|
-
testCompile "com.kamatama41:embulk-test-helpers:0.1.3"
|
|
37
|
-
// Uncomment when using local embulk-test-helpers (and settings.gradle as well)
|
|
38
|
-
//testCompile project(':embulk-test-helpers')
|
|
39
|
-
}
|
|
40
|
-
|
|
41
|
-
task classpath(type: Copy, dependsOn: ["jar"]) {
|
|
42
|
-
doFirst { file("classpath").deleteDir() }
|
|
43
|
-
from (configurations.runtime - configurations.provided + files(jar.archivePath))
|
|
44
|
-
into "classpath"
|
|
28
|
+
compile "org.jetbrains.kotlin:kotlin-stdlib:$kotlinVersion"
|
|
29
|
+
testCompile "com.github.kamatama41:embulk-test-helpers:0.3.2"
|
|
45
30
|
}
|
|
46
|
-
clean { delete "classpath" }
|
|
47
31
|
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
task gemPush(type: JRubyExec, dependsOn: ["gem"]) {
|
|
55
|
-
jrubyArgs "-rrubygems/gem_runner", "-eGem::GemRunner.new.run(ARGV)", "push"
|
|
56
|
-
script "pkg/${project.name}-${project.version}.gem"
|
|
57
|
-
}
|
|
58
|
-
|
|
59
|
-
task "package"(dependsOn: ["gemspec", "classpath"]) << {
|
|
60
|
-
println "> Build succeeded."
|
|
61
|
-
println "> You can run embulk with '-L ${file(".").absolutePath}' argument."
|
|
62
|
-
}
|
|
63
|
-
|
|
64
|
-
task gemspec {
|
|
65
|
-
ext.gemspecFile = file("${project.name}.gemspec")
|
|
66
|
-
inputs.file "build.gradle"
|
|
67
|
-
outputs.file gemspecFile
|
|
68
|
-
doLast { gemspecFile.write($/
|
|
69
|
-
Gem::Specification.new do |spec|
|
|
70
|
-
spec.name = "${project.name}"
|
|
71
|
-
spec.version = "${project.version}"
|
|
72
|
-
spec.authors = ["Shinichi Ishimura"]
|
|
73
|
-
spec.summary = %[Hash filter plugin for Embulk]
|
|
74
|
-
spec.description = %[Embulk filter plugin to convert a input to hash.]
|
|
75
|
-
spec.email = ["shiketaudonko41@gmail.com"]
|
|
76
|
-
spec.licenses = ["MIT"]
|
|
77
|
-
spec.homepage = "https://github.com/kamatama41/embulk-filter-hash"
|
|
78
|
-
|
|
79
|
-
spec.files = `git ls-files`.split("\n") + Dir["classpath/*.jar"]
|
|
80
|
-
spec.test_files = spec.files.grep(%r"^(test|spec)/")
|
|
81
|
-
spec.require_paths = ["lib"]
|
|
82
|
-
|
|
83
|
-
#spec.add_dependency 'YOUR_GEM_DEPENDENCY', ['~> YOUR_GEM_DEPENDENCY_VERSION']
|
|
84
|
-
spec.add_development_dependency 'bundler', ['~> 1.0']
|
|
85
|
-
spec.add_development_dependency 'rake', ['>= 10.0']
|
|
86
|
-
end
|
|
87
|
-
/$)
|
|
88
|
-
}
|
|
32
|
+
embulk {
|
|
33
|
+
version = "0.8.18"
|
|
34
|
+
category = "filter"
|
|
35
|
+
name = "hash"
|
|
36
|
+
homepage = "https://github.com/kamatama41/embulk-filter-hash"
|
|
89
37
|
}
|
|
90
|
-
clean { delete "${project.name}.gemspec" }
|
|
@@ -0,0 +1,128 @@
|
|
|
1
|
+
<?xml version="1.0" encoding="UTF-8"?>
|
|
2
|
+
<!DOCTYPE module PUBLIC
|
|
3
|
+
"-//Puppy Crawl//DTD Check Configuration 1.3//EN"
|
|
4
|
+
"http://www.puppycrawl.com/dtds/configuration_1_3.dtd">
|
|
5
|
+
<module name="Checker">
|
|
6
|
+
<!-- https://github.com/facebook/presto/blob/master/src/checkstyle/checks.xml -->
|
|
7
|
+
<module name="FileTabCharacter"/>
|
|
8
|
+
<module name="NewlineAtEndOfFile">
|
|
9
|
+
<property name="lineSeparator" value="lf"/>
|
|
10
|
+
</module>
|
|
11
|
+
<module name="RegexpMultiline">
|
|
12
|
+
<property name="format" value="\r"/>
|
|
13
|
+
<property name="message" value="Line contains carriage return"/>
|
|
14
|
+
</module>
|
|
15
|
+
<module name="RegexpMultiline">
|
|
16
|
+
<property name="format" value=" \n"/>
|
|
17
|
+
<property name="message" value="Line has trailing whitespace"/>
|
|
18
|
+
</module>
|
|
19
|
+
<module name="RegexpMultiline">
|
|
20
|
+
<property name="format" value="\{\n\n"/>
|
|
21
|
+
<property name="message" value="Blank line after opening brace"/>
|
|
22
|
+
</module>
|
|
23
|
+
<module name="RegexpMultiline">
|
|
24
|
+
<property name="format" value="\n\n\s*\}"/>
|
|
25
|
+
<property name="message" value="Blank line before closing brace"/>
|
|
26
|
+
</module>
|
|
27
|
+
<module name="RegexpMultiline">
|
|
28
|
+
<property name="format" value="\n\n\n"/>
|
|
29
|
+
<property name="message" value="Multiple consecutive blank lines"/>
|
|
30
|
+
</module>
|
|
31
|
+
<module name="RegexpMultiline">
|
|
32
|
+
<property name="format" value="\n\n\Z"/>
|
|
33
|
+
<property name="message" value="Blank line before end of file"/>
|
|
34
|
+
</module>
|
|
35
|
+
<module name="RegexpMultiline">
|
|
36
|
+
<property name="format" value="Preconditions\.checkNotNull"/>
|
|
37
|
+
<property name="message" value="Use of checkNotNull"/>
|
|
38
|
+
</module>
|
|
39
|
+
|
|
40
|
+
<module name="TreeWalker">
|
|
41
|
+
<module name="EmptyBlock">
|
|
42
|
+
<property name="option" value="text"/>
|
|
43
|
+
<property name="tokens" value="
|
|
44
|
+
LITERAL_DO, LITERAL_ELSE, LITERAL_FINALLY, LITERAL_IF,
|
|
45
|
+
LITERAL_FOR, LITERAL_TRY, LITERAL_WHILE, INSTANCE_INIT, STATIC_INIT"/>
|
|
46
|
+
</module>
|
|
47
|
+
<module name="EmptyStatement"/>
|
|
48
|
+
<module name="EmptyForInitializerPad"/>
|
|
49
|
+
<module name="EmptyForIteratorPad">
|
|
50
|
+
<property name="option" value="space"/>
|
|
51
|
+
</module>
|
|
52
|
+
<module name="MethodParamPad">
|
|
53
|
+
<property name="allowLineBreaks" value="true"/>
|
|
54
|
+
<property name="option" value="nospace"/>
|
|
55
|
+
</module>
|
|
56
|
+
<module name="ParenPad"/>
|
|
57
|
+
<module name="TypecastParenPad"/>
|
|
58
|
+
<module name="NeedBraces"/>
|
|
59
|
+
<module name="LeftCurly">
|
|
60
|
+
<property name="option" value="nl"/>
|
|
61
|
+
<property name="tokens" value="CLASS_DEF, CTOR_DEF, INTERFACE_DEF, METHOD_DEF"/>
|
|
62
|
+
</module>
|
|
63
|
+
<module name="LeftCurly">
|
|
64
|
+
<property name="option" value="eol"/>
|
|
65
|
+
<property name="tokens" value="
|
|
66
|
+
LITERAL_CATCH, LITERAL_DO, LITERAL_ELSE, LITERAL_FINALLY, LITERAL_FOR,
|
|
67
|
+
LITERAL_IF, LITERAL_SWITCH, LITERAL_SYNCHRONIZED, LITERAL_TRY, LITERAL_WHILE"/>
|
|
68
|
+
</module>
|
|
69
|
+
<module name="RightCurly">
|
|
70
|
+
<property name="option" value="alone"/>
|
|
71
|
+
</module>
|
|
72
|
+
<module name="GenericWhitespace"/>
|
|
73
|
+
<module name="WhitespaceAfter"/>
|
|
74
|
+
<module name="NoWhitespaceBefore"/>
|
|
75
|
+
|
|
76
|
+
<module name="UpperEll"/>
|
|
77
|
+
<module name="DefaultComesLast"/>
|
|
78
|
+
<module name="ArrayTypeStyle"/>
|
|
79
|
+
<module name="MultipleVariableDeclarations"/>
|
|
80
|
+
<module name="ModifierOrder"/>
|
|
81
|
+
<module name="OneStatementPerLine"/>
|
|
82
|
+
<module name="StringLiteralEquality"/>
|
|
83
|
+
<module name="MutableException"/>
|
|
84
|
+
<module name="EqualsHashCode"/>
|
|
85
|
+
<module name="InnerAssignment"/>
|
|
86
|
+
<module name="InterfaceIsType"/>
|
|
87
|
+
<module name="HideUtilityClassConstructor"/>
|
|
88
|
+
|
|
89
|
+
<module name="MemberName"/>
|
|
90
|
+
<module name="LocalVariableName"/>
|
|
91
|
+
<module name="LocalFinalVariableName"/>
|
|
92
|
+
<module name="TypeName"/>
|
|
93
|
+
<module name="PackageName"/>
|
|
94
|
+
<module name="ParameterName"/>
|
|
95
|
+
<module name="StaticVariableName"/>
|
|
96
|
+
<module name="ClassTypeParameterName">
|
|
97
|
+
<property name="format" value="^[A-Z][0-9]?$"/>
|
|
98
|
+
</module>
|
|
99
|
+
<module name="MethodTypeParameterName">
|
|
100
|
+
<property name="format" value="^[A-Z][0-9]?$"/>
|
|
101
|
+
</module>
|
|
102
|
+
|
|
103
|
+
<module name="AvoidStarImport"/>
|
|
104
|
+
<module name="RedundantImport"/>
|
|
105
|
+
<module name="UnusedImports"/>
|
|
106
|
+
<module name="ImportOrder">
|
|
107
|
+
<property name="groups" value="*,javax,java"/>
|
|
108
|
+
<property name="separated" value="true"/>
|
|
109
|
+
<property name="option" value="bottom"/>
|
|
110
|
+
<property name="sortStaticImportsAlphabetically" value="true"/>
|
|
111
|
+
</module>
|
|
112
|
+
|
|
113
|
+
<module name="WhitespaceAround">
|
|
114
|
+
<property name="allowEmptyConstructors" value="true"/>
|
|
115
|
+
<property name="allowEmptyMethods" value="true"/>
|
|
116
|
+
<property name="ignoreEnhancedForColon" value="false"/>
|
|
117
|
+
<property name="tokens" value="
|
|
118
|
+
ASSIGN, BAND, BAND_ASSIGN, BOR, BOR_ASSIGN, BSR, BSR_ASSIGN,
|
|
119
|
+
BXOR, BXOR_ASSIGN, COLON, DIV, DIV_ASSIGN, EQUAL, GE, GT, LAND, LE,
|
|
120
|
+
LITERAL_ASSERT, LITERAL_CATCH, LITERAL_DO, LITERAL_ELSE,
|
|
121
|
+
LITERAL_FINALLY, LITERAL_FOR, LITERAL_IF, LITERAL_RETURN,
|
|
122
|
+
LITERAL_SYNCHRONIZED, LITERAL_TRY, LITERAL_WHILE,
|
|
123
|
+
LOR, LT, MINUS, MINUS_ASSIGN, MOD, MOD_ASSIGN, NOT_EQUAL,
|
|
124
|
+
PLUS, PLUS_ASSIGN, QUESTION, SL, SLIST, SL_ASSIGN, SR, SR_ASSIGN,
|
|
125
|
+
STAR, STAR_ASSIGN, TYPE_EXTENSION_AND"/>
|
|
126
|
+
</module>
|
|
127
|
+
</module>
|
|
128
|
+
</module>
|
|
@@ -0,0 +1,108 @@
|
|
|
1
|
+
<?xml version="1.0" encoding="UTF-8"?>
|
|
2
|
+
<!DOCTYPE module PUBLIC
|
|
3
|
+
"-//Puppy Crawl//DTD Check Configuration 1.3//EN"
|
|
4
|
+
"http://www.puppycrawl.com/dtds/configuration_1_3.dtd">
|
|
5
|
+
<!--
|
|
6
|
+
This is a subset of ./checkstyle.xml which allows some loose styles
|
|
7
|
+
-->
|
|
8
|
+
<module name="Checker">
|
|
9
|
+
<module name="FileTabCharacter"/>
|
|
10
|
+
<module name="NewlineAtEndOfFile">
|
|
11
|
+
<property name="lineSeparator" value="lf"/>
|
|
12
|
+
</module>
|
|
13
|
+
<module name="RegexpMultiline">
|
|
14
|
+
<property name="format" value="\r"/>
|
|
15
|
+
<property name="message" value="Line contains carriage return"/>
|
|
16
|
+
</module>
|
|
17
|
+
<module name="RegexpMultiline">
|
|
18
|
+
<property name="format" value=" \n"/>
|
|
19
|
+
<property name="message" value="Line has trailing whitespace"/>
|
|
20
|
+
</module>
|
|
21
|
+
<module name="RegexpMultiline">
|
|
22
|
+
<property name="format" value="\n\n\n"/>
|
|
23
|
+
<property name="message" value="Multiple consecutive blank lines"/>
|
|
24
|
+
</module>
|
|
25
|
+
<module name="RegexpMultiline">
|
|
26
|
+
<property name="format" value="\n\n\Z"/>
|
|
27
|
+
<property name="message" value="Blank line before end of file"/>
|
|
28
|
+
</module>
|
|
29
|
+
|
|
30
|
+
<module name="TreeWalker">
|
|
31
|
+
<module name="EmptyBlock">
|
|
32
|
+
<property name="option" value="text"/>
|
|
33
|
+
<property name="tokens" value="
|
|
34
|
+
LITERAL_DO, LITERAL_ELSE, LITERAL_FINALLY, LITERAL_IF,
|
|
35
|
+
LITERAL_FOR, LITERAL_TRY, LITERAL_WHILE, INSTANCE_INIT, STATIC_INIT"/>
|
|
36
|
+
</module>
|
|
37
|
+
<module name="EmptyStatement"/>
|
|
38
|
+
<module name="EmptyForInitializerPad"/>
|
|
39
|
+
<module name="EmptyForIteratorPad">
|
|
40
|
+
<property name="option" value="space"/>
|
|
41
|
+
</module>
|
|
42
|
+
<module name="MethodParamPad">
|
|
43
|
+
<property name="allowLineBreaks" value="true"/>
|
|
44
|
+
<property name="option" value="nospace"/>
|
|
45
|
+
</module>
|
|
46
|
+
<module name="ParenPad"/>
|
|
47
|
+
<module name="TypecastParenPad"/>
|
|
48
|
+
<module name="NeedBraces"/>
|
|
49
|
+
<module name="LeftCurly">
|
|
50
|
+
<property name="option" value="nl"/>
|
|
51
|
+
<property name="tokens" value="CLASS_DEF, CTOR_DEF, INTERFACE_DEF, METHOD_DEF"/>
|
|
52
|
+
</module>
|
|
53
|
+
<module name="LeftCurly">
|
|
54
|
+
<property name="option" value="eol"/>
|
|
55
|
+
<property name="tokens" value="
|
|
56
|
+
LITERAL_CATCH, LITERAL_DO, LITERAL_ELSE, LITERAL_FINALLY, LITERAL_FOR,
|
|
57
|
+
LITERAL_IF, LITERAL_SWITCH, LITERAL_SYNCHRONIZED, LITERAL_TRY, LITERAL_WHILE"/>
|
|
58
|
+
</module>
|
|
59
|
+
<module name="RightCurly">
|
|
60
|
+
<property name="option" value="alone"/>
|
|
61
|
+
</module>
|
|
62
|
+
<module name="GenericWhitespace"/>
|
|
63
|
+
<module name="WhitespaceAfter"/>
|
|
64
|
+
<module name="NoWhitespaceBefore"/>
|
|
65
|
+
|
|
66
|
+
<module name="UpperEll"/>
|
|
67
|
+
<module name="DefaultComesLast"/>
|
|
68
|
+
<module name="ArrayTypeStyle"/>
|
|
69
|
+
<module name="MultipleVariableDeclarations"/>
|
|
70
|
+
<module name="ModifierOrder"/>
|
|
71
|
+
<module name="OneStatementPerLine"/>
|
|
72
|
+
<module name="StringLiteralEquality"/>
|
|
73
|
+
<module name="MutableException"/>
|
|
74
|
+
<module name="EqualsHashCode"/>
|
|
75
|
+
<module name="InnerAssignment"/>
|
|
76
|
+
<module name="InterfaceIsType"/>
|
|
77
|
+
<module name="HideUtilityClassConstructor"/>
|
|
78
|
+
|
|
79
|
+
<module name="MemberName"/>
|
|
80
|
+
<module name="LocalVariableName"/>
|
|
81
|
+
<module name="LocalFinalVariableName"/>
|
|
82
|
+
<module name="TypeName"/>
|
|
83
|
+
<module name="PackageName"/>
|
|
84
|
+
<module name="ParameterName"/>
|
|
85
|
+
<module name="StaticVariableName"/>
|
|
86
|
+
<module name="ClassTypeParameterName">
|
|
87
|
+
<property name="format" value="^[A-Z][0-9]?$"/>
|
|
88
|
+
</module>
|
|
89
|
+
<module name="MethodTypeParameterName">
|
|
90
|
+
<property name="format" value="^[A-Z][0-9]?$"/>
|
|
91
|
+
</module>
|
|
92
|
+
|
|
93
|
+
<module name="WhitespaceAround">
|
|
94
|
+
<property name="allowEmptyConstructors" value="true"/>
|
|
95
|
+
<property name="allowEmptyMethods" value="true"/>
|
|
96
|
+
<property name="ignoreEnhancedForColon" value="false"/>
|
|
97
|
+
<property name="tokens" value="
|
|
98
|
+
ASSIGN, BAND, BAND_ASSIGN, BOR, BOR_ASSIGN, BSR, BSR_ASSIGN,
|
|
99
|
+
BXOR, BXOR_ASSIGN, COLON, DIV, DIV_ASSIGN, EQUAL, GE, GT, LAND, LE,
|
|
100
|
+
LITERAL_ASSERT, LITERAL_CATCH, LITERAL_DO, LITERAL_ELSE,
|
|
101
|
+
LITERAL_FINALLY, LITERAL_FOR, LITERAL_IF, LITERAL_RETURN,
|
|
102
|
+
LITERAL_SYNCHRONIZED, LITERAL_TRY, LITERAL_WHILE,
|
|
103
|
+
LOR, LT, MINUS, MINUS_ASSIGN, MOD, MOD_ASSIGN, NOT_EQUAL,
|
|
104
|
+
PLUS, PLUS_ASSIGN, QUESTION, SL, SLIST, SL_ASSIGN, SR, SR_ASSIGN,
|
|
105
|
+
STAR, STAR_ASSIGN, TYPE_EXTENSION_AND"/>
|
|
106
|
+
</module>
|
|
107
|
+
</module>
|
|
108
|
+
</module>
|
|
@@ -19,35 +19,33 @@ import org.embulk.spi.type.Types
|
|
|
19
19
|
import java.security.MessageDigest
|
|
20
20
|
|
|
21
21
|
class HashFilterPlugin : FilterPlugin {
|
|
22
|
-
|
|
23
22
|
interface PluginTask : Task {
|
|
24
|
-
@Config("columns")
|
|
25
|
-
|
|
23
|
+
@get:Config("columns")
|
|
24
|
+
val columns: List<HashColumn>
|
|
26
25
|
}
|
|
27
26
|
|
|
28
27
|
interface HashColumn : Task {
|
|
29
|
-
@Config("name")
|
|
30
|
-
|
|
28
|
+
@get:Config("name")
|
|
29
|
+
val name: String
|
|
31
30
|
|
|
32
|
-
@Config("algorithm")
|
|
33
|
-
@ConfigDefault("\"SHA-256\"")
|
|
34
|
-
|
|
31
|
+
@get:Config("algorithm")
|
|
32
|
+
@get:ConfigDefault("\"SHA-256\"")
|
|
33
|
+
val algorithm: Optional<String>
|
|
35
34
|
|
|
36
|
-
@Config("new_name")
|
|
37
|
-
@ConfigDefault("null")
|
|
38
|
-
|
|
35
|
+
@get:Config("new_name")
|
|
36
|
+
@get:ConfigDefault("null")
|
|
37
|
+
val newName: Optional<String>
|
|
39
38
|
}
|
|
40
39
|
|
|
41
40
|
override fun transaction(config: ConfigSource, inputSchema: Schema, control: FilterPlugin.Control) {
|
|
42
|
-
|
|
43
|
-
val
|
|
44
|
-
val hashColumnMap = convertHashColumnListToMap(task.getColumns())
|
|
41
|
+
val task: PluginTask = config.loadConfig()
|
|
42
|
+
val hashColumnMap = convertHashColumnListToMap(task.columns)
|
|
45
43
|
|
|
46
44
|
val builder = Schema.builder()
|
|
47
45
|
inputSchema.columns.forEach { column ->
|
|
48
46
|
val hashColumn = hashColumnMap[column.name]
|
|
49
47
|
if (hashColumn != null) {
|
|
50
|
-
builder.add(hashColumn.
|
|
48
|
+
builder.add(hashColumn.newName.or(column.name), Types.STRING)
|
|
51
49
|
} else {
|
|
52
50
|
builder.add(column.name, column.type)
|
|
53
51
|
}
|
|
@@ -57,9 +55,8 @@ class HashFilterPlugin : FilterPlugin {
|
|
|
57
55
|
|
|
58
56
|
override fun open(taskSource: TaskSource, inputSchema: Schema,
|
|
59
57
|
outputSchema: Schema, output: PageOutput): PageOutput {
|
|
60
|
-
|
|
61
|
-
val
|
|
62
|
-
val hashColumnMap = convertHashColumnListToMap(task.getColumns())
|
|
58
|
+
val task: PluginTask = taskSource.loadTask()
|
|
59
|
+
val hashColumnMap = convertHashColumnListToMap(task.columns)
|
|
63
60
|
val outputColumnMap = convertColumnListToMap(outputSchema.columns)
|
|
64
61
|
|
|
65
62
|
return object : PageOutput {
|
|
@@ -82,7 +79,7 @@ class HashFilterPlugin : FilterPlugin {
|
|
|
82
79
|
}
|
|
83
80
|
|
|
84
81
|
// Write the original data
|
|
85
|
-
val inputValue
|
|
82
|
+
val inputValue: Any = when (inputColumn.type) {
|
|
86
83
|
Types.STRING -> {
|
|
87
84
|
reader.getString(inputColumn).apply { builder.setString(inputColumn, this) }
|
|
88
85
|
}
|
|
@@ -100,15 +97,16 @@ class HashFilterPlugin : FilterPlugin {
|
|
|
100
97
|
}
|
|
101
98
|
Types.JSON -> {
|
|
102
99
|
reader.getJson(inputColumn).apply { builder.setJson(inputColumn, this) }
|
|
103
|
-
}
|
|
100
|
+
}
|
|
101
|
+
else -> {
|
|
104
102
|
throw DataException("Unexpected type:" + inputColumn.type)
|
|
105
103
|
}
|
|
106
104
|
}
|
|
107
105
|
|
|
108
106
|
// Overwrite the column if it's hash column.
|
|
109
107
|
hashColumnMap[inputColumn.name]?.let { hashColumn ->
|
|
110
|
-
val outputColumn = outputColumnMap[hashColumn.
|
|
111
|
-
val hashedValue = generateHash(inputValue.toString(), hashColumn.
|
|
108
|
+
val outputColumn = outputColumnMap[hashColumn.newName.or(inputColumn.name)]
|
|
109
|
+
val hashedValue = generateHash(inputValue.toString(), hashColumn.algorithm.get())
|
|
112
110
|
builder.setString(outputColumn, hashedValue)
|
|
113
111
|
}
|
|
114
112
|
}
|
|
@@ -131,7 +129,7 @@ class HashFilterPlugin : FilterPlugin {
|
|
|
131
129
|
}
|
|
132
130
|
|
|
133
131
|
private fun convertHashColumnListToMap(hashColumns: List<HashColumn>?): Map<String, HashColumn> {
|
|
134
|
-
return hashColumns!!.associate { Pair(it.
|
|
132
|
+
return hashColumns!!.associate { Pair(it.name, it) }
|
|
135
133
|
}
|
|
136
134
|
|
|
137
135
|
private fun convertColumnListToMap(columns: List<Column>?): Map<String, Column> {
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
package org.embulk.filter.hash
|
|
2
|
+
|
|
3
|
+
import org.embulk.config.ConfigSource
|
|
4
|
+
import org.embulk.config.TaskSource
|
|
5
|
+
|
|
6
|
+
inline fun <reified T : Any> ConfigSource.loadConfig() = loadConfig(T::class.java)!!
|
|
7
|
+
|
|
8
|
+
inline fun <reified T : Any> TaskSource.loadTask() = loadTask(T::class.java)!!
|
|
@@ -1,38 +1,35 @@
|
|
|
1
1
|
package org.embulk.filter.hash
|
|
2
2
|
|
|
3
|
-
import org.embulk.spi.FilterPlugin
|
|
4
3
|
import org.embulk.test.EmbulkPluginTest
|
|
5
|
-
import org.embulk.test.TestingEmbulk
|
|
6
4
|
import org.junit.Test
|
|
7
5
|
|
|
8
6
|
import org.embulk.spi.type.Types.STRING
|
|
9
|
-
import org.embulk.test.TestOutputPlugin.assertRecords
|
|
10
|
-
import org.embulk.test.TestOutputPlugin.assertSchema
|
|
11
|
-
import org.embulk.test.
|
|
12
|
-
import org.embulk.test.
|
|
7
|
+
import org.embulk.test.TestOutputPlugin.Matcher.assertRecords
|
|
8
|
+
import org.embulk.test.TestOutputPlugin.Matcher.assertSchema
|
|
9
|
+
import org.embulk.test.record
|
|
10
|
+
import org.embulk.test.registerPlugins
|
|
11
|
+
import org.embulk.test.set
|
|
12
|
+
import org.junit.Before
|
|
13
13
|
|
|
14
14
|
class TestHashFilterPlugin : EmbulkPluginTest() {
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
builder.registerPlugin(FilterPlugin::class.java, "hash", HashFilterPlugin::class.java)
|
|
15
|
+
@Before fun setup() {
|
|
16
|
+
builder.registerPlugins(HashFilterPlugin::class)
|
|
18
17
|
}
|
|
19
18
|
|
|
20
19
|
@Test fun specifiedColumnIsHashedAndRenamed() {
|
|
21
|
-
val
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
.set("new_name", "hashed_age")
|
|
29
|
-
))
|
|
20
|
+
val config = config().set(
|
|
21
|
+
"type" to "hash",
|
|
22
|
+
"columns" to listOf(config().set(
|
|
23
|
+
"name" to "age",
|
|
24
|
+
"algorithm" to "MD5",
|
|
25
|
+
"new_name" to "hashed_age"
|
|
26
|
+
)))
|
|
30
27
|
|
|
31
|
-
runFilter(config, inConfigPath)
|
|
28
|
+
runFilter(config, inConfigPath = "yaml/input_basic.yml")
|
|
32
29
|
|
|
33
30
|
assertSchema(
|
|
34
|
-
|
|
35
|
-
|
|
31
|
+
"username" to STRING,
|
|
32
|
+
"hashed_age" to STRING
|
|
36
33
|
)
|
|
37
34
|
|
|
38
35
|
assertRecords(
|
|
@@ -41,28 +38,25 @@ class TestHashFilterPlugin : EmbulkPluginTest() {
|
|
|
41
38
|
}
|
|
42
39
|
|
|
43
40
|
@Test fun allColumnTypesAreHashed() {
|
|
44
|
-
val
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
newConfig().set("name", "created_at"),
|
|
54
|
-
newConfig().set("name", "options")
|
|
41
|
+
val config = config().set(
|
|
42
|
+
"type" to "hash",
|
|
43
|
+
"columns" to listOf(
|
|
44
|
+
config().set("name" to "username"),
|
|
45
|
+
config().set("name" to "age"),
|
|
46
|
+
config().set("name" to "weight"),
|
|
47
|
+
config().set("name" to "active"),
|
|
48
|
+
config().set("name" to "created_at"),
|
|
49
|
+
config().set("name" to "options")
|
|
55
50
|
))
|
|
56
|
-
|
|
57
|
-
runFilter(config, inConfigPath)
|
|
51
|
+
runFilter(config, inConfigPath = "yaml/input_column_types.yml")
|
|
58
52
|
|
|
59
53
|
assertSchema(
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
54
|
+
"username" to STRING,
|
|
55
|
+
"age" to STRING,
|
|
56
|
+
"weight" to STRING,
|
|
57
|
+
"active" to STRING,
|
|
58
|
+
"created_at" to STRING,
|
|
59
|
+
"options" to STRING
|
|
66
60
|
)
|
|
67
61
|
|
|
68
62
|
assertRecords(
|
|
@@ -78,16 +72,14 @@ class TestHashFilterPlugin : EmbulkPluginTest() {
|
|
|
78
72
|
}
|
|
79
73
|
|
|
80
74
|
@Test fun columnIsNull() {
|
|
81
|
-
val
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
newConfig().set("name", "username"),
|
|
87
|
-
newConfig().set("name", "age")
|
|
75
|
+
val config = config().set(
|
|
76
|
+
"type" to "hash",
|
|
77
|
+
"columns" to listOf(
|
|
78
|
+
config().set("name" to "username"),
|
|
79
|
+
config().set("name" to "age")
|
|
88
80
|
))
|
|
89
81
|
|
|
90
|
-
runFilter(config, inConfigPath)
|
|
82
|
+
runFilter(config, inConfigPath = "yaml/input_null_column.yml")
|
|
91
83
|
|
|
92
84
|
assertRecords(
|
|
93
85
|
record(null, "f5ca38f748a1d6eaf726b8a42fb575c3c71f1864a8143301782de13da2d9202b")
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: embulk-filter-hash
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.3.
|
|
4
|
+
version: 0.3.1
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Shinichi Ishimura
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: bin
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2017-
|
|
11
|
+
date: 2017-03-18 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
requirement: !ruby/object:Gem::Requirement
|
|
@@ -38,7 +38,7 @@ dependencies:
|
|
|
38
38
|
- - '>='
|
|
39
39
|
- !ruby/object:Gem::Version
|
|
40
40
|
version: '10.0'
|
|
41
|
-
description:
|
|
41
|
+
description: hash
|
|
42
42
|
email:
|
|
43
43
|
- shiketaudonko41@gmail.com
|
|
44
44
|
executables: []
|
|
@@ -50,20 +50,22 @@ files:
|
|
|
50
50
|
- LICENSE.txt
|
|
51
51
|
- README.md
|
|
52
52
|
- build.gradle
|
|
53
|
+
- config/checkstyle/checkstyle.xml
|
|
54
|
+
- config/checkstyle/default.xml
|
|
53
55
|
- gradle/wrapper/gradle-wrapper.jar
|
|
54
56
|
- gradle/wrapper/gradle-wrapper.properties
|
|
55
57
|
- gradlew
|
|
56
58
|
- gradlew.bat
|
|
57
59
|
- lib/embulk/filter/hash.rb
|
|
58
|
-
- settings.gradle
|
|
59
60
|
- src/main/kotlin/org/embulk/filter/hash/HashFilterPlugin.kt
|
|
61
|
+
- src/main/kotlin/org/embulk/filter/hash/extensions.kt
|
|
60
62
|
- src/test/kotlin/org/embulk/filter/hash/TestHashFilterPlugin.kt
|
|
61
63
|
- src/test/resources/yaml/input_basic.yml
|
|
62
64
|
- src/test/resources/yaml/input_column_types.yml
|
|
63
65
|
- src/test/resources/yaml/input_null_column.yml
|
|
64
|
-
- classpath/
|
|
65
|
-
- classpath/
|
|
66
|
-
- classpath/kotlin-stdlib-1.
|
|
66
|
+
- classpath/annotations-13.0.jar
|
|
67
|
+
- classpath/embulk-filter-hash-0.3.1.jar
|
|
68
|
+
- classpath/kotlin-stdlib-1.1.1.jar
|
|
67
69
|
homepage: https://github.com/kamatama41/embulk-filter-hash
|
|
68
70
|
licenses:
|
|
69
71
|
- MIT
|
data/settings.gradle
DELETED