embulk-output-aerospike 0.3.6 → 0.4.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (12) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +47 -0
  3. data/build.gradle +26 -8
  4. data/config/checkstyle/checkstyle.xml +128 -0
  5. data/config/checkstyle/default.xml +108 -0
  6. data/example/config-single.yml +2 -0
  7. data/gradle/wrapper/gradle-wrapper.jar +0 -0
  8. data/gradle/wrapper/gradle-wrapper.properties +2 -2
  9. data/gradlew +43 -35
  10. data/src/main/scala/org/embulk/output/aerospike/AerospikeOutputPlugin.scala +4 -0
  11. data/src/main/scala/org/embulk/output/aerospike/AerospikePageOutput.scala +79 -131
  12. metadata +8 -45
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 447ce33198bb056d596d406946663042ae85bdb8
4
- data.tar.gz: ea106a3b51fe7e3c6a08a5ad4e87036469cfb80d
3
+ metadata.gz: 30a28dadd37024bf334610813246641e86317d10
4
+ data.tar.gz: 6c4b17c737ecab4470eb9448285fbc95ac88f9a7
5
5
  SHA512:
6
- metadata.gz: 8124652e505ef016b5d05d30d219620a22cef451b10b3b02e99fc05fa1d321b47ed3f501eb62d6fb6baf67f15e934eb8411fd61efd7582955eb0ca85587b4d2c
7
- data.tar.gz: 9c3f6a2d431919e7acc18645b068d73779ff2c9764c2ee6ea809cc37ecb60652296c5653538ea8357cd7a4e0f775a1b85566b31d31fe5eb4fcb2ef125b58747d
6
+ metadata.gz: 51905afa86d6586976d60b7273a52584e6b8ab5338f817a3ff67879da55ecd2599b5e3ce55b2dc56353bf76b41d8a0db64c3c26c5437fe5f89a7e6e52d415594
7
+ data.tar.gz: d89da31f6b708df469222831df485325c7bf8f0845eb0bfacc883bedc17d51cdcc0d448c5dd7f017cd1abd92345af8acd071120ef4e015455f331b2c11706eac
data/README.md CHANGED
@@ -25,6 +25,7 @@ Aerospike output plugins for Embulk loads records to databases using [aerospiker
25
25
  - **password**: user password (string, default: `null`)
26
26
  - **timeout**: command timeout (int, default: `conform to aerospike`)
27
27
  - **max_threads**: max thread numbers (int, default: `conform to aerospike`)
28
+ - **max_conns_per_node**: max connections allowed per server node (int, default: `conform to aerospike`)
28
29
  - **max_socket_idle**: max socket idel numbers (int, default: `conform to aerospike`)
29
30
  - **tend_interval**: tend interval numbers (int, default: `conform to aerospike`)
30
31
  - **fail_if_not_connected**: fail if not connected (boolean, default: `conform to aerospike`)
@@ -84,3 +85,49 @@ out:
84
85
  ```sh
85
86
  ./gradlew gem # -t to watch change of files and rebuild continuously
86
87
  ```
88
+
89
+ ## Run example
90
+
91
+ First, start the aerospike-server
92
+
93
+ ```sh
94
+ docker run --rm -ti --name aerospike -p 3000:3000 -p 3001:3001 -p 3002:3002 -p 3003:3003 aerospike/aerospike-server
95
+ ```
96
+
97
+ Then, run embulk with [example config](https://github.com/tkrs/embulk-output-aerospike/blob/master/example/config.yml).
98
+
99
+
100
+ ```sh
101
+ λ embulk -J-O -R--dev run -I lib example/config.yml
102
+ 2017-11-30 23:49:24.598 +0900: Embulk v0.8.30
103
+ 2017-11-30 23:49:38.484 +0900 [INFO] (0001:transaction): Loaded plugin embulk/output/aerospike from a load path
104
+ 2017-11-30 23:49:38.503 +0900 [INFO] (0001:transaction): Listing local files at directory 'example' filtering filename by prefix 'sample.csv'
105
+ 2017-11-30 23:49:38.505 +0900 [INFO] (0001:transaction): "follow_symlinks" is set false. Note that symbolic links to directories are skipped.
106
+ 2017-11-30 23:49:38.508 +0900 [INFO] (0001:transaction): Loading files [example/sample.csv]
107
+ 2017-11-30 23:49:38.544 +0900 [INFO] (0001:transaction): Using local thread executor with max_threads=8 / output tasks 4 = input tasks 1 * 4
108
+ 2017-11-30 23:49:38.567 +0900 [INFO] (0001:transaction): {done: 0 / 1, running: 0}
109
+ 2017-11-30 23:49:39.134 +0900 [INFO] (0013:task-0000): finish put ok[6] ng[0]
110
+ 2017-11-30 23:49:39.135 +0900 [INFO] (0013:task-0000): finish put ok[0] ng[0]
111
+ 2017-11-30 23:49:39.135 +0900 [INFO] (0013:task-0000): finish put ok[0] ng[0]
112
+ 2017-11-30 23:49:39.135 +0900 [INFO] (0013:task-0000): finish put ok[0] ng[0]
113
+ 2017-11-30 23:49:39.278 +0900 [INFO] (0001:transaction): {done: 1 / 1, running: 0}
114
+ 2017-11-30 23:49:39.284 +0900 [INFO] (main): Committed.
115
+ 2017-11-30 23:49:39.284 +0900 [INFO] (main): Next config diff: {"in":{"last_path":"example/sample.csv"},"out":{"rans":6,"failures":"{}"}}
116
+ ```
117
+
118
+ Let's check it.
119
+
120
+ ```sh
121
+ docker exec -it aerospike aql -c "select * from test"
122
+ +---------------------------------------+-----+
123
+ | user_name | age |
124
+ +---------------------------------------+-----+
125
+ | LIST('["Bomani", "Archaman"]') | 20 |
126
+ | LIST('["Ritsuka", "Fujimura"]') | 30 |
127
+ | LIST('["Fou"]') | 999 |
128
+ | LIST('["Mash", "Kyrielight"]') | 20 |
129
+ | LIST('["Olgamally", "Animusphere"]') | 10 |
130
+ | LIST('["Lev", "Lainur"]') | 45 |
131
+ +---------------------------------------+-----+
132
+ 6 rows in set (0.167 secs)
133
+ ```
data/build.gradle CHANGED
@@ -1,6 +1,7 @@
1
1
  plugins {
2
2
  id "com.jfrog.bintray" version "1.1"
3
3
  id "com.github.jruby-gradle.base" version "0.1.5"
4
+ id "checkstyle"
4
5
  id 'scala'
5
6
  }
6
7
  import com.github.jrubygradle.JRubyExec
@@ -16,16 +17,17 @@ configurations {
16
17
  provided
17
18
  }
18
19
 
19
- version = "0.3.6"
20
+ version = "0.4.0"
21
+
22
+ sourceCompatibility = 1.7
23
+
24
+ targetCompatibility = 1.7
20
25
 
21
26
  dependencies {
22
- compile "org.embulk:embulk-core:0.8.8"
23
- provided "org.embulk:embulk-core:0.8.8"
24
- compile 'org.scala-lang:scala-library:2.11.7'
25
- compile 'org.scalaz.stream:scalaz-stream_2.11:0.8'
26
- compile 'com.github.tkrs:aerospiker-core_2.11:0.4.0'
27
- compile 'com.github.tkrs:aerospiker-msgpack_2.11:0.4.0'
28
- compile 'com.github.tkrs:aerospiker-task_2.11:0.4.0'
27
+ compile "org.embulk:embulk-core:0.8.38"
28
+ provided "org.embulk:embulk-core:0.8.38"
29
+ compile 'org.scala-lang:scala-library:2.11.11'
30
+ compile 'com.aerospike:aerospike-client:4.1.0'
29
31
  testCompile "junit:junit:4.+"
30
32
  }
31
33
 
@@ -36,6 +38,22 @@ task classpath(type: Copy, dependsOn: ["jar"]) {
36
38
  }
37
39
  clean { delete "classpath" }
38
40
 
41
+ checkstyle {
42
+ configFile = file("${project.rootDir}/config/checkstyle/checkstyle.xml")
43
+ toolVersion = '6.14.1'
44
+ }
45
+ checkstyleMain {
46
+ configFile = file("${project.rootDir}/config/checkstyle/default.xml")
47
+ ignoreFailures = true
48
+ }
49
+ checkstyleTest {
50
+ configFile = file("${project.rootDir}/config/checkstyle/default.xml")
51
+ ignoreFailures = true
52
+ }
53
+ task checkstyle(type: Checkstyle) {
54
+ classpath = sourceSets.main.output + sourceSets.test.output
55
+ source = sourceSets.main.allJava + sourceSets.test.allJava
56
+ }
39
57
  task gem(type: JRubyExec, dependsOn: ["gemspec", "classpath"]) {
40
58
  jrubyArgs "-rrubygems/gem_runner", "-eGem::GemRunner.new.run(ARGV)", "build"
41
59
  script "${project.name}.gemspec"
data/config/checkstyle/checkstyle.xml ADDED
@@ -0,0 +1,128 @@
1
+ <?xml version="1.0" encoding="UTF-8"?>
2
+ <!DOCTYPE module PUBLIC
3
+ "-//Puppy Crawl//DTD Check Configuration 1.3//EN"
4
+ "http://www.puppycrawl.com/dtds/configuration_1_3.dtd">
5
+ <module name="Checker">
6
+ <!-- https://github.com/facebook/presto/blob/master/src/checkstyle/checks.xml -->
7
+ <module name="FileTabCharacter"/>
8
+ <module name="NewlineAtEndOfFile">
9
+ <property name="lineSeparator" value="lf"/>
10
+ </module>
11
+ <module name="RegexpMultiline">
12
+ <property name="format" value="\r"/>
13
+ <property name="message" value="Line contains carriage return"/>
14
+ </module>
15
+ <module name="RegexpMultiline">
16
+ <property name="format" value=" \n"/>
17
+ <property name="message" value="Line has trailing whitespace"/>
18
+ </module>
19
+ <module name="RegexpMultiline">
20
+ <property name="format" value="\{\n\n"/>
21
+ <property name="message" value="Blank line after opening brace"/>
22
+ </module>
23
+ <module name="RegexpMultiline">
24
+ <property name="format" value="\n\n\s*\}"/>
25
+ <property name="message" value="Blank line before closing brace"/>
26
+ </module>
27
+ <module name="RegexpMultiline">
28
+ <property name="format" value="\n\n\n"/>
29
+ <property name="message" value="Multiple consecutive blank lines"/>
30
+ </module>
31
+ <module name="RegexpMultiline">
32
+ <property name="format" value="\n\n\Z"/>
33
+ <property name="message" value="Blank line before end of file"/>
34
+ </module>
35
+ <module name="RegexpMultiline">
36
+ <property name="format" value="Preconditions\.checkNotNull"/>
37
+ <property name="message" value="Use of checkNotNull"/>
38
+ </module>
39
+
40
+ <module name="TreeWalker">
41
+ <module name="EmptyBlock">
42
+ <property name="option" value="text"/>
43
+ <property name="tokens" value="
44
+ LITERAL_DO, LITERAL_ELSE, LITERAL_FINALLY, LITERAL_IF,
45
+ LITERAL_FOR, LITERAL_TRY, LITERAL_WHILE, INSTANCE_INIT, STATIC_INIT"/>
46
+ </module>
47
+ <module name="EmptyStatement"/>
48
+ <module name="EmptyForInitializerPad"/>
49
+ <module name="EmptyForIteratorPad">
50
+ <property name="option" value="space"/>
51
+ </module>
52
+ <module name="MethodParamPad">
53
+ <property name="allowLineBreaks" value="true"/>
54
+ <property name="option" value="nospace"/>
55
+ </module>
56
+ <module name="ParenPad"/>
57
+ <module name="TypecastParenPad"/>
58
+ <module name="NeedBraces"/>
59
+ <module name="LeftCurly">
60
+ <property name="option" value="nl"/>
61
+ <property name="tokens" value="CLASS_DEF, CTOR_DEF, INTERFACE_DEF, METHOD_DEF"/>
62
+ </module>
63
+ <module name="LeftCurly">
64
+ <property name="option" value="eol"/>
65
+ <property name="tokens" value="
66
+ LITERAL_CATCH, LITERAL_DO, LITERAL_ELSE, LITERAL_FINALLY, LITERAL_FOR,
67
+ LITERAL_IF, LITERAL_SWITCH, LITERAL_SYNCHRONIZED, LITERAL_TRY, LITERAL_WHILE"/>
68
+ </module>
69
+ <module name="RightCurly">
70
+ <property name="option" value="alone"/>
71
+ </module>
72
+ <module name="GenericWhitespace"/>
73
+ <module name="WhitespaceAfter"/>
74
+ <module name="NoWhitespaceBefore"/>
75
+
76
+ <module name="UpperEll"/>
77
+ <module name="DefaultComesLast"/>
78
+ <module name="ArrayTypeStyle"/>
79
+ <module name="MultipleVariableDeclarations"/>
80
+ <module name="ModifierOrder"/>
81
+ <module name="OneStatementPerLine"/>
82
+ <module name="StringLiteralEquality"/>
83
+ <module name="MutableException"/>
84
+ <module name="EqualsHashCode"/>
85
+ <module name="InnerAssignment"/>
86
+ <module name="InterfaceIsType"/>
87
+ <module name="HideUtilityClassConstructor"/>
88
+
89
+ <module name="MemberName"/>
90
+ <module name="LocalVariableName"/>
91
+ <module name="LocalFinalVariableName"/>
92
+ <module name="TypeName"/>
93
+ <module name="PackageName"/>
94
+ <module name="ParameterName"/>
95
+ <module name="StaticVariableName"/>
96
+ <module name="ClassTypeParameterName">
97
+ <property name="format" value="^[A-Z][0-9]?$"/>
98
+ </module>
99
+ <module name="MethodTypeParameterName">
100
+ <property name="format" value="^[A-Z][0-9]?$"/>
101
+ </module>
102
+
103
+ <module name="AvoidStarImport"/>
104
+ <module name="RedundantImport"/>
105
+ <module name="UnusedImports"/>
106
+ <module name="ImportOrder">
107
+ <property name="groups" value="*,javax,java"/>
108
+ <property name="separated" value="true"/>
109
+ <property name="option" value="bottom"/>
110
+ <property name="sortStaticImportsAlphabetically" value="true"/>
111
+ </module>
112
+
113
+ <module name="WhitespaceAround">
114
+ <property name="allowEmptyConstructors" value="true"/>
115
+ <property name="allowEmptyMethods" value="true"/>
116
+ <property name="ignoreEnhancedForColon" value="false"/>
117
+ <property name="tokens" value="
118
+ ASSIGN, BAND, BAND_ASSIGN, BOR, BOR_ASSIGN, BSR, BSR_ASSIGN,
119
+ BXOR, BXOR_ASSIGN, COLON, DIV, DIV_ASSIGN, EQUAL, GE, GT, LAND, LE,
120
+ LITERAL_ASSERT, LITERAL_CATCH, LITERAL_DO, LITERAL_ELSE,
121
+ LITERAL_FINALLY, LITERAL_FOR, LITERAL_IF, LITERAL_RETURN,
122
+ LITERAL_SYNCHRONIZED, LITERAL_TRY, LITERAL_WHILE,
123
+ LOR, LT, MINUS, MINUS_ASSIGN, MOD, MOD_ASSIGN, NOT_EQUAL,
124
+ PLUS, PLUS_ASSIGN, QUESTION, SL, SLIST, SL_ASSIGN, SR, SR_ASSIGN,
125
+ STAR, STAR_ASSIGN, TYPE_EXTENSION_AND"/>
126
+ </module>
127
+ </module>
128
+ </module>
data/config/checkstyle/default.xml ADDED
@@ -0,0 +1,108 @@
1
+ <?xml version="1.0" encoding="UTF-8"?>
2
+ <!DOCTYPE module PUBLIC
3
+ "-//Puppy Crawl//DTD Check Configuration 1.3//EN"
4
+ "http://www.puppycrawl.com/dtds/configuration_1_3.dtd">
5
+ <!--
6
+ This is a subset of ./checkstyle.xml which allows some loose styles
7
+ -->
8
+ <module name="Checker">
9
+ <module name="FileTabCharacter"/>
10
+ <module name="NewlineAtEndOfFile">
11
+ <property name="lineSeparator" value="lf"/>
12
+ </module>
13
+ <module name="RegexpMultiline">
14
+ <property name="format" value="\r"/>
15
+ <property name="message" value="Line contains carriage return"/>
16
+ </module>
17
+ <module name="RegexpMultiline">
18
+ <property name="format" value=" \n"/>
19
+ <property name="message" value="Line has trailing whitespace"/>
20
+ </module>
21
+ <module name="RegexpMultiline">
22
+ <property name="format" value="\n\n\n"/>
23
+ <property name="message" value="Multiple consecutive blank lines"/>
24
+ </module>
25
+ <module name="RegexpMultiline">
26
+ <property name="format" value="\n\n\Z"/>
27
+ <property name="message" value="Blank line before end of file"/>
28
+ </module>
29
+
30
+ <module name="TreeWalker">
31
+ <module name="EmptyBlock">
32
+ <property name="option" value="text"/>
33
+ <property name="tokens" value="
34
+ LITERAL_DO, LITERAL_ELSE, LITERAL_FINALLY, LITERAL_IF,
35
+ LITERAL_FOR, LITERAL_TRY, LITERAL_WHILE, INSTANCE_INIT, STATIC_INIT"/>
36
+ </module>
37
+ <module name="EmptyStatement"/>
38
+ <module name="EmptyForInitializerPad"/>
39
+ <module name="EmptyForIteratorPad">
40
+ <property name="option" value="space"/>
41
+ </module>
42
+ <module name="MethodParamPad">
43
+ <property name="allowLineBreaks" value="true"/>
44
+ <property name="option" value="nospace"/>
45
+ </module>
46
+ <module name="ParenPad"/>
47
+ <module name="TypecastParenPad"/>
48
+ <module name="NeedBraces"/>
49
+ <module name="LeftCurly">
50
+ <property name="option" value="nl"/>
51
+ <property name="tokens" value="CLASS_DEF, CTOR_DEF, INTERFACE_DEF, METHOD_DEF"/>
52
+ </module>
53
+ <module name="LeftCurly">
54
+ <property name="option" value="eol"/>
55
+ <property name="tokens" value="
56
+ LITERAL_CATCH, LITERAL_DO, LITERAL_ELSE, LITERAL_FINALLY, LITERAL_FOR,
57
+ LITERAL_IF, LITERAL_SWITCH, LITERAL_SYNCHRONIZED, LITERAL_TRY, LITERAL_WHILE"/>
58
+ </module>
59
+ <module name="RightCurly">
60
+ <property name="option" value="alone"/>
61
+ </module>
62
+ <module name="GenericWhitespace"/>
63
+ <module name="WhitespaceAfter"/>
64
+ <module name="NoWhitespaceBefore"/>
65
+
66
+ <module name="UpperEll"/>
67
+ <module name="DefaultComesLast"/>
68
+ <module name="ArrayTypeStyle"/>
69
+ <module name="MultipleVariableDeclarations"/>
70
+ <module name="ModifierOrder"/>
71
+ <module name="OneStatementPerLine"/>
72
+ <module name="StringLiteralEquality"/>
73
+ <module name="MutableException"/>
74
+ <module name="EqualsHashCode"/>
75
+ <module name="InnerAssignment"/>
76
+ <module name="InterfaceIsType"/>
77
+ <module name="HideUtilityClassConstructor"/>
78
+
79
+ <module name="MemberName"/>
80
+ <module name="LocalVariableName"/>
81
+ <module name="LocalFinalVariableName"/>
82
+ <module name="TypeName"/>
83
+ <module name="PackageName"/>
84
+ <module name="ParameterName"/>
85
+ <module name="StaticVariableName"/>
86
+ <module name="ClassTypeParameterName">
87
+ <property name="format" value="^[A-Z][0-9]?$"/>
88
+ </module>
89
+ <module name="MethodTypeParameterName">
90
+ <property name="format" value="^[A-Z][0-9]?$"/>
91
+ </module>
92
+
93
+ <module name="WhitespaceAround">
94
+ <property name="allowEmptyConstructors" value="true"/>
95
+ <property name="allowEmptyMethods" value="true"/>
96
+ <property name="ignoreEnhancedForColon" value="false"/>
97
+ <property name="tokens" value="
98
+ ASSIGN, BAND, BAND_ASSIGN, BOR, BOR_ASSIGN, BSR, BSR_ASSIGN,
99
+ BXOR, BXOR_ASSIGN, COLON, DIV, DIV_ASSIGN, EQUAL, GE, GT, LAND, LE,
100
+ LITERAL_ASSERT, LITERAL_CATCH, LITERAL_DO, LITERAL_ELSE,
101
+ LITERAL_FINALLY, LITERAL_FOR, LITERAL_IF, LITERAL_RETURN,
102
+ LITERAL_SYNCHRONIZED, LITERAL_TRY, LITERAL_WHILE,
103
+ LOR, LT, MINUS, MINUS_ASSIGN, MOD, MOD_ASSIGN, NOT_EQUAL,
104
+ PLUS, PLUS_ASSIGN, QUESTION, SL, SLIST, SL_ASSIGN, SR, SR_ASSIGN,
105
+ STAR, STAR_ASSIGN, TYPE_EXTENSION_AND"/>
106
+ </module>
107
+ </module>
108
+ </module>
data/example/config-single.yml CHANGED
@@ -25,3 +25,5 @@ out:
25
25
  single_bin_name: record
26
26
  splitters:
27
27
  user_name: {separator: '\t', element_type: string}
28
+ write_policy:
29
+ expiration: 64000
data/gradle/wrapper/gradle-wrapper.properties CHANGED
@@ -1,6 +1,6 @@
1
- #Tue Oct 13 15:46:01 JST 2015
1
+ #Fri Dec 01 03:21:39 JST 2017
2
2
  distributionBase=GRADLE_USER_HOME
3
3
  distributionPath=wrapper/dists
4
4
  zipStoreBase=GRADLE_USER_HOME
5
5
  zipStorePath=wrapper/dists
6
- distributionUrl=https\://services.gradle.org/distributions/gradle-2.6-all.zip
6
+ distributionUrl=https\://services.gradle.org/distributions/gradle-4.1-all.zip
data/gradlew CHANGED
@@ -1,4 +1,4 @@
1
- #!/usr/bin/env bash
1
+ #!/usr/bin/env sh
2
2
 
3
3
  ##############################################################################
4
4
  ##
@@ -6,20 +6,38 @@
6
6
  ##
7
7
  ##############################################################################
8
8
 
9
- # Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
10
- DEFAULT_JVM_OPTS=""
9
+ # Attempt to set APP_HOME
10
+ # Resolve links: $0 may be a link
11
+ PRG="$0"
12
+ # Need this for relative symlinks.
13
+ while [ -h "$PRG" ] ; do
14
+ ls=`ls -ld "$PRG"`
15
+ link=`expr "$ls" : '.*-> \(.*\)$'`
16
+ if expr "$link" : '/.*' > /dev/null; then
17
+ PRG="$link"
18
+ else
19
+ PRG=`dirname "$PRG"`"/$link"
20
+ fi
21
+ done
22
+ SAVED="`pwd`"
23
+ cd "`dirname \"$PRG\"`/" >/dev/null
24
+ APP_HOME="`pwd -P`"
25
+ cd "$SAVED" >/dev/null
11
26
 
12
27
  APP_NAME="Gradle"
13
28
  APP_BASE_NAME=`basename "$0"`
14
29
 
30
+ # Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
31
+ DEFAULT_JVM_OPTS=""
32
+
15
33
  # Use the maximum available, or set MAX_FD != -1 to use that value.
16
34
  MAX_FD="maximum"
17
35
 
18
- warn ( ) {
36
+ warn () {
19
37
  echo "$*"
20
38
  }
21
39
 
22
- die ( ) {
40
+ die () {
23
41
  echo
24
42
  echo "$*"
25
43
  echo
@@ -30,6 +48,7 @@ die ( ) {
30
48
  cygwin=false
31
49
  msys=false
32
50
  darwin=false
51
+ nonstop=false
33
52
  case "`uname`" in
34
53
  CYGWIN* )
35
54
  cygwin=true
@@ -40,31 +59,11 @@ case "`uname`" in
40
59
  MINGW* )
41
60
  msys=true
42
61
  ;;
62
+ NONSTOP* )
63
+ nonstop=true
64
+ ;;
43
65
  esac
44
66
 
45
- # For Cygwin, ensure paths are in UNIX format before anything is touched.
46
- if $cygwin ; then
47
- [ -n "$JAVA_HOME" ] && JAVA_HOME=`cygpath --unix "$JAVA_HOME"`
48
- fi
49
-
50
- # Attempt to set APP_HOME
51
- # Resolve links: $0 may be a link
52
- PRG="$0"
53
- # Need this for relative symlinks.
54
- while [ -h "$PRG" ] ; do
55
- ls=`ls -ld "$PRG"`
56
- link=`expr "$ls" : '.*-> \(.*\)$'`
57
- if expr "$link" : '/.*' > /dev/null; then
58
- PRG="$link"
59
- else
60
- PRG=`dirname "$PRG"`"/$link"
61
- fi
62
- done
63
- SAVED="`pwd`"
64
- cd "`dirname \"$PRG\"`/" >&-
65
- APP_HOME="`pwd -P`"
66
- cd "$SAVED" >&-
67
-
68
67
  CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar
69
68
 
70
69
  # Determine the Java command to use to start the JVM.
@@ -90,7 +89,7 @@ location of your Java installation."
90
89
  fi
91
90
 
92
91
  # Increase the maximum file descriptors if we can.
93
- if [ "$cygwin" = "false" -a "$darwin" = "false" ] ; then
92
+ if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then
94
93
  MAX_FD_LIMIT=`ulimit -H -n`
95
94
  if [ $? -eq 0 ] ; then
96
95
  if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then
@@ -114,6 +113,7 @@ fi
114
113
  if $cygwin ; then
115
114
  APP_HOME=`cygpath --path --mixed "$APP_HOME"`
116
115
  CLASSPATH=`cygpath --path --mixed "$CLASSPATH"`
116
+ JAVACMD=`cygpath --unix "$JAVACMD"`
117
117
 
118
118
  # We build the pattern for arguments to be converted via cygpath
119
119
  ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null`
@@ -154,11 +154,19 @@ if $cygwin ; then
154
154
  esac
155
155
  fi
156
156
 
157
- # Split up the JVM_OPTS And GRADLE_OPTS values into an array, following the shell quoting and substitution rules
158
- function splitJvmOpts() {
159
- JVM_OPTS=("$@")
157
+ # Escape application args
158
+ save () {
159
+ for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done
160
+ echo " "
160
161
  }
161
- eval splitJvmOpts $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS
162
- JVM_OPTS[${#JVM_OPTS[*]}]="-Dorg.gradle.appname=$APP_BASE_NAME"
162
+ APP_ARGS=$(save "$@")
163
+
164
+ # Collect all arguments for the java command, following the shell quoting and substitution rules
165
+ eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS"
166
+
167
+ # by default we should be in the correct project dir, but when run from Finder on Mac, the cwd is wrong
168
+ if [ "$(uname)" = "Darwin" ] && [ "$HOME" = "$PWD" ]; then
169
+ cd "$(dirname "$0")"
170
+ fi
163
171
 
164
- exec "$JAVACMD" "${JVM_OPTS[@]}" -classpath "$CLASSPATH" org.gradle.wrapper.GradleWrapperMain "$@"
172
+ exec "$JAVACMD" "$@"
data/src/main/scala/org/embulk/output/aerospike/AerospikeOutputPlugin.scala CHANGED
@@ -59,9 +59,13 @@ object AerospikeOutputPlugin {
59
59
  @Config("timeout")
60
60
  @ConfigDefault("0") def getTimeout: Optional[Integer]
61
61
 
62
+ @Deprecated
62
63
  @Config("max_threads")
63
64
  @ConfigDefault("300") def getMaxThreads: Optional[Integer]
64
65
 
66
+ @Config("max_conns_per_node")
67
+ @ConfigDefault("300") def getMaxConnsPerNode: Optional[Integer]
68
+
65
69
  @Config("max_socket_idle")
66
70
  @ConfigDefault("14") def getMaxSocketIdle: Optional[Integer]
67
71
 
data/src/main/scala/org/embulk/output/aerospike/AerospikePageOutput.scala CHANGED
@@ -1,13 +1,9 @@
1
1
  package org.embulk.output.aerospike
2
2
 
3
- import java.util.concurrent.{ CountDownLatch, ConcurrentLinkedQueue }
4
3
  import java.util.concurrent.atomic.AtomicLong
5
4
 
6
- import aerospiker._
7
- import aerospiker.policy.{ ClientPolicy, WritePolicy }
8
- import aerospiker.task.{ DeleteError, PutError, Aerospike }
9
- import cats.data.Xor, Xor._
10
- import io.circe._, io.circe.syntax._
5
+ import com.aerospike.client.{AerospikeClient, Bin, Host, Key}
6
+ import com.aerospike.client.policy.{ClientPolicy, WritePolicy}
11
7
  import org.embulk.config.TaskReport
12
8
  import org.embulk.config.TaskSource
13
9
  import org.embulk.spi._
@@ -15,11 +11,8 @@ import org.embulk.spi.`type`.Type
15
11
  import org.embulk.spi.time.Timestamp
16
12
 
17
13
  import scala.collection.concurrent.TrieMap
18
- import scala.collection.mutable.{ Map => MMap, ListBuffer }
19
14
  import scala.collection.JavaConversions._
20
- import scalaz.{ \/-, -\/ }
21
- import scalaz.concurrent.Task
22
- import scalaz.stream._
15
+ import scala.util.{Failure, Success, Try}
23
16
 
24
17
  class AerospikePageOutput(taskSource: TaskSource, schema: Schema, taskIndex: Int) extends TransactionalPageOutput {
25
18
 
@@ -34,176 +27,131 @@ class AerospikePageOutput(taskSource: TaskSource, schema: Schema, taskIndex: Int
34
27
  private[this] val failures = TrieMap.empty[String, String]
35
28
 
36
29
  private[this] val wp: WritePolicy = {
30
+ val wp = new WritePolicy()
37
31
  if (tsk.getWritePolicy.isPresent) {
38
32
  val wpTask: WritePolicyTask = tsk.getWritePolicy.get
39
- WritePolicy(
40
- sendKey = wpTask.getSendKey.get,
41
- expiration = wpTask.getExpiration.get,
42
- maxRetries = wpTask.getMaxRetries.get,
43
- generation = wpTask.getGeneration.get,
44
- sleepBetweenRetries = wpTask.getSleepBetweenRetries.get
45
- )
46
- } else {
47
- WritePolicy()
33
+ wp.sendKey = wpTask.getSendKey.get
34
+ wp.expiration = wpTask.getExpiration.get
35
+ wp.maxRetries = wpTask.getMaxRetries.get
36
+ wp.generation = wpTask.getGeneration.get
37
+ wp.sleepBetweenRetries = wpTask.getSleepBetweenRetries.get
48
38
  }
39
+ wp
49
40
  }
50
41
 
51
42
  implicit val policy: ClientPolicy = {
43
+ val cp = new ClientPolicy()
52
44
  if (tsk.getClientPolicy.isPresent) {
53
45
  val cpTask: ClientPolicyTask = tsk.getClientPolicy.get
54
- ClientPolicy(
55
- failIfNotConnected = cpTask.getFailIfNotConnected.get,
56
- maxThreads = cpTask.getMaxThreads.get,
57
- maxSocketIdle = cpTask.getMaxSocketIdle.get,
58
- password = cpTask.getPassword.orNull,
59
- user = cpTask.getUser.orNull,
60
- timeout = cpTask.getTimeout.get,
61
- tendInterval = cpTask.getTendInterval.get,
62
- writePolicyDefault = wp
63
- )
64
- } else {
65
- ClientPolicy(writePolicyDefault = wp)
46
+ cp.failIfNotConnected = cpTask.getFailIfNotConnected.get
47
+ cp.maxConnsPerNode =cpTask.getMaxConnsPerNode.get()
48
+ cp.maxSocketIdle = cpTask.getMaxSocketIdle.get
49
+ cp.password = cpTask.getPassword.orNull
50
+ cp.user = cpTask.getUser.orNull
51
+ cp.timeout = cpTask.getTimeout.get
52
+ cp.tendInterval = cpTask.getTendInterval.get
66
53
  }
54
+ cp.writePolicyDefault = wp
55
+ cp
67
56
  }
68
57
 
69
- private[this] val hosts: Seq[Host] = tsk.getHosts.map(host => new Host(host.getName, host.getPort))
70
- private[this] val executor = AsyncCommandExecutor(AsyncClient(hosts: _*))
71
- private[this] val aerospike = new Aerospike(executor) {
72
- override protected def namespace: String = tsk.getNamespace
73
- override protected def setName: String = tsk.getSetName
74
- }
58
+ val namespace: String = tsk.getNamespace
59
+ val setName: String = tsk.getSetName
75
60
 
76
- private[this] def toJson(a: Any): Json = a match {
77
- case v: Boolean => v.asJson
78
- case v: Int => v.asJson
79
- case v: Long => v.asJson
80
- case v: Double => v.asJson
81
- case v: String => v.asJson
82
- case v: Seq[Any] => Json.array(v.map(x => toJson(x)): _*)
83
- case v: Map[String, Any] => Json.fromFields(v.map { case (k, va) => (k, toJson(va)) } toSeq)
84
- case null => Json.empty
85
- case _ => log.error(s"Unsupported class[${a.getClass}]"); throw new RuntimeException(s"Unsupported class[${a.getClass}]")
86
- }
61
+ private[this] val hosts: Seq[Host] = tsk.getHosts.map(host => new Host(host.getName, host.getPort))
87
62
 
88
- implicit val encoder = Encoder.instance[Any](toJson)
63
+ private[this] val aerospike = new AerospikeClient(policy, hosts: _*)
89
64
 
90
65
  implicit private[this] val reader: PageReader = new PageReader(schema)
91
66
 
92
- val createRecords: Page => Process[Task, Seq[Seq[Col]]] = { page =>
67
+ def createRecords(page: Page): Iterator[Seq[Col]] = {
93
68
  reader.setPage(page)
94
- val records: ListBuffer[Seq[Col]] = ListBuffer.empty
95
- while (reader.nextRecord()) {
96
- val columns = schema.getColumns.toList
97
- val convs = columns.map(Col.of)
98
- records += convs
99
- }
100
- Process.eval(Task.now(records))
69
+ Iterator.continually(())
70
+ .takeWhile(_ => reader.nextRecord())
71
+ .map(_ => schema.getColumns.toList)
72
+ .map(_.map(Col.of))
101
73
  }
102
74
 
103
- val toRecords: Seq[Seq[Col]] => Seq[Map[String, Any]] = _ map { row =>
104
- val rec: MMap[String, Any] = MMap.empty
75
+ val toRecords: Seq[Col] => java.util.Map[String, Object] = { row =>
76
+ val rec: java.util.Map[String, Object] = new java.util.HashMap()
105
77
  row foreach {
106
- case DoubleColumn(i, n, v) => rec += n -> v
107
- case LongColumn(i, n, v) => rec += n -> v
78
+ case DoubleColumn(i, n, v) => rec += n -> (v: java.lang.Double)
79
+ case LongColumn(i, n, v) => rec += n -> (v: java.lang.Long)
108
80
  case StringColumn(i, n, v) =>
109
81
  if (tsk.getSplitters.isPresent) {
110
82
  val sps = tsk.getSplitters.get.toMap
111
83
  sps.get(n) match {
112
- case None => //
113
- rec += n -> v
84
+ case None =>
85
+ rec += n -> (v: java.lang.String)
114
86
  case Some(sp) =>
115
87
  val sep = sp.getSeparator
116
88
  sp.getElementType match {
117
89
  case "long" =>
118
- val x = v.split(sep).toSeq.map(s => if (s.isEmpty) "0" else s).map(_.toLong)
119
- rec += n -> x
90
+ val xs = new java.util.ArrayList[Long]
91
+ v.split(sep).map(s => if (s.isEmpty) "0" else s).foreach(x => xs.add(x.toLong: java.lang.Long))
92
+ rec += n -> xs
120
93
  case "double" =>
121
- val x = v.split(sep).toSeq.map(s => if (s.isEmpty) "0" else s).map(_.toDouble)
122
- rec += n -> x
94
+ val xs = new java.util.ArrayList[Double]
95
+ v.split(sep).map(s => if (s.isEmpty) "0" else s).foreach(x => xs.add(x.toDouble: java.lang.Double))
96
+ rec += n -> xs
123
97
  case "string" =>
124
- val x = v.split(sep).toSeq
125
- rec += n -> x
98
+ val xs = new java.util.ArrayList[String]
99
+ val x = v.split(sep).foreach(xs.add)
100
+ rec += n -> xs
126
101
  }
127
102
  }
128
103
  } else {
129
104
  rec += n -> v
130
105
  }
131
- case BooleanColumn(i, n, v) => rec += n -> v
132
- case TimestampColumn(i, n, v) => rec += n -> v
106
+ case BooleanColumn(i, n, v) => rec += n -> (v: java.lang.Boolean)
107
+ case TimestampColumn(i, n, v) => rec += n -> (v.toEpochMilli: java.lang.Long)
133
108
  case NullColumn(i, n, t) => // nop
134
109
  }
135
- rec.toMap
110
+ rec
136
111
  }
137
112
 
138
- val updater: Sink[Task, Seq[Map[String, Any]]] = sink.lift[Task, Seq[Map[String, Any]]] { records =>
139
- val latch = new CountDownLatch(records.size)
140
- val queue = new ConcurrentLinkedQueue[Throwable Xor String]()
141
- records foreach { record =>
142
- val keyObj = record.getOrElse(tsk.getKeyName.get, "")
143
- val deRec = record - tsk.getKeyName.get
113
+ def updater(record: java.util.Map[String, Object]): Unit = {
114
+ val keyObj = record.getOrElse(tsk.getKeyName.get, "")
115
+ val key = new Key(namespace, setName, keyObj.toString)
116
+ record.remove(tsk.getKeyName.get)
117
+ Try {
144
118
  if (tsk.getSingleBinName.isPresent) {
145
- aerospike.put(keyObj.toString, Map(tsk.getSingleBinName.get() -> deRec)) runAsync {
146
- case -\/(e) => queue.add(Xor.left(e)); latch.countDown()
147
- case \/-(r) => queue.add(r); latch.countDown()
148
- }
119
+ val bin = new Bin(tsk.getSingleBinName.get(), record)
120
+ aerospike.put(wp, key, bin)
149
121
  } else {
150
- aerospike.put(keyObj.toString, deRec) runAsync {
151
- case -\/(e) => queue.add(Xor.left(e)); latch.countDown()
152
- case \/-(r) => queue.add(r); latch.countDown()
153
- }
154
- }
155
- }
156
-
157
- latch.await()
158
-
159
- Task.delay {
160
- queue foreach {
161
- case Left(e @ PutError(key, cause)) =>
162
- log.error(e.toString, e)
163
- failures += key -> cause.getMessage
164
- failCount.addAndGet(1L)
165
- case Left(e) =>
166
- log.error(e.toString, e)
167
- failures += e.getMessage -> e.getMessage
168
- failCount.addAndGet(1L)
169
- case Right(_) =>
170
- successCount.addAndGet(1L)
122
+ val bins = record.map { case (k, v) => new Bin(k, v) }
123
+ aerospike.put(wp, key, bins.toSeq: _*)
171
124
  }
125
+ } match {
126
+ case Failure(e) =>
127
+ log.error(e.toString, e)
128
+ failures += keyObj.toString -> e.getMessage
129
+ failCount.addAndGet(1L)
130
+ case Success(r) => ()
131
+ successCount.addAndGet(1L)
172
132
  }
173
133
  }
174
134
 
175
- val deleter: Sink[Task, Seq[Map[String, Any]]] = sink.lift[Task, Seq[Map[String, Any]]] { records =>
176
- val latch = new CountDownLatch(records.size)
177
- val queue = new ConcurrentLinkedQueue[DeleteError Xor Boolean]()
178
- records foreach { record =>
179
- val keyObj = record.getOrElse(tsk.getKeyName.get, "")
180
- val key = keyObj.toString
181
- aerospike.delete(key) runAsync {
182
- case -\/(e) => queue.add(Xor.left(DeleteError(key, e))); latch.countDown()
183
- case \/-(r) => queue.add(r); latch.countDown()
184
- }
185
- }
186
-
187
- latch.await()
188
-
189
- Task.delay {
190
- queue foreach {
191
- case Left(DeleteError(key, cause)) =>
192
- log.error(key, cause)
193
- failures += key -> cause.getMessage
194
- failCount.addAndGet(1L)
195
- case Right(_) =>
196
- successCount.addAndGet(1L)
197
- }
135
+ val deleter: java.util.Map[String, Object] => Unit = { record =>
136
+ val keyObj = record.getOrElse(tsk.getKeyName.get, "")
137
+ val k = keyObj.toString
138
+ val key = new Key(namespace, setName, k)
139
+ Try(aerospike.delete(wp, key)) match {
140
+ case Failure(e) =>
141
+ log.error(k, e)
142
+ failures += k -> e.getMessage
143
+ failCount.addAndGet(1L)
144
+ case Success(_) =>
145
+ successCount.addAndGet(1L)
198
146
  }
199
147
  }
200
148
 
201
149
  def add(page: Page): Unit = {
202
150
  tsk.getCommand match {
203
151
  case "put" =>
204
- createRecords(page).takeWhile(_.nonEmpty).map(toRecords).to(updater).run.run
152
+ createRecords(page).map(toRecords).foreach(updater)
205
153
  case "delete" =>
206
- createRecords(page).takeWhile(_.nonEmpty).map(toRecords).to(deleter).run.run
154
+ createRecords(page).map(toRecords).foreach(deleter)
207
155
  }
208
156
  }
209
157
 
@@ -211,7 +159,7 @@ class AerospikePageOutput(taskSource: TaskSource, schema: Schema, taskIndex: Int
211
159
 
212
160
  def close(): Unit = {
213
161
  reader.close()
214
- executor.close
162
+ aerospike.close()
215
163
  }
216
164
 
217
165
  def abort(): Unit = log.error(s"abort ${tsk.getCommand} ok[${successCount.longValue}] ng[${failCount.longValue()}]")
@@ -219,7 +167,7 @@ class AerospikePageOutput(taskSource: TaskSource, schema: Schema, taskIndex: Int
219
167
  def commit: TaskReport = {
220
168
  var r = Exec.newTaskReport
221
169
  r.set("rans", successCount.longValue() + failCount.longValue())
222
- r.set("failures", failures.toMap.asJson.pretty(Printer.noSpaces))
170
+ r.set("failures", failures)
223
171
  r
224
172
  }
225
173
  }
@@ -229,7 +177,7 @@ object ops {
229
177
  sealed trait Col
230
178
 
231
179
  object Col {
232
- def of(c: Column)(implicit r: PageReader) =
180
+ def of(c: Column)(implicit r: PageReader): Col =
233
181
  if (r isNull c) NullColumn(c.getIndex, c.getName, c.getType)
234
182
  else c.getType.getName match {
235
183
  case "string" =>
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: embulk-output-aerospike
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.3.6
4
+ version: 0.4.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Takeru Sato
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2017-08-26 00:00:00.000000000 Z
11
+ date: 2017-12-01 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  requirement: !ruby/object:Gem::Requirement
@@ -49,6 +49,8 @@ files:
49
49
  - LICENSE.txt
50
50
  - README.md
51
51
  - build.gradle
52
+ - config/checkstyle/checkstyle.xml
53
+ - config/checkstyle/default.xml
52
54
  - example/config-single.yml
53
55
  - example/config.yml
54
56
  - example/sample.csv
@@ -60,51 +62,12 @@ files:
60
62
  - src/main/scala/org/embulk/output/aerospike/AerospikeOutputPlugin.scala
61
63
  - src/main/scala/org/embulk/output/aerospike/AerospikePageOutput.scala
62
64
  - src/test/java/org/embulk/output/aerospike/TestAerospikeOutputPlugin.java
63
- - classpath/aerospike-client-3.1.4.jar
64
- - classpath/aerospiker-core_2.11-0.4.0.jar
65
- - classpath/aerospiker-msgpack_2.11-0.4.0.jar
66
- - classpath/aerospiker-task_2.11-0.4.0.jar
67
- - classpath/algebra-laws_2.11-0.2.1.jar
68
- - classpath/algebra-std_2.11-0.2.1.jar
69
- - classpath/algebra_2.11-0.2.1.jar
70
- - classpath/asm-5.0.3.jar
71
- - classpath/cats-core_2.11-0.1.2.jar
72
- - classpath/cats-free_2.11-0.1.2.jar
73
- - classpath/cats-laws_2.11-0.1.2.jar
74
- - classpath/cats-macros_2.11-0.1.2.jar
75
- - classpath/cats-state_2.11-0.1.2.jar
76
- - classpath/cats-std_2.11-0.1.2.jar
77
- - classpath/cats_2.11-0.1.2.jar
78
- - classpath/circe-core_2.11-0.1.1.jar
79
- - classpath/circe-generic_2.11-0.1.1.jar
80
- - classpath/circe-jawn_2.11-0.1.1.jar
81
- - classpath/commons-math3-3.2.jar
82
- - classpath/discipline_2.11-0.3.jar
83
- - classpath/embulk-output-aerospike-0.3.6.jar
65
+ - classpath/embulk-output-aerospike-0.4.0.jar
66
+ - classpath/scala-library-2.11.11.jar
84
67
  - classpath/gnu-crypto-2.0.1.jar
85
- - classpath/jawn-parser_2.11-0.8.0.jar
86
- - classpath/jbcrypt-0.3m.jar
87
- - classpath/jmh-core-1.9.1.jar
88
- - classpath/jmh-generator-asm-1.9.1.jar
89
- - classpath/jmh-generator-bytecode-1.9.1.jar
90
- - classpath/jmh-generator-reflection-1.9.1.jar
91
- - classpath/jopt-simple-4.6.jar
92
68
  - classpath/luaj-jse-3.0.jar
93
- - classpath/machinist_2.11-0.3.0.jar
94
- - classpath/scala-library-2.11.7.jar
95
- - classpath/scala-logging_2.11-3.1.0.jar
96
- - classpath/scala-parser-combinators_2.11-1.0.4.jar
97
- - classpath/scala-reflect-2.11.2.jar
98
- - classpath/scala-xml_2.11-1.0.4.jar
99
- - classpath/scalacheck_2.11-1.12.4.jar
100
- - classpath/scalaz-concurrent_2.11-7.1.4.jar
101
- - classpath/scalaz-core_2.11-7.1.4.jar
102
- - classpath/scalaz-effect_2.11-7.1.4.jar
103
- - classpath/scalaz-stream_2.11-0.8.jar
104
- - classpath/scodec-bits_2.11-1.0.9.jar
105
- - classpath/shapeless_2.11-2.2.5.jar
106
- - classpath/simulacrum_2.11-0.3.0.jar
107
- - classpath/test-interface-1.0.jar
69
+ - classpath/jbcrypt-0.3m.jar
70
+ - classpath/aerospike-client-4.1.0.jar
108
71
  homepage: https://github.com/tkrs/embulk-output-aerospike
109
72
  licenses:
110
73
  - MIT