embulk-output-aerospike 0.3.6 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (12) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +47 -0
  3. data/build.gradle +26 -8
  4. data/config/checkstyle/checkstyle.xml +128 -0
  5. data/config/checkstyle/default.xml +108 -0
  6. data/example/config-single.yml +2 -0
  7. data/gradle/wrapper/gradle-wrapper.jar +0 -0
  8. data/gradle/wrapper/gradle-wrapper.properties +2 -2
  9. data/gradlew +43 -35
  10. data/src/main/scala/org/embulk/output/aerospike/AerospikeOutputPlugin.scala +4 -0
  11. data/src/main/scala/org/embulk/output/aerospike/AerospikePageOutput.scala +79 -131
  12. metadata +8 -45
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 447ce33198bb056d596d406946663042ae85bdb8
4
- data.tar.gz: ea106a3b51fe7e3c6a08a5ad4e87036469cfb80d
3
+ metadata.gz: 30a28dadd37024bf334610813246641e86317d10
4
+ data.tar.gz: 6c4b17c737ecab4470eb9448285fbc95ac88f9a7
5
5
  SHA512:
6
- metadata.gz: 8124652e505ef016b5d05d30d219620a22cef451b10b3b02e99fc05fa1d321b47ed3f501eb62d6fb6baf67f15e934eb8411fd61efd7582955eb0ca85587b4d2c
7
- data.tar.gz: 9c3f6a2d431919e7acc18645b068d73779ff2c9764c2ee6ea809cc37ecb60652296c5653538ea8357cd7a4e0f775a1b85566b31d31fe5eb4fcb2ef125b58747d
6
+ metadata.gz: 51905afa86d6586976d60b7273a52584e6b8ab5338f817a3ff67879da55ecd2599b5e3ce55b2dc56353bf76b41d8a0db64c3c26c5437fe5f89a7e6e52d415594
7
+ data.tar.gz: d89da31f6b708df469222831df485325c7bf8f0845eb0bfacc883bedc17d51cdcc0d448c5dd7f017cd1abd92345af8acd071120ef4e015455f331b2c11706eac
data/README.md CHANGED
@@ -25,6 +25,7 @@ Aerospike output plugins for Embulk loads records to databases using [aerospiker
25
25
  - **password**: user password (string, default: `null`)
26
26
  - **timeout**: command timeout (int, default: `conform to aerospike`)
27
27
  - **max_threads**: max thread numbers (int, default: `conform to aerospike`)
28
+ - **max_conns_per_node**: max connections allowed per server node (int, default: `conform to aerospike`)
28
29
  - **max_socket_idle**: max socket idel numbers (int, default: `conform to aerospike`)
29
30
  - **tend_interval**: tend interval numbers (int, default: `conform to aerospike`)
30
31
  - **fail_if_not_connected**: fail if not connected (boolean, default: `conform to aerospike`)
@@ -84,3 +85,49 @@ out:
84
85
  ```sh
85
86
  ./gradlew gem # -t to watch change of files and rebuild continuously
86
87
  ```
88
+
89
+ ## Run example
90
+
91
+ First, start the aerospike-server
92
+
93
+ ```sh
94
+ docker run --rm -ti --name aerospike -p 3000:3000 -p 3001:3001 -p 3002:3002 -p 3003:3003 aerospike/aerospike-server
95
+ ```
96
+
97
+ Then, run embulk with [example config](https://github.com/tkrs/embulk-output-aerospike/blob/master/example/config.yml).
98
+
99
+
100
+ ```sh
101
+ λ embulk -J-O -R--dev run -I lib example/config.yml
102
+ 2017-11-30 23:49:24.598 +0900: Embulk v0.8.30
103
+ 2017-11-30 23:49:38.484 +0900 [INFO] (0001:transaction): Loaded plugin embulk/output/aerospike from a load path
104
+ 2017-11-30 23:49:38.503 +0900 [INFO] (0001:transaction): Listing local files at directory 'example' filtering filename by prefix 'sample.csv'
105
+ 2017-11-30 23:49:38.505 +0900 [INFO] (0001:transaction): "follow_symlinks" is set false. Note that symbolic links to directories are skipped.
106
+ 2017-11-30 23:49:38.508 +0900 [INFO] (0001:transaction): Loading files [example/sample.csv]
107
+ 2017-11-30 23:49:38.544 +0900 [INFO] (0001:transaction): Using local thread executor with max_threads=8 / output tasks 4 = input tasks 1 * 4
108
+ 2017-11-30 23:49:38.567 +0900 [INFO] (0001:transaction): {done: 0 / 1, running: 0}
109
+ 2017-11-30 23:49:39.134 +0900 [INFO] (0013:task-0000): finish put ok[6] ng[0]
110
+ 2017-11-30 23:49:39.135 +0900 [INFO] (0013:task-0000): finish put ok[0] ng[0]
111
+ 2017-11-30 23:49:39.135 +0900 [INFO] (0013:task-0000): finish put ok[0] ng[0]
112
+ 2017-11-30 23:49:39.135 +0900 [INFO] (0013:task-0000): finish put ok[0] ng[0]
113
+ 2017-11-30 23:49:39.278 +0900 [INFO] (0001:transaction): {done: 1 / 1, running: 0}
114
+ 2017-11-30 23:49:39.284 +0900 [INFO] (main): Committed.
115
+ 2017-11-30 23:49:39.284 +0900 [INFO] (main): Next config diff: {"in":{"last_path":"example/sample.csv"},"out":{"rans":6,"failures":"{}"}}
116
+ ```
117
+
118
+ Let's check it.
119
+
120
+ ```sh
121
+ docker exec -it aerospike aql -c "select * from test"
122
+ +---------------------------------------+-----+
123
+ | user_name | age |
124
+ +---------------------------------------+-----+
125
+ | LIST('["Bomani", "Archaman"]') | 20 |
126
+ | LIST('["Ritsuka", "Fujimura"]') | 30 |
127
+ | LIST('["Fou"]') | 999 |
128
+ | LIST('["Mash", "Kyrielight"]') | 20 |
129
+ | LIST('["Olgamally", "Animusphere"]') | 10 |
130
+ | LIST('["Lev", "Lainur"]') | 45 |
131
+ +---------------------------------------+-----+
132
+ 6 rows in set (0.167 secs)
133
+ ```
data/build.gradle CHANGED
@@ -1,6 +1,7 @@
1
1
  plugins {
2
2
  id "com.jfrog.bintray" version "1.1"
3
3
  id "com.github.jruby-gradle.base" version "0.1.5"
4
+ id "checkstyle"
4
5
  id 'scala'
5
6
  }
6
7
  import com.github.jrubygradle.JRubyExec
@@ -16,16 +17,17 @@ configurations {
16
17
  provided
17
18
  }
18
19
 
19
- version = "0.3.6"
20
+ version = "0.4.0"
21
+
22
+ sourceCompatibility = 1.7
23
+
24
+ targetCompatibility = 1.7
20
25
 
21
26
  dependencies {
22
- compile "org.embulk:embulk-core:0.8.8"
23
- provided "org.embulk:embulk-core:0.8.8"
24
- compile 'org.scala-lang:scala-library:2.11.7'
25
- compile 'org.scalaz.stream:scalaz-stream_2.11:0.8'
26
- compile 'com.github.tkrs:aerospiker-core_2.11:0.4.0'
27
- compile 'com.github.tkrs:aerospiker-msgpack_2.11:0.4.0'
28
- compile 'com.github.tkrs:aerospiker-task_2.11:0.4.0'
27
+ compile "org.embulk:embulk-core:0.8.38"
28
+ provided "org.embulk:embulk-core:0.8.38"
29
+ compile 'org.scala-lang:scala-library:2.11.11'
30
+ compile 'com.aerospike:aerospike-client:4.1.0'
29
31
  testCompile "junit:junit:4.+"
30
32
  }
31
33
 
@@ -36,6 +38,22 @@ task classpath(type: Copy, dependsOn: ["jar"]) {
36
38
  }
37
39
  clean { delete "classpath" }
38
40
 
41
+ checkstyle {
42
+ configFile = file("${project.rootDir}/config/checkstyle/checkstyle.xml")
43
+ toolVersion = '6.14.1'
44
+ }
45
+ checkstyleMain {
46
+ configFile = file("${project.rootDir}/config/checkstyle/default.xml")
47
+ ignoreFailures = true
48
+ }
49
+ checkstyleTest {
50
+ configFile = file("${project.rootDir}/config/checkstyle/default.xml")
51
+ ignoreFailures = true
52
+ }
53
+ task checkstyle(type: Checkstyle) {
54
+ classpath = sourceSets.main.output + sourceSets.test.output
55
+ source = sourceSets.main.allJava + sourceSets.test.allJava
56
+ }
39
57
  task gem(type: JRubyExec, dependsOn: ["gemspec", "classpath"]) {
40
58
  jrubyArgs "-rrubygems/gem_runner", "-eGem::GemRunner.new.run(ARGV)", "build"
41
59
  script "${project.name}.gemspec"
data/config/checkstyle/checkstyle.xml ADDED
@@ -0,0 +1,128 @@
1
+ <?xml version="1.0" encoding="UTF-8"?>
2
+ <!DOCTYPE module PUBLIC
3
+ "-//Puppy Crawl//DTD Check Configuration 1.3//EN"
4
+ "http://www.puppycrawl.com/dtds/configuration_1_3.dtd">
5
+ <module name="Checker">
6
+ <!-- https://github.com/facebook/presto/blob/master/src/checkstyle/checks.xml -->
7
+ <module name="FileTabCharacter"/>
8
+ <module name="NewlineAtEndOfFile">
9
+ <property name="lineSeparator" value="lf"/>
10
+ </module>
11
+ <module name="RegexpMultiline">
12
+ <property name="format" value="\r"/>
13
+ <property name="message" value="Line contains carriage return"/>
14
+ </module>
15
+ <module name="RegexpMultiline">
16
+ <property name="format" value=" \n"/>
17
+ <property name="message" value="Line has trailing whitespace"/>
18
+ </module>
19
+ <module name="RegexpMultiline">
20
+ <property name="format" value="\{\n\n"/>
21
+ <property name="message" value="Blank line after opening brace"/>
22
+ </module>
23
+ <module name="RegexpMultiline">
24
+ <property name="format" value="\n\n\s*\}"/>
25
+ <property name="message" value="Blank line before closing brace"/>
26
+ </module>
27
+ <module name="RegexpMultiline">
28
+ <property name="format" value="\n\n\n"/>
29
+ <property name="message" value="Multiple consecutive blank lines"/>
30
+ </module>
31
+ <module name="RegexpMultiline">
32
+ <property name="format" value="\n\n\Z"/>
33
+ <property name="message" value="Blank line before end of file"/>
34
+ </module>
35
+ <module name="RegexpMultiline">
36
+ <property name="format" value="Preconditions\.checkNotNull"/>
37
+ <property name="message" value="Use of checkNotNull"/>
38
+ </module>
39
+
40
+ <module name="TreeWalker">
41
+ <module name="EmptyBlock">
42
+ <property name="option" value="text"/>
43
+ <property name="tokens" value="
44
+ LITERAL_DO, LITERAL_ELSE, LITERAL_FINALLY, LITERAL_IF,
45
+ LITERAL_FOR, LITERAL_TRY, LITERAL_WHILE, INSTANCE_INIT, STATIC_INIT"/>
46
+ </module>
47
+ <module name="EmptyStatement"/>
48
+ <module name="EmptyForInitializerPad"/>
49
+ <module name="EmptyForIteratorPad">
50
+ <property name="option" value="space"/>
51
+ </module>
52
+ <module name="MethodParamPad">
53
+ <property name="allowLineBreaks" value="true"/>
54
+ <property name="option" value="nospace"/>
55
+ </module>
56
+ <module name="ParenPad"/>
57
+ <module name="TypecastParenPad"/>
58
+ <module name="NeedBraces"/>
59
+ <module name="LeftCurly">
60
+ <property name="option" value="nl"/>
61
+ <property name="tokens" value="CLASS_DEF, CTOR_DEF, INTERFACE_DEF, METHOD_DEF"/>
62
+ </module>
63
+ <module name="LeftCurly">
64
+ <property name="option" value="eol"/>
65
+ <property name="tokens" value="
66
+ LITERAL_CATCH, LITERAL_DO, LITERAL_ELSE, LITERAL_FINALLY, LITERAL_FOR,
67
+ LITERAL_IF, LITERAL_SWITCH, LITERAL_SYNCHRONIZED, LITERAL_TRY, LITERAL_WHILE"/>
68
+ </module>
69
+ <module name="RightCurly">
70
+ <property name="option" value="alone"/>
71
+ </module>
72
+ <module name="GenericWhitespace"/>
73
+ <module name="WhitespaceAfter"/>
74
+ <module name="NoWhitespaceBefore"/>
75
+
76
+ <module name="UpperEll"/>
77
+ <module name="DefaultComesLast"/>
78
+ <module name="ArrayTypeStyle"/>
79
+ <module name="MultipleVariableDeclarations"/>
80
+ <module name="ModifierOrder"/>
81
+ <module name="OneStatementPerLine"/>
82
+ <module name="StringLiteralEquality"/>
83
+ <module name="MutableException"/>
84
+ <module name="EqualsHashCode"/>
85
+ <module name="InnerAssignment"/>
86
+ <module name="InterfaceIsType"/>
87
+ <module name="HideUtilityClassConstructor"/>
88
+
89
+ <module name="MemberName"/>
90
+ <module name="LocalVariableName"/>
91
+ <module name="LocalFinalVariableName"/>
92
+ <module name="TypeName"/>
93
+ <module name="PackageName"/>
94
+ <module name="ParameterName"/>
95
+ <module name="StaticVariableName"/>
96
+ <module name="ClassTypeParameterName">
97
+ <property name="format" value="^[A-Z][0-9]?$"/>
98
+ </module>
99
+ <module name="MethodTypeParameterName">
100
+ <property name="format" value="^[A-Z][0-9]?$"/>
101
+ </module>
102
+
103
+ <module name="AvoidStarImport"/>
104
+ <module name="RedundantImport"/>
105
+ <module name="UnusedImports"/>
106
+ <module name="ImportOrder">
107
+ <property name="groups" value="*,javax,java"/>
108
+ <property name="separated" value="true"/>
109
+ <property name="option" value="bottom"/>
110
+ <property name="sortStaticImportsAlphabetically" value="true"/>
111
+ </module>
112
+
113
+ <module name="WhitespaceAround">
114
+ <property name="allowEmptyConstructors" value="true"/>
115
+ <property name="allowEmptyMethods" value="true"/>
116
+ <property name="ignoreEnhancedForColon" value="false"/>
117
+ <property name="tokens" value="
118
+ ASSIGN, BAND, BAND_ASSIGN, BOR, BOR_ASSIGN, BSR, BSR_ASSIGN,
119
+ BXOR, BXOR_ASSIGN, COLON, DIV, DIV_ASSIGN, EQUAL, GE, GT, LAND, LE,
120
+ LITERAL_ASSERT, LITERAL_CATCH, LITERAL_DO, LITERAL_ELSE,
121
+ LITERAL_FINALLY, LITERAL_FOR, LITERAL_IF, LITERAL_RETURN,
122
+ LITERAL_SYNCHRONIZED, LITERAL_TRY, LITERAL_WHILE,
123
+ LOR, LT, MINUS, MINUS_ASSIGN, MOD, MOD_ASSIGN, NOT_EQUAL,
124
+ PLUS, PLUS_ASSIGN, QUESTION, SL, SLIST, SL_ASSIGN, SR, SR_ASSIGN,
125
+ STAR, STAR_ASSIGN, TYPE_EXTENSION_AND"/>
126
+ </module>
127
+ </module>
128
+ </module>
data/config/checkstyle/default.xml ADDED
@@ -0,0 +1,108 @@
1
+ <?xml version="1.0" encoding="UTF-8"?>
2
+ <!DOCTYPE module PUBLIC
3
+ "-//Puppy Crawl//DTD Check Configuration 1.3//EN"
4
+ "http://www.puppycrawl.com/dtds/configuration_1_3.dtd">
5
+ <!--
6
+ This is a subset of ./checkstyle.xml which allows some loose styles
7
+ -->
8
+ <module name="Checker">
9
+ <module name="FileTabCharacter"/>
10
+ <module name="NewlineAtEndOfFile">
11
+ <property name="lineSeparator" value="lf"/>
12
+ </module>
13
+ <module name="RegexpMultiline">
14
+ <property name="format" value="\r"/>
15
+ <property name="message" value="Line contains carriage return"/>
16
+ </module>
17
+ <module name="RegexpMultiline">
18
+ <property name="format" value=" \n"/>
19
+ <property name="message" value="Line has trailing whitespace"/>
20
+ </module>
21
+ <module name="RegexpMultiline">
22
+ <property name="format" value="\n\n\n"/>
23
+ <property name="message" value="Multiple consecutive blank lines"/>
24
+ </module>
25
+ <module name="RegexpMultiline">
26
+ <property name="format" value="\n\n\Z"/>
27
+ <property name="message" value="Blank line before end of file"/>
28
+ </module>
29
+
30
+ <module name="TreeWalker">
31
+ <module name="EmptyBlock">
32
+ <property name="option" value="text"/>
33
+ <property name="tokens" value="
34
+ LITERAL_DO, LITERAL_ELSE, LITERAL_FINALLY, LITERAL_IF,
35
+ LITERAL_FOR, LITERAL_TRY, LITERAL_WHILE, INSTANCE_INIT, STATIC_INIT"/>
36
+ </module>
37
+ <module name="EmptyStatement"/>
38
+ <module name="EmptyForInitializerPad"/>
39
+ <module name="EmptyForIteratorPad">
40
+ <property name="option" value="space"/>
41
+ </module>
42
+ <module name="MethodParamPad">
43
+ <property name="allowLineBreaks" value="true"/>
44
+ <property name="option" value="nospace"/>
45
+ </module>
46
+ <module name="ParenPad"/>
47
+ <module name="TypecastParenPad"/>
48
+ <module name="NeedBraces"/>
49
+ <module name="LeftCurly">
50
+ <property name="option" value="nl"/>
51
+ <property name="tokens" value="CLASS_DEF, CTOR_DEF, INTERFACE_DEF, METHOD_DEF"/>
52
+ </module>
53
+ <module name="LeftCurly">
54
+ <property name="option" value="eol"/>
55
+ <property name="tokens" value="
56
+ LITERAL_CATCH, LITERAL_DO, LITERAL_ELSE, LITERAL_FINALLY, LITERAL_FOR,
57
+ LITERAL_IF, LITERAL_SWITCH, LITERAL_SYNCHRONIZED, LITERAL_TRY, LITERAL_WHILE"/>
58
+ </module>
59
+ <module name="RightCurly">
60
+ <property name="option" value="alone"/>
61
+ </module>
62
+ <module name="GenericWhitespace"/>
63
+ <module name="WhitespaceAfter"/>
64
+ <module name="NoWhitespaceBefore"/>
65
+
66
+ <module name="UpperEll"/>
67
+ <module name="DefaultComesLast"/>
68
+ <module name="ArrayTypeStyle"/>
69
+ <module name="MultipleVariableDeclarations"/>
70
+ <module name="ModifierOrder"/>
71
+ <module name="OneStatementPerLine"/>
72
+ <module name="StringLiteralEquality"/>
73
+ <module name="MutableException"/>
74
+ <module name="EqualsHashCode"/>
75
+ <module name="InnerAssignment"/>
76
+ <module name="InterfaceIsType"/>
77
+ <module name="HideUtilityClassConstructor"/>
78
+
79
+ <module name="MemberName"/>
80
+ <module name="LocalVariableName"/>
81
+ <module name="LocalFinalVariableName"/>
82
+ <module name="TypeName"/>
83
+ <module name="PackageName"/>
84
+ <module name="ParameterName"/>
85
+ <module name="StaticVariableName"/>
86
+ <module name="ClassTypeParameterName">
87
+ <property name="format" value="^[A-Z][0-9]?$"/>
88
+ </module>
89
+ <module name="MethodTypeParameterName">
90
+ <property name="format" value="^[A-Z][0-9]?$"/>
91
+ </module>
92
+
93
+ <module name="WhitespaceAround">
94
+ <property name="allowEmptyConstructors" value="true"/>
95
+ <property name="allowEmptyMethods" value="true"/>
96
+ <property name="ignoreEnhancedForColon" value="false"/>
97
+ <property name="tokens" value="
98
+ ASSIGN, BAND, BAND_ASSIGN, BOR, BOR_ASSIGN, BSR, BSR_ASSIGN,
99
+ BXOR, BXOR_ASSIGN, COLON, DIV, DIV_ASSIGN, EQUAL, GE, GT, LAND, LE,
100
+ LITERAL_ASSERT, LITERAL_CATCH, LITERAL_DO, LITERAL_ELSE,
101
+ LITERAL_FINALLY, LITERAL_FOR, LITERAL_IF, LITERAL_RETURN,
102
+ LITERAL_SYNCHRONIZED, LITERAL_TRY, LITERAL_WHILE,
103
+ LOR, LT, MINUS, MINUS_ASSIGN, MOD, MOD_ASSIGN, NOT_EQUAL,
104
+ PLUS, PLUS_ASSIGN, QUESTION, SL, SLIST, SL_ASSIGN, SR, SR_ASSIGN,
105
+ STAR, STAR_ASSIGN, TYPE_EXTENSION_AND"/>
106
+ </module>
107
+ </module>
108
+ </module>
data/example/config-single.yml CHANGED
@@ -25,3 +25,5 @@ out:
25
25
  single_bin_name: record
26
26
  splitters:
27
27
  user_name: {separator: '\t', element_type: string}
28
+ write_policy:
29
+ expiration: 64000
data/gradle/wrapper/gradle-wrapper.properties CHANGED
@@ -1,6 +1,6 @@
1
- #Tue Oct 13 15:46:01 JST 2015
1
+ #Fri Dec 01 03:21:39 JST 2017
2
2
  distributionBase=GRADLE_USER_HOME
3
3
  distributionPath=wrapper/dists
4
4
  zipStoreBase=GRADLE_USER_HOME
5
5
  zipStorePath=wrapper/dists
6
- distributionUrl=https\://services.gradle.org/distributions/gradle-2.6-all.zip
6
+ distributionUrl=https\://services.gradle.org/distributions/gradle-4.1-all.zip
data/gradlew CHANGED
@@ -1,4 +1,4 @@
1
- #!/usr/bin/env bash
1
+ #!/usr/bin/env sh
2
2
 
3
3
  ##############################################################################
4
4
  ##
@@ -6,20 +6,38 @@
6
6
  ##
7
7
  ##############################################################################
8
8
 
9
- # Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
10
- DEFAULT_JVM_OPTS=""
9
+ # Attempt to set APP_HOME
10
+ # Resolve links: $0 may be a link
11
+ PRG="$0"
12
+ # Need this for relative symlinks.
13
+ while [ -h "$PRG" ] ; do
14
+ ls=`ls -ld "$PRG"`
15
+ link=`expr "$ls" : '.*-> \(.*\)$'`
16
+ if expr "$link" : '/.*' > /dev/null; then
17
+ PRG="$link"
18
+ else
19
+ PRG=`dirname "$PRG"`"/$link"
20
+ fi
21
+ done
22
+ SAVED="`pwd`"
23
+ cd "`dirname \"$PRG\"`/" >/dev/null
24
+ APP_HOME="`pwd -P`"
25
+ cd "$SAVED" >/dev/null
11
26
 
12
27
  APP_NAME="Gradle"
13
28
  APP_BASE_NAME=`basename "$0"`
14
29
 
30
+ # Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
31
+ DEFAULT_JVM_OPTS=""
32
+
15
33
  # Use the maximum available, or set MAX_FD != -1 to use that value.
16
34
  MAX_FD="maximum"
17
35
 
18
- warn ( ) {
36
+ warn () {
19
37
  echo "$*"
20
38
  }
21
39
 
22
- die ( ) {
40
+ die () {
23
41
  echo
24
42
  echo "$*"
25
43
  echo
@@ -30,6 +48,7 @@ die ( ) {
30
48
  cygwin=false
31
49
  msys=false
32
50
  darwin=false
51
+ nonstop=false
33
52
  case "`uname`" in
34
53
  CYGWIN* )
35
54
  cygwin=true
@@ -40,31 +59,11 @@ case "`uname`" in
40
59
  MINGW* )
41
60
  msys=true
42
61
  ;;
62
+ NONSTOP* )
63
+ nonstop=true
64
+ ;;
43
65
  esac
44
66
 
45
- # For Cygwin, ensure paths are in UNIX format before anything is touched.
46
- if $cygwin ; then
47
- [ -n "$JAVA_HOME" ] && JAVA_HOME=`cygpath --unix "$JAVA_HOME"`
48
- fi
49
-
50
- # Attempt to set APP_HOME
51
- # Resolve links: $0 may be a link
52
- PRG="$0"
53
- # Need this for relative symlinks.
54
- while [ -h "$PRG" ] ; do
55
- ls=`ls -ld "$PRG"`
56
- link=`expr "$ls" : '.*-> \(.*\)$'`
57
- if expr "$link" : '/.*' > /dev/null; then
58
- PRG="$link"
59
- else
60
- PRG=`dirname "$PRG"`"/$link"
61
- fi
62
- done
63
- SAVED="`pwd`"
64
- cd "`dirname \"$PRG\"`/" >&-
65
- APP_HOME="`pwd -P`"
66
- cd "$SAVED" >&-
67
-
68
67
  CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar
69
68
 
70
69
  # Determine the Java command to use to start the JVM.
@@ -90,7 +89,7 @@ location of your Java installation."
90
89
  fi
91
90
 
92
91
  # Increase the maximum file descriptors if we can.
93
- if [ "$cygwin" = "false" -a "$darwin" = "false" ] ; then
92
+ if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then
94
93
  MAX_FD_LIMIT=`ulimit -H -n`
95
94
  if [ $? -eq 0 ] ; then
96
95
  if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then
@@ -114,6 +113,7 @@ fi
114
113
  if $cygwin ; then
115
114
  APP_HOME=`cygpath --path --mixed "$APP_HOME"`
116
115
  CLASSPATH=`cygpath --path --mixed "$CLASSPATH"`
116
+ JAVACMD=`cygpath --unix "$JAVACMD"`
117
117
 
118
118
  # We build the pattern for arguments to be converted via cygpath
119
119
  ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null`
@@ -154,11 +154,19 @@ if $cygwin ; then
154
154
  esac
155
155
  fi
156
156
 
157
- # Split up the JVM_OPTS And GRADLE_OPTS values into an array, following the shell quoting and substitution rules
158
- function splitJvmOpts() {
159
- JVM_OPTS=("$@")
157
+ # Escape application args
158
+ save () {
159
+ for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done
160
+ echo " "
160
161
  }
161
- eval splitJvmOpts $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS
162
- JVM_OPTS[${#JVM_OPTS[*]}]="-Dorg.gradle.appname=$APP_BASE_NAME"
162
+ APP_ARGS=$(save "$@")
163
+
164
+ # Collect all arguments for the java command, following the shell quoting and substitution rules
165
+ eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS"
166
+
167
+ # by default we should be in the correct project dir, but when run from Finder on Mac, the cwd is wrong
168
+ if [ "$(uname)" = "Darwin" ] && [ "$HOME" = "$PWD" ]; then
169
+ cd "$(dirname "$0")"
170
+ fi
163
171
 
164
- exec "$JAVACMD" "${JVM_OPTS[@]}" -classpath "$CLASSPATH" org.gradle.wrapper.GradleWrapperMain "$@"
172
+ exec "$JAVACMD" "$@"
data/src/main/scala/org/embulk/output/aerospike/AerospikeOutputPlugin.scala CHANGED
@@ -59,9 +59,13 @@ object AerospikeOutputPlugin {
59
59
  @Config("timeout")
60
60
  @ConfigDefault("0") def getTimeout: Optional[Integer]
61
61
 
62
+ @Deprecated
62
63
  @Config("max_threads")
63
64
  @ConfigDefault("300") def getMaxThreads: Optional[Integer]
64
65
 
66
+ @Config("max_conns_per_node")
67
+ @ConfigDefault("300") def getMaxConnsPerNode: Optional[Integer]
68
+
65
69
  @Config("max_socket_idle")
66
70
  @ConfigDefault("14") def getMaxSocketIdle: Optional[Integer]
67
71
 
data/src/main/scala/org/embulk/output/aerospike/AerospikePageOutput.scala CHANGED
@@ -1,13 +1,9 @@
1
1
  package org.embulk.output.aerospike
2
2
 
3
- import java.util.concurrent.{ CountDownLatch, ConcurrentLinkedQueue }
4
3
  import java.util.concurrent.atomic.AtomicLong
5
4
 
6
- import aerospiker._
7
- import aerospiker.policy.{ ClientPolicy, WritePolicy }
8
- import aerospiker.task.{ DeleteError, PutError, Aerospike }
9
- import cats.data.Xor, Xor._
10
- import io.circe._, io.circe.syntax._
5
+ import com.aerospike.client.{AerospikeClient, Bin, Host, Key}
6
+ import com.aerospike.client.policy.{ClientPolicy, WritePolicy}
11
7
  import org.embulk.config.TaskReport
12
8
  import org.embulk.config.TaskSource
13
9
  import org.embulk.spi._
@@ -15,11 +11,8 @@ import org.embulk.spi.`type`.Type
15
11
  import org.embulk.spi.time.Timestamp
16
12
 
17
13
  import scala.collection.concurrent.TrieMap
18
- import scala.collection.mutable.{ Map => MMap, ListBuffer }
19
14
  import scala.collection.JavaConversions._
20
- import scalaz.{ \/-, -\/ }
21
- import scalaz.concurrent.Task
22
- import scalaz.stream._
15
+ import scala.util.{Failure, Success, Try}
23
16
 
24
17
  class AerospikePageOutput(taskSource: TaskSource, schema: Schema, taskIndex: Int) extends TransactionalPageOutput {
25
18
 
@@ -34,176 +27,131 @@ class AerospikePageOutput(taskSource: TaskSource, schema: Schema, taskIndex: Int
34
27
  private[this] val failures = TrieMap.empty[String, String]
35
28
 
36
29
  private[this] val wp: WritePolicy = {
30
+ val wp = new WritePolicy()
37
31
  if (tsk.getWritePolicy.isPresent) {
38
32
  val wpTask: WritePolicyTask = tsk.getWritePolicy.get
39
- WritePolicy(
40
- sendKey = wpTask.getSendKey.get,
41
- expiration = wpTask.getExpiration.get,
42
- maxRetries = wpTask.getMaxRetries.get,
43
- generation = wpTask.getGeneration.get,
44
- sleepBetweenRetries = wpTask.getSleepBetweenRetries.get
45
- )
46
- } else {
47
- WritePolicy()
33
+ wp.sendKey = wpTask.getSendKey.get
34
+ wp.expiration = wpTask.getExpiration.get
35
+ wp.maxRetries = wpTask.getMaxRetries.get
36
+ wp.generation = wpTask.getGeneration.get
37
+ wp.sleepBetweenRetries = wpTask.getSleepBetweenRetries.get
48
38
  }
39
+ wp
49
40
  }
50
41
 
51
42
  implicit val policy: ClientPolicy = {
43
+ val cp = new ClientPolicy()
52
44
  if (tsk.getClientPolicy.isPresent) {
53
45
  val cpTask: ClientPolicyTask = tsk.getClientPolicy.get
54
- ClientPolicy(
55
- failIfNotConnected = cpTask.getFailIfNotConnected.get,
56
- maxThreads = cpTask.getMaxThreads.get,
57
- maxSocketIdle = cpTask.getMaxSocketIdle.get,
58
- password = cpTask.getPassword.orNull,
59
- user = cpTask.getUser.orNull,
60
- timeout = cpTask.getTimeout.get,
61
- tendInterval = cpTask.getTendInterval.get,
62
- writePolicyDefault = wp
63
- )
64
- } else {
65
- ClientPolicy(writePolicyDefault = wp)
46
+ cp.failIfNotConnected = cpTask.getFailIfNotConnected.get
47
+ cp.maxConnsPerNode =cpTask.getMaxConnsPerNode.get()
48
+ cp.maxSocketIdle = cpTask.getMaxSocketIdle.get
49
+ cp.password = cpTask.getPassword.orNull
50
+ cp.user = cpTask.getUser.orNull
51
+ cp.timeout = cpTask.getTimeout.get
52
+ cp.tendInterval = cpTask.getTendInterval.get
66
53
  }
54
+ cp.writePolicyDefault = wp
55
+ cp
67
56
  }
68
57
 
69
- private[this] val hosts: Seq[Host] = tsk.getHosts.map(host => new Host(host.getName, host.getPort))
70
- private[this] val executor = AsyncCommandExecutor(AsyncClient(hosts: _*))
71
- private[this] val aerospike = new Aerospike(executor) {
72
- override protected def namespace: String = tsk.getNamespace
73
- override protected def setName: String = tsk.getSetName
74
- }
58
+ val namespace: String = tsk.getNamespace
59
+ val setName: String = tsk.getSetName
75
60
 
76
- private[this] def toJson(a: Any): Json = a match {
77
- case v: Boolean => v.asJson
78
- case v: Int => v.asJson
79
- case v: Long => v.asJson
80
- case v: Double => v.asJson
81
- case v: String => v.asJson
82
- case v: Seq[Any] => Json.array(v.map(x => toJson(x)): _*)
83
- case v: Map[String, Any] => Json.fromFields(v.map { case (k, va) => (k, toJson(va)) } toSeq)
84
- case null => Json.empty
85
- case _ => log.error(s"Unsupported class[${a.getClass}]"); throw new RuntimeException(s"Unsupported class[${a.getClass}]")
86
- }
61
+ private[this] val hosts: Seq[Host] = tsk.getHosts.map(host => new Host(host.getName, host.getPort))
87
62
 
88
- implicit val encoder = Encoder.instance[Any](toJson)
63
+ private[this] val aerospike = new AerospikeClient(policy, hosts: _*)
89
64
 
90
65
  implicit private[this] val reader: PageReader = new PageReader(schema)
91
66
 
92
- val createRecords: Page => Process[Task, Seq[Seq[Col]]] = { page =>
67
+ def createRecords(page: Page): Iterator[Seq[Col]] = {
93
68
  reader.setPage(page)
94
- val records: ListBuffer[Seq[Col]] = ListBuffer.empty
95
- while (reader.nextRecord()) {
96
- val columns = schema.getColumns.toList
97
- val convs = columns.map(Col.of)
98
- records += convs
99
- }
100
- Process.eval(Task.now(records))
69
+ Iterator.continually(())
70
+ .takeWhile(_ => reader.nextRecord())
71
+ .map(_ => schema.getColumns.toList)
72
+ .map(_.map(Col.of))
101
73
  }
102
74
 
103
- val toRecords: Seq[Seq[Col]] => Seq[Map[String, Any]] = _ map { row =>
104
- val rec: MMap[String, Any] = MMap.empty
75
+ val toRecords: Seq[Col] => java.util.Map[String, Object] = { row =>
76
+ val rec: java.util.Map[String, Object] = new java.util.HashMap()
105
77
  row foreach {
106
- case DoubleColumn(i, n, v) => rec += n -> v
107
- case LongColumn(i, n, v) => rec += n -> v
78
+ case DoubleColumn(i, n, v) => rec += n -> (v: java.lang.Double)
79
+ case LongColumn(i, n, v) => rec += n -> (v: java.lang.Long)
108
80
  case StringColumn(i, n, v) =>
109
81
  if (tsk.getSplitters.isPresent) {
110
82
  val sps = tsk.getSplitters.get.toMap
111
83
  sps.get(n) match {
112
- case None => //
113
- rec += n -> v
84
+ case None =>
85
+ rec += n -> (v: java.lang.String)
114
86
  case Some(sp) =>
115
87
  val sep = sp.getSeparator
116
88
  sp.getElementType match {
117
89
  case "long" =>
118
- val x = v.split(sep).toSeq.map(s => if (s.isEmpty) "0" else s).map(_.toLong)
119
- rec += n -> x
90
+ val xs = new java.util.ArrayList[Long]
91
+ v.split(sep).map(s => if (s.isEmpty) "0" else s).foreach(x => xs.add(x.toLong: java.lang.Long))
92
+ rec += n -> xs
120
93
  case "double" =>
121
- val x = v.split(sep).toSeq.map(s => if (s.isEmpty) "0" else s).map(_.toDouble)
122
- rec += n -> x
94
+ val xs = new java.util.ArrayList[Double]
95
+ v.split(sep).map(s => if (s.isEmpty) "0" else s).foreach(x => xs.add(x.toDouble: java.lang.Double))
96
+ rec += n -> xs
123
97
  case "string" =>
124
- val x = v.split(sep).toSeq
125
- rec += n -> x
98
+ val xs = new java.util.ArrayList[String]
99
+ val x = v.split(sep).foreach(xs.add)
100
+ rec += n -> xs
126
101
  }
127
102
  }
128
103
  } else {
129
104
  rec += n -> v
130
105
  }
131
- case BooleanColumn(i, n, v) => rec += n -> v
132
- case TimestampColumn(i, n, v) => rec += n -> v
106
+ case BooleanColumn(i, n, v) => rec += n -> (v: java.lang.Boolean)
107
+ case TimestampColumn(i, n, v) => rec += n -> (v.toEpochMilli: java.lang.Long)
133
108
  case NullColumn(i, n, t) => // nop
134
109
  }
135
- rec.toMap
110
+ rec
136
111
  }
137
112
 
138
- val updater: Sink[Task, Seq[Map[String, Any]]] = sink.lift[Task, Seq[Map[String, Any]]] { records =>
139
- val latch = new CountDownLatch(records.size)
140
- val queue = new ConcurrentLinkedQueue[Throwable Xor String]()
141
- records foreach { record =>
142
- val keyObj = record.getOrElse(tsk.getKeyName.get, "")
143
- val deRec = record - tsk.getKeyName.get
113
+ def updater(record: java.util.Map[String, Object]): Unit = {
114
+ val keyObj = record.getOrElse(tsk.getKeyName.get, "")
115
+ val key = new Key(namespace, setName, keyObj.toString)
116
+ record.remove(tsk.getKeyName.get)
117
+ Try {
144
118
  if (tsk.getSingleBinName.isPresent) {
145
- aerospike.put(keyObj.toString, Map(tsk.getSingleBinName.get() -> deRec)) runAsync {
146
- case -\/(e) => queue.add(Xor.left(e)); latch.countDown()
147
- case \/-(r) => queue.add(r); latch.countDown()
148
- }
119
+ val bin = new Bin(tsk.getSingleBinName.get(), record)
120
+ aerospike.put(wp, key, bin)
149
121
  } else {
150
- aerospike.put(keyObj.toString, deRec) runAsync {
151
- case -\/(e) => queue.add(Xor.left(e)); latch.countDown()
152
- case \/-(r) => queue.add(r); latch.countDown()
153
- }
154
- }
155
- }
156
-
157
- latch.await()
158
-
159
- Task.delay {
160
- queue foreach {
161
- case Left(e @ PutError(key, cause)) =>
162
- log.error(e.toString, e)
163
- failures += key -> cause.getMessage
164
- failCount.addAndGet(1L)
165
- case Left(e) =>
166
- log.error(e.toString, e)
167
- failures += e.getMessage -> e.getMessage
168
- failCount.addAndGet(1L)
169
- case Right(_) =>
170
- successCount.addAndGet(1L)
122
+ val bins = record.map { case (k, v) => new Bin(k, v) }
123
+ aerospike.put(wp, key, bins.toSeq: _*)
171
124
  }
125
+ } match {
126
+ case Failure(e) =>
127
+ log.error(e.toString, e)
128
+ failures += keyObj.toString -> e.getMessage
129
+ failCount.addAndGet(1L)
130
+ case Success(r) => ()
131
+ successCount.addAndGet(1L)
172
132
  }
173
133
  }
174
134
 
175
- val deleter: Sink[Task, Seq[Map[String, Any]]] = sink.lift[Task, Seq[Map[String, Any]]] { records =>
176
- val latch = new CountDownLatch(records.size)
177
- val queue = new ConcurrentLinkedQueue[DeleteError Xor Boolean]()
178
- records foreach { record =>
179
- val keyObj = record.getOrElse(tsk.getKeyName.get, "")
180
- val key = keyObj.toString
181
- aerospike.delete(key) runAsync {
182
- case -\/(e) => queue.add(Xor.left(DeleteError(key, e))); latch.countDown()
183
- case \/-(r) => queue.add(r); latch.countDown()
184
- }
185
- }
186
-
187
- latch.await()
188
-
189
- Task.delay {
190
- queue foreach {
191
- case Left(DeleteError(key, cause)) =>
192
- log.error(key, cause)
193
- failures += key -> cause.getMessage
194
- failCount.addAndGet(1L)
195
- case Right(_) =>
196
- successCount.addAndGet(1L)
197
- }
135
+ val deleter: java.util.Map[String, Object] => Unit = { record =>
136
+ val keyObj = record.getOrElse(tsk.getKeyName.get, "")
137
+ val k = keyObj.toString
138
+ val key = new Key(namespace, setName, k)
139
+ Try(aerospike.delete(wp, key)) match {
140
+ case Failure(e) =>
141
+ log.error(k, e)
142
+ failures += k -> e.getMessage
143
+ failCount.addAndGet(1L)
144
+ case Success(_) =>
145
+ successCount.addAndGet(1L)
198
146
  }
199
147
  }
200
148
 
201
149
  def add(page: Page): Unit = {
202
150
  tsk.getCommand match {
203
151
  case "put" =>
204
- createRecords(page).takeWhile(_.nonEmpty).map(toRecords).to(updater).run.run
152
+ createRecords(page).map(toRecords).foreach(updater)
205
153
  case "delete" =>
206
- createRecords(page).takeWhile(_.nonEmpty).map(toRecords).to(deleter).run.run
154
+ createRecords(page).map(toRecords).foreach(deleter)
207
155
  }
208
156
  }
209
157
 
@@ -211,7 +159,7 @@ class AerospikePageOutput(taskSource: TaskSource, schema: Schema, taskIndex: Int
211
159
 
212
160
  def close(): Unit = {
213
161
  reader.close()
214
- executor.close
162
+ aerospike.close()
215
163
  }
216
164
 
217
165
  def abort(): Unit = log.error(s"abort ${tsk.getCommand} ok[${successCount.longValue}] ng[${failCount.longValue()}]")
@@ -219,7 +167,7 @@ class AerospikePageOutput(taskSource: TaskSource, schema: Schema, taskIndex: Int
219
167
  def commit: TaskReport = {
220
168
  var r = Exec.newTaskReport
221
169
  r.set("rans", successCount.longValue() + failCount.longValue())
222
- r.set("failures", failures.toMap.asJson.pretty(Printer.noSpaces))
170
+ r.set("failures", failures)
223
171
  r
224
172
  }
225
173
  }
@@ -229,7 +177,7 @@ object ops {
229
177
  sealed trait Col
230
178
 
231
179
  object Col {
232
- def of(c: Column)(implicit r: PageReader) =
180
+ def of(c: Column)(implicit r: PageReader): Col =
233
181
  if (r isNull c) NullColumn(c.getIndex, c.getName, c.getType)
234
182
  else c.getType.getName match {
235
183
  case "string" =>
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: embulk-output-aerospike
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.3.6
4
+ version: 0.4.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Takeru Sato
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2017-08-26 00:00:00.000000000 Z
11
+ date: 2017-12-01 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  requirement: !ruby/object:Gem::Requirement
@@ -49,6 +49,8 @@ files:
49
49
  - LICENSE.txt
50
50
  - README.md
51
51
  - build.gradle
52
+ - config/checkstyle/checkstyle.xml
53
+ - config/checkstyle/default.xml
52
54
  - example/config-single.yml
53
55
  - example/config.yml
54
56
  - example/sample.csv
@@ -60,51 +62,12 @@ files:
60
62
  - src/main/scala/org/embulk/output/aerospike/AerospikeOutputPlugin.scala
61
63
  - src/main/scala/org/embulk/output/aerospike/AerospikePageOutput.scala
62
64
  - src/test/java/org/embulk/output/aerospike/TestAerospikeOutputPlugin.java
63
- - classpath/aerospike-client-3.1.4.jar
64
- - classpath/aerospiker-core_2.11-0.4.0.jar
65
- - classpath/aerospiker-msgpack_2.11-0.4.0.jar
66
- - classpath/aerospiker-task_2.11-0.4.0.jar
67
- - classpath/algebra-laws_2.11-0.2.1.jar
68
- - classpath/algebra-std_2.11-0.2.1.jar
69
- - classpath/algebra_2.11-0.2.1.jar
70
- - classpath/asm-5.0.3.jar
71
- - classpath/cats-core_2.11-0.1.2.jar
72
- - classpath/cats-free_2.11-0.1.2.jar
73
- - classpath/cats-laws_2.11-0.1.2.jar
74
- - classpath/cats-macros_2.11-0.1.2.jar
75
- - classpath/cats-state_2.11-0.1.2.jar
76
- - classpath/cats-std_2.11-0.1.2.jar
77
- - classpath/cats_2.11-0.1.2.jar
78
- - classpath/circe-core_2.11-0.1.1.jar
79
- - classpath/circe-generic_2.11-0.1.1.jar
80
- - classpath/circe-jawn_2.11-0.1.1.jar
81
- - classpath/commons-math3-3.2.jar
82
- - classpath/discipline_2.11-0.3.jar
83
- - classpath/embulk-output-aerospike-0.3.6.jar
65
+ - classpath/embulk-output-aerospike-0.4.0.jar
66
+ - classpath/scala-library-2.11.11.jar
84
67
  - classpath/gnu-crypto-2.0.1.jar
85
- - classpath/jawn-parser_2.11-0.8.0.jar
86
- - classpath/jbcrypt-0.3m.jar
87
- - classpath/jmh-core-1.9.1.jar
88
- - classpath/jmh-generator-asm-1.9.1.jar
89
- - classpath/jmh-generator-bytecode-1.9.1.jar
90
- - classpath/jmh-generator-reflection-1.9.1.jar
91
- - classpath/jopt-simple-4.6.jar
92
68
  - classpath/luaj-jse-3.0.jar
93
- - classpath/machinist_2.11-0.3.0.jar
94
- - classpath/scala-library-2.11.7.jar
95
- - classpath/scala-logging_2.11-3.1.0.jar
96
- - classpath/scala-parser-combinators_2.11-1.0.4.jar
97
- - classpath/scala-reflect-2.11.2.jar
98
- - classpath/scala-xml_2.11-1.0.4.jar
99
- - classpath/scalacheck_2.11-1.12.4.jar
100
- - classpath/scalaz-concurrent_2.11-7.1.4.jar
101
- - classpath/scalaz-core_2.11-7.1.4.jar
102
- - classpath/scalaz-effect_2.11-7.1.4.jar
103
- - classpath/scalaz-stream_2.11-0.8.jar
104
- - classpath/scodec-bits_2.11-1.0.9.jar
105
- - classpath/shapeless_2.11-2.2.5.jar
106
- - classpath/simulacrum_2.11-0.3.0.jar
107
- - classpath/test-interface-1.0.jar
69
+ - classpath/jbcrypt-0.3m.jar
70
+ - classpath/aerospike-client-4.1.0.jar
108
71
  homepage: https://github.com/tkrs/embulk-output-aerospike
109
72
  licenses:
110
73
  - MIT