embulk-output-aerospike 0.3.4 → 0.3.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/build.gradle +1 -1
- data/example/config.yml +27 -0
- data/example/sample.csv +8 -0
- data/src/main/scala/org/embulk/output/aerospike/AerospikePageOutput.scala +9 -5
- metadata +5 -3
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA1:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 5165733ebd93b91c24d7dac2b5883dca977199d2
|
|
4
|
+
data.tar.gz: 20733ff320951d2bea8f74ed339e6aa94925dbb8
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 49e08e482db22e71316cd014a8e8e5d0041922e978573a07d9dbaaa29aea36576199d19b894be6f99bacd7807a7f0a888d04cb0f4c96ebdac629bb817b1554f6
|
|
7
|
+
data.tar.gz: 0190062a79c6e053e5431d5594ee0b2bfdff216a7338fac2fc4bbe28eaf1fece7d1c603ca5ce6d7c4e2277f3b5175edec2b59a58ed9315539496b307dbe53720
|
data/build.gradle
CHANGED
data/example/config.yml
ADDED
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
in:
|
|
2
|
+
type: file
|
|
3
|
+
path_prefix: './example/sample.csv'
|
|
4
|
+
parser:
|
|
5
|
+
charset: UTF-8
|
|
6
|
+
newline: CRLF
|
|
7
|
+
type: csv
|
|
8
|
+
delimiter: ','
|
|
9
|
+
quote: '"'
|
|
10
|
+
escape: '"'
|
|
11
|
+
null_string: 'NULL'
|
|
12
|
+
skip_header_lines: 1
|
|
13
|
+
columns:
|
|
14
|
+
- {name: user_id, type: long}
|
|
15
|
+
- {name: user_name, type: string}
|
|
16
|
+
out:
|
|
17
|
+
type: aerospike
|
|
18
|
+
hosts:
|
|
19
|
+
- {name: 'localhost', port: 3000}
|
|
20
|
+
command: put
|
|
21
|
+
namespace: test
|
|
22
|
+
set_name: set
|
|
23
|
+
key_name: user_id
|
|
24
|
+
single_bin_name: record
|
|
25
|
+
splitters:
|
|
26
|
+
column1: {separator: '\.*', element_type: string}
|
|
27
|
+
column2: {separator: '\t', element_type: long}
|
data/example/sample.csv
ADDED
|
@@ -92,8 +92,11 @@ class AerospikePageOutput(taskSource: TaskSource, schema: Schema, taskIndex: Int
|
|
|
92
92
|
val createRecords: Page => Process[Task, Seq[Seq[Col]]] = { page =>
|
|
93
93
|
reader.setPage(page)
|
|
94
94
|
val records: ListBuffer[Seq[Col]] = ListBuffer.empty
|
|
95
|
-
while (reader.nextRecord())
|
|
96
|
-
|
|
95
|
+
while (reader.nextRecord()) {
|
|
96
|
+
val columns = schema.getColumns.toList
|
|
97
|
+
val convs = columns.map(Col.of)
|
|
98
|
+
records += convs
|
|
99
|
+
}
|
|
97
100
|
Process.eval(Task.now(records))
|
|
98
101
|
}
|
|
99
102
|
|
|
@@ -138,16 +141,17 @@ class AerospikePageOutput(taskSource: TaskSource, schema: Schema, taskIndex: Int
|
|
|
138
141
|
records foreach { record =>
|
|
139
142
|
val keyObj = record.getOrElse(tsk.getKeyName.get, "")
|
|
140
143
|
val deRec = record - tsk.getKeyName.get
|
|
141
|
-
if (tsk.getSingleBinName.isPresent)
|
|
144
|
+
if (tsk.getSingleBinName.isPresent) {
|
|
142
145
|
aerospike.put(keyObj.toString, Map(tsk.getSingleBinName.get() -> deRec)) runAsync {
|
|
143
146
|
case -\/(e) => queue.add(Xor.left(e)); latch.countDown()
|
|
144
147
|
case \/-(r) => queue.add(r); latch.countDown()
|
|
145
148
|
}
|
|
146
|
-
else
|
|
149
|
+
} else {
|
|
147
150
|
aerospike.put(keyObj.toString, deRec) runAsync {
|
|
148
151
|
case -\/(e) => queue.add(Xor.left(e)); latch.countDown()
|
|
149
152
|
case \/-(r) => queue.add(r); latch.countDown()
|
|
150
153
|
}
|
|
154
|
+
}
|
|
151
155
|
}
|
|
152
156
|
|
|
153
157
|
latch.await()
|
|
@@ -194,7 +198,7 @@ class AerospikePageOutput(taskSource: TaskSource, schema: Schema, taskIndex: Int
|
|
|
194
198
|
}
|
|
195
199
|
}
|
|
196
200
|
|
|
197
|
-
def add(page: Page) {
|
|
201
|
+
def add(page: Page): Unit = {
|
|
198
202
|
tsk.getCommand match {
|
|
199
203
|
case "put" =>
|
|
200
204
|
createRecords(page).takeWhile(_.nonEmpty).map(toRecords).to(updater).run.run
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: embulk-output-aerospike
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.3.
|
|
4
|
+
version: 0.3.5
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Takeru Sato
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: bin
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date:
|
|
11
|
+
date: 2017-08-26 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
requirement: !ruby/object:Gem::Requirement
|
|
@@ -49,6 +49,8 @@ files:
|
|
|
49
49
|
- LICENSE.txt
|
|
50
50
|
- README.md
|
|
51
51
|
- build.gradle
|
|
52
|
+
- example/config.yml
|
|
53
|
+
- example/sample.csv
|
|
52
54
|
- gradle/wrapper/gradle-wrapper.jar
|
|
53
55
|
- gradle/wrapper/gradle-wrapper.properties
|
|
54
56
|
- gradlew
|
|
@@ -77,7 +79,7 @@ files:
|
|
|
77
79
|
- classpath/circe-jawn_2.11-0.1.1.jar
|
|
78
80
|
- classpath/commons-math3-3.2.jar
|
|
79
81
|
- classpath/discipline_2.11-0.3.jar
|
|
80
|
-
- classpath/embulk-output-aerospike-0.3.
|
|
82
|
+
- classpath/embulk-output-aerospike-0.3.5.jar
|
|
81
83
|
- classpath/gnu-crypto-2.0.1.jar
|
|
82
84
|
- classpath/jawn-parser_2.11-0.8.0.jar
|
|
83
85
|
- classpath/jbcrypt-0.3m.jar
|