embulk-output-key_to_redis 0.1.1 → 0.1.2
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/build.gradle +1 -1
- data/src/main/scala/org/embulk/output/key_to_redis/KeyToRedisOutputPlugin.scala +6 -1
- data/src/main/scala/org/embulk/output/key_to_redis/PageOutput.scala +8 -11
- data/src/main/scala/org/embulk/output/key_to_redis/redis/Redis.scala +21 -3
- data/src/main/scala/org/embulk/output/key_to_redis/redis/Sender.scala +34 -0
- metadata +4 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 05c65ae37d4a10e9ac98304434c2f9ab8421e3f2
|
4
|
+
data.tar.gz: e92b05472e3366ee478ddd04c80558053a32027d
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: e92c0012e6e0d9b4e6853fdb5c538908bc4d5e53c86a82cf9c552fdbe43d770805efe658e615bbb5af62e93083adf058472656e146ac7090a01f303f29560922
|
7
|
+
data.tar.gz: 6dba520261ff5f67242dca2d391fe1b5b00c4c01544f3d69eda24280ccbf99e6185f7dc028030c961a5b943f0c1c111bf11b28a1d24489e0aa5430df1bf008cd
|
data/build.gradle
CHANGED
@@ -14,6 +14,7 @@ class KeyToRedisOutputPlugin extends OutputPlugin {
|
|
14
14
|
control: OutputPlugin.Control): ConfigDiff = {
|
15
15
|
val task = config.loadConfig(classOf[PluginTask])
|
16
16
|
KeyToRedisOutputPlugin.createRedisInstance(task)
|
17
|
+
KeyToRedisOutputPlugin.taskCountOpt = Some(taskCount)
|
17
18
|
if (task.getFlushOnStart) {
|
18
19
|
KeyToRedisOutputPlugin.redis.foreach(_.flush())
|
19
20
|
}
|
@@ -44,13 +45,17 @@ class KeyToRedisOutputPlugin extends OutputPlugin {
|
|
44
45
|
case None => // for map reduce executor.
|
45
46
|
KeyToRedisOutputPlugin.createRedisInstance(task)
|
46
47
|
}
|
47
|
-
PageOutput(taskSource,
|
48
|
+
PageOutput(taskSource,
|
49
|
+
schema,
|
50
|
+
KeyToRedisOutputPlugin.taskCountOpt,
|
51
|
+
task.getPutAsMD5)
|
48
52
|
}
|
49
53
|
|
50
54
|
}
|
51
55
|
|
52
56
|
object KeyToRedisOutputPlugin {
|
53
57
|
var redis: Option[Redis] = None
|
58
|
+
var taskCountOpt: Option[Int] = None
|
54
59
|
|
55
60
|
def createRedisInstance(task: PluginTask): Unit = {
|
56
61
|
KeyToRedisOutputPlugin.redis = Some(
|
@@ -11,12 +11,10 @@ import org.bouncycastle.util.encoders.Hex
|
|
11
11
|
import org.embulk.output.key_to_redis.redis.Redis
|
12
12
|
|
13
13
|
import scala.collection.JavaConverters._
|
14
|
-
import scala.concurrent.duration._
|
15
|
-
import scala.collection.mutable.ListBuffer
|
16
|
-
import scala.concurrent.{Await, Future}
|
17
14
|
|
18
15
|
case class PageOutput(taskSource: TaskSource,
|
19
16
|
schema: Schema,
|
17
|
+
taskCountOpt: Option[Int],
|
20
18
|
putAsMD5: Boolean)
|
21
19
|
extends TransactionalPageOutput {
|
22
20
|
val task: PluginTask = taskSource.loadTask(classOf[PluginTask])
|
@@ -25,7 +23,6 @@ case class PageOutput(taskSource: TaskSource,
|
|
25
23
|
def timestampFormatter(): TimestampFormatter =
|
26
24
|
new TimestampFormatter(task, Optional.absent())
|
27
25
|
|
28
|
-
val buffer = new ListBuffer[Future[Long]]
|
29
26
|
val redis: Redis =
|
30
27
|
KeyToRedisOutputPlugin.redis.getOrElse(sys.error("could not find redis."))
|
31
28
|
|
@@ -45,21 +42,21 @@ case class PageOutput(taskSource: TaskSource,
|
|
45
42
|
if (putAsMD5) {
|
46
43
|
val hash = Hex.toHexString(
|
47
44
|
digestMd5.digest(setValueVisitor.getValue.getBytes()))
|
48
|
-
|
45
|
+
redis.sadd(hash)
|
49
46
|
} else {
|
50
|
-
|
47
|
+
redis.sadd(setValueVisitor.getValue)
|
51
48
|
}
|
52
49
|
}
|
53
50
|
}
|
54
51
|
reader.close()
|
55
52
|
}
|
56
|
-
|
57
53
|
override def finish(): Unit = {
|
58
|
-
|
59
|
-
|
60
|
-
|
54
|
+
// for map/reduce executor.
|
55
|
+
if (taskCountOpt.isEmpty) {
|
56
|
+
// close immediately.
|
57
|
+
redis.close()
|
58
|
+
}
|
61
59
|
}
|
62
|
-
|
63
60
|
override def close(): Unit = ()
|
64
61
|
override def commit(): TaskReport = Exec.newTaskReport
|
65
62
|
override def abort(): Unit = ()
|
@@ -1,5 +1,8 @@
|
|
1
1
|
package org.embulk.output.key_to_redis.redis
|
2
2
|
|
3
|
+
import akka.actor._
|
4
|
+
import akka.pattern.ask
|
5
|
+
import akka.util.Timeout
|
3
6
|
import redis.RedisClient
|
4
7
|
|
5
8
|
import scala.concurrent._
|
@@ -10,7 +13,9 @@ case class Redis(setKey: String, host: String, port: Int, db: Option[Int]) {
|
|
10
13
|
implicit val actorSystem = akka.actor.ActorSystem(
|
11
14
|
"redis-client",
|
12
15
|
classLoader = Some(this.getClass.getClassLoader))
|
16
|
+
|
13
17
|
val redis = RedisClient(host, port, db = db)
|
18
|
+
val sender: ActorRef = actorSystem.actorOf(Props(Sender(setKey, redis)))
|
14
19
|
|
15
20
|
def ping(): String = {
|
16
21
|
import scala.concurrent.ExecutionContext.Implicits.global
|
@@ -23,8 +28,9 @@ case class Redis(setKey: String, host: String, port: Int, db: Option[Int]) {
|
|
23
28
|
}
|
24
29
|
Await.result(s, 10.minute)
|
25
30
|
}
|
26
|
-
|
27
|
-
|
31
|
+
|
32
|
+
def sadd(value: String): Unit = {
|
33
|
+
sender ! Message(value)
|
28
34
|
}
|
29
35
|
|
30
36
|
def flush(): Boolean = {
|
@@ -32,8 +38,20 @@ case class Redis(setKey: String, host: String, port: Int, db: Option[Int]) {
|
|
32
38
|
}
|
33
39
|
|
34
40
|
def close(): Unit = {
|
41
|
+
sender ! Close
|
42
|
+
implicit val timeout: Timeout = Timeout(1000.seconds)
|
43
|
+
var finished = false
|
44
|
+
while (!finished) {
|
45
|
+
val f = sender ? GetStatus
|
46
|
+
val result = Await.result(f.mapTo[Result], Duration.Inf)
|
47
|
+
if (!result.finished) {
|
48
|
+
Thread.sleep(1000)
|
49
|
+
} else {
|
50
|
+
finished = result.finished
|
51
|
+
}
|
52
|
+
}
|
35
53
|
redis.stop()
|
36
|
-
// wait for
|
54
|
+
// wait for redis stop.
|
37
55
|
Thread.sleep(1000)
|
38
56
|
actorSystem.shutdown()
|
39
57
|
}
|
@@ -0,0 +1,34 @@
|
|
1
|
+
package org.embulk.output.key_to_redis.redis
|
2
|
+
|
3
|
+
import akka.actor.{Actor, ActorSystem}
|
4
|
+
import redis.RedisClient
|
5
|
+
|
6
|
+
import scala.collection.mutable.ListBuffer
|
7
|
+
import scala.concurrent.Future
|
8
|
+
|
9
|
+
case class Sender(setKey: String, redis: RedisClient) extends Actor {
|
10
|
+
|
11
|
+
implicit val actorSystem: ActorSystem = context.system
|
12
|
+
val buffer = new ListBuffer[String]
|
13
|
+
val command = new ListBuffer[Future[Long]]
|
14
|
+
|
15
|
+
override def receive: Receive = {
|
16
|
+
case Message(v) =>
|
17
|
+
buffer.append(v)
|
18
|
+
// bulk insert
|
19
|
+
if (buffer.size == 10000) {
|
20
|
+
command.append(redis.sadd(setKey, buffer: _*))
|
21
|
+
buffer.clear()
|
22
|
+
}
|
23
|
+
case Close =>
|
24
|
+
command.append(redis.sadd(setKey, buffer: _*))
|
25
|
+
buffer.clear()
|
26
|
+
case GetStatus =>
|
27
|
+
sender() ! Result(command.forall(_.isCompleted))
|
28
|
+
}
|
29
|
+
}
|
30
|
+
|
31
|
+
case object GetStatus
|
32
|
+
case class Result(finished: Boolean) extends AnyVal
|
33
|
+
case object Close
|
34
|
+
case class Message(v: String) extends AnyVal
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: embulk-output-key_to_redis
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.
|
4
|
+
version: 0.1.2
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- smdmts
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2017-08-
|
11
|
+
date: 2017-08-17 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
requirement: !ruby/object:Gem::Requirement
|
@@ -65,6 +65,7 @@ files:
|
|
65
65
|
- src/main/scala/org/embulk/output/key_to_redis/column/SetValueColumnVisitor.scala
|
66
66
|
- src/main/scala/org/embulk/output/key_to_redis/json/JsonParser.scala
|
67
67
|
- src/main/scala/org/embulk/output/key_to_redis/redis/Redis.scala
|
68
|
+
- src/main/scala/org/embulk/output/key_to_redis/redis/Sender.scala
|
68
69
|
- src/test/scala/org/embulk/output/key_to_redis/.gitkeep
|
69
70
|
- classpath/akka-actor_2.11-2.3.6.jar
|
70
71
|
- classpath/bcpkix-jdk15on-1.57.jar
|
@@ -78,7 +79,7 @@ files:
|
|
78
79
|
- classpath/circe-numbers_2.11-0.8.0.jar
|
79
80
|
- classpath/circe-parser_2.11-0.8.0.jar
|
80
81
|
- classpath/config-1.2.1.jar
|
81
|
-
- classpath/embulk-output-key_to_redis-0.1.
|
82
|
+
- classpath/embulk-output-key_to_redis-0.1.2.jar
|
82
83
|
- classpath/jawn-parser_2.11-0.10.4.jar
|
83
84
|
- classpath/machinist_2.11-0.6.1.jar
|
84
85
|
- classpath/macro-compat_2.11-1.1.1.jar
|