kafka_etl_base 0.0.7 → 0.0.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +8 -8
- data/lib/kafka_etl_base/base.rb +35 -19
- data/lib/kafka_etl_base/version.rb +1 -1
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,15 +1,15 @@
|
|
1
1
|
---
|
2
2
|
!binary "U0hBMQ==":
|
3
3
|
metadata.gz: !binary |-
|
4
|
-
|
4
|
+
MzQ3ODEwOTY1Y2NhZDhiZDgwZmE0N2YzNDkwMGI0NWRkMTNhZGY5NA==
|
5
5
|
data.tar.gz: !binary |-
|
6
|
-
|
6
|
+
M2FjYWI5ZGViM2I0ZDRjMDcwYzdkZThhMmJiNzA1N2RkMjIxYjlhNw==
|
7
7
|
SHA512:
|
8
8
|
metadata.gz: !binary |-
|
9
|
-
|
10
|
-
|
11
|
-
|
9
|
+
MzExZGU4ODczNmI2YjE3YThmMjYwNTBhNDAwMjRiZTkyOTA2Njk0NTkwMGY4
|
10
|
+
NDVmZTViMjAyNWI4ZjhiYTAyOTg3MDIwODM5NjU4YWZjZDg1NWQxZDc3NzA5
|
11
|
+
ODU3OWVmOGEzZGM0NzRkY2JhYzA2NzMwNDMxZTc5YzIxOTdhNDM=
|
12
12
|
data.tar.gz: !binary |-
|
13
|
-
|
14
|
-
|
15
|
-
|
13
|
+
ZDcyYzVlMTg0Njk3NjU0YjliNDIyODg4ZmY0MTY0MDBmYTQ4MjE1MjFlZjNk
|
14
|
+
ZjE4ZGIxMDc4NDRkYzE0MzRmMjA2ZDBmN2E0NzlkODMwMzFiYTQwZjc1NzY2
|
15
|
+
MzdhYWFmYjA3NDI5YTQ4NTExODM3NmY0ZDhmMTA1N2ExZGI5NjM=
|
data/lib/kafka_etl_base/base.rb
CHANGED
@@ -38,33 +38,49 @@ module KafkaETLBase
|
|
38
38
|
@total_procs = 0
|
39
39
|
|
40
40
|
zk = ZK.new(@zookeeper)
|
41
|
-
zk.create("/", ignore: :node_exists)
|
42
|
-
|
43
|
-
if zk.exists?("/stop_etl")
|
44
|
-
$log.info("zk: /stop_etl exist.")
|
45
|
-
$log.info("stop message processing.")
|
46
|
-
return
|
47
|
-
end
|
48
|
-
|
49
41
|
begin
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
-
|
42
|
+
#zk.create("/", ignore: :node_exists)
|
43
|
+
if zk.exists?("/stop_etl")
|
44
|
+
$log.info("zk: /stop_etl exist.")
|
45
|
+
$log.info("stop message processing.")
|
46
|
+
return
|
47
|
+
end
|
48
|
+
ensure
|
49
|
+
zk.close!
|
50
|
+
end
|
51
|
+
|
52
|
+
seq = [ * 0 ... @kafka_part_num ]
|
53
|
+
seq.shuffle! if @partition_shuffle == true
|
54
|
+
|
55
|
+
r = Parallel.each(seq, :in_threads => @num_threads) do |part_no|
|
56
|
+
zk_th = nil
|
57
|
+
begin
|
58
|
+
zk_th = ZK.new(@zookeeper)
|
54
59
|
zk_lock = "lock_hdfs_part_#{part_no}"
|
55
|
-
locker =
|
60
|
+
locker = zk_th.locker(zk_lock)
|
56
61
|
begin
|
57
62
|
if locker.lock!
|
58
|
-
remain = proccess_thread(
|
63
|
+
remain = proccess_thread(zk_th, part_no)
|
59
64
|
else
|
60
65
|
$log.info("part: #{part_no} is already locked skip")
|
61
66
|
end
|
62
67
|
ensure
|
63
|
-
|
68
|
+
begin
|
69
|
+
locker.unlock!
|
70
|
+
rescue => e
|
71
|
+
$log.error(e.inspect)
|
72
|
+
$log.error(e.backtrace)
|
73
|
+
end
|
74
|
+
end
|
75
|
+
rescue ZK::Exceptions::ConnectionLoss => e
|
76
|
+
$log.error(e.inspect)
|
77
|
+
$log.error(e.backtrace)
|
78
|
+
ensure
|
79
|
+
begin
|
80
|
+
zk_th.close! if ! zk_th.nil?
|
81
|
+
rescue
|
64
82
|
end
|
65
83
|
end
|
66
|
-
ensure
|
67
|
-
zk.close
|
68
84
|
end
|
69
85
|
$log.info "total procs: #{@total_procs}"
|
70
86
|
end
|
@@ -145,10 +161,10 @@ module KafkaETLBase
|
|
145
161
|
end
|
146
162
|
rescue Poseidon::Errors::OffsetOutOfRange => e
|
147
163
|
zk.set(zk_part_node, "0")
|
148
|
-
$log.error e.
|
164
|
+
$log.error e.inspect
|
149
165
|
return 0
|
150
166
|
rescue Poseidon::Errors::NotLeaderForPartition, Poseidon::Errors::UnableToFetchMetadata => e
|
151
|
-
$log.error e.
|
167
|
+
$log.error e.inspect
|
152
168
|
return 0
|
153
169
|
end
|
154
170
|
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: kafka_etl_base
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.9
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Shinji Ikeda
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2014-11-
|
11
|
+
date: 2014-11-27 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|