kafka_etl_base 0.0.7 → 0.0.9

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,15 +1,15 @@
1
1
  ---
2
2
  !binary "U0hBMQ==":
3
3
  metadata.gz: !binary |-
4
- NDU5NWVkZDk5OTBmNDUzNTdkMjhhZTkxMDEyYmY1M2Q0NTU4OTk0ZQ==
4
+ MzQ3ODEwOTY1Y2NhZDhiZDgwZmE0N2YzNDkwMGI0NWRkMTNhZGY5NA==
5
5
  data.tar.gz: !binary |-
6
- ZmRkN2UzMjQ5NzNkZDQ5YmFjYjRjNTA4YjEzZTNmN2ZjNjcwN2VlOQ==
6
+ M2FjYWI5ZGViM2I0ZDRjMDcwYzdkZThhMmJiNzA1N2RkMjIxYjlhNw==
7
7
  SHA512:
8
8
  metadata.gz: !binary |-
9
- MzIyMjJlNzk1ODA5ZjVhNTRmNmJkM2ZjNTg0NzY1ZGEzY2M3MTVmZWY1N2I4
10
- ODcxMTMxZTkwNmU4MThmNDhjNTkwZjViN2MzMjA2MDY5N2RlYmIzYzljZmQz
11
- MjU0YzU3NGJhMzA4OTQ3YzljN2M1YmRjYTdkNDc1N2NjN2JjMzY=
9
+ MzExZGU4ODczNmI2YjE3YThmMjYwNTBhNDAwMjRiZTkyOTA2Njk0NTkwMGY4
10
+ NDVmZTViMjAyNWI4ZjhiYTAyOTg3MDIwODM5NjU4YWZjZDg1NWQxZDc3NzA5
11
+ ODU3OWVmOGEzZGM0NzRkY2JhYzA2NzMwNDMxZTc5YzIxOTdhNDM=
12
12
  data.tar.gz: !binary |-
13
- YzU2NWIxM2Y1Mjg0NmQ4OTVjNTBhOGM3YTk1NGI5MmMyYWY0MzYxN2ZiZjQ5
14
- MmFlM2VjNzk3NWNmZWZiMmFjNWUxM2ZhYzNiNzYwNjA2Y2U3MzEwMjA1ZmMw
15
- ZGY0NzQ1ZTJjOGViMmFmNjA0ZDY0ZGVlNDRiMjdjNzA3ZjgzNmY=
13
+ ZDcyYzVlMTg0Njk3NjU0YjliNDIyODg4ZmY0MTY0MDBmYTQ4MjE1MjFlZjNk
14
+ ZjE4ZGIxMDc4NDRkYzE0MzRmMjA2ZDBmN2E0NzlkODMwMzFiYTQwZjc1NzY2
15
+ MzdhYWFmYjA3NDI5YTQ4NTExODM3NmY0ZDhmMTA1N2ExZGI5NjM=
@@ -38,33 +38,49 @@ module KafkaETLBase
38
38
  @total_procs = 0
39
39
 
40
40
  zk = ZK.new(@zookeeper)
41
- zk.create("/", ignore: :node_exists)
42
-
43
- if zk.exists?("/stop_etl")
44
- $log.info("zk: /stop_etl exist.")
45
- $log.info("stop message processing.")
46
- return
47
- end
48
-
49
41
  begin
50
- seq = [ * 0 ... @kafka_part_num ]
51
- seq.shuffle! if @partition_shuffle == true
52
-
53
- r = Parallel.each(seq, :in_threads => @num_threads) do |part_no|
42
+ #zk.create("/", ignore: :node_exists)
43
+ if zk.exists?("/stop_etl")
44
+ $log.info("zk: /stop_etl exist.")
45
+ $log.info("stop message processing.")
46
+ return
47
+ end
48
+ ensure
49
+ zk.close!
50
+ end
51
+
52
+ seq = [ * 0 ... @kafka_part_num ]
53
+ seq.shuffle! if @partition_shuffle == true
54
+
55
+ r = Parallel.each(seq, :in_threads => @num_threads) do |part_no|
56
+ zk_th = nil
57
+ begin
58
+ zk_th = ZK.new(@zookeeper)
54
59
  zk_lock = "lock_hdfs_part_#{part_no}"
55
- locker = zk.locker(zk_lock)
60
+ locker = zk_th.locker(zk_lock)
56
61
  begin
57
62
  if locker.lock!
58
- remain = proccess_thread(zk, part_no)
63
+ remain = proccess_thread(zk_th, part_no)
59
64
  else
60
65
  $log.info("part: #{part_no} is already locked skip")
61
66
  end
62
67
  ensure
63
- locker.unlock!
68
+ begin
69
+ locker.unlock!
70
+ rescue => e
71
+ $log.error(e.inspect)
72
+ $log.error(e.backtrace)
73
+ end
74
+ end
75
+ rescue ZK::Exceptions::ConnectionLoss => e
76
+ $log.error(e.inspect)
77
+ $log.error(e.backtrace)
78
+ ensure
79
+ begin
80
+ zk_th.close! if ! zk_th.nil?
81
+ rescue
64
82
  end
65
83
  end
66
- ensure
67
- zk.close
68
84
  end
69
85
  $log.info "total procs: #{@total_procs}"
70
86
  end
@@ -145,10 +161,10 @@ module KafkaETLBase
145
161
  end
146
162
  rescue Poseidon::Errors::OffsetOutOfRange => e
147
163
  zk.set(zk_part_node, "0")
148
- $log.error e.to_s
164
+ $log.error e.inspect
149
165
  return 0
150
166
  rescue Poseidon::Errors::NotLeaderForPartition, Poseidon::Errors::UnableToFetchMetadata => e
151
- $log.error e.to_s
167
+ $log.error e.inspect
152
168
  return 0
153
169
  end
154
170
 
@@ -1,3 +1,3 @@
1
1
  module KafkaETLBase
2
- VERSION = "0.0.7"
2
+ VERSION = "0.0.9"
3
3
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: kafka_etl_base
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.0.7
4
+ version: 0.0.9
5
5
  platform: ruby
6
6
  authors:
7
7
  - Shinji Ikeda
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2014-11-21 00:00:00.000000000 Z
11
+ date: 2014-11-27 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: bundler