kafka_etl_base 0.0.7 → 0.0.9

checksums.yaml

@@ -1,15 +1,15 @@
 ---
 !binary "U0hBMQ==":
   metadata.gz: !binary |-
-    NDU5NWVkZDk5OTBmNDUzNTdkMjhhZTkxMDEyYmY1M2Q0NTU4OTk0ZQ==
+    MzQ3ODEwOTY1Y2NhZDhiZDgwZmE0N2YzNDkwMGI0NWRkMTNhZGY5NA==
   data.tar.gz: !binary |-
-    ZmRkN2UzMjQ5NzNkZDQ5YmFjYjRjNTA4YjEzZTNmN2ZjNjcwN2VlOQ==
+    M2FjYWI5ZGViM2I0ZDRjMDcwYzdkZThhMmJiNzA1N2RkMjIxYjlhNw==
 SHA512:
   metadata.gz: !binary |-
-    MzIyMjJlNzk1ODA5ZjVhNTRmNmJkM2ZjNTg0NzY1ZGEzY2M3MTVmZWY1N2I4
-    ODcxMTMxZTkwNmU4MThmNDhjNTkwZjViN2MzMjA2MDY5N2RlYmIzYzljZmQz
-    MjU0YzU3NGJhMzA4OTQ3YzljN2M1YmRjYTdkNDc1N2NjN2JjMzY=
+    MzExZGU4ODczNmI2YjE3YThmMjYwNTBhNDAwMjRiZTkyOTA2Njk0NTkwMGY4
+    NDVmZTViMjAyNWI4ZjhiYTAyOTg3MDIwODM5NjU4YWZjZDg1NWQxZDc3NzA5
+    ODU3OWVmOGEzZGM0NzRkY2JhYzA2NzMwNDMxZTc5YzIxOTdhNDM=
   data.tar.gz: !binary |-
-    YzU2NWIxM2Y1Mjg0NmQ4OTVjNTBhOGM3YTk1NGI5MmMyYWY0MzYxN2ZiZjQ5
-    MmFlM2VjNzk3NWNmZWZiMmFjNWUxM2ZhYzNiNzYwNjA2Y2U3MzEwMjA1ZmMw
-    ZGY0NzQ1ZTJjOGViMmFmNjA0ZDY0ZGVlNDRiMjdjNzA3ZjgzNmY=
+    ZDcyYzVlMTg0Njk3NjU0YjliNDIyODg4ZmY0MTY0MDBmYTQ4MjE1MjFlZjNk
+    ZjE4ZGIxMDc4NDRkYzE0MzRmMjA2ZDBmN2E0NzlkODMwMzFiYTQwZjc1NzY2
+    MzdhYWFmYjA3NDI5YTQ4NTExODM3NmY0ZDhmMTA1N2ExZGI5NjM=

data/lib/kafka_etl_base/base.rb

@@ -38,33 +38,49 @@ module KafkaETLBase
       @total_procs = 0
       
       zk = ZK.new(@zookeeper)
-      zk.create("/", ignore: :node_exists)
-     
-      if zk.exists?("/stop_etl")
-        $log.info("zk: /stop_etl exist.")
-        $log.info("stop message processing.")
-        return
-      end
-
       begin
-        seq = [ * 0 ... @kafka_part_num ]
-        seq.shuffle! if @partition_shuffle == true
-        
-        r = Parallel.each(seq, :in_threads => @num_threads) do |part_no|
+        #zk.create("/", ignore: :node_exists)
+        if zk.exists?("/stop_etl")
+          $log.info("zk: /stop_etl exist.")
+          $log.info("stop message processing.")
+          return
+        end
+      ensure
+        zk.close!
+      end
+      
+      seq = [ * 0 ... @kafka_part_num ]
+      seq.shuffle! if @partition_shuffle == true
+      
+      r = Parallel.each(seq, :in_threads => @num_threads) do |part_no|
+        zk_th = nil
+        begin
+          zk_th = ZK.new(@zookeeper)
           zk_lock = "lock_hdfs_part_#{part_no}"
-          locker = zk.locker(zk_lock)
+          locker = zk_th.locker(zk_lock)
           begin
             if locker.lock!
-              remain = proccess_thread(zk, part_no)
+              remain = proccess_thread(zk_th, part_no)
             else
               $log.info("part: #{part_no} is already locked skip")
             end
           ensure
-            locker.unlock!
+            begin
+              locker.unlock!
+            rescue => e
+              $log.error(e.inspect)
+              $log.error(e.backtrace)
+            end
+          end
+        rescue ZK::Exceptions::ConnectionLoss => e
+          $log.error(e.inspect)
+          $log.error(e.backtrace)
+        ensure
+          begin
+            zk_th.close! if ! zk_th.nil?
+          rescue
           end
         end
-      ensure
-        zk.close
       end
       $log.info "total procs: #{@total_procs}"
     end
@@ -145,10 +161,10 @@ module KafkaETLBase
       end
     rescue Poseidon::Errors::OffsetOutOfRange => e
       zk.set(zk_part_node, "0")
-      $log.error e.to_s
+      $log.error e.inspect
       return 0
     rescue Poseidon::Errors::NotLeaderForPartition, Poseidon::Errors::UnableToFetchMetadata => e
-      $log.error e.to_s
+      $log.error e.inspect
       return 0
     end
     

data/lib/kafka_etl_base/version.rb

@@ -1,3 +1,3 @@
 module KafkaETLBase
-  VERSION = "0.0.7"
+  VERSION = "0.0.9"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: kafka_etl_base
 version: !ruby/object:Gem::Version
-  version: 0.0.7
+  version: 0.0.9
 platform: ruby
 authors:
 - Shinji Ikeda
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-11-21 00:00:00.000000000 Z
+date: 2014-11-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler