kafka_etl_base 0.0.6 → 0.0.7

checksums.yaml

@@ -1,15 +1,15 @@
 ---
 !binary "U0hBMQ==":
   metadata.gz: !binary |-
-    NjJhOWJiMTA0NzYxZDM4OGY0NmQ1ZGI0NzkzOTU4NTcwYWJhYjYzYQ==
+    NDU5NWVkZDk5OTBmNDUzNTdkMjhhZTkxMDEyYmY1M2Q0NTU4OTk0ZQ==
   data.tar.gz: !binary |-
-    MTRhYmIxNjZkMTZiNzIyMDJhMzVkNDg2MDdiODU2YmEzMTNjNDc3Nw==
+    ZmRkN2UzMjQ5NzNkZDQ5YmFjYjRjNTA4YjEzZTNmN2ZjNjcwN2VlOQ==
 SHA512:
   metadata.gz: !binary |-
-    MDY0Y2YwN2E5ODg3NzJiNDBjY2NlZjBhMTAxNjBiNDlkNDY5NmZiMjE3N2E3
-    ZmQ0YzhjMTQwODhkYmQ2ZTBmZWYzNjMzY2JlODQzNjZkZmVjMmNmZDNjMzNj
-    NTQ5YWRlZThiNTQwNDVlYjAyNDIxODIyNzhmN2U4NzA3YjJjYTQ=
+    MzIyMjJlNzk1ODA5ZjVhNTRmNmJkM2ZjNTg0NzY1ZGEzY2M3MTVmZWY1N2I4
+    ODcxMTMxZTkwNmU4MThmNDhjNTkwZjViN2MzMjA2MDY5N2RlYmIzYzljZmQz
+    MjU0YzU3NGJhMzA4OTQ3YzljN2M1YmRjYTdkNDc1N2NjN2JjMzY=
   data.tar.gz: !binary |-
-    OTZhNzM1NmVkODE4MDVkNmRlN2U1ODliN2JkYTA1NzE4NGVlYjA1ZmEwNTRj
-    MmM4MTE0MmMwZjZhMDFkYzFlZTBjMjM0Y2EwYmIwMTMyZDYxYzQzYWUzZDk2
-    OWM0NjQyNDJiZDRmMzM0ZjQ1YjlkNDg2ZDYyMjI1NTA5YTczZWE=
+    YzU2NWIxM2Y1Mjg0NmQ4OTVjNTBhOGM3YTk1NGI5MmMyYWY0MzYxN2ZiZjQ5
+    MmFlM2VjNzk3NWNmZWZiMmFjNWUxM2ZhYzNiNzYwNjA2Y2U3MzEwMjA1ZmMw
+    ZGY0NzQ1ZTJjOGViMmFmNjA0ZDY0ZGVlNDRiMjdjNzA3ZjgzNmY=

data/lib/kafka_etl_base/base.rb

@@ -52,19 +52,15 @@ module KafkaETLBase
         
         r = Parallel.each(seq, :in_threads => @num_threads) do |part_no|
           zk_lock = "lock_hdfs_part_#{part_no}"
-          3.times.each do | n |
-            locker = zk.locker(zk_lock)
-            begin
-              if locker.lock!
-                remain = proccess_thread(zk, part_no)
-                break if remain < 1000
-              else
-                $log.info("part: #{part_no} is already locked skip")
-                break
-              end
-            ensure
-              locker.unlock!
+          locker = zk.locker(zk_lock)
+          begin
+            if locker.lock!
+              remain = proccess_thread(zk, part_no)
+            else
+              $log.info("part: #{part_no} is already locked skip")
             end
+          ensure
+            locker.unlock!
           end
         end
       ensure
@@ -74,25 +70,21 @@ module KafkaETLBase
     end
     
     def process_partition(part_no)
-      3.times.each do | n |
-        zk = ZK.new(@zookeeper)
+      zk = ZK.new(@zookeeper)
+      begin
+        zk_lock = "lock_hdfs_part_#{part_no}"
+        locker = zk.locker(zk_lock)
         begin
-           zk_lock = "lock_hdfs_part_#{part_no}"
-          locker = zk.locker(zk_lock)
-          begin
-            if locker.lock!
-               remain = proccess_thread(zk, part_no)
-               break if remain < 1000
-            else
-              $log.info("part: #{part_no} is already locked skip")
-              break
-            end
-          ensure
-            locker.unlock!
+          if locker.lock!
+            remain = proccess_thread(zk, part_no)
+          else
+            $log.info("part: #{part_no} is already locked skip")
           end
         ensure
-          zk.close
+          locker.unlock!
         end
+      ensure
+        zk.close
       end
     end
     
@@ -142,13 +134,6 @@ module KafkaETLBase
         # set next offset to zookeper
         zk.set(zk_part_node, next_offset.to_s) if next_offset >= offset
         return last_offset - next_offset
-      rescue Poseidon::Errors::NotLeaderForPartition => e
-        $log.error "Skip: Not Leader For Partition"
-        return 0
-      rescue Poseidon::Errors::OffsetOutOfRange => e
-        $log.error e.to_s
-        zk.set(zk_part_node, "0")
-        return 0
       rescue Poseidon::Connection::ConnectionFailedError
         $log.error "kafka connection failed"
         return 0

data/lib/kafka_etl_base/version.rb

@@ -1,3 +1,3 @@
 module KafkaETLBase
-  VERSION = "0.0.6"
+  VERSION = "0.0.7"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: kafka_etl_base
 version: !ruby/object:Gem::Version
-  version: 0.0.6
+  version: 0.0.7
 platform: ruby
 authors:
 - Shinji Ikeda