triglav-agent-hdfs 1.0.0.rc1-java → 1.0.0.rc2-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 4c845bf8090ce7c6f5d05a175b69c6e5dd7e1463
-  data.tar.gz: aecee4e6cae2b28b44c996a858e0663b31f3a954
+  metadata.gz: 9a91b6b3c3c5eaed6e47873427df60d5cd527035
+  data.tar.gz: 03e5d911634698e925262c56dce8410fc2ecebd7
 SHA512:
-  metadata.gz: af65de933dfd17d9bc10f9a45102754941947174266dd3bcef06860a7604049858aab2ee3922b09e354fe8b6e420e3a3f17131f5dc0e53b44fe70f2628b78109
-  data.tar.gz: 8eaf92c5d9a1818782563a67d2042d4eacc2e9ef1fd314487735a9e687c1c0b763710561dbb39adcfacf91c1eda218724a7cf2d68ff4fb0234152f4789df6017
+  metadata.gz: 10942b411143434c33d50dbcb675a5780db045f5010d5be544414cecd4bb96cb48645b2e50c538bc2d951139f39bb525a1100596bf1cc55d1657f93cad64d01b
+  data.tar.gz: 4e61d0f7dcbfd67f7fdf87e7955e2e0977824d2a20a3bf0548af6b1a71c5cb3911ba831cce640e98d6f9371923e19182389e6d2d0dc77a55484270f46653edb2

data/lib/triglav/agent/hdfs/monitor.rb

@@ -29,7 +29,6 @@ module Triglav::Agent
           $logger.warn { "Broken resource: #{resource.to_s}" }
           return nil
         end
-
         $logger.debug { "Start process #{resource.uri}" }
 
         events, new_last_modification_times = get_events
@@ -45,9 +44,9 @@ module Triglav::Agent
       private
 
       def get_events
-        latest_files = fetch_latest_files
+        new_last_modification_times = get_new_last_modification_times
+        latest_files = select_latest_files(new_last_modification_times)
         events = build_events(latest_files)
-        new_last_modification_times = build_last_modification_times(latest_files)
         [events, new_last_modification_times]
       rescue => e
         $logger.warn { "#{e.class} #{e.message} #{e.backtrace.join("\n  ")}" }
@@ -135,32 +134,29 @@ module Triglav::Agent
         @paths = paths
       end
 
-      def fetch_latest_files
-        latest_files = {}
+      def get_new_last_modification_times
+        new_last_modification_times = {}
         paths.each do |path, date_hour|
           latest_file = connection.get_latest_file_under(path.to_s)
           unless latest_file
             $logger.debug { "get_latest_file_under(\"#{path.to_s}\") #=> does not exist" }
             next
           end
-          is_newer = latest_file.modification_time > last_modification_times[path]
-          $logger.debug { "get_latest_file_under(\"#{path.to_s}\") #=> latest_modification_time:#{latest_file.modification_time}, is_newer:#{is_newer}" }
-          next unless is_newer
-          latest_files[path.to_sym] = latest_file
+          new_last_modification_times[path.to_sym] = latest_file.modification_time
         end
-        latest_files
+        new_last_modification_times
       end
 
-      def build_last_modification_times(latest_files)
-        last_modification_times = {}
-        latest_files.map do |path, latest_file|
-          last_modification_times[path.to_sym] = latest_file.modification_time
+      def select_latest_files(new_last_modification_times)
+        new_last_modification_times.select do |path, new_last_modification_time|
+          is_newer = new_last_modification_time > (last_modification_times[path] || 0)
+          $logger.debug { "#{path.to_s} #=> last_modification_time:#{new_last_modification_time}, is_newer:#{is_newer}" }
+          is_newer
         end
-        last_modification_times
       end
 
       def build_events(latest_files)
-        latest_files.map do |path, latest_file|
+        latest_files.map do |path, last_modification_time|
           date, hour = date_hour = paths[path]
           {
             uuid: SecureRandom.uuid,
@@ -168,7 +164,7 @@ module Triglav::Agent
             resource_unit: resource.unit,
             resource_time: date_hour_to_i(date, hour, resource.timezone),
             resource_timezone: resource.timezone,
-            payload: {path: latest_file.path.to_s, modification_time: latest_file.modification_time}.to_json, # msec
+            payload: {path: path.to_s, modification_time: last_modification_time}.to_json, # msec
           }
         end
       end

data/lib/triglav/agent/hdfs/version.rb

@@ -1,7 +1,7 @@
 module Triglav
   module Agent
     module Hdfs
-      VERSION = "1.0.0.rc1"
+      VERSION = "1.0.0.rc2"
     end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: triglav-agent-hdfs
 version: !ruby/object:Gem::Version
-  version: 1.0.0.rc1
+  version: 1.0.0.rc2
 platform: java
 authors:
 - Triglav Team
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2017-03-14 00:00:00.000000000 Z
+date: 2017-03-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement