cloud_watch_logs_poller 0.2.0 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: fc88f8d5963c66a36bc794f6f7a101d080342aba5d7a2577d956b39417a93e76
-  data.tar.gz: 9d4f3209935a3b0ceab85ab8e5842681c3804e7c8d0d99f528e5f1064c30331e
+  metadata.gz: 4641f0ec6db0d868154a40d3ac82a525c00d57ad3b51297a2078341397455a85
+  data.tar.gz: 486c7f3b586fbca8b079231df5f8bf979c5112aab5d04110a1ad983adee87570
 SHA512:
-  metadata.gz: ae1401221a8a7e900ae59fbc14b744a06184e2f58a03300d8114850c7f8d2dfcdac67b0256ef82acca352302d0fc647f28b488a0956903791a78fcd9a1b30497
-  data.tar.gz: e50f5a485af34912b8225f57177efa6cb5507337f81477a2be6d5ad937ba70f5597adf56b845a974daff8a72068884bc7444ede716bdb6e69dd953e4d49cca7c
+  metadata.gz: 35d9d853b1ee7192cca26901b1f4ae8a72a21652c3935b090f7a98dc757f5abcd7903a65bdf480c5654c5c7596db7520331289383a2a2c92a7bab0840193a989
+  data.tar.gz: f146bb5eb28ce03faf8198ceed4df78db65ea57e8d716d0096dc19b09b3c325c1405482fa91c5eaad5a8d319dfb209acc18e6fff787b5d88b93ba67c0b8e7332

data/lib/cloud_watch_logs_poller/process.rb

@@ -4,7 +4,7 @@ module CloudWatchLogsPoller
       @client = Aws::CloudWatchLogs::Client.new
       @interval = interval
       @debug = debug
-      @event_ids = []
+      @event_ids_by_timestamp = {}
     end
 
     def execute(log_group_name:, log_stream_name_prefix: nil, filter_pattern: nil, start_time: Time.now, &block)
@@ -16,24 +16,37 @@ module CloudWatchLogsPoller
         interleaved: true
       }
 
+      debug_log("start_time = #{params[:start_time]}")
+
       loop do
         loop do
           result = @client.filter_log_events(params)
           result.events.each do |event|
-            next if @event_ids.include?(event.event_id)
+            timestamp = event.timestamp&.to_i
+            event_id = event.event_id
+
+            debug_log("timestamp = #{timestamp}")
 
-            @event_ids << event.event_id
+            @event_ids_by_timestamp[timestamp] ||= Set.new
+            next if @event_ids_by_timestamp[timestamp].include?(event_id)
+
+            @event_ids_by_timestamp[timestamp] << event_id
             block.call(Event.convert_from_filtered_log_event(event))
           end
 
           debug_log(params)
-          debug_log(result)
 
+          @event_ids_by_timestamp = get_latest_events_and_timestamp
           break unless result.next_token
 
           params[:next_token] = result.next_token
         end
 
+        newest_timestamp = @event_ids_by_timestamp.keys.max
+        params[:start_time] = newest_timestamp unless newest_timestamp.nil?
+        params[:next_token] = nil
+        @event_ids_by_timestamp = {}
+
         sleep(@interval)
       end
     rescue Interrupt
@@ -42,6 +55,13 @@ module CloudWatchLogsPoller
 
     private
 
+    def get_latest_events_and_timestamp
+      newest_timestamp = @event_ids_by_timestamp.keys.max
+      return {} if newest_timestamp.nil?
+
+      {newest_timestamp => @event_ids_by_timestamp[newest_timestamp]}
+    end
+
     def debug_log(message)
       return unless @debug
 

data/lib/cloud_watch_logs_poller/version.rb

@@ -1,3 +1,3 @@
 module CloudWatchLogsPoller
-  VERSION = "0.2.0"
+  VERSION = "0.3.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: cloud_watch_logs_poller
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.3.0
 platform: ruby
 authors:
 - Takahiro Ooishi
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2019-12-23 00:00:00.000000000 Z
+date: 2020-06-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aws-sdk-cloudwatchlogs
@@ -104,7 +104,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
+rubygems_version: 3.1.2
 signing_key: 
 specification_version: 4
 summary: Polling Cloud Watch Logs