aws-logs 0.3.1 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 93209b1d08b94e59613794379fbc6da21b69113c7ef5df6bf008fae348d1d129
-  data.tar.gz: a3d4f821990b9fcf41747df792595cffa1c8ba8cb41bcbceb2f915d4a08f68e4
+  metadata.gz: f80b63c48f228ee6420c2d97226ac8fe1fab65fe42b847b15bfe80fd3f72197a
+  data.tar.gz: de12e2063cb6e25ee6505fa48ffb8d6d695b0bfa33afc9c3334a4cc132d82c00
 SHA512:
-  metadata.gz: 98436a84cb82b97d6872e65ac07a2c9cb1ff8eac228288df4df28b85e0dddb4139641f5bacdd66977355b1b789aea60bc1d21dc7dda8c0e4ce4af220a15c4874
-  data.tar.gz: 6877115a9d916ab5ac7f84ae92b1d89ad364e807e3e3f126d2ffdcf0699d2fade8e0e5d8e661a7d55635f98bad8a7b345ed6a047dedefde4fd0de26ac80fc977
+  metadata.gz: ec6276ffea2146c914704fb1021a0b3b4cc9b595640ef83cf444ef5217dbdb672504084f12cc3773afdb9ddf02018579512286c8ae25eb7399bf437b60d26c5c
+  data.tar.gz: 8af7b076de6042316a374321278dadee5d27dea00fcf08d97f58b5c1578fa07dd151b2cbb65f074d5457903c2fcb8ea77f7aa3c01af2a0ffff02ac8e28178050

data/CHANGELOG.md

@@ -3,6 +3,20 @@
 All notable changes to this project will be documented in this file.
 This project *tries* to adhere to [Semantic Versioning](http://semver.org/), even before v1.0.
 
+## [0.4.0] - 2022-01-07
+- [#3](https://github.com/tongueroo/aws-logs/pull/3) streams cli command and data method
+- fix activesupport require
+- streams cli command and data method
+
+## [0.3.4]
+- set next_token for large filter_log_events responses
+
+## [0.3.3]
+- #2 add overlap to window to account for delayed logs received at the same time
+
+## [0.3.2]
+- #1 clean up end_loop_signal logic
+
 ## [0.3.1]
 - fix --no-follow option
 

data/README.md

@@ -1,6 +1,6 @@
 # aws-logs
 
-[![Gem Version](https://badge.fury.io/rb/GEMNAME.png)](http://badge.fury.io/rb/GEMNAME)
+[![Gem Version](https://badge.fury.io/rb/aws-logs.png)](http://badge.fury.io/rb/aws-logs)
 
 [![BoltOps Badge](https://img.boltops.com/boltops/badges/boltops-badge.png)](https://www.boltops.com)
 

data/lib/aws_logs/base.rb

@@ -0,0 +1,10 @@
+module AwsLogs
+  class Base
+    include AwsServices
+
+    def initialize(options={})
+      @options = options
+      @log_group_name = options[:log_group_name]
+    end
+  end
+end

data/lib/aws_logs/cli.rb

@@ -13,6 +13,14 @@ module AwsLogs
       Tail.new(options.merge(log_group_name: log_group_name)).run
     end
 
+    desc "streams LOG_GROUP", "Show the log group stream names. Limits on only one page of results."
+    long_desc Help.text(:streams)
+    option :descending, desc: "True by default if order-by is LastEventTime, false if order-by is LogStreamName"
+    option :order_by, default: "LastEventTime", desc: "accepts LogStreamName, LastEventTime"
+    def streams(log_group_name)
+      Streams.new(options.merge(log_group_name: log_group_name)).run
+    end
+
     desc "completion *PARAMS", "Prints words for auto-completion."
     long_desc Help.text(:completion)
     def completion(*params)

data/lib/aws_logs/command.rb

@@ -77,6 +77,13 @@ module AwsLogs
       def website
         ""
       end
+
+      # https://github.com/erikhuda/thor/issues/244
+      # Deprecation warning: Thor exit with status 0 on errors. To keep this behavior, you must define `exit_on_failure?` in `Lono::CLI`
+      # You can silence deprecations warning by setting the environment variable THOR_SILENCE_DEPRECATION.
+      def exit_on_failure?
+        true
+      end
     end
   end
 end

data/lib/aws_logs/help/tail.md

@@ -52,6 +52,6 @@ To match terms with spaces in it, you'll need quotes around it. Otherise, the ma
 
     aws-logs tail /aws/codebuild/demo --filter-pattern '"Wed Nov 27 23"' --since 3h --no-follow
 
-Here's an example of matching with an exclude patter using the `-` (minus sign).
+Here's an example of matching with an exclude pattern using the `-` (minus sign).
 
     aws-logs tail /aws/codebuild/demo --filter-pattern '"ERROR" - "Exiting"' --since 3h --no-follow

data/lib/aws_logs/since.rb

@@ -1,3 +1,4 @@
+require "active_support"
 require "active_support/core_ext/integer"
 require "time"
 

data/lib/aws_logs/streams.rb

@@ -0,0 +1,24 @@
+module AwsLogs
+  class Streams < Base
+    def run
+      resp = cloudwatchlogs.describe_log_streams(
+        log_group_name: @log_group_name,
+        descending: descending,
+        order_by: order_by,
+      )
+      names = resp.log_streams.map { |s| s.log_stream_name }
+      puts names
+    end
+
+  private
+    # True by default if order-by is LastEventTime, false if order-by is LogStreamName
+    def descending
+      return @options[:descending] unless @options[:descending].nil?
+      order_by == "LastEventTime"
+    end
+
+    def order_by
+      @options[:order_by] || "LastEventTime" # accepts LogStreamName, LastEventTime
+    end
+  end
+end

data/lib/aws_logs/tail.rb

@@ -1,12 +1,9 @@
 require "json"
 
 module AwsLogs
-  class Tail
-    include AwsServices
-
+  class Tail < Base
     def initialize(options={})
-      @options = options
-      @log_group_name = options[:log_group_name]
+      super
       # Setting to ensure matches default CLI option
       @follow = @options[:follow].nil? ? true : @options[:follow]
 
@@ -16,6 +13,15 @@ module AwsLogs
       set_trap
     end
 
+    def data(since="1d")
+      since, now = Since.new(since).to_i, current_now
+      resp = filter_log_events(since, now)
+      resp.events
+    rescue Aws::CloudWatchLogs::Errors::ResourceNotFoundException => e
+      puts "WARN: #{e.class}: #{e.message}"
+      []
+    end
+
     def reset
       @events = [] # constantly replaced with recent events
       @last_shown_event_id = nil
@@ -35,13 +41,17 @@ module AwsLogs
         return
       end
 
+      # We overlap the sliding window because CloudWatch logs can receive or send the logs out of order.
+      # For example, a bunch of logs can all come in at the same second, but they haven't registered to CloudWatch logs
+      # yet. If we don't overlap the sliding window then we'll miss the logs that were delayed in registering.
+      overlap = 60*1000 # overlap the sliding window by a minute
       since, now = initial_since, current_now
       until end_loop?
         refresh_events(since, now)
         display
-        since, now = now, current_now
+        since, now = now-overlap, current_now
         loop_count!
-        sleep 5 if @follow && !@@end_loop_signal && !ENV["AWS_LOGS_TEST"]
+        sleep 5 if @follow && !ENV["AWS_LOGS_TEST"]
       end
       # Refresh and display a final time in case the end_loop gets interrupted by stop_follow!
       refresh_events(since, now)
@@ -57,17 +67,7 @@ module AwsLogs
 
       # TODO: can hit throttle limit if there are lots of pages
       while next_token
-        options = {
-          log_group_name: @log_group_name, # required
-          start_time: start_time,
-          end_time: end_time,
-          # limit: 10,
-        }
-        options[:log_stream_names] = @options[:log_stream_names] if @options[:log_stream_names]
-        options[:log_stream_name_prefix] = @options[:log_stream_name_prefix] if @options[:log_stream_name_prefix]
-        options[:filter_pattern] = @options[:filter_pattern] if @options[:filter_pattern]
-        resp = cloudwatchlogs.filter_log_events(options)
-
+        resp = filter_log_events(start_time, end_time, next_token)
         @events += resp.events
         next_token = resp.next_token
       end
@@ -75,7 +75,24 @@ module AwsLogs
       @events
     end
 
-    # Events canduplicated as events can be written to the exact same timestamp.
+    def filter_log_events(start_time, end_time, next_token=nil)
+      options = {
+        log_group_name: @log_group_name, # required
+        start_time: start_time,
+        end_time: end_time,
+        # limit: 1000,
+        # interleaved: true,
+      }
+
+      options[:log_stream_names] = @options[:log_stream_names] if @options[:log_stream_names]
+      options[:log_stream_name_prefix] = @options[:log_stream_name_prefix] if @options[:log_stream_name_prefix]
+      options[:filter_pattern] = @options[:filter_pattern] if @options[:filter_pattern]
+      options[:next_token] = next_token if next_token != :start && !next_token.nil?
+
+      cloudwatchlogs.filter_log_events(options)
+    end
+
+    # There can be duplicated events as events can be written to the exact same timestamp.
     # So also track the last_shown_event_id and prevent duplicate log lines from re-appearing.
     def display
       new_events = @events
@@ -89,7 +106,7 @@ module AwsLogs
         line = [time.to_s.color(:green), e.message]
         format = @options[:format] || "detailed"
         line.insert(1, e.log_stream_name.color(:purple)) if format == "detailed"
-        say line.join(' ')
+        say line.join(' ') unless @options[:silence]
       end
       @last_shown_event_id = @events.last&.event_id
       check_follow_until!
@@ -100,9 +117,7 @@ module AwsLogs
       return unless follow_until
 
       messages = @events.map(&:message)
-      if messages.detect { |m| m.include?(follow_until) }
-        @@end_loop_signal = true
-      end
+      @@end_loop_signal = messages.detect { |m| m.include?(follow_until) }
     end
 
     def say(text)
@@ -113,15 +128,17 @@ module AwsLogs
       @output.join("\n") + "\n"
     end
 
-    @@end_loop_signal = false
     def set_trap
       Signal.trap("INT") {
         puts "\nCtrl-C detected. Exiting..."
-        @@end_loop_signal = true  # useful to control loop
         exit # immediate exit
       }
     end
 
+    # The stop_follow! results in a little waiting because it signals to break the polling loop.
+    # Since it's in the middle of the loop process, the loop will finish the sleep 5 first.
+    # So it can pause from 0-5 seconds.
+    @@end_loop_signal = false
     def self.stop_follow!
       @@end_loop_signal = true
     end

data/lib/aws_logs/version.rb

@@ -1,3 +1,3 @@
 module AwsLogs
-  VERSION = "0.3.1"
+  VERSION = "0.4.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: aws-logs
 version: !ruby/object:Gem::Version
-  version: 0.3.1
+  version: 0.4.0
 platform: ruby
 authors:
 - Tung Nguyen
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2019-11-28 00:00:00.000000000 Z
+date: 2022-01-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -186,6 +186,7 @@ files:
 - lib/aws_logs.rb
 - lib/aws_logs/autoloader.rb
 - lib/aws_logs/aws_services.rb
+- lib/aws_logs/base.rb
 - lib/aws_logs/cli.rb
 - lib/aws_logs/command.rb
 - lib/aws_logs/completer.rb
@@ -196,6 +197,7 @@ files:
 - lib/aws_logs/help/completion_script.md
 - lib/aws_logs/help/tail.md
 - lib/aws_logs/since.rb
+- lib/aws_logs/streams.rb
 - lib/aws_logs/tail.rb
 - lib/aws_logs/version.rb
 - spec/fixtures/typical/events-1.json
@@ -225,7 +227,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.6
+rubygems_version: 3.2.32
 signing_key: 
 specification_version: 4
 summary: Tail AWS CloudWatch Logs