aws-logs 0.3.2

data/lib/aws_logs/completer/script.rb

@@ -0,0 +1,6 @@
+class AwsLogs::Completer::Script
+  def self.generate
+    bash_script = File.expand_path("script.sh", File.dirname(__FILE__))
+    puts "source #{bash_script}"
+  end
+end

data/lib/aws_logs/completer/script.sh

@@ -0,0 +1,10 @@
+_aws-logs() {
+  COMPREPLY=()
+  local word="${COMP_WORDS[COMP_CWORD]}"
+  local words=("${COMP_WORDS[@]}")
+  unset words[0]
+  local completion=$(aws-logs completion ${words[@]})
+  COMPREPLY=( $(compgen -W "$completion" -- "$word") )
+}
+
+complete -F _aws-logs aws-logs

data/lib/aws_logs/help.rb

@@ -0,0 +1,9 @@
+module AwsLogs::Help
+  class << self
+    def text(namespaced_command)
+      path = namespaced_command.to_s.gsub(':','/')
+      path = File.expand_path("../help/#{path}.md", __FILE__)
+      IO.read(path) if File.exist?(path)
+    end
+  end
+end

data/lib/aws_logs/help/completion.md

@@ -0,0 +1,20 @@
+## Examples
+
+    aws-logs completion
+
+Prints words for TAB auto-completion.
+
+    aws-logs completion
+    aws-logs completion hello
+    aws-logs completion hello name
+
+To enable, TAB auto-completion add the following to your profile:
+
+    eval $(aws-logs completion_script)
+
+Auto-completion example usage:
+
+    aws-logs [TAB]
+    aws-logs hello [TAB]
+    aws-logs hello name [TAB]
+    aws-logs hello name --[TAB]

data/lib/aws_logs/help/completion_script.md

@@ -0,0 +1,3 @@
+To use, add the following to your `~/.bashrc` or `~/.profile`
+
+    eval $(aws-logs completion_script)

data/lib/aws_logs/help/tail.md

@@ -0,0 +1,57 @@
+## Examples
+
+    aws-logs tail /aws/codebuild/demo --since 60m
+    aws-logs tail /aws/codebuild/demo --since "2018-08-08 08:00:00"
+    aws-logs tail /aws/codebuild/demo --no-follow
+    aws-logs tail /aws/codebuild/demo --format simple
+    aws-logs tail /aws/codebuild/demo --filter-pattern Wed
+
+## Examples with Output
+
+Using `--since`
+
+    $ aws-logs tail /aws/codebuild/demo --since 60m --no-follow
+    2019-11-27 22:56:05 UTC 8cb8b7fd-3662-4120-95bc-efff637c7220 Wed Nov 27 22:56:04 UTC 2019
+    2019-11-27 22:56:16 UTC 8cb8b7fd-3662-4120-95bc-efff637c7220
+    2019-11-27 22:56:16 UTC 8cb8b7fd-3662-4120-95bc-efff637c7220 [Container] 2019/11/27 22:56:14 Phase complete: BUILD State: SUCCEEDED
+    2019-11-27 22:56:16 UTC 8cb8b7fd-3662-4120-95bc-efff637c7220 [Container] 2019/11/27 22:56:14 Phase context status code:  Message:
+    2019-11-27 22:56:16 UTC 8cb8b7fd-3662-4120-95bc-efff637c7220 [Container] 2019/11/27 22:56:14 Entering phase POST_BUILD
+    2019-11-27 22:56:16 UTC 8cb8b7fd-3662-4120-95bc-efff637c7220 [Container] 2019/11/27 22:56:14 Phase complete: POST_BUILD State: SUCCEEDED
+    2019-11-27 22:56:16 UTC 8cb8b7fd-3662-4120-95bc-efff637c7220 [Container] 2019/11/27 22:56:14 Phase context status code:  Message:
+    $
+
+Using `--filter-pattern`.
+
+    $ aws-logs tail /aws/codebuild/demo --filter-pattern Wed --since 60m
+    2019-11-27 22:19:41 UTC 0d933e8f-c15b-41af-a5c7-36b54530cb17 Wed Nov 27 22:19:37 UTC 2019
+    2019-11-27 22:19:49 UTC 0d933e8f-c15b-41af-a5c7-36b54530cb17 Wed Nov 27 22:19:47 UTC 2019
+    2019-11-27 22:19:59 UTC 0d933e8f-c15b-41af-a5c7-36b54530cb17 Wed Nov 27 22:19:57 UTC 2019
+    2019-11-27 22:20:09 UTC 0d933e8f-c15b-41af-a5c7-36b54530cb17 Wed Nov 27 22:20:07 UTC 2019
+    2019-11-27 22:20:19 UTC 0d933e8f-c15b-41af-a5c7-36b54530cb17 Wed Nov 27 22:20:17 UTC 2019
+    2019-11-27 22:20:29 UTC 0d933e8f-c15b-41af-a5c7-36b54530cb17 Wed Nov 27 22:20:27 UTC 2019
+    2019-11-27 22:20:39 UTC 0d933e8f-c15b-41af-a5c7-36b54530cb17 Wed Nov 27 22:20:37 UTC 2019
+
+## Since Formats
+
+Since supports these formats:
+
+* s - seconds
+* m - minutes
+* h - hours
+* d - days
+* w - weeks
+
+Since does not support combining the formats. IE: 5m30s.
+
+## Filter Pattern
+
+The `--filter-pattern` option is quite powerful as CloudWatch supports a full
+[Filter and Pattern Syntax](https://docs.aws.amazon.com/AmazonCloudWatch/latest/logs/FilterAndPatternSyntax.html).
+
+To match terms with spaces in it, you'll need quotes around it. Otherise, the match will be an OR of the terms. Example:
+
+    aws-logs tail /aws/codebuild/demo --filter-pattern '"Wed Nov 27 23"' --since 3h --no-follow
+
+Here's an example of matching with an exclude pattern using the `-` (minus sign).
+
+    aws-logs tail /aws/codebuild/demo --filter-pattern '"ERROR" - "Exiting"' --since 3h --no-follow

data/lib/aws_logs/since.rb

@@ -0,0 +1,82 @@
+require "active_support/core_ext/integer"
+require "time"
+
+module AwsLogs
+  class Since
+    DEFAULT = 10.minutes.to_i
+
+    def initialize(str)
+      @str = str
+    end
+
+    def to_i
+      if iso8601_format?
+        iso8601_seconds
+      elsif friendly_format?
+        friendly_seconds
+      else
+        puts warning
+        return DEFAULT
+      end
+    end
+
+    ISO8601_REGEXP = /\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}/
+    def iso8601_format?
+      !!@str.match(ISO8601_REGEXP)
+    end
+
+    def iso8601_seconds
+      # https://stackoverflow.com/questions/3775544/how-do-you-convert-an-iso-8601-date-to-a-unix-timestamp-in-ruby
+      Time.iso8601(@str.sub(/ /,'T')).to_i
+    end
+
+    FRIENDLY_REGEXP = /(\d+)(\w+)/
+    def friendly_format?
+      !!@str.match(FRIENDLY_REGEXP)
+    end
+
+    def friendly_seconds
+      number, unit = find_match(FRIENDLY_REGEXP)
+      unless number && unit
+        puts warning
+        return DEFAULT
+      end
+
+      meth = shorthand(unit)
+      if number.respond_to?(meth)
+        number.send(meth).to_i
+      else
+        puts warning
+        return DEFAULT
+      end
+    end
+
+    def find_match(regexp)
+      md = @str.match(regexp)
+      if md
+        number, unit = md[1].to_i, md[2]
+      end
+      [number, unit]
+    end
+
+    def warning
+      "WARN: since is not in a supported format. Falling back to 10m".color(:yellow)
+    end
+
+    # s - seconds
+    # m - minutes
+    # h - hours
+    # d - days
+    # w - weeks
+    def shorthand(k)
+      map = {
+        s: :seconds,
+        m: :minutes,
+        h: :hours,
+        d: :days,
+        w: :weeks,
+      }
+      map[k.to_sym] || k
+    end
+  end
+end

data/lib/aws_logs/tail.rb

@@ -0,0 +1,151 @@
+require "json"
+
+module AwsLogs
+  class Tail
+    include AwsServices
+
+    def initialize(options={})
+      @options = options
+      @log_group_name = options[:log_group_name]
+      # Setting to ensure matches default CLI option
+      @follow = @options[:follow].nil? ? true : @options[:follow]
+
+      @loop_count = 0
+      @output = [] # for specs
+      reset
+      set_trap
+    end
+
+    def reset
+      @events = [] # constantly replaced with recent events
+      @last_shown_event_id = nil
+      @completed = nil
+    end
+
+    # The start and end time is useful to limit results and make the API fast.  We'll leverage it like so:
+    #
+    #     1. load all events from an initial since time
+    #     2. after that load events pass that first window
+    #
+    # It's a sliding window of time we're using.
+    #
+    def run
+      if ENV['AWS_LOGS_NOOP']
+        puts "Noop test"
+        return
+      end
+
+      since, now = initial_since, current_now
+      until end_loop?
+        refresh_events(since, now)
+        display
+        since, now = now, current_now
+        loop_count!
+        sleep 5 if @follow && !ENV["AWS_LOGS_TEST"]
+      end
+      # Refresh and display a final time in case the end_loop gets interrupted by stop_follow!
+      refresh_events(since, now)
+      display
+    rescue Aws::CloudWatchLogs::Errors::ResourceNotFoundException => e
+      puts "ERROR: #{e.class}: #{e.message}".color(:red)
+      puts "Log group #{@log_group_name} not found."
+    end
+
+    def refresh_events(start_time, end_time)
+      @events = []
+      next_token = :start
+
+      # TODO: can hit throttle limit if there are lots of pages
+      while next_token
+        options = {
+          log_group_name: @log_group_name, # required
+          start_time: start_time,
+          end_time: end_time,
+          # limit: 10,
+        }
+        options[:log_stream_names] = @options[:log_stream_names] if @options[:log_stream_names]
+        options[:log_stream_name_prefix] = @options[:log_stream_name_prefix] if @options[:log_stream_name_prefix]
+        options[:filter_pattern] = @options[:filter_pattern] if @options[:filter_pattern]
+        resp = cloudwatchlogs.filter_log_events(options)
+
+        @events += resp.events
+        next_token = resp.next_token
+      end
+
+      @events
+    end
+
+    # Events canduplicated as events can be written to the exact same timestamp.
+    # So also track the last_shown_event_id and prevent duplicate log lines from re-appearing.
+    def display
+      new_events = @events
+      shown_index = new_events.find_index { |e| e.event_id == @last_shown_event_id }
+      if shown_index
+        new_events = @events[shown_index+1..-1] || []
+      end
+
+      new_events.each do |e|
+        time = Time.at(e.timestamp/1000).utc
+        line = [time.to_s.color(:green), e.message]
+        format = @options[:format] || "detailed"
+        line.insert(1, e.log_stream_name.color(:purple)) if format == "detailed"
+        say line.join(' ')
+      end
+      @last_shown_event_id = @events.last&.event_id
+      check_follow_until!
+    end
+
+    def check_follow_until!
+      follow_until = @options[:follow_until]
+      return unless follow_until
+
+      messages = @events.map(&:message)
+      @@end_loop_signal = messages.detect { |m| m.include?(follow_until) }
+    end
+
+    def say(text)
+      ENV["AWS_LOGS_TEST"] ? @output << text : puts(text)
+    end
+
+    def output
+      @output.join("\n") + "\n"
+    end
+
+    def set_trap
+      Signal.trap("INT") {
+        puts "\nCtrl-C detected. Exiting..."
+        exit # immediate exit
+      }
+    end
+
+    @@end_loop_signal = false
+    def self.stop_follow!
+      @@end_loop_signal = true
+    end
+
+  private
+    def initial_since
+      since = @options[:since]
+      seconds = since ? Since.new(since).to_i : Since::DEFAULT
+      (Time.now.to_i - seconds) * 1000 # past 10 minutes in milliseconds
+    end
+
+    def current_now
+      (Time.now.to_i) * 1000 # now in milliseconds
+    end
+
+    def end_loop?
+      return true if @@end_loop_signal
+      max_loop_count && @loop_count >= max_loop_count
+    end
+
+    def loop_count!
+      @loop_count += 1
+    end
+
+    # Useful for specs
+    def max_loop_count
+      @follow ? nil : 1
+    end
+  end
+end

data/lib/aws_logs/version.rb

@@ -0,0 +1,3 @@
+module AwsLogs
+  VERSION = "0.3.2"
+end

data/spec/fixtures/typical/events-1.json

@@ -0,0 +1,11 @@
+{
+  "events": [
+    {
+      "log_stream_name": "stream-name",
+      "timestamp": 1574888810000,
+      "message": "message1",
+      "event_id": "event1"
+    }
+  ],
+  "next_token": 2
+}

data/spec/fixtures/typical/events-2.json

@@ -0,0 +1,17 @@
+{
+  "events": [
+    {
+      "log_stream_name": "stream-name",
+      "timestamp": 1574888810000,
+      "message": "message1",
+      "event_id": "event1"
+    },
+    {
+      "log_stream_name": "stream-name",
+      "timestamp": 1574888820000,
+      "message": "message2",
+      "event_id": "event2"
+    }
+  ],
+  "next_token": 3
+}

data/spec/fixtures/typical/events-3.json

@@ -0,0 +1,23 @@
+{
+  "events": [
+    {
+      "log_stream_name": "stream-name",
+      "timestamp": 1574888810000,
+      "message": "message1",
+      "event_id": "event1"
+    },
+    {
+      "log_stream_name": "stream-name",
+      "timestamp": 1574888820000,
+      "message": "message2",
+      "event_id": "event2"
+    },
+    {
+      "log_stream_name": "stream-name",
+      "timestamp": 1574888830000,
+      "message": "message3",
+      "event_id": "event3"
+    }
+  ],
+  "next_token": 4
+}

data/spec/fixtures/typical/events-4.json

@@ -0,0 +1,28 @@
+{
+  "events": [
+    {
+      "log_stream_name": "stream-name",
+      "timestamp": 1574888810000,
+      "message": "message1",
+      "event_id": "event1"
+    },
+    {
+      "log_stream_name": "stream-name",
+      "timestamp": 1574888820000,
+      "message": "message2",
+      "event_id": "event2"
+    },
+    {
+      "log_stream_name": "stream-name",
+      "timestamp": 1574888830000,
+      "message": "message3",
+      "event_id": "event3"
+    },
+    {
+      "log_stream_name": "stream-name",
+      "timestamp": 1574888840000,
+      "message": "message4",
+      "event_id": "event4"
+    }
+  ]
+}

data/spec/lib/cli_spec.rb

@@ -0,0 +1,8 @@
+describe AwsLogs::CLI do
+  describe "aws-logs" do
+    it "tail" do
+      out = execute("AWS_LOGS_NOOP=1 exe/aws-logs tail LOG_GROUP")
+      expect(out).to include("Noop test")
+    end
+  end
+end