stax 0.0.3 → 0.0.4

data/lib/stax/mixin/dynamodb.rb

@@ -40,7 +40,9 @@ module Stax
       def tables
         print_table stack_tables.map { |r|
           t = Aws::DynamoDB.table(r.physical_resource_id)
-          [ t.table_name, color(t.table_status, COLORS), t.item_count, t.table_size_bytes, t.creation_date_time ]
+          g = Aws::DynamoDB.global_table(r.physical_resource_id)
+          regions = g.nil? ? '-' : g.replication_group.map(&:region_name).sort.join(',')
+          [ t.table_name, color(t.table_status, COLORS), t.item_count, t.table_size_bytes, t.creation_date_time, regions ]
         }
       end
 
@@ -94,6 +96,20 @@ module Stax
         )
       end
 
+      desc 'put ID', 'put items from stdin to table'
+      method_option :verbose, aliases: '-v', type: :boolean, default: false, desc: 'show progress'
+      def put(id)
+        table = my.resource(id)
+        count = 0
+        $stdin.each do |line|
+          Aws::DynamoDB.put(table_name: table, item: JSON.parse(line))
+          print '.' if options[:verbose]
+          count += 1
+        end
+        print "\n" if options[:verbose]
+        puts "put #{count} items to #{table}"
+      end
+
     end
   end
 end

data/lib/stax/mixin/ec2.rb

@@ -11,9 +11,14 @@ module Stax
   module Cmd
     class Ec2 < SubCommand
       COLORS = {
+        ## instances
         running:    :green,
         stopped:    :yellow,
         terminated: :red,
+        ## images
+        available: :green,
+        pending:   :yellow,
+        failed:    :red,
       }
 
       desc 'ls', 'list instances for stack'
@@ -31,7 +36,7 @@ module Stax
           ]
         }
       end
-    end
 
+    end
   end
 end

data/lib/stax/mixin/ecr.rb

@@ -0,0 +1,68 @@
+require 'base64'
+require 'stax/aws/ecr'
+
+module Stax
+  module Ecr
+    def self.included(thor)
+      thor.desc(:ecr, 'ECR subcommands')
+      thor.subcommand(:ecr, Cmd::Ecr)
+    end
+
+    def ecr_registry
+      @_ecr_registry ||= "#{aws_account_id}.dkr.ecr.#{aws_region}.amazonaws.com"
+    end
+
+    def ecr_repositories
+      @_ecr_repositories ||= Aws::Cfn.resources_by_type(stack_name, 'AWS::ECR::Repository')
+    end
+
+    def ecr_repository_names
+      @_ecr_repository_names ||= ecr_repositories.map(&:physical_resource_id)
+    end
+
+    ## override to set an explicit repo name
+    def ecr_repository_name
+      @_ecr_repository_name ||= (ecr_repository_names&.first || app_name)
+    end
+  end
+
+  module Cmd
+    class Ecr < SubCommand
+
+      desc 'registry', 'show ECR registry'
+      def registry
+        puts my.ecr_registry
+      end
+
+      ## TODO: reimplement using --password-stdin
+      desc 'login', 'login to ECR registry'
+      def login
+        Aws::Ecr.auth.each do |auth|
+          debug("Login to ECR registry #{auth.proxy_endpoint}")
+          user, pass = Base64.decode64(auth.authorization_token).split(':')
+          system "docker login -u #{user} -p #{pass} #{auth.proxy_endpoint}"
+        end
+      end
+
+      desc 'repositories', 'list ECR repositories'
+      def repositories
+        print_table Aws::Ecr.repositories(repository_names: my.ecr_repository_names).map { |r|
+          [r.repository_name, r.repository_uri, r.created_at]
+        }
+      end
+
+      desc 'images', 'list ECR images'
+      method_option :repositories, aliases: '-r', type: :array, default: nil, desc: 'list of repos'
+      def images
+        (options[:repositories] || my.ecr_repository_names).each do |repo|
+          debug("Images in repo #{repo}")
+          print_table Aws::Ecr.images(repository_name: repo).map { |i|
+            [i.image_tags.join(' '), i.image_digest, human_bytes(i.image_size_in_bytes), i.image_pushed_at]
+          }
+        end
+      end
+
+    end
+  end
+
+end

data/lib/stax/mixin/ecs.rb

@@ -1,4 +1,5 @@
 require 'stax/aws/ecs'
+require_relative 'ecs/deploy'
 
 module Stax
   module Ecs
@@ -7,8 +8,12 @@ module Stax
       thor.subcommand(:ecs, Cmd::Ecs)
     end
 
+    def ecs_clusters
+      @_ecs_clusters ||= Aws::Cfn.resources_by_type(stack_name, 'AWS::ECS::Cluster')
+    end
+
     def ecs_cluster_name
-      'default'
+      @_ecs_cluster_name ||= (ecs_clusters&.first&.physical_resource_id || 'default')
     end
 
     def ecs_services
@@ -19,11 +24,22 @@ module Stax
       @_ecs_task_definitions ||= Aws::Cfn.resources_by_type(stack_name, 'AWS::ECS::TaskDefinition')
     end
 
+    ## mangle taskdef arn into family name
+    def ecs_task_families
+      ecs_task_definitions.map do |r|
+        r.physical_resource_id.split(':')[-2].split('/').last
+      end
+    end
+
+    def ecs_service_names
+      @_ecs_service_names ||= ecs_services.map(&:physical_resource_id)
+    end
+
     def ecs_service_objects
-      Aws::Ecs.services(ecs_cluster_name, ecs_services.map(&:physical_resource_id))
+      Aws::Ecs.services(ecs_cluster_name, ecs_service_names)
     end
 
-    ## register a new revision of existing task definition
+    ## deprecated: register a new revision of existing task definition
     def ecs_update_taskdef(id)
       taskdef = Aws::Ecs.task_definition(resource(id))
       debug("Registering new revision of #{taskdef.family}")
@@ -33,7 +49,7 @@ module Stax
       end
     end
 
-    ## update service to use a new task definition
+    ## deprecated: update service to use a new task definition
     def ecs_update_service(id, taskdef)
       service_name = resource(id).split('/').last
       taskdef_name = taskdef.task_definition_arn.split('/').last
@@ -42,6 +58,7 @@ module Stax
         puts s.task_definition
       end
     end
+
   end
 
   module Cmd
@@ -57,6 +74,10 @@ module Stax
         def ecs_task_definition(id)
           Aws::Cfn.id(my.stack_name, id)
         end
+
+        def print_event(e)
+          puts "#{set_color(e.created_at, :green)}  #{e.message}"
+        end
       end
 
       desc 'clusters', 'ECS cluster for stack'
@@ -84,9 +105,26 @@ module Stax
       def events
         my.ecs_service_objects.each do |s|
           debug("Events for #{s.service_name}")
-          print_table s.events.first(options[:number]).map { |e|
-            [set_color(e.created_at, :green), e.message]
-          }.reverse
+          s.events.first(options[:number]).reverse.map(&method(:print_event))
+        end
+      end
+
+      desc 'tail [SERVICE]', 'tail ECS events'
+      def tail(service = nil)
+        trap('SIGINT', 'EXIT')    # clean exit with ctrl-c
+        service ||= my.ecs_service_names.first
+        latest_event = Aws::Ecs.services(my.ecs_cluster_name, [service]).first.events.first
+        print_event(latest_event)
+        last_seen = latest_event.id
+        loop do
+          sleep 5
+          unseen = []
+          Aws::Ecs.services(my.ecs_cluster_name, [service]).first.events.each do |e|
+            break if e.id == last_seen
+            unseen.unshift(e)
+          end
+          unseen.each(&method(:print_event))
+          last_seen = unseen.last.id unless unseen.empty?
         end
       end
 
@@ -109,39 +147,65 @@ module Stax
         }
       end
 
+      desc 'env', 'env vars for latest rev of task families'
+      def env
+        my.ecs_task_families.each do |family|
+          Aws::Ecs.task_definition(family).container_definitions.each do |c|
+            debug("Environment for #{family} #{c.name}")
+            print_table c.environment.map { |e|
+              [e.name, e.value]
+            }
+          end
+        end
+      end
+
       desc 'tasks', 'ECS tasks for stack'
       method_option :status, aliases: '-s', type: :string, default: 'RUNNING', desc: 'status to list'
       def tasks
-        print_table Aws::Ecs.tasks(my.ecs_cluster_name, options[:status].upcase).map { |t|
-          [
-            t.task_arn.split('/').last,
-            t.task_definition_arn.split('/').last,
-            t.container_instance_arn&.split('/')&.last || '--',
-            color(t.last_status, COLORS),
-            "(#{t.desired_status})",
-            t.started_by,
-          ]
-        }
+        my.ecs_services.each do |s|
+          name = s.physical_resource_id.split('/').last
+          debug("Tasks for service #{name}")
+          Aws::Ecs.tasks(
+            cluster: my.ecs_cluster_name,
+            service_name: s.physical_resource_id,
+            desired_status: options[:status].upcase,
+          ).map { |t|
+            [
+              t.task_arn.split('/').last,
+              t.task_definition_arn.split('/').last,
+              t.container_instance_arn&.split('/')&.last || '--',
+              color(t.last_status, COLORS),
+              "(#{t.desired_status})",
+              t.started_by,
+            ]
+          }.tap(&method(:print_table))
+        end
       end
 
       desc 'containers', 'containers for running tasks'
       method_option :status, aliases: '-s', type: :string, default: 'RUNNING', desc: 'status to list'
       def containers
-        debug("Containers for cluster #{my.ecs_cluster_name}")
-        print_table Aws::Ecs.tasks(my.ecs_cluster_name, options[:status].upcase).map { |task|
-          task_defn = task.task_definition_arn.split('/').last
-          task.containers.map { |c|
-            [
-              c.container_arn.split('/').last,
-              c.name,
-              color(c.last_status, COLORS),
-              c.network_interfaces.map(&:private_ipv_4_address).join(','),
-              task_defn,
-              c.exit_code,
-              c.reason,
-            ]
-          }
-        }.flatten(1)
+        my.ecs_services.each do |s|
+          Aws::Ecs.tasks(
+            cluster: my.ecs_cluster_name,
+            service_name: s.physical_resource_id,
+            desired_status: options[:status].upcase,
+          ).each do |t|
+            task = t.task_arn.split('/').last
+            debug("Containers for task #{task}")
+            print_table t.containers.map { |c|
+              [
+                c.name,
+                c.container_arn.split('/').last,
+                color(c.last_status, COLORS),
+                c.network_interfaces.map(&:private_ipv_4_address).join(','),
+                t.task_definition_arn.split('/').last,
+                c.exit_code,
+                c.reason,
+              ]
+            }
+          end
+        end
       end
 
       desc 'instances', 'ECS instances'
@@ -178,8 +242,9 @@ module Stax
       method_option :desired, aliases: '-d', type: :numeric, default: nil, desc: 'desired container count'
       def scale
         my.ecs_services.each do |s|
-          debug("Scaling service #{s.logical_resource_id}")
+          debug("Scaling service #{s.physical_resource_id.split('/').last}")
           Aws::Ecs.update_service(
+            cluster: my.ecs_cluster_name,
             service: s.physical_resource_id,
             desired_count: options[:desired],
           ).tap do |s|

data/lib/stax/mixin/ecs/deploy.rb

@@ -0,0 +1,49 @@
+module Stax
+  module Ecs
+
+    ## convert to hash for registering new taskdef
+    def taskdef_to_hash(taskdef)
+      args = %i[family cpu memory requires_compatibilities task_role_arn execution_role_arn network_mode container_definitions volumes placement_constraints]
+      taskdef.to_hash.slice(*args)
+    end
+
+    def get_taskdef(service)
+      debug("Current task definition for #{service.service_name}")
+      Aws::Ecs.task_definition(service.task_definition).tap do |t|
+        puts t.task_definition_arn
+      end
+    end
+
+    def register_taskdef(hash)
+      debug("Registering new revision")
+      Aws::Ecs.client.register_task_definition(hash).task_definition.tap do |t|
+        puts t.task_definition_arn
+      end
+    end
+
+    def update_service(service, taskdef)
+      debug("Updating #{service.service_name} to new revision")
+      Aws::Ecs.update_service(
+        service:         service.service_name,
+        task_definition: taskdef.task_definition_arn
+      ).tap do |s|
+        puts s.deployments.first.id
+      end
+    end
+
+    ## update taskdef for a service, triggering a deploy
+    ## modify current taskdef in block
+    def ecs_deploy(id, &block)
+      service = Aws::Ecs.services(ecs_cluster_name, [resource(id)]).first
+      taskdef = get_taskdef(service)
+
+      ## convert to a hash and modify in block
+      hash = taskdef_to_hash(taskdef)
+      yield(hash) if block_given?
+
+      taskdef = register_taskdef(hash)
+      update_service(service, taskdef)
+    end
+
+  end
+end

data/lib/stax/mixin/logs.rb

@@ -62,12 +62,12 @@ module Stax
         end
       end
 
-      desc 'tail [STREAM]', 'tail latest/given log stream'
+      desc 'events [STREAM]', 'show events just from latest/given log stream'
       method_option :group,    aliases: '-g', type: :string,  default: nil,   desc: 'log group to tail'
       method_option :numlines, aliases: '-n', type: :numeric, default: 10,    desc: 'number of lines to show'
       method_option :follow,   aliases: '-f', type: :boolean, default: false, desc: 'follow log output'
       method_option :sleep,    aliases: '-s', type: :numeric, default: 1,     desc: 'seconds to sleep between poll for new data'
-      def tail(stream = nil)
+      def events(stream = nil)
         trap('SIGINT', 'EXIT')    # clean exit with ctrl-c
         group  = ((g = options[:group]) ? log_groups[g] : log_groups.values.first).log_group_name
         stream ||= latest_stream(group).log_stream_name
@@ -89,6 +89,77 @@ module Stax
         end
       end
 
+      desc 'tail', 'tail all events from log group'
+      method_option :group,   aliases: '-g', type: :string, default: nil, desc: 'log group to tail'
+      method_option :streams, aliases: '-s', type: :array,  default: nil, desc: 'limit to given streams'
+      def tail
+        trap('SIGINT', 'EXIT')    # clean exit with ctrl-c
+        group  = ((g = options[:group]) ? log_groups[g] : log_groups.values.first).log_group_name
+
+        debug("Log group #{group}")
+        token = nil
+        start_time = Time.now.to_i - 30 # start 30 sec ago
+
+        loop do
+          end_time = Time.now.to_i
+          resp = Aws::Logs.client.filter_log_events(
+            log_group_name: group,
+            log_stream_names: options[:streams],
+            start_time: start_time * 1000, # aws needs msec
+            end_time: end_time * 1000,
+            next_token: token,
+            interleaved: true,
+          )
+
+          ## pretty-print the events
+          resp.events.each do |e|
+            puts("#{set_color(human_time(e.timestamp).utc, :green)}  #{set_color(e.log_stream_name, :blue)}  #{e.message}")
+          end
+
+          ## token means more data available from this request, so loop and get it right away
+          token = resp.next_token
+
+          ## no token, so sleep and start next request from end time of this one
+          unless token
+            start_time = end_time
+            sleep 10
+          end
+        end
+      end
+
+      desc 'filter', 'filter events from log group'
+      method_option :group,   aliases: '-g', type: :string, default: nil, desc: 'log group to filter'
+      method_option :streams, aliases: '-s', type: :array,  default: nil, desc: 'limit to given streams'
+      method_option :pattern, aliases: '-p', type: :string, default: nil, desc: 'pattern to filter logs'
+      method_option :start,   aliases: '-t', type: :string, default: nil, desc: 'start time'
+      method_option :end,     aliases: '-e', type: :string, default: nil, desc: 'end time'
+      def filter
+        trap('SIGINT', 'EXIT')    # clean exit with ctrl-c
+        group = ((g = options[:group]) ? log_groups[g] : log_groups.values.first).log_group_name
+        debug("Log group #{group}")
+
+        start_time = options[:start] ? Time.parse(options[:start]).to_i*1000 : nil
+        end_time   = options[:end]   ? Time.parse(options[:end]).to_i*1000   : nil
+        token = nil
+        loop do
+          resp = Aws::Logs.client.filter_log_events(
+            log_group_name: group,
+            log_stream_names: options[:streams],
+            next_token: token,
+            start_time: start_time,
+            end_time: end_time,
+            filter_pattern: options[:pattern],
+          )
+          resp.events.each do |e|
+            time   = set_color(human_time(e.timestamp).utc, :green)
+            stream = set_color(e.log_stream_name, :blue)
+            puts("#{time}  #{stream}  #{e.message}")
+          end
+          token = resp.next_token
+          break unless token
+        end
+      end
+
       ## lambdas create their own log groups, and when we delete stack they are left behind;
       ## this task looks up their names by stack prefix, and deletes them
       desc 'cleanup', 'cleanup lambda log groups named for stack'