lsaws 0.2.0 → 0.3.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: '09f8ec552161ab82ab45a5c57e50e7756b434974088dadd90e8417d97c317ecc'
-  data.tar.gz: c77880939cc6058a9f8120e95f3df65392ddfe4105783fac7183277edd19c31a
+  metadata.gz: e2b87621670d3747c4c1b6cb9979c7401f7e736ceb8c257819a42dc31b190471
+  data.tar.gz: ab3227e58e62f14a718858242bd2cdd5ffd8f6a5f15acc38550605a01aea2352
 SHA512:
-  metadata.gz: df5f2f83c723a75be975af1c96e38b91e14661d920b62407817ee05a8cce33da9e6be2b59bce23bc0d5638f5bd6be821e2ea674df8991f6902233e22b5d3b2f2
-  data.tar.gz: 7e07bfb922870d91b12cd8f94ef9ddb354817e921cb34e269cf6a4d85d3c84737fa69dd69eec104d18cffd6709a2a77767500613312d0caf0dde4430f6d9cef6
+  metadata.gz: 4eeb107b1267e40dadb27fccb5dc619f53807410c68c3b85cda9c5e63e3c57da9ab731b422f1c236a0b3bc7a2701e4bffee989257676a9e3cff20cfc9ce2b07c
+  data.tar.gz: a725ba9efec0bf251516ca30a8841dac825ca3dacaf2adf00dcef453f2806e22d946512a6b4f6e70bb6f72547f2204e474065f623f86bc258e0a48624c8e9aef

data/Gemfile

@@ -8,8 +8,8 @@ gemspec
 group :development, :test do
   gem "activesupport", "~> 7.0"
   gem "aws-sdk"
+  gem "pry"
   gem "rake", "~> 13.0"
   gem "rspec", "~> 3.0"
   gem "rubocop", "~> 1.21"
-  gem "debug"
 end

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    lsaws (0.1.0)
+    lsaws (0.3.0)
       aws-sdk-core (~> 3.131)
       tabulo (~> 2.8)
 
@@ -1379,27 +1379,24 @@ GEM
     aws-sigv2 (1.1.0)
     aws-sigv4 (1.5.2)
       aws-eventstream (~> 1, >= 1.0.2)
+    coderay (1.1.3)
     concurrent-ruby (1.2.0)
-    debug (1.7.1)
-      irb (>= 1.5.0)
-      reline (>= 0.3.1)
     diff-lcs (1.5.0)
     i18n (1.12.0)
       concurrent-ruby (~> 1.0)
-    io-console (0.6.0)
-    irb (1.6.2)
-      reline (>= 0.3.0)
     jmespath (1.6.2)
     json (2.6.3)
+    method_source (1.0.0)
     minitest (5.17.0)
     parallel (1.22.1)
     parser (3.2.0.0)
       ast (~> 2.4.1)
+    pry (0.14.2)
+      coderay (~> 1.1)
+      method_source (~> 1.0)
     rainbow (3.1.1)
     rake (13.0.6)
     regexp_parser (2.6.2)
-    reline (0.3.2)
-      io-console (~> 0.5)
     rexml (3.2.5)
     rspec (3.12.0)
       rspec-core (~> 3.12.0)
@@ -1441,8 +1438,8 @@ PLATFORMS
 DEPENDENCIES
   activesupport (~> 7.0)
   aws-sdk
-  debug
   lsaws!
+  pry
   rake (~> 13.0)
   rspec (~> 3.0)
   rubocop (~> 1.21)

data/README.md

@@ -23,6 +23,88 @@ User-friendly AWS resources listing tool
         -A, --all                        List all entity types within SDK
 
 
+### Listing installed SDKs
+
+    # lsaws -L
+
+    accessanalyzer
+    account
+    acm
+    acmpca
+    alexaforbusiness
+    amplify
+    amplifybackend
+    amplifyuibuilder
+    apigateway
+    apigatewaymanagementapi
+    apigatewayv2
+    appconfig
+    appconfigdata
+    appflow
+    appintegrationsservice
+    applicationautoscaling
+    applicationcostprofiler
+    applicationdiscoveryservice
+    applicationinsights
+    appmesh
+    appregistry
+    apprunner
+    appstream
+    appsync
+    arczonalshift
+    ...
+
+
+### Listing listable entities from EC2 sdk
+
+    # lsaws ec2 -L
+
+    account_attributes
+    address_transfers
+    addresses
+    availability_zones
+    aws_network_performance_metric_subscriptions
+    bundle_tasks
+    capacity_reservation_fleets
+    capacity_reservations
+    carrier_gateways
+    classic_link_instances
+    client_vpn_endpoints
+    coip_pools
+    conversion_tasks
+    customer_gateways
+    dhcp_options
+    egress_only_internet_gateways
+    elastic_gpus
+    export_image_tasks
+    export_tasks
+    fast_launch_images
+    fast_snapshot_restores
+    fleets
+    flow_logs
+    fpga_images
+    host_reservation_offerings
+    ...
+
+
+### Listing EC2 instances
+
+    # lsaws ec2
+
+    ┌─────────────────────┬─────────────┬───────────────────────┬────────────────────┬─────────────────────────┐
+    │ instance_id         │ name        │ vpc_id                │ private_ip_address │ launch_time             │
+    ├─────────────────────┼─────────────┼───────────────────────┼────────────────────┼─────────────────────────┤
+    │ i-1234567890abcdef0 │ my-instance │ vpc-1234567890abcdef0 │ 10-0-0-157         │ 2022-11-15 10:48:59 UTC │
+    └─────────────────────┴─────────────┴───────────────────────┴────────────────────┴─────────────────────────┘
+
+### Listing specific columns of EC2 images as json-stream
+
+    # lsaws ec2 images -c image_id,creation_date -o js
+
+    {"image_id":"ami-1234e6197567838b4","creation_date":"2023-01-11T13:02:00.000Z"}
+    {"image_id":"ami-1234f58a167898416","creation_date":"2023-01-11T02:02:00.000Z"}
+    {"image_id":"ami-12341a1fd567897a8","creation_date":"2023-01-01T01:02:00.000Z"}
+
 ## License
 
 The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).

data/README.md.tpl

@@ -6,6 +6,32 @@ User-friendly AWS resources listing tool
 
 % lsaws
 
+### Listing installed SDKs
+
+% lsaws -L
+
+### Listing listable entities from EC2 sdk
+
+% lsaws ec2 -L
+
+### Listing EC2 instances
+
+    # lsaws ec2
+
+    ┌─────────────────────┬─────────────┬───────────────────────┬────────────────────┬─────────────────────────┐
+    │ instance_id         │ name        │ vpc_id                │ private_ip_address │ launch_time             │
+    ├─────────────────────┼─────────────┼───────────────────────┼────────────────────┼─────────────────────────┤
+    │ i-1234567890abcdef0 │ my-instance │ vpc-1234567890abcdef0 │ 10-0-0-157         │ 2022-11-15 10:48:59 UTC │
+    └─────────────────────┴─────────────┴───────────────────────┴────────────────────┴─────────────────────────┘
+
+### Listing specific columns of EC2 images as json-stream
+
+    # lsaws ec2 images -c image_id,creation_date -o js
+
+    {"image_id":"ami-0245e6197bf3138b4","creation_date":"2023-01-19T12:03:38.000Z"}
+    {"image_id":"ami-0bc3f58a172118416","creation_date":"2023-01-20T09:37:32.000Z"}
+    {"image_id":"ami-07e71a1fdf49957a8","creation_date":"2023-02-01T02:06:29.000Z"}
+
 ## License
 
 The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).

data/lib/lsaws/lister.rb

@@ -3,20 +3,31 @@
 require "aws-sdk-core"
 require "json"
 require "tabulo"
-require "yaml"
 
 module Lsaws
   class Lister
-    CONFIG = YAML.load_file(File.join(Lsaws.root, "lsaws.yml"))
-
     include Utils
 
+    NEXT_PAGE_FIELDS = %i[next_token next_marker next_page_token page_token].freeze
+
     def initialize(options)
       @options = options
+      @options[:max_width] = nil if @options[:max_width].to_i.zero?
+    end
+
+    def _abort_with_list(sdk, type)
+      if type == "default"
+        warn "[!] no default entity type set for #{sdk.inspect} SDK"
+      else
+        warn "[!] #{sdk.inspect} SDK does not have #{type.inspect} entity type"
+      end
+      puts "Known entity types are:"
+      list_entity_types(sdk)
+      exit 1
     end
 
     def _prepare_entities(sdk, type, &block)
-      edef = CONFIG.dig(sdk, type)
+      edef = Lsaws.config.dig(sdk, type)
       return _prepare_entities(sdk, edef, &block) if edef.is_a?(String) # redirect like 'default' -> 'instances'
 
       edef ||= {}
@@ -33,38 +44,43 @@ module Lsaws
       sdkp = SDKParser.new(sdk)
       client_class = edef["client_class"] || sdkp.client_class_name
       client = Kernel.const_get(client_class).new
-      method_name = edef["method"] || (client.respond_to?("describe_#{type}") ? "describe_#{type}" : "list_#{type}")
-      unless client.respond_to?(method_name)
-        if type == "default"
-          warn "[!] no default entity type set for #{sdk.inspect} SDK"
-        else
-          warn "[!] #{sdk.inspect} SDK does not have #{type.inspect} entity type"
-        end
-        puts "Known entity types are:"
-        list_entity_types(sdk)
-        exit 1
-      end
+      method_name = edef["method"] || sdkp.etype2method(type)
+      _abort_with_list(sdk, type) unless method_name
+      warn "[d] #{method_name} #{params}" if @options[:debug]
       results = client.send(method_name, params)
 
-      pp results if @options[:debug]
+      warn "[d] #{File.basename(__FILE__)}:#{__LINE__} results:\n#{results.pretty_inspect}" if @options[:debug]
 
       if !edef["result_keys"] && results.any?
         r = results.first
         r = r.last if r.is_a?(Array)
-        edef["result_keys"] = if r.respond_to?(type)
-                                [type]
-                              else
-                                [(r.members - [:next_token]).first]
-                              end
+        edef["result_keys"] =
+          if r.respond_to?(type)
+            [type]
+          elsif NEXT_PAGE_FIELDS.any? { |t| r.respond_to?(t) }
+            data_members = r.members - NEXT_PAGE_FIELDS
+            if data_members.size == 1
+              [data_members[0]]
+            else
+              # XXX what if there's more than one array?
+              [data_members.find { |key| r[key].is_a?(Array) }].compact
+            end
+          else
+            []
+          end
       end
+
       edef["result_keys"].each do |key|
         results = if results.is_a?(Array)
-                    results.map(&key.to_sym).flatten
+                    results.map(&key.to_sym).flatten # TODO: is flatten necessary?
                   else
-                    results.send(key)
+                    results[key]
                   end
       end
+
+      warn "[d] #{File.basename(__FILE__)}:#{__LINE__} results:\n#{results.pretty_inspect}" if @options[:debug]
       edef["cols"] = @options[:show_cols] if @options[:show_cols].any?
+      warn "[d] edef: #{edef}" if @options[:debug]
 
       col_defs = {}
       Array(edef["cols"]).each do |r|
@@ -77,10 +93,12 @@ module Lsaws
           raise Error, "unexpected #{r.inspect}"
         end
       end
+      # TODO: check with all types
       col_defs["tags"] = _convert_tags_proc if @options[:show_tags] || col_defs["tags"]
 
       results ||= []
-      if results.any? && !results.first.respond_to?(:name)
+      warn "[d] #{results.inspect}" if @options[:debug]
+      if results.respond_to?(:any?) && results.any? && !results.first.respond_to?(:name) && results.first.respond_to?(:tags)
         results.first.class.class_eval do
           def name
             tags.find { |tag| tag.key == "Name" }&.value
@@ -88,9 +106,19 @@ module Lsaws
         end
       end
 
-      # ec2 instance_event_notification_attributes
-      # 'Array(results)' doesn't work here
-      results = [results] unless results.is_a?(Array)
+      case results
+      when Hash
+        col_defs = {
+          key: :first,
+          value: :last
+        }
+      when Array
+        # ok
+      else
+        # ec2 instance_event_notification_attributes
+        # 'Array(results)' doesn't work here
+        results = [results]
+      end
 
       if block_given?
         results.map do |entity|
@@ -106,6 +134,9 @@ module Lsaws
       if rows.is_a?(Array) && rows[0].is_a?(String)
         # sqs
         cols = { value: proc { |entity| entity } }
+      elsif rows.is_a?(Array) && rows[0].is_a?(Hash)
+        # securitylake:log_sources
+        cols = { value: proc { |entity| entity } }
       elsif rows.respond_to?(:members)
         rows = [rows]
       end
@@ -168,12 +199,23 @@ module Lsaws
       _list_array SDKParser.new(sdk).entity_types
     end
 
+    # elasticache:global_replication_groups has 'members' as a column, so cannot just use `rows.members`
+    # assuming row is always subclass of Struct here
+    def _get_cols(row)
+      if row.is_a?(Struct)
+        Struct.instance_method(:members).bind_call(row)
+      else
+        row.members
+      end
+    end
+
     def _tabulo_guess_max_cols(rows, _cols)
-      max_cols = rows[0].members.size
+      all_cols = _get_cols(rows[0])
+      max_cols = all_cols.size
       return max_cols if max_cols < 4 || !@options[:max_width]
 
       4.upto(max_cols) do |ncols|
-        tbl = Tabulo::Table.new(rows[0, 100], *rows[0].members[0, ncols])
+        tbl = Tabulo::Table.new(rows[0, 100], *all_cols[0, ncols])
         tbl.autosize_columns
         tbl_width = tbl.column_registry.values.map { |c| c.padded_width + 1 }.inject(:+) + 1
         return ncols - 1 if tbl_width >= @options[:max_width]
@@ -187,7 +229,7 @@ module Lsaws
       elsif type == :list
         return list_entity_types(sdk)
       elsif type == :all
-        get_entity_types(sdk).each do |etype|
+        SDKParser.new(sdk).entity_types.each do |etype|
           puts "#{etype}:"
           process_command(sdk, etype)
         end
@@ -208,7 +250,7 @@ module Lsaws
           cols.each { |name, func| tbl.add_column(name, &func) }
         else
           max_cols = _tabulo_guess_max_cols(rows, cols)
-          rows[0].members[0, max_cols].each { |col| tbl.add_column(col) }
+          _get_cols(rows[0])[0, max_cols].each { |col| tbl.add_column(col) }
         end
         puts tbl.pack(max_table_width: @options[:max_width])
       when :json

data/lib/lsaws/sdk_parser.rb

@@ -2,6 +2,11 @@
 
 module Lsaws
   class SDKParser
+    IGNORED_SDKS = [
+      "core", "resources", # do not contain any resource listing methods
+      "s3control" # requires account_id param for all requests
+    ].freeze
+
     def self.get_sdks
       r = []
       Gem.path.each do |p|
@@ -12,7 +17,7 @@ module Lsaws
           a.size == 4 ? a[2] : nil
         end)
       end
-      r.compact.uniq.sort - ["core"]
+      r.compact.uniq.sort - IGNORED_SDKS
     end
 
     def initialize(sdk)
@@ -23,6 +28,7 @@ module Lsaws
     def client_class_name
       @client_class_name ||=
         begin
+          # TODO: use constants from gems/aws-sdk-resources-x.xxx
           c = Aws.constants.find { |x| x.to_s.downcase == @sdk }
           "Aws::#{c}::Client"
         end
@@ -32,10 +38,25 @@ module Lsaws
       @client_class ||= Kernel.const_get(client_class_name)
     end
 
+    # order is important!
+    LIST_METHOD_PREFIXES = %w[list describe get].freeze
+
+    def etype2method(etype)
+      LIST_METHOD_PREFIXES.each do |prefix|
+        m = "#{prefix}_#{etype}"
+        return m if client_class.public_method_defined?(m)
+      end
+      nil
+    end
+
+    def method2etype(method)
+      method.to_s.sub(/^(?:#{LIST_METHOD_PREFIXES.join("|")})_/, "")
+    end
+
     def entity_types
       methods = client_class
                 .instance_methods
-                .find_all { |m| m =~ /^(describe|list)_.+s$/ && m !~ /(status|access)$/ }
+                .find_all { |m| m =~ /^(?:#{LIST_METHOD_PREFIXES.join("|")})_.+s$/ && m !~ /(?:status|access)$/ }
 
       return [] if methods.empty?
 
@@ -44,10 +65,10 @@ module Lsaws
         next(true) unless rdoc
 
         required_params = rdoc.scan(/^\s+# @option params \[required, (.+?)\] :(\w+)/)
-        required_params.any?
+        required_params.any? || Lsaws.config.dig(@sdk, method2etype(m), "required_params")
       end
 
-      methods.map { |m| m.to_s.sub(/^(describe|list)_/, "") }.sort
+      methods.map { |m| method2etype(m) }.uniq.sort
     end
 
     def get_method_rdoc(method)
@@ -64,5 +85,10 @@ module Lsaws
       end
       chunk[chunk.rindex("\n\n") + 2..]
     end
+
+    def get_method_api(method)
+      # calling private API!
+      client_class.api.operation(method)
+    end
   end
 end

data/lib/lsaws/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Lsaws
-  VERSION = "0.2.0"
+  VERSION = "0.3.1"
 end

data/lib/lsaws.rb

@@ -1,11 +1,17 @@
 # frozen_string_literal: true
 
+require "yaml"
+
 module Lsaws
   class Error < StandardError; end
 
   def self.root
     File.dirname(File.dirname(File.expand_path(__FILE__)))
   end
+
+  def self.config
+    @config ||= YAML.load_file(File.join(Lsaws.root, "lsaws.yml"))
+  end
 end
 
 require_relative "lsaws/version"

data/lsaws.yml

@@ -1,3 +1,8 @@
+cloudformation:
+  stack_resources:
+    required_params:
+      - [ stack_name ]
+      - [ physical_resource_id ]
 ec2:
   default: instances
   images:

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: lsaws
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.3.1
 platform: ruby
 authors:
 - Andrey "Zed" Zaikin
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2023-02-10 00:00:00.000000000 Z
+date: 2023-02-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aws-sdk-core