lsaws 0.1.0 → 0.3.0

data/lib/lsaws/lister.rb

@@ -3,23 +3,22 @@
 require "aws-sdk-core"
 require "json"
 require "tabulo"
-require "yaml"
 
 module Lsaws
   class Lister
-    CONFIG = YAML.load_file(File.join(Lsaws.root, "lsaws.yml"))
+    include Utils
+
+    NEXT_PAGE_FIELDS = %i[next_token next_marker next_page_token page_token].freeze
 
     def initialize(options)
       @options = options
+      @options[:max_width] = nil if @options[:max_width].to_i.zero?
     end
 
-    def _prepare_entities(sdk, type)
-      edef = CONFIG.dig(sdk, type)
-      if edef.is_a?(String)
-        type = edef
-        edef = CONFIG[sdk][type] # should exist
-        raise Error, "unknown entity type: #{type}" unless edef
-      end
+    def _prepare_entities(sdk, type, &block)
+      edef = Lsaws.config.dig(sdk, type)
+      return _prepare_entities(sdk, edef, &block) if edef.is_a?(String) # redirect like 'default' -> 'instances'
+
       edef ||= {}
       require(edef["require"] || "aws-sdk-#{sdk}")
 
@@ -31,9 +30,10 @@ module Lsaws
       end
       params[:max_results] = @options[:max_results] if @options[:max_results]
 
-      client_class = edef["client_class"] || guess_client_class(sdk)
+      sdkp = SDKParser.new(sdk)
+      client_class = edef["client_class"] || sdkp.client_class_name
       client = Kernel.const_get(client_class).new
-      method_name = edef["method"] || (client.respond_to?("describe_#{type}") ? "describe_#{type}" : "list_#{type}")
+      method_name = edef["method"] || sdkp.etype2method(type)
       unless client.respond_to?(method_name)
         if type == "default"
           warn "[!] no default entity type set for #{sdk.inspect} SDK"
@@ -44,27 +44,41 @@ module Lsaws
         list_entity_types(sdk)
         exit 1
       end
+      warn "[d] #{method_name} #{params}" if @options[:debug]
       results = client.send(method_name, params)
 
-      pp results if @options[:debug]
+      warn "[d] #{File.basename(__FILE__)}:#{__LINE__} results:\n#{results.pretty_inspect}" if @options[:debug]
 
       if !edef["result_keys"] && results.any?
         r = results.first
         r = r.last if r.is_a?(Array)
-        edef["result_keys"] = if r.respond_to?(type)
-                                [type]
-                              else
-                                [(r.members - [:next_token]).first]
-                              end
+        edef["result_keys"] =
+          if r.respond_to?(type)
+            [type]
+          elsif NEXT_PAGE_FIELDS.any? { |t| r.respond_to?(t) }
+            data_members = r.members - NEXT_PAGE_FIELDS
+            if data_members.size == 1
+              [data_members[0]]
+            else
+              # XXX what if there's more than one array?
+              [data_members.find { |key| r[key].is_a?(Array) }].compact
+            end
+          else
+            []
+          end
       end
+
       edef["result_keys"].each do |key|
         results = if results.is_a?(Array)
-                    results.map(&key.to_sym).flatten
+                    results.map(&key.to_sym).flatten # TODO: is flatten necessary?
                   else
-                    results.send(key)
+                    results[key]
                   end
       end
+
+      warn "[d] #{File.basename(__FILE__)}:#{__LINE__} results:\n#{results.pretty_inspect}" if @options[:debug]
       edef["cols"] = @options[:show_cols] if @options[:show_cols].any?
+      warn "[d] edef: #{edef}" if @options[:debug]
 
       col_defs = {}
       Array(edef["cols"]).each do |r|
@@ -77,10 +91,12 @@ module Lsaws
           raise Error, "unexpected #{r.inspect}"
         end
       end
+      # TODO: check with all types
       col_defs["tags"] = _convert_tags_proc if @options[:show_tags] || col_defs["tags"]
 
       results ||= []
-      if results.any? && !results.first.respond_to?(:name)
+      warn "[d] #{results.inspect}" if @options[:debug]
+      if results.respond_to?(:any?) && results.any? && !results.first.respond_to?(:name) && results.first.respond_to?(:tags)
         results.first.class.class_eval do
           def name
             tags.find { |tag| tag.key == "Name" }&.value
@@ -88,9 +104,19 @@ module Lsaws
         end
       end
 
-      # ec2 instance_event_notification_attributes
-      # 'Array(results)' doesn't work here
-      results = [results] unless results.is_a?(Array)
+      case results
+      when Hash
+        col_defs = {
+          key: :first,
+          value: :last
+        }
+      when Array
+        # ok
+      else
+        # ec2 instance_event_notification_attributes
+        # 'Array(results)' doesn't work here
+        results = [results]
+      end
 
       if block_given?
         results.map do |entity|
@@ -106,6 +132,9 @@ module Lsaws
       if rows.is_a?(Array) && rows[0].is_a?(String)
         # sqs
         cols = { value: proc { |entity| entity } }
+      elsif rows.is_a?(Array) && rows[0].is_a?(Hash)
+        # securitylake:log_sources
+        cols = { value: proc { |entity| entity } }
       elsif rows.respond_to?(:members)
         rows = [rows]
       end
@@ -149,21 +178,8 @@ module Lsaws
       proc { |entity| entity.tags.map { |tag| "#{tag.key}=#{tag.value}" }.join(", ") }
     end
 
-    def get_sdks
-      r = []
-      Gem.path.each do |p|
-        next unless Dir.exist?(p)
-
-        r.append(*Dir[File.join(p, "gems/aws-sdk-*")].map do |gem_dir|
-          a = File.basename(gem_dir).split("-")
-          a.size == 4 ? a[2] : nil
-        end)
-      end
-      r.compact.uniq.sort - ["core"]
-    end
-
     def list_sdks
-      _list_array get_sdks
+      _list_array SDKParser.get_sdks
     end
 
     def _list_array(a)
@@ -177,55 +193,27 @@ module Lsaws
       end
     end
 
-    def guess_client_class(sdk)
-      c = Aws.constants.find { |x| x.to_s.downcase == sdk }
-      "Aws::#{c}::Client"
-    end
-
-    def _get_method_rdoc(data, method)
-      pos = data =~ /^\s+def\s+#{method}\s*\(/
-      return nil unless pos
-
-      chunk = ""
-      bs = 4096
-      until chunk["\n\n"]
-        chunk = data[pos - bs..pos]
-        bs *= 2
-      end
-      chunk[chunk.rindex("\n\n") + 2..]
+    def list_entity_types(sdk)
+      _list_array SDKParser.new(sdk).entity_types
     end
 
-    def get_entity_types(sdk)
-      require "aws-sdk-#{sdk}"
-      client_class = Kernel.const_get(guess_client_class(sdk))
-      methods = client_class
-                .instance_methods
-                .find_all { |m| m =~ /^(describe|list)_.+s$/ && m !~ /(status|access)$/ }
-
-      return [] if methods.empty?
-
-      data = File.read(client_class.instance_method(methods[0]).source_location[0])
-      methods.delete_if do |m|
-        rdoc = _get_method_rdoc(data, m)
-        next unless rdoc
-
-        required_params = rdoc.scan(/^\s+# @option params \[required, (.+?)\] :(\w+)/)
-        required_params.any?
+    # elasticache:global_replication_groups has 'members' as a column, so cannot just use `rows.members`
+    # assuming row is always subclass of Struct here
+    def _get_cols(row)
+      if row.is_a?(Struct)
+        Struct.instance_method(:members).bind_call(row)
+      else
+        row.members
       end
-
-      methods.map { |m| m.to_s.sub(/^(describe|list)_/, "") }.sort
-    end
-
-    def list_entity_types(sdk)
-      _list_array get_entity_types sdk
     end
 
     def _tabulo_guess_max_cols(rows, _cols)
-      max_cols = rows[0].members.size
+      all_cols = _get_cols(rows[0])
+      max_cols = all_cols.size
       return max_cols if max_cols < 4 || !@options[:max_width]
 
       4.upto(max_cols) do |ncols|
-        tbl = Tabulo::Table.new(rows[0, 100], *rows[0].members[0, ncols])
+        tbl = Tabulo::Table.new(rows[0, 100], *all_cols[0, ncols])
         tbl.autosize_columns
         tbl_width = tbl.column_registry.values.map { |c| c.padded_width + 1 }.inject(:+) + 1
         return ncols - 1 if tbl_width >= @options[:max_width]
@@ -239,7 +227,7 @@ module Lsaws
       elsif type == :list
         return list_entity_types(sdk)
       elsif type == :all
-        get_entity_types(sdk).each do |etype|
+        SDKParser.new(sdk).entity_types.each do |etype|
           puts "#{etype}:"
           process_command(sdk, etype)
         end
@@ -260,7 +248,7 @@ module Lsaws
           cols.each { |name, func| tbl.add_column(name, &func) }
         else
           max_cols = _tabulo_guess_max_cols(rows, cols)
-          rows[0].members[0, max_cols].each { |col| tbl.add_column(col) }
+          _get_cols(rows[0])[0, max_cols].each { |col| tbl.add_column(col) }
         end
         puts tbl.pack(max_table_width: @options[:max_width])
       when :json
@@ -273,8 +261,7 @@ module Lsaws
         end
       when :yaml
         rows = entities2hashes(sdk, type)
-        rows.each { |row| Utils._deep_transform_keys_in_object!(row, &:to_s) }
-        puts rows.to_yaml
+        puts rows.map { |row| _deep_transform_keys_in_object(row, &:to_s) }.to_yaml
       else
         warn "[!] unknown format: #{@options[:format]}"
         exit 1

data/lib/lsaws/sdk_parser.rb

@@ -0,0 +1,94 @@
+# frozen_string_literal: true
+
+module Lsaws
+  class SDKParser
+    IGNORED_SDKS = [
+      "core", "resources", # do not contain any resource listing methods
+      "s3control" # requires account_id param for all requests
+    ].freeze
+
+    def self.get_sdks
+      r = []
+      Gem.path.each do |p|
+        next unless Dir.exist?(p)
+
+        r.append(*Dir[File.join(p, "gems/aws-sdk-*")].map do |gem_dir|
+          a = File.basename(gem_dir).split("-")
+          a.size == 4 ? a[2] : nil
+        end)
+      end
+      r.compact.uniq.sort - IGNORED_SDKS
+    end
+
+    def initialize(sdk)
+      @sdk = sdk
+      require "aws-sdk-#{sdk}"
+    end
+
+    def client_class_name
+      @client_class_name ||=
+        begin
+          # TODO: use constants from gems/aws-sdk-resources-x.xxx
+          c = Aws.constants.find { |x| x.to_s.downcase == @sdk }
+          "Aws::#{c}::Client"
+        end
+    end
+
+    def client_class
+      @client_class ||= Kernel.const_get(client_class_name)
+    end
+
+    # order is important!
+    LIST_METHOD_PREFIXES = %w[list describe get].freeze
+
+    def etype2method(etype)
+      LIST_METHOD_PREFIXES.each do |prefix|
+        m = "#{prefix}_#{etype}"
+        return m if client_class.public_method_defined?(m)
+      end
+      nil
+    end
+
+    def method2etype(method)
+      method.to_s.sub(/^(?:#{LIST_METHOD_PREFIXES.join("|")})_/, "")
+    end
+
+    def entity_types
+      methods = client_class
+                .instance_methods
+                .find_all { |m| m =~ /^(?:#{LIST_METHOD_PREFIXES.join("|")})_.+s$/ && m !~ /(?:status|access)$/ }
+
+      return [] if methods.empty?
+
+      methods.delete_if do |m|
+        rdoc = get_method_rdoc(m)
+        next(true) unless rdoc
+
+        required_params = rdoc.scan(/^\s+# @option params \[required, (.+?)\] :(\w+)/)
+        required_params.any? || Lsaws.config.dig(@sdk, method2etype(m), "required_params")
+      end
+
+      methods.map { |m| method2etype(m) }.sort
+    end
+
+    def get_method_rdoc(method)
+      @source ||= File.read(client_class.instance_method(method).source_location[0])
+
+      pos = @source =~ /^\s+def\s+#{method}\s*\(/
+      return nil unless pos
+
+      chunk = ""
+      bs = 4096
+      until chunk["\n\n"]
+        chunk = @source[pos - bs..pos]
+        bs *= 2
+      end
+      chunk[chunk.rindex("\n\n") + 2..]
+    end
+
+    def get_method_api(method)
+      # calling private API!
+      client_class.api.operation(method)
+    end
+  end
+end

data/lib/lsaws/utils.rb

@@ -3,7 +3,7 @@
 module Lsaws
   module Utils
     # copypasted from lib/active_support/core_ext/hash/keys.rb
-    def self._deep_transform_keys_in_object!(object, &block)
+    def _deep_transform_keys_in_object!(object, &block)
       case object
       when Hash
         object.each_key do |key|
@@ -17,5 +17,28 @@ module Lsaws
         object
       end
     end
+
+    def _deep_transform_keys_in_object(object, &block)
+      case object
+      when Hash
+        object.each_with_object(object.class.new) do |(key, value), result|
+          result[yield(key)] = _deep_transform_keys_in_object(value, &block)
+        end
+      when Array
+        object.map { |e| _deep_transform_keys_in_object(e, &block) }
+      else
+        object
+      end
+    end
+
+    # "2022-11-15 10:49:00 UTC" => "2022-11-15T10:49:00+00:00"
+    def to_iso8601(s)
+      s.gsub(/([0-9-]{10}) ([0-9:]+{8}) UTC/, '\1T\2+00:00')
+    end
+
+    # "2022-11-15T10:49:00+00:00" => "2022-11-15 10:49:00 UTC"
+    def from_iso8601(s)
+      s.gsub(/([0-9-]{10})T([0-9:]+{8})\+00:00/, '\1 \2 UTC')
+    end
   end
 end

data/lib/lsaws/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Lsaws
-  VERSION = "0.1.0"
+  VERSION = "0.3.0"
 end

data/lib/lsaws.rb

@@ -1,13 +1,21 @@
 # frozen_string_literal: true
 
+require "yaml"
+
 module Lsaws
   class Error < StandardError; end
 
   def self.root
     File.dirname(File.dirname(File.expand_path(__FILE__)))
   end
+
+  def self.config
+    @config ||= YAML.load_file(File.join(Lsaws.root, "lsaws.yml"))
+  end
 end
 
 require_relative "lsaws/version"
+require_relative "lsaws/sdk_parser"
 require_relative "lsaws/cli"
+require_relative "lsaws/utils"
 require_relative "lsaws/lister"

data/lsaws.yml

@@ -1,3 +1,8 @@
+cloudformation:
+  stack_resources:
+    required_params:
+      - [ stack_name ]
+      - [ physical_resource_id ]
 ec2:
   default: instances
   images:
@@ -39,3 +44,5 @@ kms:
     - alias_name
     - target_key_id
     - last_updated_date
+s3:
+  default: buckets

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: lsaws
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.3.0
 platform: ruby
 authors:
 - Andrey "Zed" Zaikin
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2023-02-09 00:00:00.000000000 Z
+date: 2023-02-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aws-sdk-core
@@ -59,6 +59,7 @@ files:
 - lib/lsaws.rb
 - lib/lsaws/cli.rb
 - lib/lsaws/lister.rb
+- lib/lsaws/sdk_parser.rb
 - lib/lsaws/utils.rb
 - lib/lsaws/version.rb
 - lsaws.gemspec