miam 0.1.1 → 0.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: dac584f8c0f3829974bbf66a703a0514b0bba56b
-  data.tar.gz: 4c1554986b97d252d2e6fea8c17dbdee4e7c0072
+  metadata.gz: 959a0bf3fb3d32d91bebccb4cb858b7ab9be503f
+  data.tar.gz: a31ededb7c84f9d188e15f9c19a20defbbbfcca8
 SHA512:
-  metadata.gz: a197de1545bb2bfdaa906a215d23e61eb4ff63759875929a6f01605a38c6a3c4d5bd3dfa753a6e77af7c4521e0ef5f22ac36a70c1d049c8ada9e4a211f969601
-  data.tar.gz: cad4724a8a76be2bfc6c92ebc5ef97bf95a5e2b08f97469ce2f1213b95cc8ac42c89d694d71cf2805b3ce7e93258a3212354e076455a1852351c81e1a1bb9c59
+  metadata.gz: f2ed7296253deed56af8e3c2e4628772a6c84691c078e655b1a9a95d58b5e201c68fc0817b71c1212927c16b9ade6383edeaa65832560dbdab07863a05970a6a
+  data.tar.gz: 8be72aa39cbd4bc73c27ae1ed75e8b5c74e367996c2933a76e00e8e21b478b2d9c829348b6786043b87420e5de648ad1a56f64c6f13b5ae4becbe50411768124

data/README.md

@@ -52,6 +52,8 @@ Usage: miam [options]
     -e, --export
     -o, --output FILE
         --split
+        --split-more
+        --export-concurrency N
         --no-color
         --no-progress
         --debug

data/bin/miam

@@ -2,6 +2,7 @@
 $: << File.expand_path("#{File.dirname __FILE__}/../lib")
 require 'rubygems'
 require 'miam'
+require 'fileutils'
 require 'optparse'
 
 Version = Miam::VERSION
@@ -29,22 +30,23 @@ ARGV.options do |opt|
     profile_name = nil
     credentials_path = nil
 
-    opt.on('-p', '--profile PROFILE_NAME')  {|v| profile_name               = v                                     }
-    opt.on(''  , '--credentials-path PATH') {|v| credentials_path           = v                                     }
-    opt.on('-k', '--access-key ACCESS_KEY') {|v| access_key                 = v                                     }
-    opt.on('-s', '--secret-key SECRET_KEY') {|v| secret_key                 = v                                     }
-    opt.on('-r', '--region REGION')         {|v| region                     = v                                     }
-    opt.on('-a', '--apply')                 {    mode                       = :apply                                }
-    opt.on('-f', '--file FILE')             {|v| file                       = v                                     }
-    opt.on('',   '--dry-run')               {    options[:dry_run]          = true                                  }
-    opt.on(''  , '--account-output FILE')   {|v| options[:password_manager] = Miam::PasswordManager.new(v, options) }
-    opt.on(''  , '--split')                 {    split                      = true                                  }
-    opt.on('-e', '--export')                {    mode                       = :export                               }
-    opt.on('-o', '--output FILE')           {|v| output_file                = v                                     }
-    opt.on(''  , '--split')                 {    split                      = true                                  }
-    opt.on(''  , '--no-color')              {    options[:color]            = false                                 }
-    opt.on(''  , '--no-progress')           {    options[:no_progress]      = true                                  }
-    opt.on(''  , '--debug')                 {    options[:debug]            = true                                  }
+    opt.on('-p', '--profile PROFILE_NAME')          {|v| profile_name                 = v                                     }
+    opt.on(''  , '--credentials-path PATH')         {|v| credentials_path             = v                                     }
+    opt.on('-k', '--access-key ACCESS_KEY')         {|v| access_key                   = v                                     }
+    opt.on('-s', '--secret-key SECRET_KEY')         {|v| secret_key                   = v                                     }
+    opt.on('-r', '--region REGION')                 {|v| region                       = v                                     }
+    opt.on('-a', '--apply')                         {    mode                         = :apply                                }
+    opt.on('-f', '--file FILE')                     {|v| file                         = v                                     }
+    opt.on(''  , '--dry-run')                       {    options[:dry_run]            = true                                  }
+    opt.on(''  , '--account-output FILE')           {|v| options[:password_manager]   = Miam::PasswordManager.new(v, options) }
+    opt.on('-e', '--export')                        {    mode                         = :export                               }
+    opt.on('-o', '--output FILE')                   {|v| output_file                  = v                                     }
+    opt.on(''  , '--split')                         {    split                        = true                                  }
+    opt.on(''  , '--split-more')                    {    split                        = :more                                 }
+    opt.on(''  , '--export-concurrency N', Integer) {|v| options[:export_concurrency] = v                                     }
+    opt.on(''  , '--no-color')                      {    options[:color]              = false                                 }
+    opt.on(''  , '--no-progress')                   {    options[:no_progress]        = true                                  }
+    opt.on(''  , '--debug')                         {    options[:debug]              = true                                  }
     opt.parse!
 
     aws_opts = {}
@@ -94,10 +96,26 @@ begin
       output_file = DEFAULT_FILENAME if output_file == '-'
       requires = []
 
-      client.export do |type, dsl|
+      client.export(:split_more => (split == :more)) do |args|
+        type, dsl = args.values_at(:type, :dsl)
         next if dsl.strip.empty?
-        iam_file = File.join(File.dirname(output_file), "#{type}.iam")
-        requires << iam_file
+
+        type = type.to_s
+        dir = File.dirname(output_file)
+
+        if split == :more
+          name = args[:name]
+          dir = File.join(dir, type)
+          FileUtils.mkdir_p(dir)
+          iam_filename =  "#{name}.iam"
+          iam_file = File.join(dir, iam_filename)
+          requires << File.join(type, iam_filename)
+        else
+          iam_filename = "#{type}.iam"
+          iam_file = File.join(dir, iam_filename)
+          requires << iam_filename
+        end
+
         logger.info("  write `#{iam_file}`")
 
         open(iam_file, 'wb') do |f|
@@ -109,7 +127,7 @@ begin
 
       open(output_file, 'wb') do |f|
         requires.each do |iam_file|
-          f.puts "require '#{File.basename iam_file}'"
+          f.puts "require '#{iam_file}'"
         end
       end
     else

data/lib/miam.rb

@@ -3,9 +3,11 @@ require 'json'
 require 'logger'
 require 'pp'
 require 'singleton'
+require 'thread'
 
 require 'aws-sdk-core'
 require 'ruby-progressbar'
+require 'parallel'
 require 'term/ansicolor'
 
 module Miam; end

data/lib/miam/client.rb

@@ -9,16 +9,24 @@ class Miam::Client
     @password_manager = options[:password_manager] || Miam::PasswordManager.new('-', options)
   end
 
-  def export
-    exported, group_users, instance_profile_roles = Miam::Exporter.export(@iam, @options) do |export_options|
-      progress(*export_options.values_at(:progress_total, :progress))
-    end
+  def export(export_options = {})
+    exported, group_users, instance_profile_roles = Miam::Exporter.export(@iam, @options)
 
     if block_given?
       [:users, :groups, :roles, :instance_profiles].each do |type|
         splitted = {:users => {}, :groups => {}, :roles => {}, :instance_profiles => {}}
-        splitted[type] = exported[type]
-        yield(type, Miam::DSL.convert(splitted, @options).strip)
+
+        if export_options[:split_more]
+          exported[type].each do |name, attrs|
+            more_splitted = splitted.dup
+            more_splitted[type] = {}
+            more_splitted[type][name] = attrs
+            yield(:type => type, :name => name, :dsl => Miam::DSL.convert(more_splitted, @options).strip)
+          end
+        else
+          splitted[type] = exported[type]
+          yield(:type => type, :dsl => Miam::DSL.convert(splitted, @options).strip)
+        end
       end
     else
       Miam::DSL.convert(exported, @options)
@@ -34,10 +42,7 @@ class Miam::Client
   def walk(file)
     expected = load_file(file)
 
-    actual, group_users, instance_profile_roles = Miam::Exporter.export(@iam, @options) do |export_options|
-      progress(*export_options.values_at(:progress_total, :progress))
-    end
-
+    actual, group_users, instance_profile_roles = Miam::Exporter.export(@iam, @options)
     updated = walk_groups(expected[:groups], actual[:groups], actual[:users], group_users)
     updated = walk_users(expected[:users], actual[:users], group_users) || updated
     updated = walk_instance_profiles(expected[:instance_profiles], actual[:instance_profiles], actual[:roles], instance_profile_roles) || updated
@@ -379,14 +384,4 @@ class Miam::Client
       raise TypeError, "can't convert #{file} into File"
     end
   end
-
-  def progress(total, n)
-    return if @options[:no_progress]
-
-    unless @progressbar
-      @progressbar = ProgressBar.create(:title => "Loading", :total => total, :output => $stderr)
-    end
-
-    @progressbar.progress = n
-  end
 end

data/lib/miam/dsl/converter.rb

@@ -131,9 +131,13 @@ instance_profile #{instance_profile_name.inspect}, #{Miam::Utils.unbrace(instanc
   end
 
   def output_policies(policies)
-    policies.map {|policy_name, policy_document|
-      output_policy(policy_name, policy_document)
-    }.join("\n\n  ").strip
+    if policies.empty?
+      "# no policy"
+    else
+      policies.map {|policy_name, policy_document|
+        output_policy(policy_name, policy_document)
+      }.join("\n\n  ").strip
+    end
   end
 
   def output_policy(policy_name, policy_document)

data/lib/miam/exporter.rb

@@ -1,14 +1,16 @@
 class Miam::Exporter
-  def self.export(iam, options = {}, &block)
-    self.new(iam, options).export(&block)
+  def self.export(iam, options = {})
+    self.new(iam, options).export
   end
 
   def initialize(iam, options = {})
     @iam = iam
     @options = options
+    @mutex = Mutex.new
+    @concurrency = options[:export_concurrency] || 16
   end
 
-  def export(&block)
+  def export
     users = list_users
     groups = list_groups
     roles = list_roles
@@ -16,16 +18,16 @@ class Miam::Exporter
     group_users = {}
     instance_profile_roles = {}
 
-    export_options = {
-      :progress_total => (users.length + groups.length + roles.length + instance_profiles.length),
-      :progress => 0,
-    }
+    unless @options[:no_progress]
+      progress_total = users.length + groups.length + roles.length + instance_profiles.length
+      @progressbar = ProgressBar.create(:title => "Loading", :total => progress_total, :output => $stderr)
+    end
 
     expected = {
-      :users => export_users(users, group_users, export_options, &block),
-      :groups => export_groups(groups, export_options, &block),
-      :roles => export_roles(roles, instance_profile_roles, export_options, &block),
-      :instance_profiles => export_instance_profiles(instance_profiles, export_options, &block),
+      :users => export_users(users, group_users),
+      :groups => export_groups(groups),
+      :roles => export_roles(roles, instance_profile_roles),
+      :instance_profiles => export_instance_profiles(instance_profiles),
     }
 
     [expected, group_users, instance_profile_roles]
@@ -33,33 +35,33 @@ class Miam::Exporter
 
   private
 
-  def export_users(users, group_users, export_options = {})
+  def export_users(users, group_users)
     result = {}
 
-    users.each do |user|
+    Parallel.each(users, :in_threads => @concurrency) do |user|
       user_name = user.user_name
-
       groups = export_user_groups(user_name)
+      policies = export_user_policies(user_name)
+      login_profile = export_login_profile(user_name)
 
-      groups.each do |group_name|
-        group_users[group_name] ||= []
-        group_users[group_name] << user_name
-      end
+      @mutex.synchronize do
+        groups.each do |group_name|
+          group_users[group_name] ||= []
+          group_users[group_name] << user_name
+        end
 
-      result[user_name] = {
-        :path => user.path,
-        :groups => groups,
-        :policies => export_user_policies(user_name),
-      }
+        result[user_name] = {
+          :path => user.path,
+          :groups => groups,
+          :policies => policies,
+        }
 
-      login_profile = export_login_profile(user_name)
+        if login_profile
+          result[user_name][:login_profile] = login_profile
+        end
 
-      if login_profile
-        result[user_name][:login_profile] = login_profile
+        progress
       end
-
-      export_options[:progress] += 1
-      yield(export_options) if block_given?
     end
 
     result
@@ -96,19 +98,21 @@ class Miam::Exporter
     end
   end
 
-  def export_groups(groups, export_options = {})
+  def export_groups(groups)
     result = {}
 
-    groups.each do |group|
+    Parallel.each(groups, :in_threads => @concurrency) do |group|
       group_name = group.group_name
+      policies = export_group_policies(group_name)
 
-      result[group_name] = {
-        :path => group.path,
-        :policies => export_group_policies(group_name),
-      }
+      @mutex.synchronize do
+        result[group_name] = {
+          :path => group.path,
+          :policies => policies,
+        }
 
-      export_options[:progress] += 1
-      yield(export_options) if block_given?
+        progress
+      end
     end
 
     result
@@ -128,30 +132,31 @@ class Miam::Exporter
     result
   end
 
-  def export_roles(roles, instance_profile_roles, export_options = {})
+  def export_roles(roles, instance_profile_roles)
     result = {}
 
-    roles.each do |role|
+    Parallel.each(roles, :in_threads => @concurrency) do |role|
       role_name = role.role_name
-
       instance_profiles = export_role_instance_profiles(role_name)
+      policies = export_role_policies(role_name)
 
-      instance_profiles.each do |instance_profile_name|
-        instance_profile_roles[instance_profile_name] ||= []
-        instance_profile_roles[instance_profile_name] << role_name
-      end
+      @mutex.synchronize do
+        instance_profiles.each do |instance_profile_name|
+          instance_profile_roles[instance_profile_name] ||= []
+          instance_profile_roles[instance_profile_name] << role_name
+        end
 
-      document = CGI.unescape(role.assume_role_policy_document)
+        document = CGI.unescape(role.assume_role_policy_document)
 
-      result[role_name] = {
-        :path => role.path,
-        :assume_role_policy_document => JSON.parse(document),
-        :instance_profiles => instance_profiles,
-        :policies => export_role_policies(role_name),
-      }
+        result[role_name] = {
+          :path => role.path,
+          :assume_role_policy_document => JSON.parse(document),
+          :instance_profiles => instance_profiles,
+          :policies => policies,
+        }
 
-      export_options[:progress] += 1
-      yield(export_options) if block_given?
+        progress
+      end
     end
 
     result
@@ -179,18 +184,19 @@ class Miam::Exporter
     result
   end
 
-  def export_instance_profiles(instance_profiles, export_options = {})
+  def export_instance_profiles(instance_profiles)
     result = {}
 
-    instance_profiles.each do |instance_profile|
+    Parallel.each(instance_profiles, :in_threads => @concurrency) do |instance_profile|
       instance_profile_name = instance_profile.instance_profile_name
 
-      result[instance_profile_name] = {
-        :path => instance_profile.path,
-      }
+      @mutex.synchronize do
+        result[instance_profile_name] = {
+          :path => instance_profile.path,
+        }
 
-      export_options[:progress] += 1
-      yield(export_options) if block_given?
+        progress
+      end
     end
 
     result
@@ -227,4 +233,8 @@ class Miam::Exporter
       resp.instance_profiles.to_a
     }.flatten
   end
+
+  def progress
+    @progressbar.increment if @progressbar
+  end
 end

data/lib/miam/version.rb

@@ -1,3 +1,3 @@
 module Miam
-  VERSION = '0.1.1'
+  VERSION = '0.1.2'
 end

data/miam.gemspec

@@ -22,6 +22,7 @@ Gem::Specification.new do |spec|
 
   spec.add_dependency 'aws-sdk-core', '~> 2.0.3'
   spec.add_dependency 'ruby-progressbar'
+  spec.add_dependency 'parallel'
   spec.add_dependency 'term-ansicolor'
   spec.add_development_dependency 'bundler', '~> 1.7'
   spec.add_development_dependency 'rake', '~> 10.0'

data/spec/spec_helper.rb

@@ -76,6 +76,7 @@ def apply(cli = client)
 end
 
 def export(options = {})
+  options = {no_progress: true}.merge(options)
   cli = options.delete(:client) || Aws::IAM::Client.new
   Miam::Exporter.export(cli, options)[0]
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: miam
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.1.2
 platform: ruby
 authors:
 - Genki Sugawara
@@ -38,6 +38,20 @@ dependencies:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: parallel
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: term-ansicolor
   requirement: !ruby/object:Gem::Requirement
@@ -180,7 +194,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.1
+rubygems_version: 2.0.14
 signing_key: 
 specification_version: 4
 summary: Miam is a tool to manage IAM.