cantemo-portal-agent 1.3.3 → 1.3.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 7c2c43f547f0f607e087665ce93f3a5c63dd4d5a
-  data.tar.gz: 96c2cc23d5b30fa48abbc3d329e01287782b343e
+  metadata.gz: 657db4631f5e476906091f6126709406cff78820
+  data.tar.gz: cfdcda72a46b18b78cd626db096d2af1f3559830
 SHA512:
-  metadata.gz: 16d67112efc3e3748edbb090c97e86c5874945e18c17bedd34184ac0b9709f5771bccd7fc40b1f6c2ec382e17dc551c041c3ab33d901e6d809632ef183c9a60f
-  data.tar.gz: 8d93bcc5a774aba3a369e3dbe77c09a9c4dfe2b05d71572bfa0c5e387be9f4220c26f3beac924771e5e7541be06fb85123530fb2d9e1254cf22376f320542c28
+  metadata.gz: 3f94323823c25998c23f7823c3c105f569bc01b9363d267dcd107d4e999c8000c9878abcb8bd2ea0b40a250f85b2e7fcbd3bc323f156e088e54677a66f98cd1a
+  data.tar.gz: 863be982724502f6f7fd69d16537301c07ec6196e45d1fac3e3816f2ff1726f4260a68a20d1336dd588de6d425a9efbd653234dbf15a7964619f8daa161b90c1

data/lib/cantemo/portal/agent/cli/commands/watch_folders.rb

@@ -296,7 +296,7 @@ SERVICE_DISPLAYNAME = 'Cantemo Portal Watch Folder Agent'
 
 # You must provide at least one argument.
 abort('No argument provided.') unless sub_command
-abort('Service not installed.') unless Service.exists?(SERVICE_NAME) || sub_command == 'install'
+abort('Service is not installed.') unless Service.exists?(SERVICE_NAME) || sub_command == 'install'
 
 
 case sub_command
@@ -352,7 +352,7 @@ when 'status'
 when 'install'
   # Quote the full path to deal with possible spaces in the path name.
   ruby = File.join(CONFIG['bindir'], CONFIG['ruby_install_name']).tr('/', '\\')
-  path = __FILE__.tr('/', '\\')
+  path = File.join(CONFIG['bindir'], 'cantemo-portal-agent')
   # cmd = %Q("#{ruby}" "#{path}" runasservice --config-file-path "#{config_file_path}")
   cmd = %Q("#{ruby}" "#{path}" run-as-service)
 

data/lib/cantemo/portal/agent/version.rb

@@ -1,7 +1,7 @@
 module Cantemo
   module Portal
     class Agent
-      VERSION = '1.3.3'.freeze
+      VERSION = '1.3.4'.freeze
     end
   end
 end

data/lib/envoi/mam/agent/cli/commands.rb

@@ -23,33 +23,43 @@
 #   # Envoi
 # end
 
-def usage
-  command_list_str = ''
-  Dir.glob(File.join(@commands_dir, "*#{@command_ext}")).each do |fn|
-    command_list_str += "\n\t - #{File.basename(fn, @command_ext)}"
+@commands_dir = File.join(File.dirname(__FILE__), 'commands')
+@command_ext = '.rb'
+
+@command_list_str = ''
+@commands = [ ]
+@command_aliases = { }
+
+def build_command_aliases(commands_dir = @commands_dir)
+  Dir.glob(File.join(commands_dir, "*.rb")).each do |fn|
+    command_file_name = File.basename(fn)
+    command_name = File.basename(command_file_name, '.*')
+    command_alias = command.tr('-_', '')
+    @commands << command
+    @command_aliases[command_alias] = command_name
+    @command_list_str += "\n\t - #{command}"
   end
+end
+
+def usage
 
   <<-EOT
 Usage: #{File.basename($0)} COMMAND [ARGS]
   
   Available Commands:
-    #{command_list_str}
+    #{@command_list_str}
     
 All commands can be run with -h (or --help) for more information.
   EOT
 end
 
-@commands_dir = File.join(File.dirname(__FILE__), 'commands')
-@command_ext = '.rb'
-
-command_aliases = {
-}
 
+build_command_aliases
 if ARGV.empty?
   should_show_usage = true
 else
-  command_name = ARGV[0].clone
-  command_name = command_aliases[command_name] || command_name
+  command_name = ARGV.first.dup
+  command_name = @command_aliases[command_name] || command_name
   should_show_usage = %w(-h --help).include?(command_name.downcase)
 end
 if should_show_usage

data/lib/envoi/mam/agent/cli/commands/s3-to-eis.rb

@@ -0,0 +1,46 @@
+#! /usr/bin/env ruby
+# Installation
+# `gem install aws-sdk-s3`
+#
+require 'net/https'
+require 'json'
+require 'optparse'
+require 'pp'
+require 'csv'
+
+ARGV << '--help' if ARGV.empty?
+args = {
+    show_summary: true,
+    eis_url: 'https://ksbw8ud7v6.execute-api.us-east-1.amazonaws.com/stage/createmcjob',
+    s3_object_key_prefix: '',
+    preview_only: false
+}
+op = OptionParser.new
+op.on('--bucket-name [BUCKETNAME]', 'The bucket to get objects from.') { |v| args[:s3_bucket_name] = v }
+op.on('--key-prefix [KEYPREFIX]', 'The object key prefix to limit the results by.', 'default: /') { |v| args[:s3_object_key_prefix] = v }
+op.on('--aws-access-key [KEY]', '') { |v| args[:aws_access_key_id] = v }
+op.on('--aws-secret-key [KEY]', '') { |v| args[:aws_secret_access_key] = v }
+op.on('--aws-profile [PROFILENAME]', '') { |v| args[:aws_profile] = v }
+op.on('--aws-region [REGION]', 'The AWS region what the bucket is located.', 'default: auto-detected') { |v| args[:aws_region] = v }
+op.on('--limit [LIMIT]', 'Limit the number of results.', 'default: unlimited') { |v| args[:limit] = v }
+
+op.on('--[no-]preview', 'Output paths with summary and stop.',
+      "default: #{args[:preview_mode] ? 'true' : 'false'}") { |v| args[:preview_only] = v }
+op.on('--[no-]summary', 'Output the paths and calculate the number of assets.',
+      "default: #{args[:show_summary] ? 'true' : 'false'}",) { |v| args[:show_summary] = v }
+
+op.on('-h', '--help', 'Print help (this message) and exit') { puts op; exit }
+op.parse!
+
+
+
+
+s3_enum = S3Enumerator.new(args)
+s3_enum.run
+exit if args[:preview_only]
+s3_enum.process_files do |object|
+
+end
+
+
+

data/lib/envoi/mam/agent/cli/commands/vidispine-agent.rb

@@ -0,0 +1,60 @@
+#!/usr/bin/env ruby
+lib_path = __FILE__ == '(irb)' ? File.join(Dir.cwd, 'lib') : File.expand_path('../../../lib', __FILE__)
+$:.unshift(lib_path) unless $:.include?(lib_path) or !File.exists?(lib_path)
+require 'rubygems'
+require 'optparse'
+require 'open3'
+
+require 'envoi/mam/agent/cli'
+require 'envoi/mam/vidispine/agent'
+
+default_config_file_paths = Envoi::Mam::Agent::CLI::CONFIG_FILE_PATHS
+
+aspera_ascp_paths = Envoi::Mam::Agent::TransferClient::Aspera::ASCP_PATHS
+default_aspera_ascp_path = aspera_ascp_paths.find { |v| File.exist? v } || aspera_ascp_paths.first
+
+current_command = ARGV.shift
+ARGV << '--help' if ARGV.empty?
+
+args = {
+    :config_file_path  => default_config_file_paths,
+    :default_ascp_path => default_aspera_ascp_path,
+    :dry_run           => false,
+    :operation         => :download,
+    :preserve_path     => true,
+    :transfer_type     => '',
+}
+
+args[:config_service_app_id] = ENV['ENVOI_MAM_AGENT_APP_ID']
+args[:config_service_app_token] = ENV['ENVOI_MAM_AGENT_APP_TOKEN']
+args[:config_service_app_url] = ENV['ENVOI_MAM_AGENT_APP_URL']
+
+op = OptionParser.new
+op.on('-c', '--config-file-path FILEPATH', 'The path to the configuration file.',
+      "Default Path(s): #{args[:config_file_path]}") { |v| args[:config_file_path] = v }
+op.on('--agent-app-id ID', '') { |v| args[:config_service_app_id] = v }
+op.on('--agent-token TOKEN', '') { |v| args[:config_service_app_token] = v }
+op.on('--agent-service-api URL', '') { |v| args[:config_service_api_url] = v }
+
+op.on('-i', '--item-id ITEMID', 'The Vidispine item id.') { |v| args[:item_id] = v }
+op.on('-s', '--shape-id SHAPEID', 'The Vidispine shape id.') { |v| args[:shape_id] = v }
+op.on('-t', '--shape-tag SHAPETAG', 'The Vidispine shape tag used to find the shape when a shape id is not supplied.') { |v| args[:shape_tag] = v }
+op.on('--storage-id STORAGEID', 'The Vidispine storage id to use for the transfer.') { |v| args[:storage_id] = v}
+
+op.on('-d', '--destination-path DIRPATH', 'The destination path to download or upload to.',
+      "default: '.'") { |v| args[:destination_path] = v }
+op.on('-p','--[no-]preserve-path', "default: true") { |v| args[:preserve_path] = v }
+op.on('-p', '--operation OP', [ :download, :upload ], 'Transfer Direction.',
+      "default: #{args[:operation]}" ) { |v| args[:operation] = v }
+op.on('--transfer-type TYPE', [ :aspera, :s3 ],'Force the type of transfer to use. Ex: aspera || s3') { |v| args[:transfer_type] = v }
+op.on('--transfer-token TOKEN', '') { |v| args[:transfer_token] = v }
+op.on('--dry-run', 'Run without executing the transfer.') { |v| args[:dry_run] = v }
+op.on('--help', 'Show this message.') { puts op; exit }
+op.load
+op.parse!
+
+args[:file_path] ||= ARGV.shift
+args[:destination_path] ||= ARGV.shift unless ARGV.empty?
+
+agent = Envoi::Mam::Vidispine::Agent.load_config_and_init(args)
+agent.run_operation

data/lib/envoi/mam/agent/cli/commands/vidispine-restore.rb

@@ -0,0 +1,61 @@
+#!/usr/bin/env ruby
+lib_path = __FILE__ == '(irb)' ? File.join(Dir.cwd, 'lib') : File.expand_path('../../../lib', __FILE__)
+$:.unshift(lib_path) unless $:.include?(lib_path) or !File.exists?(lib_path)
+require 'rubygems'
+require 'optparse'
+require 'open3'
+
+require 'envoi/mam/agent/cli'
+require 'envoi/mam/vidispine/agent'
+require 'envoi/mam/vidispine/restore/watcher'
+
+default_config_file_paths = Envoi::Mam::Agent::CLI::CONFIG_FILE_PATHS
+
+aspera_ascp_paths = Envoi::Mam::Agent::TransferClient::Aspera::ASCP_PATHS
+default_aspera_ascp_path = aspera_ascp_paths.find { |v| File.exist? v } || aspera_ascp_paths.first
+
+current_command = ARGV.shift
+ARGV << '--help' if ARGV.empty?
+
+args = {
+    :config_file_path  => default_config_file_paths,
+    :default_ascp_path => default_aspera_ascp_path,
+    :dry_run           => false,
+    :operation         => :download,
+    :preserve_path     => true,
+    :transfer_type     => '',
+}
+
+args[:config_service_app_id] = ENV['ENVOI_MAM_AGENT_APP_ID']
+args[:config_service_app_token] = ENV['ENVOI_MAM_AGENT_APP_TOKEN']
+args[:config_service_app_url] = ENV['ENVOI_MAM_AGENT_APP_URL']
+
+op = OptionParser.new
+op.on('-c', '--config-file-path FILEPATH', 'The path to the configuration file.',
+      "Default Path(s): #{args[:config_file_path]}") { |v| args[:config_file_path] = v }
+op.on('--agent-app-id ID', '') { |v| args[:config_service_app_id] = v }
+op.on('--agent-token TOKEN', '') { |v| args[:config_service_app_token] = v }
+op.on('--agent-service-api URL', '') { |v| args[:config_service_api_url] = v }
+
+op.on('-i', '--item-id ITEMID', 'The Vidispine item id.') { |v| args[:item_id] = v }
+op.on('-s', '--shape-id SHAPEID', 'The Vidispine shape id.') { |v| args[:shape_id] = v }
+op.on('-t', '--shape-tag SHAPETAG', 'The Vidispine shape tag used to find the shape when a shape id is not supplied.') { |v| args[:shape_tag] = v }
+op.on('--storage-id STORAGEID', 'The Vidispine storage id to use for the transfer.') { |v| args[:storage_id] = v}
+
+op.on('-d', '--destination-path DIRPATH', 'The destination path to download or upload to.',
+      "default: '.'") { |v| args[:destination_path] = v }
+op.on('-p','--[no-]preserve-path', "default: true") { |v| args[:preserve_path] = v }
+op.on('-p', '--operation OP', [ :download, :upload ], 'Transfer Direction.',
+      "default: #{args[:operation]}" ) { |v| args[:operation] = v }
+op.on('--transfer-type TYPE', [ :aspera, :s3 ],'Force the type of transfer to use. Ex: aspera || s3') { |v| args[:transfer_type] = v }
+op.on('--transfer-token TOKEN', '') { |v| args[:transfer_token] = v }
+op.on('--dry-run', 'Run without executing the transfer.') { |v| args[:dry_run] = v }
+op.on('--help', 'Show this message.') { puts op; exit }
+op.load
+op.parse!
+
+# args[:file_path] ||= ARGV.shift
+# args[:destination_path] ||= ARGV.shift unless ARGV.empty?
+#
+# agent = Envoi::Mam::Vidispine::Restore::Watcher.load_config_and_init(args)
+# agent.run_operation

data/lib/envoi/mam/agent/cli/commands/vidispine.rb

@@ -1,60 +1,5 @@
-#!/usr/bin/env ruby
-lib_path = __FILE__ == '(irb)' ? File.join(Dir.cwd, 'lib') : File.expand_path('../../../lib', __FILE__)
-$:.unshift(lib_path) unless $:.include?(lib_path) or !File.exists?(lib_path)
-require 'rubygems'
-require 'optparse'
-require 'open3'
-
-require 'envoi/mam/agent/cli'
-require 'envoi/mam/vidispine/agent'
-
-default_config_file_paths = Envoi::Mam::Agent::CLI::CONFIG_FILE_PATHS
-
-aspera_ascp_paths = Envoi::Mam::Agent::TransferClient::Aspera::ASCP_PATHS
-default_aspera_ascp_path = aspera_ascp_paths.find { |v| File.exist? v } || aspera_ascp_paths.first
-
-current_command = ARGV.shift
-ARGV << '--help' if ARGV.empty?
-
-args = {
-    :config_file_path  => default_config_file_paths,
-    :default_ascp_path => default_aspera_ascp_path,
-    :dry_run           => false,
-    :operation         => :download,
-    :preserve_path     => true,
-    :transfer_type     => '',
-}
-
-args[:config_service_app_id] = ENV['ENVOI_MAM_AGENT_APP_ID']
-args[:config_service_app_token] = ENV['ENVOI_MAM_AGENT_APP_TOKEN']
-args[:config_service_app_url] = ENV['ENVOI_MAM_AGENT_APP_URL']
-
-op = OptionParser.new
-op.on('-c', '--config-file-path FILEPATH', 'The path to the configuration file.',
-      "Default Path(s): #{args[:config_file_path]}") { |v| args[:config_file_path] = v }
-op.on('--agent-app-id ID', '') { |v| args[:config_service_app_id] = v }
-op.on('--agent-token TOKEN', '') { |v| args[:config_service_app_token] = v }
-op.on('--agent-service-api URL', '') { |v| args[:config_service_api_url] = v }
-
-op.on('-i', '--item-id ITEMID', 'The Vidispine item id.') { |v| args[:item_id] = v }
-op.on('-s', '--shape-id SHAPEID', 'The Vidispine shape id.') { |v| args[:shape_id] = v }
-op.on('-t', '--shape-tag SHAPETAG', 'The Vidispine shape tag used to find the shape when a shape id is not supplied.') { |v| args[:shape_tag] = v }
-op.on('--storage-id STORAGEID', 'The Vidispine storage id to use for the transfer.') { |v| args[:storage_id] = v}
-
-op.on('-d', '--destination-path DIRPATH', 'The destination path to download or upload to.',
-      "default: '.'") { |v| args[:destination_path] = v }
-op.on('-p','--[no-]preserve-path', "default: true") { |v| args[:preserve_path] = v }
-op.on('-p', '--operation OP', [ :download, :upload ], 'Transfer Direction.',
-      "default: #{args[:operation]}" ) { |v| args[:operation] = v }
-op.on('--transfer-type TYPE', [ :aspera, :s3 ],'Force the type of transfer to use. Ex: aspera || s3') { |v| args[:transfer_type] = v }
-op.on('--transfer-token TOKEN', '') { |v| args[:transfer_token] = v }
-op.on('--dry-run', 'Run without executing the transfer.') { |v| args[:dry_run] = v }
-op.on('--help', 'Show this message.') { puts op; exit }
-op.load
-op.parse!
-
-args[:file_path] ||= ARGV.shift
-args[:destination_path] ||= ARGV.shift unless ARGV.empty?
-
-agent = Envoi::Mam::Vidispine::Agent.load_config_and_init(args)
-agent.run_operation
+if ARGV.map { |v| v.downcase }.include?('restore')
+  require 'envoi/mam/agent/cli/commands/vidispine-restore'
+else
+  require 'envoi/mam/agent/cli/commands/vidispine-agent'
+end

data/lib/envoi/mam/agent/daemon/{service_wrapper.rb → windows_service.rb}

@@ -6,7 +6,7 @@ module Envoi
 
       class Daemon
 
-        class ServiceWrapper
+        class WindowsService
 
         end
 

data/lib/envoi/mam/agent/daemon/windows_service_controller.rb

@@ -0,0 +1,134 @@
+# Options:
+#    install    - Installs the service.
+#    start      - Starts the service.
+#    stop       - Stops the service.
+#    pause      - Pauses the service.
+#    resume     - Resumes the service.
+#    uninstall  - Uninstalls the service.
+#    delete     - Same as uninstall.
+require 'win32/service'
+require 'rbconfig'
+
+
+
+class WindowsServiceController
+  include RbConfig
+  include Win32
+
+  SERVICE_NAME = 'cantemo_portal_watch_folder_agent'
+  SERVICE_DISPLAYNAME = 'Cantemo Portal Watch Folder Agent'
+
+  class << self
+
+
+    def run(command = nil, args = { })
+      # You must provide at least one argument.
+      raise ArgumentError, 'No argument provided.' unless command
+      unless Service.exists?(SERVICE_NAME) || command == 'install'
+        puts 'Service is not installed.'
+        return
+      end
+
+      case command
+      when 'start'; self.start
+      when 'stop'; self.stop
+      when 'pause'; self.pause
+      when 'resume'; self.resume
+      when 'status'; self.status
+      when 'install'; self.install(args)
+      when 'uninstall', 'delete'; self.uninstall
+      else
+        raise ArgumentError, 'unknown option: ' + command
+      end
+    end
+
+    def install(args = {})
+      # Quote the full path to deal with possible spaces in the path name.
+      ruby = File.join(CONFIG['bindir'], CONFIG['ruby_install_name']).tr('/', '\\')
+      path = __FILE__.tr('/', '\\')
+      cmd = %Q("#{ruby}" "#{path}")
+
+      install_args = {
+          :service_name     => SERVICE_NAME,
+          :display_name     => SERVICE_DISPLAYNAME,
+          :binary_path_name => cmd
+      }.merge!(args)
+
+      Service.new(install_args)
+      puts "Service `#{SERVICE_NAME}` installed"
+    end
+
+    def uninstall
+      if Service.status(SERVICE_NAME).current_state != 'stopped'
+        Service.stop(SERVICE_NAME)
+      end
+      while Service.status(SERVICE_NAME).current_state != 'stopped'
+        puts "One moment... #{Service.status(SERVICE_NAME).current_state}"
+        sleep 1
+      end
+      Service.delete(SERVICE_NAME)
+      puts "Service #{SERVICE_NAME} deleted"
+    end
+
+    def start
+      if Service.status(SERVICE_NAME).current_state != 'running'
+        # Service.start(SERVICE_NAME, nil, args)
+        Service.start(SERVICE_NAME)
+        while Service.status(SERVICE_NAME).current_state != 'running'
+          puts "One moment... #{Service.status(SERVICE_NAME).current_state}"
+          sleep 1
+        end
+        puts "Service `#{SERVICE_NAME}` started"
+      else
+        puts 'Already running'
+      end
+    end
+
+    def status
+      puts "Service `#{SERVICE_NAME}` #{Service.status(SERVICE_NAME).current_state}"
+    end
+
+    def stop
+      if Service.status(SERVICE_NAME).current_state != 'stopped'
+        Service.stop(SERVICE_NAME)
+        while Service.status(SERVICE_NAME).current_state != 'stopped'
+          puts "One moment... #{Service.status(SERVICE_NAME).current_state}"
+          sleep 1
+        end
+        puts "Service `#{SERVICE_NAME}` stopped"
+      else
+        puts 'Already stopped'
+      end
+    end
+
+    def pause
+      if Service.status(SERVICE_NAME).current_state != 'paused'
+        Service.pause(SERVICE_NAME)
+        while Service.status(SERVICE_NAME).current_state != 'paused'
+          puts "One moment... #{Service.status(SERVICE_NAME).current_state}"
+          sleep 1
+        end
+        puts "Service `#{SERVICE_NAME}` paused"
+      else
+        puts 'Already paused'
+      end
+    end
+
+    def resume
+      if Service.status(SERVICE_NAME).current_state != 'running'
+        Service.resume(SERVICE_NAME)
+        while Service.status(SERVICE_NAME).current_state != 'running'
+          puts "One moment... #{Service.status(SERVICE_NAME).current_state}"
+          sleep 1
+        end
+        puts "Service `#{SERVICE_NAME}` resumed"
+      else
+        puts 'Already running'
+      end
+    end
+
+  end
+
+end
+
+

data/lib/envoi/mam/vidispine/restore/watcher.rb

@@ -0,0 +1,41 @@
+require 'envoi/mam/agent/transfer_client/aspera'
+require 'envoi/mam/agent/transfer_client/s3'
+
+require 'vidispine/api/utilities'
+
+module Envoi
+
+  module Mam
+
+    class Vidispine
+
+      class Restore
+
+        class Watcher
+
+          def initialize(args = {})
+
+          end
+
+          def run
+
+
+
+          end
+
+          def self.run(args = {})
+            watcher = self.new(args)
+            watcher.run
+            watcher
+          end
+
+        end
+
+      end
+
+      # Vidispine
+    end
+
+  end
+
+end

data/lib/envoi/utils/s3_enumerator.rb

@@ -0,0 +1,204 @@
+require 'aws-sdk-s3'
+
+class S3Enumerator
+
+  attr_reader :s3, :s3_bucket_name, :s3_object_key_prefix, :s3_bucket_region
+
+  attr_reader :objects
+
+  attr_reader :preview_only, :should_show_summary
+
+  attr_reader :limit, :max_keys
+
+  attr_reader  :objects_by_ext, :objects_by_storage_class, :total_bytes, :directories, :files, :largest_file
+
+  HUMAN_READABLE_SHORT_SUFFIX = %w(Bytes KB MB GB TB PB EB ZB YB)
+
+  def initialize(args = { })
+    aws_access_key_id = args[:aws_access_key_id]
+    aws_secret_access_key = args[:aws_secret_access_key]
+    aws_region = args[:aws_region]
+
+    aws_config = {}
+    aws_config[:credentials] = (aws_access_key_id || aws_secret_access_key) ?
+                                   Aws::Credentials.new(aws_access_key_id, aws_secret_access_key) :
+                                   Aws::SharedCredentials.new(profile_name: args[:aws_profile])
+    aws_config[:region] = aws_region if aws_region
+    Aws.config.update(aws_config) unless aws_config.empty?
+
+    @preview_only = args[:preview_only]
+    @should_show_summary = args.fetch(:show_summary, preview_only)
+
+    @limit = args[:limit]
+    @limit = @limit.to_i if @limit
+
+    @max_keys = 1000
+    @max_keys = @limit if @limit && @limit < @max_keys
+
+    @s3_bucket_name = args[:s3_bucket_name] || args[:bucket_name]
+    @s3_object_key_prefix = args[:s3_object_key_prefix] || args[:object_key_prefix] || ''
+    @s3_object_key_prefix = @s3_object_key_prefix[1..-1] if @s3_object_key_prefix.start_with?('/')
+
+    @s3 = Aws::S3::Client.new
+    @s3_bucket_region = s3.get_bucket_location(bucket: @s3_bucket_name).location_constraint
+    @s3_bucket_region = 'us-east-1' if @s3_bucket_region.empty?
+
+    @objects = []
+    @files = []
+    @ignored = []
+    @directories = []
+    @objects_by_ext = Hash.new { |h, k| h[k] = [] }
+  end
+
+  def concat_objects(objects, resp)
+    objects.concat resp.contents
+    objects
+  end
+
+  def total_object_count
+    objects.length
+  end
+
+  def retrieve_objects(&block)
+    @objects = []
+
+    resp = s3.list_objects_v2(bucket: s3_bucket_name, prefix: s3_object_key_prefix, max_keys: max_keys)
+    loop do
+      concat_objects(objects, resp)
+      puts objects.length
+      break if !resp.next_page? || (limit && total_object_count >= limit)
+      resp = resp.next_page
+    end
+    @objects = objects.first(limit) if limit
+
+    @files = []
+    @directories = []
+    @objects_by_ext = Hash.new { |h, k| h[k] = [] }
+    @objects_by_storage_class = Hash.new { |h, k| h[k] = [] }
+    #
+    @total_bytes = 0
+    @largest_file = nil
+    objects.each do |object|
+      if object.key.end_with?('/')
+        @directories << object
+      else
+        filename = File.basename(object.key)
+        should_ignore = filename.start_with?('.')
+        if should_ignore
+          @ignored << object
+        else
+          @files << object
+          @total_bytes += object.size
+          @largest_file = object unless @largest_file && @largest_file.size > object.size
+          filename_ext = File.extname(filename).downcase
+          objects_by_ext[filename_ext] << object
+          objects_by_storage_class[object.storage_class] << object
+        end
+      end
+
+      yield object, self if block_given?
+    end
+  end
+
+
+  def print_table(data, options = { })
+    first_row = data.first
+    table = first_row.is_a?(Hash) ? [first_row.keys] + data.map(&:values) : data
+
+    widths = []
+    table.each do |line|
+      line.each_with_index do |col, idx|
+        cur_col_width = widths[idx]
+        if cur_col_width
+          col_len = col.to_s.length
+          widths[idx] = col_len if col_len > cur_col_width
+        else
+          widths[idx] = col.to_s.length
+        end
+      end
+    end
+
+    # header separator
+    separator_ary = widths.map { |n| '-' * n }
+    table.insert(1, separator_ary)
+    table.insert(-2, separator_ary) if options[:has_totals]
+
+    format = widths.collect { |n| "%-#{n}s"}.join(' | ')
+    table.each { |line| printf "| #{format} |\n", *line }
+  end
+
+  def run(&block)
+    retrieve_objects(&block)
+    show_summary if @should_show_summary
+  end
+
+  def human_readable_bytes_short(human_readable_number)
+    "#{human_readable_number.split(',').first} #{HUMAN_READABLE_SHORT_SUFFIX[human_readable_number.count(',')]}"
+  end
+
+  def humanize_number(number)
+    number.to_s.chars.to_a.reverse.each_slice(3).map(&:join).join(',').reverse
+  end
+
+  def grouped_data_to_table(collection, key_name)
+    table_data =  [ [ key_name, 'Count', 'Short Bytes', 'Total Bytes' ] ]
+    collection.each do |key, objects|
+      group_total_size = objects.reduce(0) { |s, o| s + o.size }
+      human_readable_group_total_size = humanize_number(group_total_size)
+      human_readable_group_total_size_short = human_readable_bytes_short(human_readable_group_total_size)
+      row_values = [ key, humanize_number(objects.length), human_readable_group_total_size_short, human_readable_group_total_size ]
+      table_data << row_values
+    end
+    table_data
+  end
+
+  def show_summary
+    objects_by_ext_table_data = grouped_data_to_table(objects_by_ext, 'File Ext')
+    objects_by_storage_class_table_data = grouped_data_to_table(objects_by_storage_class, 'Storage Class')
+
+    human_readable_total_bytes = humanize_number(total_bytes)
+    human_readable_total_bytes_short = human_readable_bytes_short(human_readable_total_bytes)
+    row_data = [ 'TOTAL', humanize_number(objects.length), human_readable_total_bytes_short,  human_readable_total_bytes]
+    objects_by_ext_table_data << row_data
+    objects_by_storage_class_table_data << row_data
+
+    puts "\n\n--- Summary ---"
+    puts "Bucket Name: #{s3_bucket_name}"
+    puts "Bucket Region: #{s3_bucket_region}"
+    puts "Object Key Prefix: #{s3_object_key_prefix}"
+    puts "Total Objects: #{humanize_number(total_object_count)}"
+    puts "Total Directories: #{humanize_number(directories.length)}"
+    puts "Total Files: #{humanize_number(files.length)}"
+    puts "Total Size (in bytes): #{human_readable_total_bytes} (#{human_readable_total_bytes_short})"
+    puts "Largest File: #{largest_file}"
+    puts "\n"
+    print_table(objects_by_ext_table_data, { has_totals: true })
+    puts "\n"
+    print_table(objects_by_storage_class_table_data, { has_totals: true })
+  end
+
+  def process_files(args = { })
+    process_objects(files, { total_bytes: total_bytes })
+  end
+
+  def process_objects(objects, args = {})
+    return objects unless block_given?
+    return objects.map { |object| yield object, self } if args[:quiet]
+
+    bytes_remaining = args[:total_bytes] || objects.reduce(0) { |s, o| s + o.size }
+    _total_object_count = objects.length
+    counter = 0
+    objects.each do |object|
+      counter += 1
+      human_readable_bytes_remaining = humanize_number(bytes_remaining)
+      human_readable_bytes_remaining_short = human_readable_bytes_short(human_readable_bytes_remaining)
+      human_readable_object_bytes = humanize_number(object.size)
+      human_readable_object_bytes_short = human_readable_bytes_short(human_readable_object_bytes)
+      puts "Processing #{humanize_number(counter)} of #{humanize_number(_total_object_count)} #{human_readable_object_bytes} (#{human_readable_object_bytes_short}) of #{human_readable_bytes_remaining} (#{human_readable_bytes_remaining_short})  #{object.key}"
+      yield object, self if block_given?
+      bytes_remaining -= object.size
+    end
+  end
+
+end
+

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: cantemo-portal-agent
 version: !ruby/object:Gem::Version
-  version: 1.3.3
+  version: 1.3.4
 platform: ruby
 authors:
 - John Whitson
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2019-01-25 00:00:00.000000000 Z
+date: 2019-01-31 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: asperalm
@@ -138,11 +138,11 @@ extra_rdoc_files: []
 files:
 - exe/cantemo-portal-agent
 - lib/cantemo/portal/agent.rb
-- lib/cantemo/portal/agent/WatchFolderUtility/foreman.rb
 - lib/cantemo/portal/agent/cli/commands/watch_folder.rb
 - lib/cantemo/portal/agent/cli/commands/watch_folders-working.rb
 - lib/cantemo/portal/agent/cli/commands/watch_folders.rb
 - lib/cantemo/portal/agent/version.rb
+- lib/cantemo/portal/agent/watch_folder_utility/foreman.rb
 - lib/cantemo/portal/api/client.rb
 - lib/envoi/aspera/watch_service/client.rb
 - lib/envoi/aspera/watch_service/snapshot.rb
@@ -155,16 +155,21 @@ files:
 - lib/envoi/mam/agent/cli/commands/cantemo-agent.rb
 - lib/envoi/mam/agent/cli/commands/cantemo-watch_folders.rb
 - lib/envoi/mam/agent/cli/commands/cantemo.rb
+- lib/envoi/mam/agent/cli/commands/envoi.rb
 - lib/envoi/mam/agent/cli/commands/iconik.rb
 - lib/envoi/mam/agent/cli/commands/mediasilo.rb
+- lib/envoi/mam/agent/cli/commands/s3-to-eis.rb
 - lib/envoi/mam/agent/cli/commands/vdms.rb
+- lib/envoi/mam/agent/cli/commands/vidispine-agent.rb
+- lib/envoi/mam/agent/cli/commands/vidispine-restore.rb
 - lib/envoi/mam/agent/cli/commands/vidispine.rb
 - lib/envoi/mam/agent/cli/commands/wiredrive.rb
 - lib/envoi/mam/agent/client/iconik.rb
 - lib/envoi/mam/agent/client/mediasilo.rb
 - lib/envoi/mam/agent/client/vidispine.rb
 - lib/envoi/mam/agent/config_service_client.rb
-- lib/envoi/mam/agent/daemon/service_wrapper.rb
+- lib/envoi/mam/agent/daemon/windows_service.rb
+- lib/envoi/mam/agent/daemon/windows_service_controller.rb
 - lib/envoi/mam/agent/generic.rb
 - lib/envoi/mam/agent/notifier.rb
 - lib/envoi/mam/agent/notifier/sns.rb
@@ -183,7 +188,9 @@ files:
 - lib/envoi/mam/mediasilo/agent.rb
 - lib/envoi/mam/vdms/agent.rb
 - lib/envoi/mam/vidispine/agent.rb
+- lib/envoi/mam/vidispine/restore/watcher.rb
 - lib/envoi/mam/wiredrive/agent.rb
+- lib/envoi/utils/s3_enumerator.rb
 - lib/envoi/watch_folder_utility/watch_folder.rb
 - lib/envoi/watch_folder_utility/watch_folder/handler/listen.rb
 homepage: http://www.github.com/XPlatform-Consulting/envoi-mam-agent