cantemo-portal-agent 1.3.3 → 1.3.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 7c2c43f547f0f607e087665ce93f3a5c63dd4d5a
4
- data.tar.gz: 96c2cc23d5b30fa48abbc3d329e01287782b343e
3
+ metadata.gz: 657db4631f5e476906091f6126709406cff78820
4
+ data.tar.gz: cfdcda72a46b18b78cd626db096d2af1f3559830
5
5
  SHA512:
6
- metadata.gz: 16d67112efc3e3748edbb090c97e86c5874945e18c17bedd34184ac0b9709f5771bccd7fc40b1f6c2ec382e17dc551c041c3ab33d901e6d809632ef183c9a60f
7
- data.tar.gz: 8d93bcc5a774aba3a369e3dbe77c09a9c4dfe2b05d71572bfa0c5e387be9f4220c26f3beac924771e5e7541be06fb85123530fb2d9e1254cf22376f320542c28
6
+ metadata.gz: 3f94323823c25998c23f7823c3c105f569bc01b9363d267dcd107d4e999c8000c9878abcb8bd2ea0b40a250f85b2e7fcbd3bc323f156e088e54677a66f98cd1a
7
+ data.tar.gz: 863be982724502f6f7fd69d16537301c07ec6196e45d1fac3e3816f2ff1726f4260a68a20d1336dd588de6d425a9efbd653234dbf15a7964619f8daa161b90c1
@@ -296,7 +296,7 @@ SERVICE_DISPLAYNAME = 'Cantemo Portal Watch Folder Agent'
296
296
 
297
297
  # You must provide at least one argument.
298
298
  abort('No argument provided.') unless sub_command
299
- abort('Service not installed.') unless Service.exists?(SERVICE_NAME) || sub_command == 'install'
299
+ abort('Service is not installed.') unless Service.exists?(SERVICE_NAME) || sub_command == 'install'
300
300
 
301
301
 
302
302
  case sub_command
@@ -352,7 +352,7 @@ when 'status'
352
352
  when 'install'
353
353
  # Quote the full path to deal with possible spaces in the path name.
354
354
  ruby = File.join(CONFIG['bindir'], CONFIG['ruby_install_name']).tr('/', '\\')
355
- path = __FILE__.tr('/', '\\')
355
+ path = File.join(CONFIG['bindir'], 'cantemo-portal-agent')
356
356
  # cmd = %Q("#{ruby}" "#{path}" runasservice --config-file-path "#{config_file_path}")
357
357
  cmd = %Q("#{ruby}" "#{path}" run-as-service)
358
358
 
@@ -1,7 +1,7 @@
1
1
  module Cantemo
2
2
  module Portal
3
3
  class Agent
4
- VERSION = '1.3.3'.freeze
4
+ VERSION = '1.3.4'.freeze
5
5
  end
6
6
  end
7
7
  end
@@ -23,33 +23,43 @@
23
23
  # # Envoi
24
24
  # end
25
25
 
26
- def usage
27
- command_list_str = ''
28
- Dir.glob(File.join(@commands_dir, "*#{@command_ext}")).each do |fn|
29
- command_list_str += "\n\t - #{File.basename(fn, @command_ext)}"
26
+ @commands_dir = File.join(File.dirname(__FILE__), 'commands')
27
+ @command_ext = '.rb'
28
+
29
+ @command_list_str = ''
30
+ @commands = [ ]
31
+ @command_aliases = { }
32
+
33
+ def build_command_aliases(commands_dir = @commands_dir)
34
+ Dir.glob(File.join(commands_dir, "*.rb")).each do |fn|
35
+ command_file_name = File.basename(fn)
36
+ command_name = File.basename(command_file_name, '.*')
37
+ command_alias = command.tr('-_', '')
38
+ @commands << command
39
+ @command_aliases[command_alias] = command_name
40
+ @command_list_str += "\n\t - #{command}"
30
41
  end
42
+ end
43
+
44
+ def usage
31
45
 
32
46
  <<-EOT
33
47
  Usage: #{File.basename($0)} COMMAND [ARGS]
34
48
 
35
49
  Available Commands:
36
- #{command_list_str}
50
+ #{@command_list_str}
37
51
 
38
52
  All commands can be run with -h (or --help) for more information.
39
53
  EOT
40
54
  end
41
55
 
42
- @commands_dir = File.join(File.dirname(__FILE__), 'commands')
43
- @command_ext = '.rb'
44
-
45
- command_aliases = {
46
- }
47
56
 
57
+ build_command_aliases
48
58
  if ARGV.empty?
49
59
  should_show_usage = true
50
60
  else
51
- command_name = ARGV[0].clone
52
- command_name = command_aliases[command_name] || command_name
61
+ command_name = ARGV.first.dup
62
+ command_name = @command_aliases[command_name] || command_name
53
63
  should_show_usage = %w(-h --help).include?(command_name.downcase)
54
64
  end
55
65
  if should_show_usage
@@ -0,0 +1,46 @@
1
+ #! /usr/bin/env ruby
2
+ # Installation
3
+ # `gem install aws-sdk-s3`
4
+ #
5
+ require 'net/https'
6
+ require 'json'
7
+ require 'optparse'
8
+ require 'pp'
9
+ require 'csv'
10
+
11
+ ARGV << '--help' if ARGV.empty?
12
+ args = {
13
+ show_summary: true,
14
+ eis_url: 'https://ksbw8ud7v6.execute-api.us-east-1.amazonaws.com/stage/createmcjob',
15
+ s3_object_key_prefix: '',
16
+ preview_only: false
17
+ }
18
+ op = OptionParser.new
19
+ op.on('--bucket-name [BUCKETNAME]', 'The bucket to get objects from.') { |v| args[:s3_bucket_name] = v }
20
+ op.on('--key-prefix [KEYPREFIX]', 'The object key prefix to limit the results by.', 'default: /') { |v| args[:s3_object_key_prefix] = v }
21
+ op.on('--aws-access-key [KEY]', '') { |v| args[:aws_access_key_id] = v }
22
+ op.on('--aws-secret-key [KEY]', '') { |v| args[:aws_secret_access_key] = v }
23
+ op.on('--aws-profile [PROFILENAME]', '') { |v| args[:aws_profile] = v }
24
+ op.on('--aws-region [REGION]', 'The AWS region what the bucket is located.', 'default: auto-detected') { |v| args[:aws_region] = v }
25
+ op.on('--limit [LIMIT]', 'Limit the number of results.', 'default: unlimited') { |v| args[:limit] = v }
26
+
27
+ op.on('--[no-]preview', 'Output paths with summary and stop.',
28
+ "default: #{args[:preview_mode] ? 'true' : 'false'}") { |v| args[:preview_only] = v }
29
+ op.on('--[no-]summary', 'Output the paths and calculate the number of assets.',
30
+ "default: #{args[:show_summary] ? 'true' : 'false'}",) { |v| args[:show_summary] = v }
31
+
32
+ op.on('-h', '--help', 'Print help (this message) and exit') { puts op; exit }
33
+ op.parse!
34
+
35
+
36
+
37
+
38
+ s3_enum = S3Enumerator.new(args)
39
+ s3_enum.run
40
+ exit if args[:preview_only]
41
+ s3_enum.process_files do |object|
42
+
43
+ end
44
+
45
+
46
+
@@ -0,0 +1,60 @@
1
+ #!/usr/bin/env ruby
2
+ lib_path = __FILE__ == '(irb)' ? File.join(Dir.cwd, 'lib') : File.expand_path('../../../lib', __FILE__)
3
+ $:.unshift(lib_path) unless $:.include?(lib_path) or !File.exists?(lib_path)
4
+ require 'rubygems'
5
+ require 'optparse'
6
+ require 'open3'
7
+
8
+ require 'envoi/mam/agent/cli'
9
+ require 'envoi/mam/vidispine/agent'
10
+
11
+ default_config_file_paths = Envoi::Mam::Agent::CLI::CONFIG_FILE_PATHS
12
+
13
+ aspera_ascp_paths = Envoi::Mam::Agent::TransferClient::Aspera::ASCP_PATHS
14
+ default_aspera_ascp_path = aspera_ascp_paths.find { |v| File.exist? v } || aspera_ascp_paths.first
15
+
16
+ current_command = ARGV.shift
17
+ ARGV << '--help' if ARGV.empty?
18
+
19
+ args = {
20
+ :config_file_path => default_config_file_paths,
21
+ :default_ascp_path => default_aspera_ascp_path,
22
+ :dry_run => false,
23
+ :operation => :download,
24
+ :preserve_path => true,
25
+ :transfer_type => '',
26
+ }
27
+
28
+ args[:config_service_app_id] = ENV['ENVOI_MAM_AGENT_APP_ID']
29
+ args[:config_service_app_token] = ENV['ENVOI_MAM_AGENT_APP_TOKEN']
30
+ args[:config_service_app_url] = ENV['ENVOI_MAM_AGENT_APP_URL']
31
+
32
+ op = OptionParser.new
33
+ op.on('-c', '--config-file-path FILEPATH', 'The path to the configuration file.',
34
+ "Default Path(s): #{args[:config_file_path]}") { |v| args[:config_file_path] = v }
35
+ op.on('--agent-app-id ID', '') { |v| args[:config_service_app_id] = v }
36
+ op.on('--agent-token TOKEN', '') { |v| args[:config_service_app_token] = v }
37
+ op.on('--agent-service-api URL', '') { |v| args[:config_service_api_url] = v }
38
+
39
+ op.on('-i', '--item-id ITEMID', 'The Vidispine item id.') { |v| args[:item_id] = v }
40
+ op.on('-s', '--shape-id SHAPEID', 'The Vidispine shape id.') { |v| args[:shape_id] = v }
41
+ op.on('-t', '--shape-tag SHAPETAG', 'The Vidispine shape tag used to find the shape when a shape id is not supplied.') { |v| args[:shape_tag] = v }
42
+ op.on('--storage-id STORAGEID', 'The Vidispine storage id to use for the transfer.') { |v| args[:storage_id] = v}
43
+
44
+ op.on('-d', '--destination-path DIRPATH', 'The destination path to download or upload to.',
45
+ "default: '.'") { |v| args[:destination_path] = v }
46
+ op.on('-p','--[no-]preserve-path', "default: true") { |v| args[:preserve_path] = v }
47
+ op.on('-p', '--operation OP', [ :download, :upload ], 'Transfer Direction.',
48
+ "default: #{args[:operation]}" ) { |v| args[:operation] = v }
49
+ op.on('--transfer-type TYPE', [ :aspera, :s3 ],'Force the type of transfer to use. Ex: aspera || s3') { |v| args[:transfer_type] = v }
50
+ op.on('--transfer-token TOKEN', '') { |v| args[:transfer_token] = v }
51
+ op.on('--dry-run', 'Run without executing the transfer.') { |v| args[:dry_run] = v }
52
+ op.on('--help', 'Show this message.') { puts op; exit }
53
+ op.load
54
+ op.parse!
55
+
56
+ args[:file_path] ||= ARGV.shift
57
+ args[:destination_path] ||= ARGV.shift unless ARGV.empty?
58
+
59
+ agent = Envoi::Mam::Vidispine::Agent.load_config_and_init(args)
60
+ agent.run_operation
@@ -0,0 +1,61 @@
1
+ #!/usr/bin/env ruby
2
+ lib_path = __FILE__ == '(irb)' ? File.join(Dir.cwd, 'lib') : File.expand_path('../../../lib', __FILE__)
3
+ $:.unshift(lib_path) unless $:.include?(lib_path) or !File.exists?(lib_path)
4
+ require 'rubygems'
5
+ require 'optparse'
6
+ require 'open3'
7
+
8
+ require 'envoi/mam/agent/cli'
9
+ require 'envoi/mam/vidispine/agent'
10
+ require 'envoi/mam/vidispine/restore/watcher'
11
+
12
+ default_config_file_paths = Envoi::Mam::Agent::CLI::CONFIG_FILE_PATHS
13
+
14
+ aspera_ascp_paths = Envoi::Mam::Agent::TransferClient::Aspera::ASCP_PATHS
15
+ default_aspera_ascp_path = aspera_ascp_paths.find { |v| File.exist? v } || aspera_ascp_paths.first
16
+
17
+ current_command = ARGV.shift
18
+ ARGV << '--help' if ARGV.empty?
19
+
20
+ args = {
21
+ :config_file_path => default_config_file_paths,
22
+ :default_ascp_path => default_aspera_ascp_path,
23
+ :dry_run => false,
24
+ :operation => :download,
25
+ :preserve_path => true,
26
+ :transfer_type => '',
27
+ }
28
+
29
+ args[:config_service_app_id] = ENV['ENVOI_MAM_AGENT_APP_ID']
30
+ args[:config_service_app_token] = ENV['ENVOI_MAM_AGENT_APP_TOKEN']
31
+ args[:config_service_app_url] = ENV['ENVOI_MAM_AGENT_APP_URL']
32
+
33
+ op = OptionParser.new
34
+ op.on('-c', '--config-file-path FILEPATH', 'The path to the configuration file.',
35
+ "Default Path(s): #{args[:config_file_path]}") { |v| args[:config_file_path] = v }
36
+ op.on('--agent-app-id ID', '') { |v| args[:config_service_app_id] = v }
37
+ op.on('--agent-token TOKEN', '') { |v| args[:config_service_app_token] = v }
38
+ op.on('--agent-service-api URL', '') { |v| args[:config_service_api_url] = v }
39
+
40
+ op.on('-i', '--item-id ITEMID', 'The Vidispine item id.') { |v| args[:item_id] = v }
41
+ op.on('-s', '--shape-id SHAPEID', 'The Vidispine shape id.') { |v| args[:shape_id] = v }
42
+ op.on('-t', '--shape-tag SHAPETAG', 'The Vidispine shape tag used to find the shape when a shape id is not supplied.') { |v| args[:shape_tag] = v }
43
+ op.on('--storage-id STORAGEID', 'The Vidispine storage id to use for the transfer.') { |v| args[:storage_id] = v}
44
+
45
+ op.on('-d', '--destination-path DIRPATH', 'The destination path to download or upload to.',
46
+ "default: '.'") { |v| args[:destination_path] = v }
47
+ op.on('-p','--[no-]preserve-path', "default: true") { |v| args[:preserve_path] = v }
48
+ op.on('-p', '--operation OP', [ :download, :upload ], 'Transfer Direction.',
49
+ "default: #{args[:operation]}" ) { |v| args[:operation] = v }
50
+ op.on('--transfer-type TYPE', [ :aspera, :s3 ],'Force the type of transfer to use. Ex: aspera || s3') { |v| args[:transfer_type] = v }
51
+ op.on('--transfer-token TOKEN', '') { |v| args[:transfer_token] = v }
52
+ op.on('--dry-run', 'Run without executing the transfer.') { |v| args[:dry_run] = v }
53
+ op.on('--help', 'Show this message.') { puts op; exit }
54
+ op.load
55
+ op.parse!
56
+
57
+ # args[:file_path] ||= ARGV.shift
58
+ # args[:destination_path] ||= ARGV.shift unless ARGV.empty?
59
+ #
60
+ # agent = Envoi::Mam::Vidispine::Restore::Watcher.load_config_and_init(args)
61
+ # agent.run_operation
@@ -1,60 +1,5 @@
1
- #!/usr/bin/env ruby
2
- lib_path = __FILE__ == '(irb)' ? File.join(Dir.cwd, 'lib') : File.expand_path('../../../lib', __FILE__)
3
- $:.unshift(lib_path) unless $:.include?(lib_path) or !File.exists?(lib_path)
4
- require 'rubygems'
5
- require 'optparse'
6
- require 'open3'
7
-
8
- require 'envoi/mam/agent/cli'
9
- require 'envoi/mam/vidispine/agent'
10
-
11
- default_config_file_paths = Envoi::Mam::Agent::CLI::CONFIG_FILE_PATHS
12
-
13
- aspera_ascp_paths = Envoi::Mam::Agent::TransferClient::Aspera::ASCP_PATHS
14
- default_aspera_ascp_path = aspera_ascp_paths.find { |v| File.exist? v } || aspera_ascp_paths.first
15
-
16
- current_command = ARGV.shift
17
- ARGV << '--help' if ARGV.empty?
18
-
19
- args = {
20
- :config_file_path => default_config_file_paths,
21
- :default_ascp_path => default_aspera_ascp_path,
22
- :dry_run => false,
23
- :operation => :download,
24
- :preserve_path => true,
25
- :transfer_type => '',
26
- }
27
-
28
- args[:config_service_app_id] = ENV['ENVOI_MAM_AGENT_APP_ID']
29
- args[:config_service_app_token] = ENV['ENVOI_MAM_AGENT_APP_TOKEN']
30
- args[:config_service_app_url] = ENV['ENVOI_MAM_AGENT_APP_URL']
31
-
32
- op = OptionParser.new
33
- op.on('-c', '--config-file-path FILEPATH', 'The path to the configuration file.',
34
- "Default Path(s): #{args[:config_file_path]}") { |v| args[:config_file_path] = v }
35
- op.on('--agent-app-id ID', '') { |v| args[:config_service_app_id] = v }
36
- op.on('--agent-token TOKEN', '') { |v| args[:config_service_app_token] = v }
37
- op.on('--agent-service-api URL', '') { |v| args[:config_service_api_url] = v }
38
-
39
- op.on('-i', '--item-id ITEMID', 'The Vidispine item id.') { |v| args[:item_id] = v }
40
- op.on('-s', '--shape-id SHAPEID', 'The Vidispine shape id.') { |v| args[:shape_id] = v }
41
- op.on('-t', '--shape-tag SHAPETAG', 'The Vidispine shape tag used to find the shape when a shape id is not supplied.') { |v| args[:shape_tag] = v }
42
- op.on('--storage-id STORAGEID', 'The Vidispine storage id to use for the transfer.') { |v| args[:storage_id] = v}
43
-
44
- op.on('-d', '--destination-path DIRPATH', 'The destination path to download or upload to.',
45
- "default: '.'") { |v| args[:destination_path] = v }
46
- op.on('-p','--[no-]preserve-path', "default: true") { |v| args[:preserve_path] = v }
47
- op.on('-p', '--operation OP', [ :download, :upload ], 'Transfer Direction.',
48
- "default: #{args[:operation]}" ) { |v| args[:operation] = v }
49
- op.on('--transfer-type TYPE', [ :aspera, :s3 ],'Force the type of transfer to use. Ex: aspera || s3') { |v| args[:transfer_type] = v }
50
- op.on('--transfer-token TOKEN', '') { |v| args[:transfer_token] = v }
51
- op.on('--dry-run', 'Run without executing the transfer.') { |v| args[:dry_run] = v }
52
- op.on('--help', 'Show this message.') { puts op; exit }
53
- op.load
54
- op.parse!
55
-
56
- args[:file_path] ||= ARGV.shift
57
- args[:destination_path] ||= ARGV.shift unless ARGV.empty?
58
-
59
- agent = Envoi::Mam::Vidispine::Agent.load_config_and_init(args)
60
- agent.run_operation
1
+ if ARGV.map { |v| v.downcase }.include?('restore')
2
+ require 'envoi/mam/agent/cli/commands/vidispine-restore'
3
+ else
4
+ require 'envoi/mam/agent/cli/commands/vidispine-agent'
5
+ end
@@ -6,7 +6,7 @@ module Envoi
6
6
 
7
7
  class Daemon
8
8
 
9
- class ServiceWrapper
9
+ class WindowsService
10
10
 
11
11
  end
12
12
 
@@ -0,0 +1,134 @@
1
+ # Options:
2
+ # install - Installs the service.
3
+ # start - Starts the service.
4
+ # stop - Stops the service.
5
+ # pause - Pauses the service.
6
+ # resume - Resumes the service.
7
+ # uninstall - Uninstalls the service.
8
+ # delete - Same as uninstall.
9
+ require 'win32/service'
10
+ require 'rbconfig'
11
+
12
+
13
+
14
+ class WindowsServiceController
15
+ include RbConfig
16
+ include Win32
17
+
18
+ SERVICE_NAME = 'cantemo_portal_watch_folder_agent'
19
+ SERVICE_DISPLAYNAME = 'Cantemo Portal Watch Folder Agent'
20
+
21
+ class << self
22
+
23
+
24
+ def run(command = nil, args = { })
25
+ # You must provide at least one argument.
26
+ raise ArgumentError, 'No argument provided.' unless command
27
+ unless Service.exists?(SERVICE_NAME) || command == 'install'
28
+ puts 'Service is not installed.'
29
+ return
30
+ end
31
+
32
+ case command
33
+ when 'start'; self.start
34
+ when 'stop'; self.stop
35
+ when 'pause'; self.pause
36
+ when 'resume'; self.resume
37
+ when 'status'; self.status
38
+ when 'install'; self.install(args)
39
+ when 'uninstall', 'delete'; self.uninstall
40
+ else
41
+ raise ArgumentError, 'unknown option: ' + command
42
+ end
43
+ end
44
+
45
+ def install(args = {})
46
+ # Quote the full path to deal with possible spaces in the path name.
47
+ ruby = File.join(CONFIG['bindir'], CONFIG['ruby_install_name']).tr('/', '\\')
48
+ path = __FILE__.tr('/', '\\')
49
+ cmd = %Q("#{ruby}" "#{path}")
50
+
51
+ install_args = {
52
+ :service_name => SERVICE_NAME,
53
+ :display_name => SERVICE_DISPLAYNAME,
54
+ :binary_path_name => cmd
55
+ }.merge!(args)
56
+
57
+ Service.new(install_args)
58
+ puts "Service `#{SERVICE_NAME}` installed"
59
+ end
60
+
61
+ def uninstall
62
+ if Service.status(SERVICE_NAME).current_state != 'stopped'
63
+ Service.stop(SERVICE_NAME)
64
+ end
65
+ while Service.status(SERVICE_NAME).current_state != 'stopped'
66
+ puts "One moment... #{Service.status(SERVICE_NAME).current_state}"
67
+ sleep 1
68
+ end
69
+ Service.delete(SERVICE_NAME)
70
+ puts "Service #{SERVICE_NAME} deleted"
71
+ end
72
+
73
+ def start
74
+ if Service.status(SERVICE_NAME).current_state != 'running'
75
+ # Service.start(SERVICE_NAME, nil, args)
76
+ Service.start(SERVICE_NAME)
77
+ while Service.status(SERVICE_NAME).current_state != 'running'
78
+ puts "One moment... #{Service.status(SERVICE_NAME).current_state}"
79
+ sleep 1
80
+ end
81
+ puts "Service `#{SERVICE_NAME}` started"
82
+ else
83
+ puts 'Already running'
84
+ end
85
+ end
86
+
87
+ def status
88
+ puts "Service `#{SERVICE_NAME}` #{Service.status(SERVICE_NAME).current_state}"
89
+ end
90
+
91
+ def stop
92
+ if Service.status(SERVICE_NAME).current_state != 'stopped'
93
+ Service.stop(SERVICE_NAME)
94
+ while Service.status(SERVICE_NAME).current_state != 'stopped'
95
+ puts "One moment... #{Service.status(SERVICE_NAME).current_state}"
96
+ sleep 1
97
+ end
98
+ puts "Service `#{SERVICE_NAME}` stopped"
99
+ else
100
+ puts 'Already stopped'
101
+ end
102
+ end
103
+
104
+ def pause
105
+ if Service.status(SERVICE_NAME).current_state != 'paused'
106
+ Service.pause(SERVICE_NAME)
107
+ while Service.status(SERVICE_NAME).current_state != 'paused'
108
+ puts "One moment... #{Service.status(SERVICE_NAME).current_state}"
109
+ sleep 1
110
+ end
111
+ puts "Service `#{SERVICE_NAME}` paused"
112
+ else
113
+ puts 'Already paused'
114
+ end
115
+ end
116
+
117
+ def resume
118
+ if Service.status(SERVICE_NAME).current_state != 'running'
119
+ Service.resume(SERVICE_NAME)
120
+ while Service.status(SERVICE_NAME).current_state != 'running'
121
+ puts "One moment... #{Service.status(SERVICE_NAME).current_state}"
122
+ sleep 1
123
+ end
124
+ puts "Service `#{SERVICE_NAME}` resumed"
125
+ else
126
+ puts 'Already running'
127
+ end
128
+ end
129
+
130
+ end
131
+
132
+ end
133
+
134
+
@@ -0,0 +1,41 @@
1
+ require 'envoi/mam/agent/transfer_client/aspera'
2
+ require 'envoi/mam/agent/transfer_client/s3'
3
+
4
+ require 'vidispine/api/utilities'
5
+
6
+ module Envoi
7
+
8
+ module Mam
9
+
10
+ class Vidispine
11
+
12
+ class Restore
13
+
14
+ class Watcher
15
+
16
+ def initialize(args = {})
17
+
18
+ end
19
+
20
+ def run
21
+
22
+
23
+
24
+ end
25
+
26
+ def self.run(args = {})
27
+ watcher = self.new(args)
28
+ watcher.run
29
+ watcher
30
+ end
31
+
32
+ end
33
+
34
+ end
35
+
36
+ # Vidispine
37
+ end
38
+
39
+ end
40
+
41
+ end
@@ -0,0 +1,204 @@
1
+ require 'aws-sdk-s3'
2
+
3
+ class S3Enumerator
4
+
5
+ attr_reader :s3, :s3_bucket_name, :s3_object_key_prefix, :s3_bucket_region
6
+
7
+ attr_reader :objects
8
+
9
+ attr_reader :preview_only, :should_show_summary
10
+
11
+ attr_reader :limit, :max_keys
12
+
13
+ attr_reader :objects_by_ext, :objects_by_storage_class, :total_bytes, :directories, :files, :largest_file
14
+
15
+ HUMAN_READABLE_SHORT_SUFFIX = %w(Bytes KB MB GB TB PB EB ZB YB)
16
+
17
+ def initialize(args = { })
18
+ aws_access_key_id = args[:aws_access_key_id]
19
+ aws_secret_access_key = args[:aws_secret_access_key]
20
+ aws_region = args[:aws_region]
21
+
22
+ aws_config = {}
23
+ aws_config[:credentials] = (aws_access_key_id || aws_secret_access_key) ?
24
+ Aws::Credentials.new(aws_access_key_id, aws_secret_access_key) :
25
+ Aws::SharedCredentials.new(profile_name: args[:aws_profile])
26
+ aws_config[:region] = aws_region if aws_region
27
+ Aws.config.update(aws_config) unless aws_config.empty?
28
+
29
+ @preview_only = args[:preview_only]
30
+ @should_show_summary = args.fetch(:show_summary, preview_only)
31
+
32
+ @limit = args[:limit]
33
+ @limit = @limit.to_i if @limit
34
+
35
+ @max_keys = 1000
36
+ @max_keys = @limit if @limit && @limit < @max_keys
37
+
38
+ @s3_bucket_name = args[:s3_bucket_name] || args[:bucket_name]
39
+ @s3_object_key_prefix = args[:s3_object_key_prefix] || args[:object_key_prefix] || ''
40
+ @s3_object_key_prefix = @s3_object_key_prefix[1..-1] if @s3_object_key_prefix.start_with?('/')
41
+
42
+ @s3 = Aws::S3::Client.new
43
+ @s3_bucket_region = s3.get_bucket_location(bucket: @s3_bucket_name).location_constraint
44
+ @s3_bucket_region = 'us-east-1' if @s3_bucket_region.empty?
45
+
46
+ @objects = []
47
+ @files = []
48
+ @ignored = []
49
+ @directories = []
50
+ @objects_by_ext = Hash.new { |h, k| h[k] = [] }
51
+ end
52
+
53
+ def concat_objects(objects, resp)
54
+ objects.concat resp.contents
55
+ objects
56
+ end
57
+
58
+ def total_object_count
59
+ objects.length
60
+ end
61
+
62
+ def retrieve_objects(&block)
63
+ @objects = []
64
+
65
+ resp = s3.list_objects_v2(bucket: s3_bucket_name, prefix: s3_object_key_prefix, max_keys: max_keys)
66
+ loop do
67
+ concat_objects(objects, resp)
68
+ puts objects.length
69
+ break if !resp.next_page? || (limit && total_object_count >= limit)
70
+ resp = resp.next_page
71
+ end
72
+ @objects = objects.first(limit) if limit
73
+
74
+ @files = []
75
+ @directories = []
76
+ @objects_by_ext = Hash.new { |h, k| h[k] = [] }
77
+ @objects_by_storage_class = Hash.new { |h, k| h[k] = [] }
78
+ #
79
+ @total_bytes = 0
80
+ @largest_file = nil
81
+ objects.each do |object|
82
+ if object.key.end_with?('/')
83
+ @directories << object
84
+ else
85
+ filename = File.basename(object.key)
86
+ should_ignore = filename.start_with?('.')
87
+ if should_ignore
88
+ @ignored << object
89
+ else
90
+ @files << object
91
+ @total_bytes += object.size
92
+ @largest_file = object unless @largest_file && @largest_file.size > object.size
93
+ filename_ext = File.extname(filename).downcase
94
+ objects_by_ext[filename_ext] << object
95
+ objects_by_storage_class[object.storage_class] << object
96
+ end
97
+ end
98
+
99
+ yield object, self if block_given?
100
+ end
101
+ end
102
+
103
+
104
+ def print_table(data, options = { })
105
+ first_row = data.first
106
+ table = first_row.is_a?(Hash) ? [first_row.keys] + data.map(&:values) : data
107
+
108
+ widths = []
109
+ table.each do |line|
110
+ line.each_with_index do |col, idx|
111
+ cur_col_width = widths[idx]
112
+ if cur_col_width
113
+ col_len = col.to_s.length
114
+ widths[idx] = col_len if col_len > cur_col_width
115
+ else
116
+ widths[idx] = col.to_s.length
117
+ end
118
+ end
119
+ end
120
+
121
+ # header separator
122
+ separator_ary = widths.map { |n| '-' * n }
123
+ table.insert(1, separator_ary)
124
+ table.insert(-2, separator_ary) if options[:has_totals]
125
+
126
+ format = widths.collect { |n| "%-#{n}s"}.join(' | ')
127
+ table.each { |line| printf "| #{format} |\n", *line }
128
+ end
129
+
130
+ def run(&block)
131
+ retrieve_objects(&block)
132
+ show_summary if @should_show_summary
133
+ end
134
+
135
+ def human_readable_bytes_short(human_readable_number)
136
+ "#{human_readable_number.split(',').first} #{HUMAN_READABLE_SHORT_SUFFIX[human_readable_number.count(',')]}"
137
+ end
138
+
139
+ def humanize_number(number)
140
+ number.to_s.chars.to_a.reverse.each_slice(3).map(&:join).join(',').reverse
141
+ end
142
+
143
+ def grouped_data_to_table(collection, key_name)
144
+ table_data = [ [ key_name, 'Count', 'Short Bytes', 'Total Bytes' ] ]
145
+ collection.each do |key, objects|
146
+ group_total_size = objects.reduce(0) { |s, o| s + o.size }
147
+ human_readable_group_total_size = humanize_number(group_total_size)
148
+ human_readable_group_total_size_short = human_readable_bytes_short(human_readable_group_total_size)
149
+ row_values = [ key, humanize_number(objects.length), human_readable_group_total_size_short, human_readable_group_total_size ]
150
+ table_data << row_values
151
+ end
152
+ table_data
153
+ end
154
+
155
+ def show_summary
156
+ objects_by_ext_table_data = grouped_data_to_table(objects_by_ext, 'File Ext')
157
+ objects_by_storage_class_table_data = grouped_data_to_table(objects_by_storage_class, 'Storage Class')
158
+
159
+ human_readable_total_bytes = humanize_number(total_bytes)
160
+ human_readable_total_bytes_short = human_readable_bytes_short(human_readable_total_bytes)
161
+ row_data = [ 'TOTAL', humanize_number(objects.length), human_readable_total_bytes_short, human_readable_total_bytes]
162
+ objects_by_ext_table_data << row_data
163
+ objects_by_storage_class_table_data << row_data
164
+
165
+ puts "\n\n--- Summary ---"
166
+ puts "Bucket Name: #{s3_bucket_name}"
167
+ puts "Bucket Region: #{s3_bucket_region}"
168
+ puts "Object Key Prefix: #{s3_object_key_prefix}"
169
+ puts "Total Objects: #{humanize_number(total_object_count)}"
170
+ puts "Total Directories: #{humanize_number(directories.length)}"
171
+ puts "Total Files: #{humanize_number(files.length)}"
172
+ puts "Total Size (in bytes): #{human_readable_total_bytes} (#{human_readable_total_bytes_short})"
173
+ puts "Largest File: #{largest_file}"
174
+ puts "\n"
175
+ print_table(objects_by_ext_table_data, { has_totals: true })
176
+ puts "\n"
177
+ print_table(objects_by_storage_class_table_data, { has_totals: true })
178
+ end
179
+
180
+ def process_files(args = { })
181
+ process_objects(files, { total_bytes: total_bytes })
182
+ end
183
+
184
+ def process_objects(objects, args = {})
185
+ return objects unless block_given?
186
+ return objects.map { |object| yield object, self } if args[:quiet]
187
+
188
+ bytes_remaining = args[:total_bytes] || objects.reduce(0) { |s, o| s + o.size }
189
+ _total_object_count = objects.length
190
+ counter = 0
191
+ objects.each do |object|
192
+ counter += 1
193
+ human_readable_bytes_remaining = humanize_number(bytes_remaining)
194
+ human_readable_bytes_remaining_short = human_readable_bytes_short(human_readable_bytes_remaining)
195
+ human_readable_object_bytes = humanize_number(object.size)
196
+ human_readable_object_bytes_short = human_readable_bytes_short(human_readable_object_bytes)
197
+ puts "Processing #{humanize_number(counter)} of #{humanize_number(_total_object_count)} #{human_readable_object_bytes} (#{human_readable_object_bytes_short}) of #{human_readable_bytes_remaining} (#{human_readable_bytes_remaining_short}) #{object.key}"
198
+ yield object, self if block_given?
199
+ bytes_remaining -= object.size
200
+ end
201
+ end
202
+
203
+ end
204
+
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: cantemo-portal-agent
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.3.3
4
+ version: 1.3.4
5
5
  platform: ruby
6
6
  authors:
7
7
  - John Whitson
8
8
  autorequire:
9
9
  bindir: exe
10
10
  cert_chain: []
11
- date: 2019-01-25 00:00:00.000000000 Z
11
+ date: 2019-01-31 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: asperalm
@@ -138,11 +138,11 @@ extra_rdoc_files: []
138
138
  files:
139
139
  - exe/cantemo-portal-agent
140
140
  - lib/cantemo/portal/agent.rb
141
- - lib/cantemo/portal/agent/WatchFolderUtility/foreman.rb
142
141
  - lib/cantemo/portal/agent/cli/commands/watch_folder.rb
143
142
  - lib/cantemo/portal/agent/cli/commands/watch_folders-working.rb
144
143
  - lib/cantemo/portal/agent/cli/commands/watch_folders.rb
145
144
  - lib/cantemo/portal/agent/version.rb
145
+ - lib/cantemo/portal/agent/watch_folder_utility/foreman.rb
146
146
  - lib/cantemo/portal/api/client.rb
147
147
  - lib/envoi/aspera/watch_service/client.rb
148
148
  - lib/envoi/aspera/watch_service/snapshot.rb
@@ -155,16 +155,21 @@ files:
155
155
  - lib/envoi/mam/agent/cli/commands/cantemo-agent.rb
156
156
  - lib/envoi/mam/agent/cli/commands/cantemo-watch_folders.rb
157
157
  - lib/envoi/mam/agent/cli/commands/cantemo.rb
158
+ - lib/envoi/mam/agent/cli/commands/envoi.rb
158
159
  - lib/envoi/mam/agent/cli/commands/iconik.rb
159
160
  - lib/envoi/mam/agent/cli/commands/mediasilo.rb
161
+ - lib/envoi/mam/agent/cli/commands/s3-to-eis.rb
160
162
  - lib/envoi/mam/agent/cli/commands/vdms.rb
163
+ - lib/envoi/mam/agent/cli/commands/vidispine-agent.rb
164
+ - lib/envoi/mam/agent/cli/commands/vidispine-restore.rb
161
165
  - lib/envoi/mam/agent/cli/commands/vidispine.rb
162
166
  - lib/envoi/mam/agent/cli/commands/wiredrive.rb
163
167
  - lib/envoi/mam/agent/client/iconik.rb
164
168
  - lib/envoi/mam/agent/client/mediasilo.rb
165
169
  - lib/envoi/mam/agent/client/vidispine.rb
166
170
  - lib/envoi/mam/agent/config_service_client.rb
167
- - lib/envoi/mam/agent/daemon/service_wrapper.rb
171
+ - lib/envoi/mam/agent/daemon/windows_service.rb
172
+ - lib/envoi/mam/agent/daemon/windows_service_controller.rb
168
173
  - lib/envoi/mam/agent/generic.rb
169
174
  - lib/envoi/mam/agent/notifier.rb
170
175
  - lib/envoi/mam/agent/notifier/sns.rb
@@ -183,7 +188,9 @@ files:
183
188
  - lib/envoi/mam/mediasilo/agent.rb
184
189
  - lib/envoi/mam/vdms/agent.rb
185
190
  - lib/envoi/mam/vidispine/agent.rb
191
+ - lib/envoi/mam/vidispine/restore/watcher.rb
186
192
  - lib/envoi/mam/wiredrive/agent.rb
193
+ - lib/envoi/utils/s3_enumerator.rb
187
194
  - lib/envoi/watch_folder_utility/watch_folder.rb
188
195
  - lib/envoi/watch_folder_utility/watch_folder/handler/listen.rb
189
196
  homepage: http://www.github.com/XPlatform-Consulting/envoi-mam-agent