envoi-mam-agent 1.1.0 → 1.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 87aae76f05847dd0841872249db7841cd0ed1a48
-  data.tar.gz: 598e977aaebf41e231ac89f3cb8f67a2cafbde2f
+  metadata.gz: 3542672342cb1a7bf58f0ab4889f5241d722592f
+  data.tar.gz: 4d74581af836bf987ada79e4250526a3d8a40b15
 SHA512:
-  metadata.gz: b8a75310a3a12544fadbc8f7aa3c7cec1c684c8d9666b604a4136f09d1b16c83507961517ebb02e376f33cc2f1ae8b7d768028d3f831f2c487ac14fe0971a39d
-  data.tar.gz: f869a550600737e3c9cb87fa88c041abf47d5f28e794755505d68b61603806bfa6df25aac8cb2ec2841fa79e677099eabba5b519ac9e18e45ea57e15a76e7885
+  metadata.gz: 5df25d66d00dd8d77dca9c76b404d3c8b2248e19be8f2df2dc1ffe4141134136559856c0df042cec913ce1c22b80f96dc813e9548b83a2911fc2e762586c609b
+  data.tar.gz: 8149920ac1f94e68d246da8b5d0758198201ccaf4d9388e8fdefd7eba90ef0081e7267fdf54f4c0d28040d768b11483ad2809fbd00d2b884a238c702dc678b48

data/Gemfile

@@ -3,5 +3,12 @@ source 'https://rubygems.org'
 local_gemfile_name = 'Gemfile.local'
 eval(IO.read(local_gemfile_name), binding) if File.exist? local_gemfile_name
 
+require 'rbconfig'
+
+case RbConfig::CONFIG['target_os']
+when /mswin|mingw|cygwin/i
+  gem 'win32/service', '~> 1'
+end
+
 # Specify your gem's dependencies in envoi-mam-agent.gemspec
-gemspec
+gemspec name: 'envoi-mam-agent'

data/docs/restore.md

@@ -0,0 +1,43 @@
+# Installation
+`gem install envoi-mam-agent`
+
+# Usage
+`envoi-mam-agent restore --config-file-path [PATH TO CONFIG FILE]`
+
+Default Config File Paths:
+```
+./envoi-mam-agent-config.json
+~/envoi-mam-agent-config.json
+```
+
+
+# Configuration
+
+```
+{
+    "restore": {
+        "sqs": {
+            "access_key_id": "<access key id>",
+            "secret_access_key": "<secret access key>",
+            "queue_url": "<SQS QUEUE URL>"
+            "s3": {
+                "<bucket name>": {
+                    "destination_path": "<destination path>",
+                    "aspera": {
+                        "host": "<host address IP or FQDN>",
+                        "username": "<password>",
+                        "password": "<username>>",
+                        "base_path": "/<bucket_name>>",
+                    },
+                    "s3": {
+                        "access_key_id": "<access key id>",
+                        "secret_access_key": "<secret access key>",
+                        "region": "us-east-1",
+                        "bucket_name": "<bucket name>"
+                    }
+                }
+            }
+        }
+    }
+}
+```

data/envoi-mam-agent.gemspec

@@ -22,20 +22,27 @@ Gem::Specification.new do |spec|
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ['lib']
 
-  spec.add_runtime_dependency 'adobe_media_encoder', '~> 0.0.1'
+  # spec.add_runtime_dependency 'adobe_media_encoder', '~> 0.0.1'
+  spec.add_runtime_dependency 'asperalm', '~> 0.9'
   spec.add_runtime_dependency 'aws-sdk-s3', '~> 1'
-  spec.add_runtime_dependency 'daemons', '~> 0'
-  spec.add_runtime_dependency 'faraday', '~> 0.15'
+  spec.add_runtime_dependency 'aws-sdk-sqs', '~> 1'
+  # spec.add_runtime_dependency 'shoryuken' '~> 4'
+  spec.add_runtime_dependency 'daemons', '~> 1.3'
+  # spec.add_runtime_dependency 'faraday', '~> 0.15'
+  spec.add_runtime_dependency 'listen', '~> 3.1'
+  # spec.add_runtime_dependency 'directory_watcher'
+  # spec.add_runtime_dependency 'file-monitor'
   spec.add_runtime_dependency 'ubiquity-envoi', '~> 1'
   spec.add_runtime_dependency 'ubiquity-iconik', '~> 1'
   spec.add_runtime_dependency 'ubiquity-mediasilo-api-v3', '~> 1'
   spec.add_runtime_dependency 'ubiquity-vdms', '~> 1'
   spec.add_runtime_dependency 'ubiquity-wiredrive', '~> 1'
   # spec.add_runtime_dependency 'vantage', '~> 1'
-  spec.add_runtime_dependency 'vidispine', '~> 1'
+  spec.add_runtime_dependency 'vidispine', '~> 1.6'
 
 
   spec.add_development_dependency 'bundler', '~> 1.16'
   spec.add_development_dependency 'minitest', '~> 5.0'
+  spec.add_development_dependency 'pry', '~> 0.11'
   spec.add_development_dependency 'rake', '~> 10.0'
 end

data/lib/cantemo/portal/agent/version.rb

@@ -0,0 +1,7 @@
+module Cantemo
+  module Portal
+    class Agent
+      VERSION = '1.3.0'.freeze
+    end
+  end
+end

data/lib/cantemo/portal/api/client.rb

@@ -0,0 +1,82 @@
+require 'vidispine/api/utilities'
+require 'cantemo/portal/agent/version'
+
+module Cantemo
+  module Portal
+    module API
+
+      class Client
+
+        def self.new(args = { })
+          api_host               = args[:host]
+          api_port               = args[:port]
+          api_host_use_ssl       = args[:ssl]
+          api_username           = args[:username]
+          api_password           = args[:password]
+          api_auth_token         = args[:api_auth_token]
+          api_base_path          = args[:api_base_path]
+          api_default_query_data = args[:default_query_data]
+
+          api_log_level = args[:log_level]
+          api_log_to = args[:log_to]
+          api_logger = args[:logger]
+
+          api_url = args[:url] || args[:uri]
+          if api_url
+            api_uri      = URI(api_url)
+            api_host     ||= api_uri.host
+            api_port     ||= api_uri.port
+            api_userinfo = api_uri.userinfo
+            if api_userinfo
+              _api_username, _api_password = api_userinfo.split(':')
+              api_username                 ||= _api_username
+              api_password                 ||= _api_password
+            end
+            api_host_use_ssl       = api_uri.scheme == 'https' if api_host_use_ssl.nil?
+            api_uri_query          = api_uri.query
+            api_default_query_data ||= Hash[api_uri_query.split('&').map { |kp| kp.split('=') }]
+            api_base_path          ||= api_uri.path
+          end
+
+          api_port      ||= (api_host_use_ssl ? 443 : 80)
+          api_base_path ||= '/'
+
+          api_endpoint_prefix        = 'VSAPI'
+          api_noauth_endpoint_prefix = 'APInoauth'
+
+          client_args                              = {}
+          client_args[:http_host_address]          = api_host if api_host
+          client_args[:http_host_port]             = api_port if api_port
+          client_args[:http_host_use_ssl]          = api_host_use_ssl if api_host_use_ssl
+          client_args[:username]                   = api_username if api_username
+          client_args[:password]                   = api_password if api_password
+          client_args[:default_base_path]          = api_base_path if api_base_path
+          client_args[:default_query_data]         = api_default_query_data if api_default_query_data
+          client_args[:api_endpoint_prefix]        = api_endpoint_prefix
+          client_args[:api_noauth_endpoint_prefix] = api_noauth_endpoint_prefix
+          client_args[:log_level]                  = api_log_level if api_log_level
+          client_args[:log_to]                     = api_log_to if api_log_to
+          client_args[:logger]                     = api_logger if api_logger
+
+          if api_auth_token
+            # Cantemo Portal supports an auth token for authentication, replace basic auth with auth-token
+            client_args[:authorization_header_key]   = 'Auth-Token'
+            client_args[:authorization_header_value] = api_auth_token
+          end
+          _client = ::Vidispine::API::Utilities.new(client_args)
+          _client.http_client.user_agent = "Cantemo Portal Agent v#{Cantemo::Portal::Agent::VERSION} - #{_client.http_client.user_agent}"
+          _client
+        end
+
+
+        # Client
+      end
+
+      # API
+    end
+
+    # Portal
+  end
+
+  # Cantemo
+end

data/lib/envoi/aspera/watch_service/client.rb

@@ -0,0 +1,397 @@
+require 'json'
+require 'logger'
+require 'open3'
+require 'shellwords'
+
+module Envoi
+  module Aspera
+    module WatchService
+
+      class Client
+
+        attr_accessor :logger, :initial_args, :config
+
+        attr_accessor :aswatchadmin_executable_path, :default_username
+
+        DEFAULT_ASWATCHADMIN_EXECUTABLE_PATH = '/Library/Aspera/bin/d'
+
+        DAEMON_NAME_FIELD_NAME          = 'name'
+        DAEMON_CONFIGURATION_FIELD_NAME = 'configuration'
+
+        DAEMONS_QUERY_LINE_PART_TO_SUB = 'configuration: '
+        DAEMONS_QUERY_LINE_START       = "#{DAEMONS_QUERY_LINE_PART_TO_SUB} {"
+
+        class Expressions
+          SNAPSHOT_CREATE_VERSION_PARSE          = /(?<version>\d*)\.$/
+          SNAPSHOT_PRINT_PARSE                   = /^\s{2}(?<path>.*), (?<stat_as_json>\{"ino":\d+,.*})$/
+          SUBSCRIPTION_CREATE_PARSE              = /(?<json>\{.*\})\s*$/
+          SUBSCRIPTION_RESUBSCRIBE_SUCCESS_PARSE = /^\[aswatchadmin resubscribe\] Successfully resubscribed subscription '[^']*'\.$/
+          SNAPSHOT_DIFFERENTIAL_PARSE            = /^event\=(?<event>[^\,]*),path\=(?<path>.*),stat\=\[type\=(?<stat_type>[A-Z_]*),(?<change>[^\]]*)\]$/
+        end
+
+        class SnapshotEntry < Hash
+
+          def initialize(data)
+            # @attributes = data.is_a?(MatchData) ? { path: data[:path], stat_as_json: data[:stat_as_json] } : data
+            merge! data
+          end
+
+          def path; self[:path] end
+
+          def stat
+            self[:stat] ||= begin
+              json = self[:stat_as_json]
+              JSON.parse(json) if json
+            end
+          end
+
+          def stat_as_json; self[:stat_as_json] end
+
+          if MatchData.method_defined?(:named_captures)
+            def self.new_from_match_data(match_data)
+              new(match_data.named_captures)
+            end
+          else
+            def self.new_from_match_data(match_data)
+              new(Hash[match_data.names.map(&:to_sym).zip(match_data.captures)])
+            end
+          end
+        end
+
+        def self.current_os_type
+          case RUBY_PLATFORM
+          when /mswin|msys|mingw|cygwin|bccwin|wince|emc/
+            return :windows
+          when /darwin|mac os/
+            return :mac
+          else  # unix family
+            return :unix
+          end
+        end
+
+        def self.detect_executable
+          executable_name = 'aswatchadmin'
+          paths = case current_os_type
+          when :windows
+            executable_name += '.exe'
+            [
+              File.join(ENV['LOCALAPPDATA'], 'Aspera', 'bin', executable_name)
+            ]
+          when :mac
+            [
+              "/Library/Aspera/bin/#{executable_name}",
+            ]
+          else
+            [
+              File.join("/opt/aspera/bin/#{executable_name}")
+            ]
+          end
+          paths.find { |p| File.executable?(p) }
+        end
+
+        def initialize(args = {})
+          @initial_args = args.clone
+          initialize_logger(args)
+
+          @config = args[:config]
+
+          @aswatchadmin_executable_path = args[:aswatchadmin_executable_path] ||
+                                          self.class.detect_executable ||
+                                          DEFAULT_ASWATCHADMIN_EXECUTABLE_PATH
+          @default_daemon_name          = args[:default_daemon_name] || args[:daemon_name] || default_daemon_name_get
+          @dry_run                      = args.fetch(:dry_run, false)
+        end
+
+        def initialize_logger(args = {})
+          @logger = args[:logger] || Logger.new(STDOUT)
+        end
+
+        def dry_run?;
+          @dry_run
+        end
+
+        def shell_execute(command, dry_run = @dry_run)
+          command = command.shelljoin if command.is_a?(Array)
+          if dry_run
+            logger.debug { "Skipping Execution of Command: '#{command}' " }
+            return
+          end
+          logger.debug { "Executing Command: '#{command}'" }
+
+          response = ''
+          Open3.popen3(command) do |stdin, stdout, stderr, thread|
+            # stdin.sync = true
+            # stdout.sync = true
+            # stderr.sync = true
+            output = ''
+            loop do
+              output << stdout.read #rescue nil
+              output << stderr.read # rescue nil
+              unless output.empty?
+                # print output
+                response << output.dup
+                output.clear
+              end
+              break if thread.stop?
+            end
+          end
+
+          logger.debug { "RESPONSE: #{response.empty? ? '' : "\n#{response}"}" }
+          response
+        end
+
+        # @return [Hash]
+        def default_daemon_get
+          daemons = daemons_query
+          return nil unless daemons.length == 1
+          default_daemon = daemons.first
+        end
+
+        # @return [String|nil]
+        def default_daemon_name_get
+          _default_daemon = default_daemon_get
+          return nil unless _default_daemon
+          _default_daemon[DAEMON_NAME_FIELD_NAME]
+        end
+
+        # @return [Array]
+        def daemons_query(args = {})
+          command  = [aswatchadmin_executable_path, 'query-daemons']
+          response = shell_execute(command)
+
+          return response unless args.fetch(:parse_response, true)
+
+          daemons_query_response_parse(response)
+        end
+
+        # @return [Array[Hash]]
+        def daemons_query_response_parse(response)
+          response_ary = response.split("\n")
+          first_line   = response_ary.shift
+          daemon       = nil
+          daemons      = response_ary.each_with_object([]) do |line, _daemons|
+            _line = line.strip
+
+            if _line.start_with?(DAEMONS_QUERY_LINE_START)
+              configuration_json                      = _line.sub(DAEMONS_QUERY_LINE_PART_TO_SUB, '')
+              configuration                           = JSON.parse(configuration_json)
+              daemon[DAEMON_CONFIGURATION_FIELD_NAME] = configuration
+            else
+              _daemons << daemon if daemon
+              daemon_name = _line
+              daemon      = { DAEMON_NAME_FIELD_NAME => daemon_name }
+            end
+          end
+          daemons << daemon if daemon
+        end
+
+        def subscription_create(args = {})
+          daemon_name = args[:daemon_name] || args[:daemon] || @default_daemon_name
+          raise ArgumentError, 'Missing argument for required parameter daemon_name' unless daemon_name
+
+          watch_folder_path = args[:watch_folder_path]
+          raise ArgumentError, 'Missing argument for required parameter watch_folder_path' unless watch_folder_path
+
+          scan_period = args[:scan_period]
+          expire_in   = args[:expire_in]
+
+          command = [aswatchadmin_executable_path, 'subscribe', daemon_name, watch_folder_path]
+          command << '--scan-period' << scan_period if scan_period
+          command << '--expire_in' << expire_in if expire_in
+          response = shell_execute(command)
+
+          return response unless args.fetch(:parse_response, true)
+
+          subscription_create_response_parse(response)
+        end
+
+        # @param [Object]  response
+        # @return [Object]
+        def subscription_create_response_parse(response)
+          subscription_json_match = Expressions::SUBSCRIPTION_CREATE_PARSE.match(response)
+          raise "Failed to parse new subscription information from response. '#{response}'" unless subscription_json_match
+          subscription_json = subscription_json_match.to_s
+          subscription      = JSON.parse(subscription_json)
+        end
+
+        def subscription_resubscribe(args = {})
+          daemon_name = args[:daemon_name] || args[:daemon] || @default_daemon_name
+          raise ArgumentError, 'Missing argument for required parameter daemon_name' unless daemon_name
+
+          subscription_id = args[:subscription_id]
+          raise ArgumentError, 'Missing argument for required parameter subscription_id' unless subscription_id
+
+          command  = [aswatchadmin_executable_path, 'resubscribe', daemon_name, subscription_id]
+          response = shell_execute(command)
+
+          return response unless args.fetch(:parse_response, true)
+
+          subscription_resubscribe_parse_response(response)
+        end
+
+        def subscription_resubscribe_parse_response(response)
+          raise response unless Expressions::SUBSCRIPTION_RESUBSCRIBE_SUCCESS_PARSE.match(response)
+          true
+        end
+
+        # Create a snapshot using the subscription
+        #
+        # ```
+        # $ /Library/Aspera/bin/aswatchadmin create-snapshot jw f090e8b5-d9e2-474b-acb1-68c4f76c8c53
+        # [aswatchadmin create-snapshot] Successfully created snapshot 0.
+        # ```
+        def subscription_snapshot_create(args = {})
+          daemon_name = args[:daemon_name] || args[:daemon] || @default_daemon_name
+          raise ArgumentError, 'Missing argument for required parameter daemon_name' unless daemon_name
+
+          subscription_id = args[:subscription_id]
+          raise ArgumentError, 'Missing argument for required parameter subscription_id' unless subscription_id
+
+          command  = [aswatchadmin_executable_path, 'create-snapshot', daemon_name, subscription_id]
+          response = shell_execute(command)
+
+          return response unless args.fetch(:parse_response, true)
+
+          subscription_snapshot_create_response_parse(response)
+        end
+
+        def subscription_snapshot_create_response_parse(response)
+          snapshot_version = Expressions::SNAPSHOT_CREATE_VERSION_PARSE.match(response)
+          raise "Failed to parse snapshot version from response. '#{response}'" unless snapshot_version
+          snapshot_version
+        end
+
+        def subscription_snapshot_differential(args = {})
+          daemon_name = args[:daemon_name] || args[:daemon] || @default_daemon_name
+          raise ArgumentError, 'Missing argument for required parameter daemon_name' unless daemon_name
+
+          subscription_id = args[:subscription_id]
+          raise ArgumentError, 'Missing argument for required parameter subscription_id' unless subscription_id
+
+          from = args[:from]
+
+          to = args[:to]
+
+          path = args[:path]
+          identifier = args[:identifier]
+          format = args[:format]
+          escape = args[:escape]
+          exclude_create = args[:exclude_create]
+          exclude_modify = args[:exclude_modify]
+          exclude_remove = args[:exclude_remove]
+          leafs_only = args[:leafs_only]
+
+          command  = [aswatchadmin_executable_path, 'snapshot-differential', daemon_name, subscription_id]
+          command << from if from
+          command << to if to
+          command << '--path' << path if path && !path.empty?
+          command << '--identifier' << identifier if identifier && !identifier.empty?
+          command << '--format' << format if format && !format.empty?
+          command << '--escape' << escape if escape && !escape.empty?
+          command << '--exclude_create' if exclude_create
+          command << '--exclude_modify' if exclude_modify
+          command << '--exclude-remove' if exclude_remove
+          command << '--leafs-only' if leafs_only
+          response = shell_execute(command)
+
+          return response unless args.fetch(:parse_response, true)
+
+          subscription_snapshot_differential_response_parse(response)
+        end
+
+        def subscription_snapshot_differential_response_parse(response)
+          response.split.map do |v|
+            m = v.match(Expressions::SNAPSHOT_DIFFERENTIAL_PARSE)
+            m ? Hash[m.names.zip(m.captures)] : v
+          end.join('\n')
+        end
+
+        def subscription_snapshot_print(args = {})
+          daemon_name = args[:daemon_name] || args[:daemon] || @default_daemon_name
+          raise ArgumentError, 'Missing argument for required parameter daemon_name' unless daemon_name
+
+          subscription_id = args[:subscription_id]
+          raise ArgumentError, 'Missing argument for required parameter subscription_id' unless subscription_id
+
+          snapshot_version = args[:snapshot_version]
+          command          = [aswatchadmin_executable_path, 'print-snapshot', daemon_name, subscription_id]
+          command << '--snapshot' << snapshot_version if snapshot_version
+
+          response = shell_execute(command)
+
+          return response unless args.fetch(:parse_response, false)
+
+          subscription_snapshot_print_response_parse(response)
+        end
+
+        def subscription_snapshot_print_response_parse(response)
+          response
+              .to_enum(:scan, Expressions::SNAPSHOT_PRINT_PARSE)
+              .map { SnapshotEntry.new_from_match_data(Regexp.last_match) }
+        end
+
+        def subscription_snapshot_entries_get(args = { })
+          response = subscription_snapshot_print(args)
+          subscription_snapshot_print_response_parse(response)
+        end
+
+
+        # List subscriptions for a specific daemon
+        # @see https://download.asperasoft.com/download/docs/entsrv/3.8.1/cs_admin_osx/webhelp/index.html#dita-watchfolder/watchd_subscriptions.html
+        def subscriptions_get(args = {})
+          daemon_name = args[:daemon_name] || args[:daemon] || @default_daemon_name
+          raise ArgumentError, 'Missing argument for required parameter daemon_name' unless daemon_name
+
+          command  = [aswatchadmin_executable_path, 'query-subscriptions', daemon_name]
+          response = shell_execute(command)
+
+          return response unless args.fetch(:parse_response, true)
+
+          subscriptions_get_response_parse(response)
+        end
+
+        def subscriptions_get_response_parse(response)
+          response_json = "[#{response.split.join(",\n")}]"
+          subscriptions = JSON.parse(response_json)
+        end
+
+        # Unsubscribe from a watch
+        #
+        # $ /Library/Aspera/bin/aswatchadmin unsubscribe daemonname 3b758f2d-97f9-4054-80d5-ed67dcaa8daa
+        # [aswatchadmin unsubscribe] Successfully unsubscribed subscription '3b758f2d-97f9-4054-80d5-ed67dcaa8daa' from daemon 'daemonname'.
+        #
+        # @see https://download.asperasoft.com/download/docs/entsrv/3.8.1/cs_admin_osx/webhelp/index.html#dita-watchfolder/watchd_subscriptions.html
+        def subscription_unsubscribe(args = {})
+          daemon_name = args[:daemon_name] || args[:daemon] || @default_daemon_name
+          raise ArgumentError, 'Missing argument for required parameter daemon_name' unless daemon_name
+
+          subscription_id = args[:subscription_id]
+          raise ArgumentError, 'Missing argument for required parameter subscription_id' unless subscription_id
+
+          command  = [aswatchadmin_executable_path, 'unsubscribe', daemon_name, subscription_id]
+          response = shell_execute(command)
+        end
+
+        def subscriptions_unsubscribe_all(args = {})
+          subscriptions = subscriptions_get(args)
+          args_out      = args.dup
+          path          = args[:path]
+          subscriptions.each do |subscription|
+            next unless subscription['path'] == path if path
+            subscription_id            = subscription['identifier']
+            args_out[:subscription_id] = subscription_id
+            subscription_unsubscribe(args_out)
+          end
+        end
+
+        def subscription_find_for_path(args = {})
+          path          = args[:path]
+          subscriptions = args[:subscriptions] || subscriptions_get(args)
+          subscriptions.keep_if { |s| s['path'] =~ /(?:\w|-)*:\/{2,3}#{Regexp.escape(path)}/ }
+        end
+
+      end
+
+    end
+  end
+end