cantemo-portal-agent 1.0.9 → 1.1.0

data/lib/envoi/mam/cantemo/agent/watch_folder_handler_aspera.rb

@@ -0,0 +1,112 @@
+require 'envoi/mam/cantemo/agent'
+require 'envoi/aspera/watch_service/watch_folder'
+
+module Envoi::Mam::Cantemo
+
+  class Agent
+
+    class WatchFolderHandler
+
+      AWF = Envoi::Aspera::WatchService::WatchFolder
+      LWF = Envoi::WatchFolderUtility::WatchFolder::Listen
+
+      attr_accessor :logger, :agent, :config, :watch_folders
+
+      def initialize(args = { })
+        initialize_logger(args)
+
+        @agent = Envoi::Mam::Cantemo::Agent.load_from_config_file(args)
+        @config = agent.config
+        cantemo_config = config[:cantemo] || config['cantemo']
+        watch_folder_defs = cantemo_config[:watch_folders] || cantemo_config['watch_folders']
+
+        @ignored_file_paths_by_watch_folder = Hash.new { |h, k| h[k] = [] }
+
+        @watch_folders = AWF.process_watch_folder_defs(watch_folder_defs)
+      end
+
+      def initialize_logger(args = { })
+        @logger = args[:logger] ||= Logger.new(args[:log_to] || STDOUT)
+        log_level = args[:log_level]
+        if log_level
+          @logger.level = log_level
+          args[:logger] = @logger
+        end
+        @logger
+      end
+
+      def add_to_ignore(wf, file)
+        @ignored_file_paths_by_watch_folder[wf] << file.path
+      end
+
+      def process_file(watch_folder, file, storage_id = nil, quarantine_directory_path = nil)
+        full_file_path = File.join(watch_folder.path, file.path)
+        if storage_id && agent.upload(file_path: full_file_path, storage_id: storage_id)
+          FileUtils.rm full_file_path
+        else
+          if Dir.exist?(quarantine_directory_path)
+            FileUtils.mv full_file_path, quarantine_directory_path
+          else
+            add_to_ignore(watch_folder, file)
+          end
+        end
+      end
+
+      def process_watch_folder(wf)
+        storage_id = wf.definition['upload_to_storage_id'] || wf.definition['storage_id']
+        quarantine_directory_path = wf.definition['quarantine_path']
+        exclude = wf.definition['exclude']
+        min_stable_poll_count = wf.definition['stable_poll_count'] || 3
+
+        maps = wf.state.details[:maps]
+        stable_paths = maps[:stable]
+        ignored_files = @ignored_file_paths_by_watch_folder[wf]
+        stable_paths.each do |fp, file|
+          if exclude
+            next if ignored_files.include?(file.path)
+            if [*exclude].find { |ep| File.fnmatch(ep, file.path) }
+              logger.debug { "Adding File to Ignore Cache: '#{file.path}'"}
+              ignored_files << file.path
+              next
+            end
+          end
+
+          # full_file_path = File.join(wf.path, file.path)
+
+          # pp file
+          # puts file_path
+          stable_poll_count = file[:stable_poll_count]
+
+          if stable_poll_count && stable_poll_count > min_stable_poll_count
+            process_file(wf, file, storage_id, quarantine_directory_path)
+          end
+        end
+
+        # process_watch_folder
+      end
+
+      def run
+        # AWF.run_once(watch_folders) { |wf| pp wf }
+        AWF.run(watch_folders) { |wf| process_watch_folder(wf) }
+      end
+
+      def run_once
+        AWF.run_once(watch_folders) { |wf| process_watch_folder(wf) }
+      end
+
+      def self.run(args)
+        w = self.new(args)
+        w.run
+      end
+
+      def self.run_as_daemon(args)
+        # ARGV.unshift 'run' unless %w(start stop restart run zap killall status).include? ARGV.first
+        require 'daemons'
+        Daemons.run_proc('cantemo-portal-watch-folders') { self.run(args) }
+      end
+
+    end
+
+  end
+
+end

data/lib/envoi/mam/cantemo/agent.rb

@@ -0,0 +1,288 @@
+require 'envoi/mam/agent/transfer_client/aspera'
+require 'envoi/mam/agent/transfer_client/s3'
+
+require 'vidispine/api/utilities'
+
+module Envoi
+
+  module Mam
+    
+    module Cantemo
+
+      class Agent < Envoi::Mam::Agent
+    
+        DEFAULT_SHAPE_TAG = 'original'
+        DEFAULT_DESTINATION_PATH = '.'
+
+        attr_accessor :default_aspera_ascp_args,
+                      :default_aspera_ascp_path,
+                      :default_vidispine_shape_tag
+    
+        def after_initialize
+          args = initial_args
+          @default_aspera_ascp_path   = args[:default_aspera_ascp_path]
+          @default_aspera_args = args[:default_ascp_args] || Envoi::Mam::Agent::TransferClient::Aspera::DEFAULT_ASCP_ARGS
+        end
+        
+        def dry_run?; @dry_run end
+    
+        def agent_config
+          @agent_config ||= config['cantemo'] || config['vidispine'] || { }
+        end
+    
+        def initialize_api_client(args = { })
+          api_config = agent_config
+          @api_client = args[:api_client] || begin
+    
+            api_host = api_config['host']
+            api_host_use_ssl = api_config['use_ssl']
+            api_port = api_config['port']
+            api_username = api_config['username']
+            api_password = api_config['password']
+            api_auth_token = api_config['api_auth_token']
+            api_base_path = api_config['api_base_path']
+            api_default_query_data = api_config['default_query_data']
+
+            api_url = api_config['url'] || api_config['uri']
+            if api_url
+              api_uri = URI(api_url)
+              api_host ||= api_uri.host
+              api_port ||= api_uri.port
+              api_userinfo = api_uri.userinfo
+              if api_userinfo
+                _api_username, _api_password = api_userinfo.split(':')
+                api_username ||= _api_username
+                api_password ||= _api_password
+              end
+              api_uri_query = api_uri.query
+              api_default_query_data ||= Hash[api_uri_query.split('&').map { |kp| kp.split('=') }]
+              api_base_path ||= api_uri.path
+            end
+
+            api_port ||= (api_host_use_ssl ? 443 : 80)
+            api_base_path ||= 'VSAPI/'
+
+            client_args = { }
+            client_args[:http_host_address] = api_host if api_host
+            client_args[:http_host_port] = api_port if api_port
+            client_args[:http_host_use_ssl] = api_host_use_ssl if api_host_use_ssl
+            client_args[:username] = api_username if api_username
+            client_args[:password] =  api_password if api_password
+            client_args[:default_base_path] = api_base_path if api_base_path
+            client_args[:default_query_data] = api_default_query_data if api_default_query_data
+
+            if api_auth_token
+              # Cantemo Portal supports an auth token for authentication, replace basic auth with auth-token
+              client_args[:authorization_header_key] = 'Auth-Token'
+              client_args[:authorization_header_value] = api_auth_token
+            end
+
+            _client = ::Vidispine::API::Utilities.new(client_args)
+            _client
+          end
+
+
+
+          @default_vidispine_shape_tag = args[:default_shape_tag] || api_config['default_shape_tag'] || api_config['shape_tag'] || DEFAULT_SHAPE_TAG
+    
+        end
+    
+        def item_get_shape_by_tag(item_id, shape_tag)
+          item_shapes_get_response = api_client.item_shapes_get(:item_id => item_id, :tag => shape_tag)
+          shape_id = item_shapes_get_response['uri'].first
+        end
+
+        def download(args = { })
+
+          item_id = args[:item_id]
+          shape_id = args[:shape_id]
+          unless shape_id && !shape_id.empty?
+            shape_tag = args[:shape_tag] || default_vidispine_shape_tag
+            shape_id = item_get_shape_by_tag(item_id, shape_tag)
+          end
+
+          logger.info { "Getting file path for Item ID: #{item_id} Shape ID: #{shape_id}"}
+          item_shape_get_response = api_client.item_shape_get(:item_id => item_id, :shape_id => shape_id)
+
+          files = item_shape_get_response['containerComponent']['file']
+          logger.debug { "Files: #{files}"}
+
+          # file = files.first
+          files = [ files.first ] # just do the first file for now
+          files.each do |file|
+            begin
+              download_file(args, file)
+            rescue => e
+              logger.warn { "Exception: #{$!}" }
+            end
+          end
+          logger.info { 'DONE' }
+        end
+
+        def download_file(args, file)
+          logger.debug { "File: #{file}"}
+          transfer_type = args[:transfer_type]
+
+          file_storage_id = file['storage']
+          file_path = file['path']
+
+          file_storage_config = agent_config['storages'][file_storage_id]
+
+          unless file_storage_config && !file_storage_config.empty?
+            raise Exception, "No configuration found for storage '#{file_storage_id}'"
+          end
+
+          logger.info { "Transferring File Path: '#{file_path}'" }
+          preserve_path = args.fetch(:preserve_path, file_storage_config.fetch('preserve_path', true))
+
+          destination_path = args[:destination_path] || file_storage_config['destination_path'] || DEFAULT_DESTINATION_PATH
+          relative_path = preserve_path ? File.dirname(file_path) : nil
+          relative_path = nil if relative_path == '.'
+
+          target_path = relative_path ? File.join(destination_path, relative_path) : destination_path
+          target_path = target_path[0..-1] if target_path.start_with?('/') && !destination_path.start_with?('/')
+
+          aspera_config = file_storage_config['aspera']
+          if (transfer_type.empty? || transfer_type == :aspera) && (aspera_config && !aspera_config.empty?)
+            client = Envoi::Mam::Agent::TransferClient::Aspera.new(agent: self)
+            return client.download(aspera_config, file_path, target_path)
+            # download_using_aspera(aspera_config, file_path, target_path)
+          end
+
+          s3_config = file_storage_config['s3']
+          if (transfer_type.empty? || transfer_type == :s3) && (s3_config && !s3_config.empty?)
+            target_path = File.expand_path(target_path) if target_path == '.'
+            target_path = File.join(target_path, File.basename(file_path))
+            client = Envoi::Mam::Agent::TransferClient::S3.new(agent: self)
+            return client.download(s3_config, file_path, target_path)
+          end
+
+          logger.warn { "No Supported TransferClient Configuration#{transfer_type && !transfer_type.empty? ? " for transfer type '#{transfer_type}' " : ''}Found in Storage Configuration." }
+        end
+
+        def upload(args = { })
+          file_path = args[:file_path]
+          raise ArgumentError, "Path not found: '#{file_path}'" unless File.exists?(file_path)
+
+          if File.directory?(file_path)
+            # Non-recursive directory upload
+            file_paths = Dir.glob(File.join(file_path, '*.*'))
+            logger.debug { "File Paths: #{file_paths}"}
+            file_paths.map { |fp| upload(args.merge(file_path: fp))}
+            return file_paths
+          end
+          logger.debug { "Preparing to upload '#{file_path}'" }
+
+          transfer_type = args[:transfer_type] || ''
+          storage_id = args[:storage_id]
+          vidispine_storage_config = agent_config['storages'][storage_id]
+    
+          unless vidispine_storage_config && !vidispine_storage_config.empty?
+            raise "No configuration found for storage '#{storage_id}'"
+          end
+
+          should_import_file = args.fetch(:import_file, vidispine_storage_config.fetch('import', true))
+
+          should_preserve_path = args.fetch(:preserve_path, vidispine_storage_config.fetch('preserve_path', true))
+    
+          destination_path = args[:destination_path] || vidispine_storage_config['destination_path'] || '/'
+          relative_path = should_preserve_path ? File.dirname(file_path) : nil
+          relative_path = File.expand_path(relative_path) if relative_path == '.'
+    
+          target_path = relative_path ? File.join(destination_path, relative_path) : destination_path
+          target_path = target_path[0..-1] if target_path.start_with?('/') && !destination_path.start_with?('/')
+    
+    
+          # upload file
+    
+          transfer_response = begin
+            response = nil
+            aspera_config = vidispine_storage_config['aspera']
+            begin
+              if (transfer_type.empty? || transfer_type == :aspera) && (aspera_config && !aspera_config.empty?)
+                client = Envoi::Mam::Agent::TransferClient::Aspera.new(agent: self)
+                response = client.upload(aspera_config, file_path, target_path)
+              end
+            rescue => e
+              logger.error { "Aspera Transfer Failed. '#{e.message}'" }
+            end
+    
+            s3_config = vidispine_storage_config['s3']
+            begin
+              if !response && (transfer_type.empty? || transfer_type == :s3) && (s3_config && !s3_config.empty?)
+                _target_path = target_path
+                _target_path = File.expand_path(_target_path) if target_path == '.'
+                _target_path = File.join(_target_path, File.basename(file_path))
+                client = Envoi::Mam::Agent::TransferClient::S3.new(agent: self)
+                response = client.upload(s3_config, file_path, _target_path)
+              end
+            rescue => e
+              logger.error { "S3 Transfer Failed. '#{e.message}'" }
+            end
+
+            response
+          end
+    
+          logger.warn { "No supported TransferClient configuration#{transfer_type && !transfer_type.empty? ? " for transfer type '#{transfer_type}' " : ''}found in storage configuration." } unless transfer_response != nil
+
+          _response = { transfer_response: transfer_response }
+
+          return _response unless should_import_file
+
+          item_id = args[:item_id]
+          shape_tag = args[:shape_tag] || default_vidispine_shape_tag
+
+          # attach file to item as shape
+          path_on_storage = File.join(target_path, File.basename(file_path))
+          file_create_response = api_client.storage_file_create storage_id: storage_id, path: path_on_storage, state: 'CLOSED'
+          file_id = file_create_response['id']
+
+          if item_id
+            item_shape_import_response = api_client.item_shape_import item_id: item_id, tag: shape_tag, fileId: file_id
+          else
+            item_shape_import_response = api_client.item_add_using_file_path storage_id: storage_id,
+                                                                         file_path: file_path,
+                                                                         file_id: file_id,
+                                                                         storage_path_map: { '/' => storage_id }
+          end
+          _response[:import_response] = item_shape_import_response
+
+          _response
+        end
+
+        def upload_transfer_spec_get(file_paths)
+
+        end
+
+
+
+        def ingest_file
+          # 1) export ASPERA_SCP_TOKEN="ATB3_AEA9dUI5dd2u1k8XBMVD0qInR-Lyylkz8AylTXLVt5_SMVGR8SO7Sdc0lj6RkoHK_DvAAEWac8bllF_Yan1NbbDTyPj_3BTA"
+          # 2) ascp -i asperaweb_id_dsa.openssh -P 33001 -l 20m /Users/nicholasstokes/Desktop/CantemoAgent.mov xfer@ats-aws-us-west-2.aspera.io:
+          #
+          # single line equivalent
+          # ascp -i asperaweb_id_dsa.openssh -W 'ATB3_AEAiBdrf-7sMQu782sPoQ4Q7Yh4LadgvYK9BP4Kt1hcVlZGR8SO7Sdc0lj6RkoHK_DvAAG5XlBBIZGEINicYOxMRE7g_3BTA' -P 33001 -l 20m /Users/nicholasstokes/Desktop/CantemoAgent.mov xfer@ats-aws-us-west-2.aspera.io:
+
+          # Aspera:
+          # `GET /aspera/ping/` - to check if Aspera upload is available
+          # `POST /vs/item/new/` to create a placeholder to upload to
+          # `POST /aspera/upload_setup/` - Aspera token
+          # ... upload the file with Aspera ...
+          # `POST /API/v1/placeholder/VX-3012/associatefile/?filepath=Screen%20Shot%202018-08-07%20at%2013.06.58.png&storageid=VX-11&platform=mac`  - associate new file to placeholder
+
+          # HTTP upload:
+          # `POST /vs/item/new/` - create a placeholder
+          # then streamind POST (?) requests to `VSAPI/item/VX-3014/shape/essence/raw?transferId=2D30030A-6EC3-4364-9598-5BF20972A218&throttle=false&filename=Screen%20Shot%202018-08-07%20at%2014.34.15.png&jobmetadata=portal_groups:StringArray%3dAdmin`  to upload the data.
+          # that end-point for HTTP is documented in http://apidoc.vidispine.com/latest/ref/item/shape.html#import-a-shape-using-the-request-body
+          #
+        end
+
+        # Agent
+      end
+    
+      # Cantemo
+    end
+    
+  end
+
+end

data/lib/envoi/mam/iconik/agent.rb

@@ -112,11 +112,13 @@ module Envoi
           if File.directory?(file_path)
             file_paths = Dir.glob(File.join(file_path, '*.*'))
             logger.debug { "File Paths: #{file_paths}"}
-            file_paths.map { |fp| upload(args.merge(file_path: fp))}
-            return file_paths
+            return file_paths.map { |fp| upload(args.merge(file_path: fp))}
           end
           logger.debug { "Preparing to upload '#{file_path}'" }
 
+          ingest_after_upload = args[:ingest_after_upload]
+          analyze_asset = args[:analyze_asset]
+
           transfer_type = args[:transfer_type] || ''
           storage_id = args[:storage_id]
           path_on_storage = nil
@@ -170,7 +172,17 @@ module Envoi
           file_size = File.exist?(file_path) ? File.size?(file_path) : 1024
           path_on_storage ||= File.join(target_path, File.basename(file_path))
 
-          api_client.asset_add_using_file_path(file_path: path_on_storage, storage_id: storage_id, file_size: file_size)
+          if ingest_after_upload
+            asset_create_response = api_client.asset_add_using_file_path(file_path: path_on_storage,
+                                                                         storage_id: storage_id,
+                                                                         file_size: file_size)
+            if analyze_asset
+              asset_id = asset_create_response[:asset]['id']
+              api_client.asset_analyze(:asset_id => asset_id)
+            end
+
+          end
+
         end
 
 

data/lib/envoi/mam/vidispine/agent.rb

@@ -34,12 +34,17 @@ module Envoi
         def initialize_api_client(args = { })
           _vidispine_config = vidispine_config
           @api_client = args[:vidispine_api_client] || begin
-    
+
             vidispine_host = _vidispine_config['host']
+            vidispine_port = _vidispine_config['port']
+            vidispine_host_use_ssl = _vidispine_config['use_ssl']
             vidispine_username = _vidispine_config['username']
             vidispine_password = _vidispine_config['password']
+
             client_args = { }
             client_args[:http_host_address] = vidispine_host if vidispine_host
+            client_args[:http_host_port] = vidispine_port if vidispine_port
+            client_args[:http_host_use_ssl] = vidispine_host_use_ssl if vidispine_host_use_ssl
             client_args[:username] = vidispine_username if vidispine_username
             client_args[:password] =  vidispine_password if vidispine_password
             ::Vidispine::API::Utilities.new(client_args)

data/lib/envoi/watch_folder_utility/watch_folder/handler/listen.rb

@@ -0,0 +1,189 @@
+require 'listen' # https://github.com/guard/listen
+require 'pp'
+module Envoi
+  module WatchFolderUtility
+    class WatchFolder
+      class Handler
+        class Listen
+
+          attr_accessor :logger, :definition,
+                        :known_path_map,
+                        :last_poll_time,
+                        :min_stable_poll_count,
+                        :paths,
+                        :poll_interval,
+
+                        :listener, :lock,
+
+
+                        :quarantine_directory_path
+
+          def initialize(args = { })
+            initialize_logger(args)
+            logger.debug { "Initializing Work Folder. Args: #{args}" }
+
+            @definition = args[:definition]
+            raise ArgumentError, "Definition is a required argument." unless definition
+
+
+
+            # @paths = [ File.expand_path('~/watch_folders/aspera') ]
+            @paths = definition['paths'] || []
+            path = definition['path']
+            @paths << path if path
+
+            @known_path_map = { }
+            @ignored_files_map = { }
+
+            @lock = Mutex.new
+            # @quarantine_directory_path = definition['quarantine_directory_path'] || definition['quarantine_path']
+            @poll_interval = definition['poll_interval'] || 15
+            @min_stable_poll_count = 3
+            @processing_limit = 10
+            @processing = { }
+
+            initialize_listener
+          end
+
+          def initialize_logger(args = { })
+            @logger = args[:logger] ||= Logger.new(args[:log_to] || STDOUT)
+            log_level = args[:log_level] || Logger::DEBUG
+            if log_level
+              @logger.level = log_level
+              args[:logger] = @logger
+            end
+            @logger
+          end
+
+          def process_add_or_modified_path(path, event_type)
+            return path.each { |path| process_add_or_modified_path(path, event_type) } if path.is_a?(Array)
+            logger.debug { "PATH #{event_type} '#{path}'" }
+            lock.synchronize { known_path_map[path] = OpenStruct.new({ type: event_type, path: path, stable_poll_count: 0, timestamp: Time.now}) }
+          end
+
+          def initialize_listener(args = { })
+            # @directories = directories.map do |directory|
+            #   Pathname.new(directory.to_s).realpath
+            # end
+
+
+            @listener = ::Listen.to(*paths, {}) do |m, a, r|
+              if !r.empty?
+                log "#{r.length} file(s) removed."
+                lock.synchronize { r.each { |f| known_path_map.delete(f) } }
+              end
+
+              if !m.empty?
+                log "#{m.length} file(s) modified."
+                lock.synchronize { process_add_or_modified_path(m, :modified) }
+              end
+
+              if !a.empty?
+                log "#{a.length} file(s) added."
+                lock.synchronize { process_add_or_modified_path(a, :added) }
+              end
+
+            end
+            _config = @listener.instance_variable_get(:@backend).instance_variable_get(:@adapter).config
+
+            _snapshots ||= {}
+            # change_config = ::Listen::Change::Config.new(_config.queue, _config.silencer)
+            _config.directories.each do |dir|
+              record = ::Listen::Record.new(dir)
+              record.build
+              record.instance_variable_get(:@tree).each do |rel_path, fstats|
+                _path = File.join(dir, rel_path)
+                process_add_or_modified_path(_path, :added)
+              end
+              # snapshot = ::Listen::Change.new(change_config, record)
+              # _config.optimize_changes(snapshot)
+              # _snapshots[dir] = snapshot
+            end
+
+            # pp _snapshots
+          end
+
+          def log(message)
+            puts "#{Time.now} #{message}"
+          end
+
+          def add_to_ignore(file)
+            @ignored_files_map[file.path] = file
+          end
+
+          # def handle_stable_file(file, quarantine_directory_path = @quarantine_directory_path)
+          #   full_file_path = file[:path]
+          #   logger.info { "Processing File Path: #{full_file_path}..." }
+          #   if true
+          #     FileUtils.rm full_file_path
+          #   else
+          #     if quarantine_directory_path
+          #       FileUtils.mv full_file_path, quarantine_directory_path
+          #     end
+          #   end
+          # end
+
+          def poll
+            @previous_poll_time = @last_poll_time
+            @last_poll_time = Time.now
+            stable_files = []
+            lock.synchronize do
+              @known_path_map.each do |fp, f|
+                logger.debug { "Incrementing Stable Count: #{f.inspect}" }
+                f[:stable_poll_count] += 1
+                if f[:stable_poll_count] >= @min_stable_poll_count
+                  stable_files << f
+                  # handle_stable_file(f)
+                end
+              end
+            end
+            stable_files
+          end
+
+          def run(options = { })
+            logger.info { "Starting handler for #{paths}" }
+            listener.start
+          end
+
+        end
+      end
+    end
+  end
+end
+
+
+# watcher = Envoi::WatchFolderUtility::WatchFolder::Handler::Listen.new
+# watcher.run
+#
+#
+#
+#
+# class WatchFolder
+#
+#   attr_accessor :definition, :state, :client, :watcher,
+#                 :previous_poll_time, :last_poll_time
+#
+#   def initialize(args = { })
+#
+#   end
+#
+#   def poll(&block)
+#
+#   end
+#
+#   def run_once(&block)
+#     poll(&block)
+#   end
+#
+#   def run(poll_interval = 15, &block)
+#     loop do
+#       begin
+#         run_once(&block)
+#         sleep poll_interval
+#       rescue SystemExit, Interrupt
+#         break
+#       end
+#     end
+#   end
+#
+# end