cantemo-portal-agent 1.0.9 → 1.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 54171a0f3c7ff03ad9ea8411d2f0df91ce7b352a
-  data.tar.gz: 7ca1ab0d8df364f79e19c29c31f315245b797217
+  metadata.gz: e52bfb1fe0a32298dab4b64bf2aca6f18fe0f6ab
+  data.tar.gz: 3d442e6e485752b2dba2c7bcdaef210dd9398163
 SHA512:
-  metadata.gz: 953df856aa882c921ec522d1bea9664fb72f8545af8f488c62b10a9f0d47b11a0dab564c6b1b61399243887263c39cf28777599860f1d4449a625e5c2d7d0450
-  data.tar.gz: ae7adea3924f9f89d70d00c5a8c07be688eb0a04498557ad078c761d5e68a4b0d82d3f9c97c19fa00bbaaa70c049a83bc1134022976f0e00afb471ec3a2176ce
+  metadata.gz: 9bd60ea14fba7a17e322228e074e948165aebb750a264e035cf2728d68d8690158674754040ffd5e1dcf26dd4a2d89df69259b138d52f802798999b8d63f6060
+  data.tar.gz: 9db178a801b0656b6a3e737cc904a46ec21ac2d0bc1d30380ed1ff85daa0e753ea1e66aec7309f7f32f9512000ce59f97171e569a2f4a7117e66e830550dff76

data/exe/cantemo-portal-agent

@@ -1,17 +1,19 @@
 #!/usr/bin/env ruby
 
-ARGV.unshift 'vidispine'
 should_retry = true
 
 begin
   require 'envoi/mam/agent/cli'
 
-  Envoi::Mam::Agent::Cli::CONFIG_FILE_PATHS.clear
-  Envoi::Mam::Agent::Cli::CONFIG_FILE_PATHS.concat [
-      "./cantemo-portal-agent-config.json",
-      "~/cantemo-portal-agent-config.json",
+  Envoi::Mam::Agent::CLI::CONFIG_FILE_PATHS.clear
+  Envoi::Mam::Agent::CLI::CONFIG_FILE_PATHS.concat [
+    './cantemo-portal-agent-config.json',
+    '~/cantemo-portal-agent-config.json',
+    './.cantemo-portal-agent-config.json',
+    '~/.cantemo-portal-agent-config.json',
   ]
-  require 'envoi/mam/agent/cli/commands/vidispine'
+
+  require 'envoi/mam/agent/cli/commands/cantemo'
 rescue LoadError => e
   lib_path = __FILE__ == '(irb)' ? File.join(Dir.cwd, 'lib') : File.expand_path('../../lib', __FILE__)
   $:.unshift(lib_path) unless $:.include?(lib_path) or !File.exist?(lib_path)

data/lib/cantemo/portal/agent/WatchFolderUtility/foreman.rb

@@ -0,0 +1,59 @@
+require 'envoi/mam/cantemo/agent'
+require 'envoi/aspera/watch_service/watch_folder'
+
+module Cantemo
+  module Portal
+    class Agent
+      class WatchFolderUtility
+        class Foreman
+
+          def self.initialize_watch_folder(watch_folder_def)
+            case  (watch_folder_def['handler'] || '').downcase.gsub(/[-_ ]/, '')
+            when 'asperawatch'
+              wf_class = Envoi::Aspera::WatchService::WatchFolder
+            end
+            wf_class.new(watch_folder_def)
+          end
+
+          def self.process_watch_folder_def(watch_folder_def)
+            initialize_watch_folder(watch_folder_def)
+          end
+
+          def self.process_watch_folder_defs(watch_folder_defs)
+            if watch_folder_defs.is_a?(Array)
+              watch_folder_defs.map { |watch_folder_def| process_watch_folder_def(watch_folder_def) }
+            elsif watch_folder_defs.is_a?(Hash)
+              watch_folder_defs.map do |name, watch_folder_def|
+                watch_folder_def['path'] ||= name; process_watch_folder_def(watch_folder_def)
+              end
+            else
+              raise "Unhandled format: #{watch_folder_defs.class.name}"
+            end
+          end
+
+          def self.poll_watch_folder(watch_folder, &block)
+            watch_folder.poll(&block)
+          end
+
+          def self.poll_watch_folders(watch_folders, &block)
+            watch_folders.each { |watch_folder| poll_watch_folder(watch_folder, &block) }
+          end
+
+          def self.run_once(watch_folders, &block)
+            poll_watch_folders(watch_folders, &block)
+          end
+
+          # Foreman
+        end
+
+        # WatchFolderUtility
+      end
+
+      # Agent
+    end
+
+    # Mam
+  end
+
+  # Envoi
+end

data/lib/cantemo/portal/agent/cli/commands/watch_folders-working.rb

@@ -0,0 +1,237 @@
+#!/usr/bin/env ruby
+lib_path = __FILE__ == '(irb)' ? File.join(Dir.cwd, 'lib') : File.expand_path('../../../lib', __FILE__)
+$:.unshift(lib_path) unless $:.include?(lib_path) or !File.exists?(lib_path)
+
+require 'rubygems'
+require 'optparse'
+require 'open3'
+require 'pp'
+
+require 'envoi/mam/agent/cli'
+require 'envoi/mam/cantemo/agent'
+require 'envoi/aspera/watch_service/watch_folder'
+
+Envoi::Mam::Agent::CLI::CONFIG_FILE_PATHS.clear
+Envoi::Mam::Agent::CLI::CONFIG_FILE_PATHS.concat [
+   './cantemo-portal-agent-config.json',
+   '~/cantemo-portal-agent-config.json',
+   './.cantemo-portal-agent-config.json',
+   '~/.cantemo-portal-agent-config.json',
+]
+default_config_file_paths = Envoi::Mam::Agent::CLI::CONFIG_FILE_PATHS
+
+aspera_ascp_paths = Envoi::Mam::Agent::TransferClient::Aspera::ASCP_PATHS
+default_aspera_ascp_path = aspera_ascp_paths.find { |v| File.exist? v } || aspera_ascp_paths.first
+
+current_command = ARGV.shift
+ARGV << 'run' if ARGV.empty?
+# ARGV << '--help' if ARGV.empty?
+
+@args = {
+    :config_file_path  => default_config_file_paths,
+    :default_ascp_path => default_aspera_ascp_path,
+    :dry_run           => false,
+    :operation         => :upload,
+    :preserve_path     => true,
+    :transfer_type     => '',
+}
+def args; @args end
+
+op = OptionParser.new
+op.on('-c', '--config-file-path FILEPATH', 'The path to the configuration file.',
+      "Default Path(s): #{args[:config_file_path]}") { |v| args[:config_file_path] = v }
+op.on('-b', '--[no-]background', 'Tells the application to run in the background.') { |v| args[:daemonize] = v }
+op.on('--help', 'Show this message.') { puts op; exit }
+op.load
+op.parse!
+
+config_file_path = args[:config_file_path]
+args[:config_file_path].map! { |v| File.expand_path(v) } if config_file_path.is_a?(Array)
+
+control_command_present = %w(start stop restart run zap killall status).include? ARGV.first
+
+daemonize = control_command_present || begin
+  _d = args[:daemonize]
+end
+
+
+# puts "#{__FILE__}:#{__LINE__ } #{args}"
+# next_command = ARGV.shift
+# puts "COMMAND: #{next_command}"
+
+
+module Envoi
+
+  module WatchFolderUtility
+
+    class WatchFolder
+
+      class State
+        attr_accessor :known_path_map
+        attr_accessor :details
+
+        def initialize
+          @known_path_map = {}
+          @details        = {}
+        end
+
+      end
+
+      class Worker
+
+        attr_accessor :name, :definition, :state, :client, :watcher,
+                      :previous_poll_time, :last_poll_time
+
+        def initialize(definition, state = nil, client = nil)
+          @definition = definition
+          @state      = state || State.new
+          @client     = @watcher = client || Client.new
+
+          process_definition(definition)
+        end
+
+      end
+
+      class Foreman
+
+        attr_accessor :watch_folder_defs
+
+        AWF = Envoi::Aspera::WatchService::WatchFolder
+
+        def initialize(args = { })
+          initialize_logger(args)
+
+          @agent = Envoi::Mam::Cantemo::Agent.load_from_config_file(args)
+          @config = agent.config
+          cantemo_config = config[:cantemo] || config['cantemo']
+          @watch_folder_defs = cantemo_config[:watch_folders] || cantemo_config['watch_folders']
+
+          @ignored_file_paths_by_watch_folder = Hash.new { |h, k| h[k] = [] }
+
+          @watch_folders = process_watch_folder_defs
+        end
+
+        def process_watch_folder_def(watch_folder_def)
+          # @TODO TYPE CHECK
+          AWF.process_watch_folder_def(watch_folder_def)
+        end
+
+        def process_watch_folder_defs(watch_folder_defs = @watch_folder_defs)
+          watch_folder_defs.map { |wfd| process_watch_folder_def(wfd) }
+        end
+
+      end
+
+    end
+
+  end
+
+end
+
+class Watcher
+
+  AWF = Envoi::Aspera::WatchService::WatchFolder
+
+  attr_accessor :logger, :agent, :config, :watch_folders
+
+  def initialize(args = { })
+    initialize_logger(args)
+
+    @agent = Envoi::Mam::Cantemo::Agent.load_from_config_file(args)
+    @config = agent.config
+    cantemo_config = config[:cantemo] || config['cantemo']
+    watch_folder_defs = cantemo_config[:watch_folders] || cantemo_config['watch_folders']
+
+    @ignored_file_paths_by_watch_folder = Hash.new { |h, k| h[k] = [] }
+
+    @watch_folders = AWF.process_watch_folder_defs(watch_folder_defs)
+    pp watch_folders
+  end
+
+  def initialize_logger(args = { })
+    @logger = args[:logger] ||= Logger.new(args[:log_to] || STDOUT)
+    log_level = args[:log_level]
+    if log_level
+      @logger.level = log_level
+      args[:logger] = @logger
+    end
+    @logger
+  end
+
+  def add_to_ignore(wf, file)
+    @ignored_file_paths_by_watch_folder[wf] << file.path
+  end
+
+  def process_file(watch_folder, full_file_path, storage_id = nil, quarantine_directory_path = nil)
+    return unless storage_id
+
+    if agent.upload(file_path: full_file_path, storage_id: storage_id)
+      FileUtils.rm full_file_path
+    else
+      FileUtils.mv full_file_path, quarantine_directory_path
+    end
+  end
+
+  def process_watch_folder(wf)
+    storage_id = wf.definition['upload_to_storage_id'] || wf.definition['storage_id']
+    quarantine_directory_path = wf.definition['quarantine_path']
+    exclude = wf.definition['exclude']
+    min_stable_poll_count = wf.definition['stable_poll_count'] || 3
+
+    maps = wf.state.details[:maps]
+    stable_paths = maps[:stable]
+    ignored_files = @ignored_file_paths_by_watch_folder[wf]
+    stable_paths.each do |fp, file|
+      if exclude
+        next if ignored_files.include?(file.path)
+        if [*exclude].find { |ep| File.fnmatch(ep, file.path) }
+          logger.debug { "Adding File to Ignore Cache: '#{file.path}'"}
+          ignored_files << file.path
+          next
+        end
+      end
+
+      full_file_path = File.join(wf.path, file.path)
+
+      # pp file
+      # puts file_path
+      stable_poll_count = file[:stable_poll_count]
+
+      if stable_poll_count && stable_poll_count > min_stable_poll_count
+        if storage_id
+          if agent.upload(file_path: full_file_path, storage_id: storage_id)
+            FileUtils.rm full_file_path
+          else
+            FileUtils.mv full_file_path, quarantine_directory_path
+          end
+        end
+
+      end
+    end
+
+    # process_watch_folder
+  end
+
+  def run
+    # AWF.run_once(watch_folders) { |wf| pp wf }
+    AWF.run(watch_folders) { |wf| process_watch_folder(wf) }
+  end
+
+  def run_once
+    AWF.run_once(watch_folders) { |wf| process_watch_folder(wf) }
+  end
+
+  def self.run(args)
+    w = self.new(args)
+    w.run
+  end
+
+  def self.run_as_daemon(args)
+    # ARGV.unshift 'run' unless %w(start stop restart run zap killall status).include? ARGV.first
+    require 'daemons'
+    Daemons.run_proc('cantemo-portal-agent-watch-folders') { self.run(args) }
+  end
+
+end
+
+# daemonize ? Watcher.run_as_daemon(args) : Watcher.run(args)

data/lib/cantemo/portal/agent/cli/commands/watch_folders.rb

@@ -0,0 +1,63 @@
+#!/usr/bin/env ruby
+lib_path = __FILE__ == '(irb)' ? File.join(Dir.cwd, 'lib') : File.expand_path('../../../lib', __FILE__)
+$:.unshift(lib_path) unless $:.include?(lib_path) or !File.exists?(lib_path)
+
+require 'rubygems'
+require 'optparse'
+require 'open3'
+require 'pp'
+
+require 'envoi/mam/agent/cli'
+require 'envoi/mam/cantemo/agent'
+# require 'envoi/aspera/watch_service/watch_folder'
+require 'envoi/mam/cantemo/agent/watch_folder_handler'
+
+Envoi::Mam::Agent::CLI::CONFIG_FILE_PATHS.clear
+Envoi::Mam::Agent::CLI::CONFIG_FILE_PATHS.concat [
+   './cantemo-portal-agent-config.json',
+   '~/cantemo-portal-agent-config.json',
+   './.cantemo-portal-agent-config.json',
+   '~/.cantemo-portal-agent-config.json',
+]
+default_config_file_paths = Envoi::Mam::Agent::CLI::CONFIG_FILE_PATHS
+
+# aspera_ascp_paths = Envoi::Mam::Agent::TransferClient::Aspera::ASCP_PATHS
+# default_aspera_ascp_path = aspera_ascp_paths.find { |v| File.exist? v } || aspera_ascp_paths.first
+
+current_command = ARGV.shift
+ARGV << 'run' if ARGV.empty?
+# ARGV << '--help' if ARGV.empty?
+
+@args = {
+    :config_file_path  => default_config_file_paths,
+    :dry_run           => false,
+    :operation         => :upload,
+    :preserve_path     => true,
+    :transfer_type     => '',
+}
+def args; @args end
+
+op = OptionParser.new
+op.on('-c', '--config-file-path FILEPATH', 'The path to the configuration file.',
+      "Default Path(s): #{args[:config_file_path]}") { |v| args[:config_file_path] = v }
+op.on('-b', '--[no-]background', 'Tells the application to run in the background.') { |v| args[:daemonize] = v }
+op.on('-v', '--version', 'Output the version and exit.') { puts "Version #{Cantemo::Portal::Agent::VERSION}"; exit }
+op.on('--help', 'Show this message.') { puts op; exit }
+op.load
+op.parse!
+
+config_file_path = args[:config_file_path]
+args[:config_file_path].map! { |v| File.expand_path(v) } if config_file_path.is_a?(Array)
+
+control_command_present = %w(start stop restart run zap killall status).include? ARGV.first
+
+daemonize = control_command_present || begin
+  _d = args[:daemonize]
+end
+
+# puts "#{__FILE__}:#{__LINE__ } #{args}"
+# next_command = ARGV.shift
+# puts "COMMAND: #{next_command}"
+
+class Watcher < Envoi::Mam::Cantemo::Agent::WatchFolderHandler; end
+daemonize ? Watcher.run_as_daemon(args) : Watcher.run(args)

data/lib/cantemo/portal/agent/version.rb

@@ -1,7 +1,7 @@
 module Cantemo
   module Portal
     class Agent
-      VERSION = '1.0.9'.freeze
+      VERSION = '1.1.0'.freeze
     end
   end
 end