envoi-mam-agent 1.1.0 → 1.2.0

data/lib/envoi/aspera/watch_service/watch_folder.rb

@@ -0,0 +1,322 @@
+require 'json'
+
+require 'envoi/aspera/watch_service/client'
+
+module Envoi
+  module Aspera
+    module WatchService
+
+      class WatchFolder
+        class State
+
+          attr_accessor :subscription
+          attr_accessor :known_path_map
+          attr_accessor :details
+
+
+          def initialize
+            @known_path_map = {}
+            @details        = {}
+          end
+
+        end
+
+        class Subscription
+
+          class Snapshot
+
+            class Entry < Hash
+
+              def initialize(data)
+                merge! data
+              end
+
+              def path;
+                self[:path]
+              end
+
+              def stat;
+                self[:stat] ||= ((json = self[:stat_as_json]) ? JSON.parse(json) : nil)
+              end
+
+              def stat_as_json;
+                self[:stat_as_json]
+              end
+
+              if MatchData.method_defined?(:named_captures)
+                def self.new_from_match_data(match_data)
+                  new(match_data.named_captures)
+                end
+              else
+                def self.new_from_match_data(match_data)
+                  new(Hash[match_data.names.map(&:to_sym).zip(match_data.captures)])
+                end
+              end
+
+              # Entry
+            end
+
+            attr_accessor :subscription, :version, :entries
+
+            def initialize(subscription, version)
+              @subscription = subscription
+              @version      = version
+              @entries      = nil
+            end
+
+            def client
+              subscription.client
+            end
+
+            def logger
+              client.logger
+            end
+
+            def entries
+              @entries ||= client.subscription_snapshot_entries_get(subscription_id:  subscription['identifier'],
+                                                                    snapshot_version: version)
+            end
+
+            def differential(from = nil)
+              (@differentials ||= {})[from] ||= differential_no_cache(from: from)
+            end
+
+            def differential_no_cache(args = {})
+              args_out                   = args.dup
+              args_out[:subscription_id] ||= subscription['identifier']
+              args_out[:to]              ||= version
+              client.subscription_snapshot_differential(args_out)
+            end
+
+            def entries_by_path
+              @entries_by_hash ||= Hash[entries.map { |e| [e[:path], e] }]
+            end
+
+            # Snapshot
+          end
+
+          attr_accessor :attributes, :client, :definition, :snapshots
+
+          def initialize(data, client, definition = nil, snapshots = { })
+            @attributes = data
+            @client     = client
+            @definition = definition
+            @snapshots  = {}
+          end
+
+          def logger
+            client.logger
+          end
+
+          def [](key)
+            attributes[key]
+          end
+
+          def identifier
+            attributes['identifier']
+          end
+
+          alias id identifier
+
+          def snapshot_create(args = {})
+            args_out                   = args.dup
+            args_out[:subscription_id] = attributes['identifier']
+            snapshot_id                = client.subscription_snapshot_create(args_out)
+            snapshots[snapshot_id]     ||= Snapshot.new(self, snapshot_id)
+          end
+
+          def resubscribe
+            client.subscription_resubscribe(subscription_id: id)
+          end
+
+
+          def self.get_or_create(client, definition, snapshots = { })
+            watch_folder_path = definition['path']
+            scan_period       = definition['scan_period']
+            expire_in         = definition['expire_in']
+
+            subscriptions          = nil
+            subscriptions_for_path = client.subscription_find_for_path(path: watch_folder_path, subscriptions: subscriptions)
+            subscriptions_for_path.delete_if { |s| s['scan_period']['sec'] != scan_period } if scan_period
+
+            if subscriptions_for_path.empty?
+              args_out               = { watch_folder_path: watch_folder_path }
+              args_out[:scan_period] = scan_period if scan_period
+              args_out[:expire_in]   = expire_in if expire_in
+              # Create Subscription
+              subscription = client.subscription_create(args_out)
+            else
+              subscription = subscriptions_for_path.first
+            end
+
+            new(subscription, client, definition, snapshots)
+          end
+
+          # Subscription
+        end
+
+        attr_accessor :definition, :state, :client, :watcher,
+                      :poll_interval, :previous_poll_time, :last_poll_time
+
+        def initialize(definition, state = nil, client = nil)
+          @definition = definition
+          @state      = state || State.new
+          @client     = @watcher = client || Client.new
+          @poll_interval = definition.fetch('poll_interval', 15)
+          process_definition(definition)
+        end
+
+        def logger
+          client.logger
+        end
+
+        def path
+          definition['path']
+        end
+
+        def process_definition(watch_folder_def)
+          state.subscription = subscription_get_or_create(watch_folder_def)
+        end
+
+        def subscription_get_or_create(watch_folder_def)
+          Subscription.get_or_create(client, watch_folder_def)
+        end
+
+        def process_stable_entry(entry)
+          logger.debug { "Stable Entry Detected: #{entry.stat}" }
+        end
+
+        def events(from, to)
+          state.subscription.client.subscription_snapshot_differential(subscription_id: state.subscription['identifier'],
+                                                                       from:            from,
+                                                                       to:              to)
+        end
+
+        def poll
+          subscription              = state.subscription
+          subscription.resubscribe
+          previous_snapshot_version = subscription.snapshots.keys.last
+
+          # Delete other snapshots
+          subscription.snapshots.delete_if { |p, e| p != previous_snapshot_version } if previous_snapshot_version
+
+          snapshot = subscription.snapshot_create
+          @previous_poll_time = @last_poll_time
+          @last_poll_time = Time.now
+
+          no_change = snapshot.version == previous_snapshot_version
+          if !no_change
+            events = snapshot.differential(previous_snapshot_version)
+            logger.debug { "Events: #{events}" }
+          end
+
+          _known_path_map = state.known_path_map.dup
+
+          logger.debug { "Subscription ID: #{subscription['identifier']}" }
+          logger.debug { "Previous Snapshot: #{previous_snapshot_version}" }
+          logger.debug { "Current  Snapshot: #{snapshot.version}" }
+          logger.debug { "Known Paths: #{_known_path_map.keys}" }
+
+          current_path_map = snapshot.entries_by_path
+          current_paths    = current_path_map.keys
+
+          new_path_map = current_path_map.dup
+
+          deleted_path_map  = {}
+          stable_path_map   = {}
+          unstable_path_map = {}
+          _known_path_map.delete_if do |p, e|
+            deleted = !no_change && !current_paths.include?(p)
+            if deleted
+              logger.debug { "DELETED '#{p}'" }
+              deleted_path_map[p] = e
+            else
+              new_path_map.delete(p) # The file is not new, so remove it from the new list
+
+              new_entry     = current_path_map[p]
+              previous_stat = e.stat.values_at('mtime', 'size')
+              new_stat      = new_entry.stat.values_at('mtime', 'size')
+
+              if no_change || new_stat === previous_stat
+                logger.debug { "UNCHANGED '#{p}' '#{new_stat}' == '#{previous_stat}'" }
+                stable_poll_count     = e[:stable_poll_count] || 0
+                stable_poll_count     += 1
+                e[:stable_poll_count] = stable_poll_count
+                stable_path_map[p]    = e
+              else
+                logger.debug { "MODIFIED '#{p}' '#{new_stat}' != '#{previous_stat}'" }
+                e[:stable_poll_count] = 0
+                unstable_path_map[p]  = new_entry
+              end
+            end
+            deleted
+          end
+
+          new_path_map.each do |p, e|
+            logger.debug { "CREATED '#{p}' #{e.stat.values_at('mtime', 'size')}" }
+            e[:stable_poll_count] = 0
+            unstable_path_map[p]  = e.dup
+          end
+
+          _known_path_map.merge! stable_path_map
+          _known_path_map.merge! unstable_path_map unless no_change
+          state.known_path_map = _known_path_map
+          state.details = {
+            previous_snapshot_version: previous_snapshot_version,
+            maps: {
+              new:      new_path_map,
+              deleted:  deleted_path_map,
+              stable:   stable_path_map,
+              unstable: unstable_path_map
+            }
+          }
+
+          logger.debug { "Known Paths: #{_known_path_map.keys}" }
+          logger.debug { "New Paths: #{new_path_map.map { |p, e| "#{p} #{e.stat}" }}" }
+
+          logger.debug { "Deleted Paths: #{deleted_path_map.keys}" }
+          logger.debug { "Unstable Paths: #{unstable_path_map.keys}" }
+          logger.debug { "Stable Paths: #{stable_path_map.map { |p, e| [p, e[:stable_poll_count]] }}" }
+
+          yield self if block_given?
+
+        end
+
+        def self.process_watch_folder_def(watch_folder_def)
+          new(watch_folder_def)
+        end
+
+        def self.process_watch_folder_defs(watch_folder_defs)
+
+
+        end
+
+        def self.process_watch_folder(watch_folder, &block)
+          watch_folder.poll(&block) if (Time.now - watch_folder.last_poll_time) >= watch_folder.poll_interval
+        end
+
+        def self.process_watch_folders(watch_folders, &block)
+          watch_folders.each { |watch_folder| process_watch_folder(watch_folder, &block) }
+        end
+
+        def self.run_once(watch_folders, &block)
+          process_watch_folders(watch_folders, &block)
+        end
+
+        def self.run(watch_folders, poll_interval = 15, &block)
+          puts 'Starting...'
+          loop do
+            begin
+              run_once(watch_folders, &block)
+              sleep 1
+            rescue SystemExit, Interrupt
+              break
+            end
+          end
+          puts 'Exiting...'
+        end
+
+      end
+    end
+  end
+end

data/lib/envoi/mam/agent.rb

@@ -12,7 +12,7 @@ module Envoi
 
       def initialize(args = { })
         @initial_args = args.clone
-        @config = args[:config]
+        @config = args[:config] || { }
         @notifiers = args[:notifiers] || [ ]
 
         @dry_run = args.fetch(:dry_run, false)
@@ -36,7 +36,13 @@ module Envoi
         return if @notifiers.empty?
         args[:level] ||= :info
         args[:message] ||= message
-        @notifiers.each { |notifier| notifier.notify(args) }
+        @notifiers.each do |notifier|
+          begin
+            notifier.notify(args) rescue nil
+          rescue => e
+            logger.warn { "Notification Failed: #{e.message} "}
+          end
+        end
       end
 
       # @param [Hash] args {}
@@ -48,13 +54,17 @@ module Envoi
         end
       end
 
+      # @param [String] command
+      # @param [Boolean] dry_run
+      # @return [Hash]
       def shell_execute(command, dry_run = @dry_run)
         if dry_run
           logger.debug { "Skipping Execution of Command: '#{command}' " }
-          return
+          return { }
         end
         logger.debug { "Executing Command: '#{command}'" }
 
+        success = false
         Open3.popen3(command) do |stdin, stdout, stderr, thread|
           # stdin.sync = true
           # stdout.sync = true
@@ -65,36 +75,46 @@ module Envoi
             output << stdout.read #rescue nil
             output << stderr.read # rescue nil
             unless output.empty?
-              print output
+              logger.debug { output }
               output.clear
             end
           end
+
+          success = thread.value == 0 ? true : false
         end
+
+        { success: success }
       end
 
-      def self.load_from_config_file(args)
-        config_file_path = args[:config_file_path]
-        config_file_path = config_file_path.find { |v| File.exists?(v) } if config_file_path.is_a?(Array)
-        abort("Missing Config File. '#{config_file_path}'") unless config_file_path && !config_file_path.empty? && File.exists?(config_file_path)
+      def self.load_config_from_file(args)
+        config_file_paths = args[:config_file_path]
+        config_file_path = [*config_file_paths].find { |v| File.exists?(File.expand_path(v)) }
+        raise "Configuration file not found. Searched '#{config_file_paths}'" unless config_file_path && !config_file_path.empty? && File.exists?(config_file_path)
 
         begin
           config = JSON.parse(File.read(config_file_path))
         rescue => e
-          abort("Config File Failed to Load. '#{$!}'")
+          raise "Failed to Load Config File. '#{config_file_path}' '#{e.message}'"
         end
-        args[:config] = config
-
-        self.new(args.merge({ :config => config }))
+        config
       end
 
-      def self.load_from_config_service(args)
+      def self.load_config_from_service(args)
         args_out = { }
         args_out[:app_id] = args[:config_service_app_id]
         args_out[:token] = args[:config_service_app_token]
         args_out[:api_url] = args[:config_service_app_url]
         config = Envoi::Mam::Agent::ConfigServiceClient.config_get(args_out)
-        args[:config] = config
+        config
+      end
 
+      def self.load_from_config_file(args)
+        config = load_config_from_file(args)
+        self.new(args.merge({ :config => config }))
+      end
+
+      def self.load_from_config_service(args)
+        config = load_config_from_service(args)
         self.new(args.merge({ :config => config }))
       end
 

data/lib/envoi/mam/agent/cli.rb

@@ -6,11 +6,11 @@ module Envoi
   module Mam
     class Agent
 
-      class Cli
+      class CLI
 
         CONFIG_FILE_PATHS = [
-          "./envoi-mam-agent-config.json",
-          "~/envoi-mam-agent-config.json",
+            './envoi-mam-agent-config.json',
+            '~/envoi-mam-agent-config.json',
         ]
         CONFIG_FILE_PATHS.map! { |v| File.expand_path(v) }
 

data/lib/envoi/mam/agent/cli/commands.rb

@@ -23,33 +23,43 @@
 #   # Envoi
 # end
 
-def usage
-  command_list_str = ''
-  Dir.glob(File.join(@commands_dir, "*#{@command_ext}")).each do |fn|
-    command_list_str += "\n\t - #{File.basename(fn, @command_ext)}"
+@commands_dir = File.join(File.dirname(__FILE__), 'commands')
+@command_ext = '.rb'
+
+@command_list_str = ''
+@commands = [ ]
+@command_aliases = { }
+
+def build_command_aliases(commands_dir = @commands_dir)
+  Dir.glob(File.join(commands_dir, "*.rb")).each do |fn|
+    command_file_name = File.basename(fn)
+    command_name = File.basename(command_file_name, '.*')
+    command_alias = command_name.tr('-_', '')
+    @commands << command_name
+    @command_aliases[command_alias] = command_name
+    @command_list_str += "\n\t - #{command_name}"
   end
+end
+
+def usage
 
   <<-EOT
 Usage: #{File.basename($0)} COMMAND [ARGS]
   
   Available Commands:
-    #{command_list_str}
+    #{@command_list_str}
     
 All commands can be run with -h (or --help) for more information.
   EOT
 end
 
-@commands_dir = File.join(File.dirname(__FILE__), 'commands')
-@command_ext = '.rb'
-
-command_aliases = {
-}
 
+build_command_aliases
 if ARGV.empty?
   should_show_usage = true
 else
-  command_name = ARGV[0].clone
-  command_name = command_aliases[command_name] || command_name
+  command_name = ARGV.first.dup
+  command_name = @command_aliases[command_name] || command_name
   should_show_usage = %w(-h --help).include?(command_name.downcase)
 end
 if should_show_usage
@@ -70,14 +80,14 @@ end
 
 
 # Execute the command if it is in object form, otherwise it's just a script and executed when we required it
-if Envoi::Mam::Agent::Cli.const_defined?('Commands')
+if Envoi::Mam::Agent::CLI.const_defined?('Commands')
   command_class_name = command_name.to_s.capitalize
-  if Envoi::Mam::Agent::Cli::Commands.const_defined?(command_class_name)
-    command_object = Envoi::Mam::Agent::Cli::Commands.const_get( command_class_name.capitalize ) rescue nil
+  if Envoi::Mam::Agent::CLI::Commands.const_defined?(command_class_name)
+    command_object = Envoi::Mam::Agent::CLI::Commands.const_get(command_class_name.capitalize ) rescue nil
   end
   command_object ||= begin
     command_class_name = command_class_name.upcase
-    Envoi::Mam::Agent::Cli::Commands.const_defined?(command_class_name) ? Envoi::Mam::Agent::Cli::Commands.const_get( command_class_name ) : false
+    Envoi::Mam::Agent::CLI::Commands.const_defined?(command_class_name) ? Envoi::Mam::Agent::CLI::Commands.const_get(command_class_name ) : false
   end
   if command_object
     command_instance = command_object.new if command_object.respond_to?(:new)