snoopit 0.0.1

data/lib/snoopit/sniffer.rb

@@ -0,0 +1,51 @@
+module Snoopit
+  class Sniffer
+
+    attr :pre_before, :before, :after, :comment, :regexp, :sniffed, :notifiers
+
+    def initialize(sniffer_params)
+      @before  = sniffer_params['lines']['before'].nil? ? 2 : sniffer_params['lines']['before']
+      @pre_before = Register.new @before
+      @after   = sniffer_params['lines']['after'].nil? ? 2 : sniffer_params['lines']['after']
+      @comment = sniffer_params['comment']
+      @regexp  = Regexp.new sniffer_params['regexp']
+      @notifiers = {}
+      setup_notifiers sniffer_params
+      @sniffed = []
+    end
+
+    def setup_notifiers(params)
+      @notifiers = params['notify'] unless params['notify'].nil?
+    end
+
+    def track(file, line_no, line)
+      matched = @regexp.match(line) do |m|
+        @sniffed << Detected.new(@comment, @pre_before, @after, line, file, line_no)
+      end
+      @pre_before.push_front line if matched.nil?
+      tracking line
+    end
+
+    def tracking(line)
+      @sniffed.each do |detected|
+        detected.track line unless detected.finished?
+      end
+    end
+
+    def as_json(options=nil)
+      {
+          before: @before,
+          after: @after,
+          comment: @comment,
+          regexp: @regexp.to_json,
+          sniffed: @sniffed,
+          notifiers: @notifiers
+      }
+    end
+
+    def to_json(*a)
+      as_json.to_json(*a)
+    end
+
+  end
+end

data/lib/snoopit/snooper.rb

@@ -0,0 +1,149 @@
+require 'json'
+module Snoopit
+  class Snooper
+
+    attr_accessor :snoopies, :notifier, :file_tracker
+
+    # Snoopies are the list of available named snoopers
+    # Snoopers are the current active invocations of selected snoopies
+    def initialize(notifications=true, db_file=nil, logger=nil, log_level=::Logger::INFO)
+      @snoopies = { }
+      @file_tracker = FileTracker.new db_file unless db_file.nil?
+      @notifier = NotificationManager.new if notifications
+      Snoopit::Logging.create_logger(logger) unless logger.nil?
+      Snoopit.logger.level = log_level
+    end
+
+    # Load the configuration from a file
+    # Calls <code>load_snoopers(json_hash)</code> and <code>load_notifiers(json_hash)</code> and
+    # @param snoopies_file [String] path to file
+    def load_file(snoopies_file)
+      raise ArgumentError.new "Invalid Snooper JSON File: #{snoopies_file}" if (snoopies_file.nil?) || (! File.exist? snoopies_file)
+      json_hash = JSON.parse(IO.read(snoopies_file))
+      load_snoopers json_hash
+      load_notifiers json_hash
+    end
+
+    # Load the configuration from a file
+    # @param json [String] json string
+    def load_json(json)
+      json_hash = JSON.parse(json)
+      load_snoopers json_hash
+      load_notifiers json_hash
+    end
+
+    # Load the configuration from a file
+    # @param json_hash [Hash]
+    def load_snoopers(json_hash)
+      snoopies_json = json_hash['snoopers']
+      snoopies_json.each do |name, snooper|
+        @snoopies[name] =  Snoopy.new(name, snooper)
+      end
+      raise ArgumentError.new 'There are no Snoopies in the JSON Snooper ' if @snoopies.size == 0
+    end
+
+    def load_notifiers(json_hash)
+      @notifier.load_notifier_config json_hash['notifiers'] unless @notifier.nil?
+    end
+
+    def register_notifier(notifier)
+      @notifier.register notifier
+    end
+
+    def unregister_notifier(notifier)
+      @notifier.unregister notifier
+    end
+
+    # Use the snoopies and start snooping
+    def snoop(names=[])
+      snoopers = get_snoopers names
+      snoopers.each do |snoopy|
+        if (!snoopy.dir.nil?) && (snoopy.dir?)
+          snoop_dir snoopy
+        else
+          snoop_file snoopy
+        end
+      end
+      @notifier.notify snoopers unless @notifier.nil?
+      snoopers
+    end
+
+    def get_snoopers(names=[])
+      snoopers = []
+      use_names = (names.size == 0 ? false : true)
+      snoopies.each do |key, snooper|
+        if use_names
+          snoopers << snooper if names.include? key
+        else
+          snoopers << snooper
+        end
+      end
+      snoopers
+    end
+
+    def snoop_dir(snoopy)
+      Snoopit.logger.debug "Snooping directory: #{snoopy.dir}"
+      get_files(snoopy).each do |file|
+        next if File.directory? file
+        sniff_it snoopy, "#{snoopy.dir}/#{file}"
+      end
+    end
+
+    def get_files(snoopy)
+      if snoopy.glob?
+        files = get_glob_list snoopy
+      else
+        files = get_file_list snoopy
+      end
+      files
+    end
+
+    def get_glob_list(snoopy)
+      Snoopit.logger.debug "Snooping glob: #{snoopy.glob}"
+      cwd = Dir.getwd
+      begin
+        Dir.chdir snoopy.dir
+        files = Dir.glob snoopy.glob
+      ensure
+        Dir.chdir cwd
+      end
+      files
+    end
+
+    def get_file_list(snoopy)
+      Snoopit.logger.debug "Snooper directory: #{snoopy.dir}"
+      Dir.entries snoopy.dir
+    end
+
+    def snoop_file(snoopy)
+      raise ArgumentError.new "Could find file #{snoopy.input}" unless File.exist? snoopy.input
+      Snoopit.logger.debug "Snooping file: #{snoopy.input} with snoopy: #{snoopy.name}"
+      sniff_it snoopy, snoopy.input
+    end
+
+    def sniff_it(snoopy, file_name)
+      Snoopit.logger.debug "Sniffing file: #{file_name} with snoopy: #{snoopy.name}"
+      unless @file_tracker.nil?
+        file_track_read(snoopy, file_name)
+      else
+        file_read(snoopy, file_name)
+      end
+    end
+
+    def file_track_read(snoopy, file_name)
+      @file_tracker.foreach file_name do |line, line_no|
+        snoopy.sniff snoopy.input, line_no, line
+      end
+    end
+
+    def file_read(snoopy, file_name)
+      line_no = 0
+      File.foreach file_name do |line|
+        snoopy.sniff snoopy.input, line_no, line
+        line_no += 1
+      end
+    end
+
+  end
+
+end

data/lib/snoopit/snoopy.rb

@@ -0,0 +1,67 @@
+module Snoopit
+  class Snoopy
+
+    attr :name, :input, :output, :dir, :glob, :sniffers
+
+    def initialize(name, params)
+      @name = name
+      @output = params['output']
+      setup_input params
+      setup_dir params unless params['dir'].nil?
+      input_check?
+      setup_sniffers params
+    end
+
+    def setup_input(params)
+      @input = params['snoop']
+    end
+
+    def setup_dir(params)
+      @dir = params['dir']['path']
+      @glob = params['dir']['glob']
+    end
+
+    def input_check?
+      return true unless @input.nil?
+      return true unless @dir.nil?
+      raise ArgumentError.new('Snooper JSON must contain either an input or dir parameter')
+    end
+
+    def setup_sniffers(params)
+      raise ArgumentError.new('Snooper JSON missing sniffers array') if params['sniffers'].nil?
+      @sniffers = []
+      params['sniffers'].each do |sniffer|
+        @sniffers << Sniffer.new(sniffer)
+      end
+    end
+
+    def dir?
+      ! @dir.nil?
+    end
+
+    def glob?
+      ! @glob.nil?
+    end
+
+    def sniff(file, line_no, line)
+      @sniffers.each do |sniffer|
+        sniffer.track file, line_no, line
+      end
+    end
+
+    def as_json(options=nil)
+      {
+          name: @name,
+          input: @input,
+          dir: @dir,
+          glob: @glob,
+          sniffers: @sniffers
+      }
+    end
+
+    def to_json(*a)
+     as_json.to_json(*a)
+    end
+
+  end
+end

data/lib/snoopit/version.rb

@@ -0,0 +1,3 @@
+module Snoopit
+  VERSION = '0.0.1'
+end

data/snoopit.gemspec

@@ -0,0 +1,27 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'snoopit/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = 'snoopit'
+  spec.version       = Snoopit::VERSION
+  spec.authors       = ['Robert Birch']
+  spec.email         = ['robdbirch@gmail.com']
+  spec.description   = %q{Snoops files for specified information via a simple configuration file}
+  spec.summary       = %q{Using regular expressions snoops files or directories for specific information. Sends events and tracks repeated invocations as not to send repeat events}
+  spec.homepage      = 'https://github.com/robdbirch/snoopit/blob/master/README.md'
+  spec.license       = 'MIT'
+
+  spec.files         = `git ls-files`.split($/)
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ['lib']
+
+  spec.add_runtime_dependency 'awesome_print', '~> 1.2'
+  spec.add_runtime_dependency 'stomp', '~> 1.3'
+
+  spec.add_development_dependency 'bundler', '~> 1.3'
+  spec.add_development_dependency 'rake', '10.1'
+  spec.add_development_dependency 'rspec', '2.14'
+end

data/spec/bin/snoopit_spec.rb

@@ -0,0 +1,258 @@
+require 'spec_helper'
+
+describe 'Snoopit' do
+
+  # *** Note ***
+  # used :skip on some tests due to captured stdout during coverage showing rvm environment only in the ide,
+  def capture_stdout(cmd)
+    IO.popen cmd, 'r+' do |pipe|
+      pipe.close_write
+      output = pipe.read
+      pipe.close_read
+      output
+    end
+  end
+
+  def cmd
+    lib = File.expand_path '../../../lib', __FILE__
+    'ruby -I ' +  lib + ' ' + File.expand_path('../../../bin/snoopit', __FILE__)
+  end
+
+  context 'command line options' do
+
+    it '--help' do
+      output = capture_stdout cmd + ' --help'
+      expect(output).to match(/Usage: snoopit \[options\]/)
+      expect(output).to match(/--help/)
+      expect(output).to match(/-h/)
+      expect(output).to match(/-v/)
+      expect(output).to match(/-verbose/)
+    end
+
+    it '-h print help message' do
+      output = capture_stdout cmd + ' --help'
+      expect(output).to match(/Usage: snoopit \[options\]/)
+      expect(output).to match(/--help/)
+      expect(output).to match(/-h/)
+      expect(output).to match(/-v/)
+      expect(output).to match(/-verbose/)
+    end
+
+    # use :skip with coverage
+    it '-t print a snoopers template to standard out' do
+      output = capture_stdout cmd + ' -t'
+      jo = JSON.parse output
+      notifiers = jo['notifiers']
+      expect(notifiers['load'].size).to eq 1
+      expect(notifiers['load']['Notifier Identifier']['file']).to eq '/path/to/mynotifier'
+      expect(notifiers).to include 'email'
+      expect(notifiers).to include 'stomp'
+      expect(notifiers).to include 'http'
+      expect(notifiers).to include 'https'
+    end
+
+    it '--snoopers file.json load specified snoopers file' do
+      file = File.expand_path '../../support/snoopies.json', __FILE__
+      output = capture_stdout cmd + ' --snoopers ' + file
+      expect(output).to match /Reading from queue: scores\:\/queue\/scores/
+      expect(output).to match /Prediction loader waiting for scores \.\.\./
+      expect(output).to include 'Non OK Status'
+      expect(output).to include 'Total Number of records'
+    end
+
+    it '-s file.json load specified snoopers file' do
+      file = File.expand_path '../../support/snoopies.json', __FILE__
+      output = capture_stdout cmd + ' -s ' + file
+      expect(output).to match /Reading from queue: scores\:\/queue\/scores/
+      expect(output).to match /Prediction loader waiting for scores \.\.\./
+      expect(output).to match /Non OK Status from AI Core\:/
+      expect(output).to include 'Non OK Status'
+      expect(output).to include 'Total Number of records'
+      expect(output).to include 'Failed to bulk load'
+    end
+
+    it '-S snooper only use the specified snooper from the snoopers file' do
+      file = File.expand_path '../../support/snoopies.json', __FILE__
+      output = capture_stdout cmd + ' -s ' + file + ' -S AppServer2'
+      expect(output).to match /Reading from queue: scores\:\/queue\/scores/
+      expect(output).to match /Prediction loader waiting for scores \.\.\./
+      expect(output).to match /Non OK Status from AI Core:/
+      expect(output).to include 'Non OK Status'
+      expect(output).not_to include 'Total Number of records'
+      expect(output).not_to include 'Failed to bulk load'
+    end
+
+    it '--snooper snooper only user the specified snooper from the snoopers file' do
+      file = File.expand_path '../../support/snoopies.json', __FILE__
+      output = capture_stdout cmd + ' -s ' + file + ' --snooper AppServer2'
+      expect(output).to match /Reading from queue: scores\:\/queue\/scores/
+      expect(output).to match /Prediction loader waiting for scores \.\.\./
+      expect(output).to match /Non OK Status from AI Core:/
+      expect(output).to include 'Non OK Status'
+      expect(output).not_to include 'Total Number of records'
+      expect(output).not_to include 'Failed to bulk load'
+    end
+
+    # use :skip with coverage
+    it '--json generate json output' do
+      file = File.expand_path '../../support/snoopies.json', __FILE__
+      output = capture_stdout cmd + ' -s ' + file + ' --snooper AppServer2 --json'
+      expect(output).to match /Reading from queue: scores\:\/queue\/scores/
+      expect(output).to match /Prediction loader waiting for scores \.\.\./
+      expect(output).to match /Non OK Status from AI Core:/
+      expect(output).to include 'Non OK Status'
+      expect(output).not_to include 'Total Number of records'
+      expect(output).not_to include 'Failed to bulk load'
+      jo = JSON.parse output
+      expect(jo.size).to eq 1
+      app2 = jo[0]
+      expect(app2['name']).to eq 'AppServer2'
+      sniffers = app2['sniffers'][0]
+      expect(sniffers['before']).to eq 2
+      expect(sniffers['after']).to eq 2
+      sniffed = sniffers['sniffed']
+      expect(sniffed.size).to eq 120
+    end
+
+    # use :skip with coverage
+    it '-j generate json output' do
+      file = File.expand_path '../../support/snoopies.json', __FILE__
+      output = capture_stdout cmd + ' -s ' + file + ' --snooper AppServer2 -j'
+      expect(output).to match /Reading from queue: scores\:\/queue\/scores/
+      expect(output).to match /Prediction loader waiting for scores \.\.\./
+      expect(output).to match /Non OK Status from AI Core:/
+      expect(output).to include 'Non OK Status'
+      expect(output).not_to include 'Total Number of records'
+      expect(output).not_to include 'Failed to bulk load'
+      jo = JSON.parse output
+      expect(jo.size).to eq 1
+      app2 = jo[0]
+      expect(app2['name']).to eq 'AppServer2'
+      sniffers = app2['sniffers'][0]
+      expect(sniffers['before']).to eq 2
+      expect(sniffers['after']).to eq 2
+      sniffed = sniffers['sniffed']
+      expect(sniffed.size).to eq 120
+    end
+
+    # use :skip with coverage
+    it '2 snoopers' do
+      file = File.expand_path '../../support/snoopies.json', __FILE__
+      ap file
+      output = capture_stdout cmd + ' -s ' + file + ' -S AppServer2  -S SnoopTest -j'
+      expect(output).to match /Reading from queue: scores\:\/queue\/scores/
+      expect(output).to match /Prediction loader waiting for scores \.\.\./
+      expect(output).to include 'Non OK Status'
+      expect(output).to include 'Total Number of records'
+      expect(output).to include 'Failed to bulk load'
+      jo = JSON.parse output
+      expect(jo.size).to eq 2
+      app = jo[0]
+      names = []
+      names << app['name']
+      sniffers = app['sniffers'][0]
+      expect(sniffers['before']).to eq 2
+      expect(sniffers['after']).to eq 2
+      sniffed = sniffers['sniffed']
+      expect(sniffed.size).to eq 120
+      app2 = jo[1]
+      names << app2['name']
+      expect(names.size).to eq 2
+      expect(names).to include 'SnoopTest'
+      expect(names).to include 'AppServer2'
+      sniffers = app2['sniffers'][0]
+      expect(sniffers['before']).to eq 2
+      expect(sniffers['after']).to eq 2
+      sniffed = sniffers['sniffed']
+      expect(sniffed.size).to eq 120
+    end
+
+    it '-T enable file tracking' do
+      db_file = File.expand_path '../../../snoopit_db.json', __FILE__
+      File.delete db_file if File.exist? db_file
+      file = File.expand_path '../../support/snoopies.json', __FILE__
+      output = capture_stdout cmd + ' -s ' + file + ' --snooper AppServer2 -T'
+      expect(output).to match /Reading from queue: scores\:\/queue\/scores/
+      expect(output).to match /Prediction loader waiting for scores \.\.\./
+      expect(output).to match /Non OK Status from AI Core:/
+      expect(output).to include 'Non OK Status'
+      expect(output).not_to include 'Total Number of records'
+      expect(output).not_to include 'Failed to bulk load'
+      expect(File.exist?(db_file)).to eq true
+      expect(File.size(db_file)).to be > 0
+      tracked = File.read db_file
+      expect(tracked).to include './spec/support/log/snoop_log.test'
+      expect(tracked).to include './spec/support/log/snoop_log_2.test'
+      #ap JSON.parse tracked
+      File.delete db_file if File.exist? db_file
+    end
+
+    it '--tracking enable file tracking' do
+      db_file = File.expand_path '../../../snoopit_db.json', __FILE__
+      ap db_file
+      File.delete db_file if File.exist? db_file
+      file = File.expand_path '../../support/snoopies.json', __FILE__
+      output = capture_stdout cmd + ' -s ' + file + ' --snooper AppServer2 --tracking'
+      expect(output).to match /Reading from queue: scores\:\/queue\/scores/
+      expect(output).to match /Prediction loader waiting for scores \.\.\./
+      expect(output).to match /Non OK Status from AI Core:/
+      expect(output).to include 'Non OK Status'
+      expect(output).not_to include 'Total Number of records'
+      expect(output).not_to include 'Failed to bulk load'
+      expect(File.exist?(db_file)).to eq true
+      expect(File.size(db_file)).to be > 0
+      tracked = File.read db_file
+      expect(tracked).to include './spec/support/log/snoop_log.test'
+      expect(tracked).to include './spec/support/log/snoop_log_2.test'
+      #ap JSON.parse tracked
+      File.delete db_file if File.exist? db_file
+    end
+
+    it '-f filename use a different named tracking file' do
+      tmp_dir = File.expand_path '../../../tmp', __FILE__
+      Dir.mkdir tmp_dir unless Dir.exist?(tmp_dir)
+      db_file = File.expand_path '../../../tmp/snoopit_db.json', __FILE__
+      File.delete db_file if File.exist? db_file
+      file = File.expand_path '../../support/snoopies.json', __FILE__
+      output = capture_stdout cmd + ' -s ' + file + ' --snooper AppServer2 -f ' + db_file
+      expect(output).to match /Reading from queue: scores\:\/queue\/scores/
+      expect(output).to match /Prediction loader waiting for scores \.\.\./
+      expect(output).to match /Non OK Status from AI Core:/
+      expect(output).to include 'Non OK Status'
+      expect(output).not_to include 'Total Number of records'
+      expect(output).not_to include 'Failed to bulk load'
+      expect(File.exist?(db_file)).to eq true
+      expect(File.size(db_file)).to be > 0
+      tracked = File.read db_file
+      expect(tracked).to include './spec/support/log/snoop_log.test'
+      expect(tracked).to include './spec/support/log/snoop_log_2.test'
+      #ap JSON.parse tracked
+      File.delete db_file if File.exist? db_file
+    end
+
+    it '--tracking-file filename use a different named tracking file' do
+      tmp_dir = File.expand_path '../../../tmp', __FILE__
+      Dir.mkdir tmp_dir unless Dir.exist?(tmp_dir)
+      db_file = File.expand_path '../../../tmp/snoopit_db.json', __FILE__
+      File.delete db_file if File.exist? db_file
+      file = File.expand_path '../../support/snoopies.json', __FILE__
+      output = capture_stdout cmd + ' -s ' + file + ' --snooper AppServer2 --tracking-file ' + db_file
+      expect(output).to match /Reading from queue: scores\:\/queue\/scores/
+      expect(output).to match /Prediction loader waiting for scores \.\.\./
+      expect(output).to match /Non OK Status from AI Core:/
+      expect(output).to include 'Non OK Status'
+      expect(output).not_to include 'Total Number of records'
+      expect(output).not_to include 'Failed to bulk load'
+      expect(File.exist?(db_file)).to eq true
+      expect(File.size(db_file)).to be > 0
+      tracked = File.read db_file
+      expect(tracked).to include './spec/support/log/snoop_log.test'
+      expect(tracked).to include './spec/support/log/snoop_log_2.test'
+      #ap JSON.parse tracked
+      File.delete db_file if File.exist? db_file
+    end
+
+  end
+
+
+end