watchtower_job 1.3.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: b074e4831eec4322589f4562fd25bd2a364f45cd
+  data.tar.gz: aaf0d321d18885196a6efe189ac9f38ea7bd52c3
+SHA512:
+  metadata.gz: df6b2d939a5524cb29afa118f37c3921fece0ee090fada993aaeff3112763a0f64aedc532c2a2bf0117af2e546107b06fa7f127167d5acedfbde4b3108b64c28
+  data.tar.gz: e4849ce069b3700028c0d6c7884a237fbe1c789752f57d8ac496ca27b6cdbd1e354787742c8ff40bd7c1202b76217c05d9d45f49091d90030692dfdf41e212b5

data/.gitignore

@@ -0,0 +1,14 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+
+*.gem
+
+# rspec failure tracking
+.rspec_status

data/.rspec

@@ -0,0 +1,2 @@
+--format progress
+--color

data/.travis.yml

@@ -0,0 +1,5 @@
+sudo: false
+language: ruby
+rvm:
+  - 2.1.5
+before_install: gem install bundler -v 1.14.3

data/Gemfile

@@ -0,0 +1,6 @@
+source 'https://rubygems.org'
+source 'http://gems.appriss.com/core/gems'
+source 'https://gems:gp9L2OCcKw@gems.prod.awstools.appriss.com'
+
+# Specify your gem's dependencies in watchtower_job.gemspec
+gemspec

data/Guardfile

@@ -0,0 +1,17 @@
+# A sample Guardfile
+# More info at https://github.com/guard/guard#readme
+
+guard :rspec, cmd: "bundle exec rspec --order rand" do
+  require "guard/rspec/dsl"
+  dsl = Guard::RSpec::Dsl.new(self)
+
+  # Feel free to open issues for suggestions and improvements
+
+  # RSpec files
+  rspec = dsl.rspec
+  watch(rspec.spec_helper) { rspec.spec_dir }
+  watch(rspec.spec_support) { rspec.spec_dir }
+  watch(rspec.spec_files)
+
+  watch(/^lib\/(.*)\.rb/) { |m| "spec/#{m[1]}_spec.rb" }
+end

data/README.md

@@ -0,0 +1,36 @@
+# WatchtowerJob
+
+This gem will read from an Interface Execution event kinesis stream and post events to Einstien.
+
+Alarm conditions described here: https://wiki.sdlc.appriss.com/display/EINSTEIN/WatchTower+Alarm+Types
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'watchtower_job'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install watchtower_job
+
+## Usage
+
+This is expected to be a long-running process.  Run the command in /exe.  Use the -h command line arg to understand what options can be used with the running of this gem for choosing the NT2 environment and which Einstein instance to use.
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+This source is kept in Appriss's BitBucket repository.
+

data/Rakefile

@@ -0,0 +1,6 @@
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new(:spec)
+
+task :default => :spec

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "watchtower_job"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start(__FILE__)

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/exe/watchtower_job

@@ -0,0 +1,63 @@
+#!/usr/bin/env ruby
+# -*- mode: ruby -*-
+$:.unshift File.join(File.dirname(__FILE__), "..", "lib")
+
+require "watchtower_job"
+require 'slop'
+require 'einstein_environment'
+require 'syslog/logger'
+
+opts = Slop.parse(ARGV, :help => true) do
+  banner <<-EOS
+*****************
+  watchtower_job listens to the NT2 Interface Execution Kinesis stream and posts Data Feed NT2 events to Einstein.  This app runs indefinitely.
+  Version: #{WatchtowerJob::VERSION}
+*****************
+Usage:
+        watchtower_job [options]
+where [options] are:
+EOS
+  on :l, :log_level=, "Set to debug, info, warn, error, or fatal.", :default => "debug"
+  on :o, :log_file=, "File to log output to.  By default logs to stdout.", :default => nil
+  on :n, :nt2_env=, "Environment from which to read NT2 Interface Execution Kinesis stream", :default => Einstein::Environment::Environment.get_appriss_environment
+  on :e, :einstein_env=, "Environment of Einstein for posting events", :default => Einstein::Environment::Environment.get_appriss_environment
+  on :c, :counts_only, 'Do not send errors, only counts of successful runs', :default => false
+  on :p, :pid_file=, "File to which the current pid will be written", :default => nil
+  on :s, :sequence_file=, "File from which the last read squence_number will be read and written", :default => nil
+  on :d, :datafeed_state_file=, "File from which the data feed state will be read and written", :default => nil
+end
+exit 0 if opts.help?
+
+if pid_file_name = opts[:pid_file]
+  if File.exists?(pid_file_name)
+    file = File.open(pid_file_name)
+    file.flock(File::LOCK_EX | File::LOCK_NB) or abort "WatchTower is already running using #{pid_file_name} to track it's status."
+  end
+  WatchtowerJob.write_pid(pid_file_name)
+end
+
+#setup logging
+log_name = opts[:log_file]
+if log_name
+  log = Syslog::Logger.new(log_name)
+else
+  log = Logger.new($stdout)
+end
+log.level = Logger.const_get(opts[:log_level].upcase)
+if log_name
+  log.formatter = proc do |severity, datetime, progname, msg|
+    datetime = datetime.strftime('%m/%d/%yT%H:%M')
+    "[#{datetime}] #{$$} #{severity} -- #{msg}\n"
+  end
+else
+  # Give a more concise format when logging to STDOUT
+  log.formatter = proc do |severity, datetime, progname, msg|
+    "#{severity}: #{msg}\n"
+  end
+end
+
+WatchtowerJob.set_logger(log)
+WatchtowerJob.set_nt2_environment(opts[:nt2_env])
+WatchtowerJob.set_einstein_environment(opts[:einstein_env])
+
+WatchtowerJob::Application.new(skip_errors:opts[:counts_only], last_run_file: opts[:sequence_file], datafeed_state_file: opts[:datafeed_state_file]).run

data/initd_script_basic

@@ -0,0 +1,42 @@
+# This script must be located within the /etc/init.d directory
+# this should be copied as watchtower_job
+
+### BEGIN INIT INFO
+# Provides:          watchtower_job
+# Required-Start:    $all
+# Required-Stop:     $local_fs $remote_fs $network
+# Default-Start:     3 5
+# Default-Stop:	     0 1 2 6
+# Short-Description: WatchTower Job
+# Description:       Generates Einstein events based on interface executions
+### END INIT INFO
+
+. /etc/rc.status
+rc_reset
+
+DUSER=root
+DGROUP=root
+
+case "$1" in
+  start)
+    echo -n 'Starting WatchTower Job: '
+    nohup env AWS_SECRET_ACCESS_KEY=qs1DA4TvNqLMxmDv5wt6GW0jBS+rb/R99j1Ubkz2 AWS_ACCESS_KEY_ID=AKIAJ3LAV3OLVBKJEMMQ watchtower_job -o watchtower_job -n prod -p /var/tmp/watchtower_job.pid -s /var/tmp/watchtower_job.last -d /var/tmp/watchtower_job_datafeed_state.yml > /dev/null 2>&1&
+    rc_status -v
+    ;;
+  stop)
+    echo -n 'Stopping WatchTower Job: '
+    kill `cat /var/tmp/watchtower_job.pid`
+    rc_status -v
+    ;;
+  status)
+    if ! ps -p `cat /var/tmp/watchtower_job.pid` > /dev/null 2>&1; then echo "not running">&2; else echo "running">&2; fi
+    ;;
+  restart)
+    $0 stop
+    $0 start
+    ;;
+  *)
+    echo "Usage: $0 {start|stop|restart|status}"
+    exit 1
+esac
+rc_exit

data/initd_script_counts

@@ -0,0 +1,42 @@
+# This script must be located within the /etc/init.d directory
+# this should be copied as watchtower_job_prep_counts
+
+### BEGIN INIT INFO
+# Provides:          watchtower_job
+# Required-Start:    $all
+# Required-Stop:     $local_fs $remote_fs $network
+# Default-Start:     3 5
+# Default-Stop:	     0 1 2 6
+# Short-Description: WatchTower Job Prep Counts
+# Description:       Generates Einstein events based on interface executions
+### END INIT INFO
+
+. /etc/rc.status
+rc_reset
+
+DUSER=root
+DGROUP=root
+
+case "$1" in
+  start)
+    echo -n 'Starting WatchTower Job Prep Counts: '
+    nohup env AWS_SECRET_ACCESS_KEY=qs1DA4TvNqLMxmDv5wt6GW0jBS+rb/R99j1Ubkz2 AWS_ACCESS_KEY_ID=AKIAJ3LAV3OLVBKJEMMQ watchtower_job -o watchtower_job_prep_counts -n prep -p /var/tmp/watchtower_job_prep_counts.pid -s /var/tmp/watchtower_job_prep_counts.last -c > /dev/null 2>&1&
+    rc_status -v
+    ;;
+  stop)
+    echo -n 'Stopping WatchTower Job Prep Counts: '
+    kill `cat /var/tmp/watchtower_job_prep_counts.pid`
+    rc_status -v
+    ;;
+  status)
+    if ! ps -p `cat /var/tmp/watchtower_job_prep_counts.pid` > /dev/null 2>&1; then echo "not running">&2; else echo "running">&2; fi
+    ;;
+  restart)
+    $0 stop
+    $0 start
+    ;;
+  *)
+    echo "Usage: $0 {start|stop|restart|status}"
+    exit 1
+esac
+rc_exit

data/lib/watchtower_job.rb

@@ -0,0 +1,90 @@
+require 'watchtower_job/version'
+require 'watchtower_job/data_feed'
+require 'watchtower_job/execution_failure'
+require 'watchtower_job/interface_execution'
+require 'watchtower_job/event_collector'
+require 'watchtower_job/vine_site_services'
+require 'watchtower_job/kinesis_reader'
+require 'watchtower_job/workflow_counts'
+require 'watchtower_job/application'
+require 'net/https'
+require 'logger'
+require 'json'
+
+module WatchtowerJob
+  class DataFeedNotCompleted < RuntimeError; end
+
+  @@nt2_env = nil
+  @@nt2_service = nil
+  @@log = nil
+  @@event_collector_url = nil
+  @@einstein_env = nil
+  @@dfr2_api_url = nil
+
+  def self.write_pid(file_name)
+    File.open(file_name, 'w') { |file| file.write(Process.pid) }
+  end
+
+  def self.nt2_service
+    return @@nt2_service unless @@nt2_service.nil?
+    raise 'Set NT2 Environment!' if @@nt2_env.nil?
+    uri = URI.parse "https://web-external-#{@@nt2_env}.nt2.appriss.com"
+    @@nt2_service = Net::HTTP.new(uri.host, uri.port)
+    @@nt2_service.use_ssl = true
+    @@nt2_service
+  end
+
+  def self.dfr2_api_url
+    raise 'Set NT2 Environment!' if @@nt2_env.nil?
+    @@dfr2_api_url ||=
+      case @@nt2_env
+      when 'dev', 'qa'
+        'https://dfr2-api.dev.awstools.appriss.com'
+      when 'prep', 'prod'
+        'https://dfr2-api.prod.awstools.appriss.com'
+      when 'test'
+        'https://dfr2-api.test.awstools.appriss.com'
+      end
+  end
+
+  def self.vine_site_service
+    raise 'Set NT2 Environment!' if @@nt2_env.nil?
+    @@vine_services ||= VineSiteServices.new
+  end
+
+  def self.nt2_env
+    @@nt2_env
+  end
+
+  def self.set_nt2_environment(env)
+    @@nt2_service = nil
+    @@vine_services = nil
+    @@dfr2_api_url = nil
+    @@nt2_env = env.downcase
+  end
+
+  def self.einstein_event_collector
+    raise 'Set Einstein Environment!' if @@einstein_env.nil?
+    @@event_collector ||= EventCollector.new("https://events-watchtower-job.#{@@einstein_env}.awstools.appriss.com/")
+  end
+
+  def self.set_einstein_environment(env)
+    @@event_collector = nil
+    @@einstein_env = env.downcase
+  end
+
+  def self.log
+    return @@log unless @@log.nil?
+    #default to stdout
+    @@log = Logger.new($stdout)
+    @@log.level = Logger::DEBUG
+    @@log.formatter = proc do |severity, datetime, progname, msg|
+      "#{severity}: #{msg}\n"
+    end
+    @@log
+  end
+
+  def self.set_logger(logger)
+    @@log = logger
+  end
+end

data/lib/watchtower_job/application.rb

@@ -0,0 +1,153 @@
+require 'einstein_aws_keys'
+require 'aws-sdk'
+require 'yaml'
+
+module WatchtowerJob
+  class Application
+    CHECK_IN_FREQUENCY = 5 * 60 #5 minutes
+    def initialize(skip_errors:false,sleep_time:30,last_run_file:nil,datafeed_state_file:nil)
+      @datafeed_state_file = datafeed_state_file
+      @data_feeds = {}
+      get_datafeed_state
+      @sleep_time = sleep_time
+      @skip_errors = skip_errors
+      @last_run_file = last_run_file
+      s3 = Aws::S3::Resource.new(Einstein::AwsKeys::AwsKeys.get_keys.merge(region:'us-east-1'))
+      config = YAML.load s3.bucket('cttools-watchtower').object("configs/kinesis_keys.yaml").get.body.string
+      @stream = KinesisReader.new(config[WatchtowerJob.nt2_env], last_sequence_number)
+      @workflow_counter = WorkflowCounts.new
+    end
+
+    def run
+      startup_time = Time.now
+      post_check_in(@last_check_in = startup_time)
+      @stream.read do |message|
+        check_in
+        if message.nil?
+          WatchtowerJob.log.debug "Read nothing, sleep for #{@sleep_time}"
+          sleep @sleep_time
+          next
+        end
+        message_data = JSON.parse(message['data'])
+
+        if (InterfaceExecution.required_values - message_data.keys).length != 0
+          WatchtowerJob.log.info "Ignoring non-InterfaceExecution message: #{message}"
+          next
+        end
+
+        interface_execution = InterfaceExecution.new(message_data)
+        if interface_execution.success? && interface_execution.end_time >= startup_time #prevent sending large counts at startup
+          @workflow_counter.count(interface_execution.workflow)
+        end
+
+        next if @skip_errors
+
+        data_feed = get_feed(interface_execution.data_feed_id)
+        if data_feed.nil?
+          #not implemented, do nothing with this execution
+          WatchtowerJob.log.debug "Data Feed #{interface_execution.data_feed_id} implementation status is not \"Completed\""
+          next
+        end
+        WatchtowerJob.log.debug "Read Interface Execution for #{interface_execution.data_feed_id}"
+        data_feed.handle_latest_execution(interface_execution)
+      end
+    rescue SignalException => e
+      WatchtowerJob.log.fatal "Received #{e.message} Signal.... Shutting down..."
+      keep_last_sequence_number
+      keep_datafeed_state
+    rescue => e
+      WatchtowerJob.log.fatal "Crashed! - #{e.class}: #{e.message}"
+      WatchtowerJob.log.debug "Backtrace:\n#{e.backtrace.join('\n')}"
+      keep_last_sequence_number
+      keep_datafeed_state
+    end
+
+    def keep_last_sequence_number
+      return unless @last_run_file
+      File.open(@last_run_file, 'w') { |file| file.write(@stream.last_sequence_number) }
+      WatchtowerJob.log.info "Saved last sequence number. (#{@stream.last_sequence_number})"
+    end
+
+    def last_sequence_number(file=@last_run_file)
+      return nil if file.nil?
+      if File.exist?(file)
+        last_sequence = File.read(file)
+        WatchtowerJob.log.info "Stream's last sequence number: (#{last_sequence})"
+        return last_sequence
+      end
+      WatchtowerJob.log.info "#{file} does not exist -- will not use a last_sequence"
+      return nil
+    end
+
+    def keep_datafeed_state
+      return unless @datafeed_state_file
+      cleared_list = @data_feeds.select do |id,data_feed|
+        data_feed.errors == 0 && !data_feed.never_been_cleared
+      end
+      File.open(@datafeed_state_file, 'w') { |file| file.write(cleared_list.keys.to_yaml) }
+      WatchtowerJob.log.info "Saved datafeed state: (#{cleared_list.keys})"
+    end
+
+    def get_datafeed_state(file=@datafeed_state_file)
+      return nil if file.nil?
+      if File.exist?(file)
+        datafeed_list = YAML.load File.read(file)
+        WatchtowerJob.log.info "found #{datafeed_list.count} datafeeds, beginning load..."
+        threads = []
+        in_groups(datafeed_list,3).each_with_index do |list,t_index|
+          threads << Thread.new do
+            list.each_with_index do |id,index|
+              WatchtowerJob.log.info "thread:#{(t_index + 1)} Fetching(#{(t_index + 1) * index}): #{id} ..."
+              get_feed(id,false)
+            end
+          end
+        end
+        threads.map(&:join)
+        WatchtowerJob.log.info "datafeed state is: (#{@data_feeds.keys})"
+      else
+        WatchtowerJob.log.info "#{file} does not exist -- will start with empty state"
+      end
+    end
+
+    def get_feed(id,never_been_cleared=true)
+      @data_feeds[id] ||= DataFeed.new(id,never_been_cleared)
+    rescue DataFeedNotCompleted
+      WatchtowerJob.log.info "data feed not in completed state, ignoring"
+      return nil
+    end
+
+    def check_in
+      check_in_time = Time.now
+      if (check_in_time - @last_check_in).to_i > CHECK_IN_FREQUENCY
+        post_check_in(check_in_time)
+        @workflow_counter.send_counts
+        @last_check_in = check_in_time
+      end
+    end
+
+    def post_check_in(check_in_time)
+      event_type = "#{WatchtowerJob.nt2_env.upcase}.check_in"
+      data = WatchtowerJob.einstein_event_collector.event_body(
+        config_item_type: 'Einstein Component',
+        config_item_name: 'Watchtower Job',
+        event_type: event_type,
+        value: 'set',
+        start_time: check_in_time
+      )
+      WatchtowerJob.einstein_event_collector.post_event(data)
+    end
+
+    def in_groups(array,number)
+      group_size = array.size / number
+      leftovers = array.size % number
+      groups = []
+      start = 0
+      number.times do |index|
+        length = group_size + (leftovers > 0 && leftovers > index ? 1 : 0)
+        groups << array.slice(start, length)
+        start += length
+      end
+      groups
+    end
+  end
+end

data/lib/watchtower_job/data_feed.rb

@@ -0,0 +1,92 @@
+module WatchtowerJob
+  class DataFeed
+    ERROR_THRESHOLD = 7
+    MAESTRO_ERROR_THRESHOLD = 4
+
+    attr_reader :errors, :id, :never_been_cleared
+
+    def initialize(id,never_been_cleared=true)
+      @id = id
+      details = JSON.parse retrieve_data_feed_details(id)
+      @data_feed_attributes = details['data']['attributes']
+      raise DataFeedNotCompleted unless @data_feed_attributes['deployed-environments'][WatchtowerJob.nt2_env]['implementation-status-cd'] == 'Completed'
+      @errors = 0
+      @never_been_cleared = never_been_cleared
+    end
+
+    def handle_latest_execution(execution)
+      WatchtowerJob.log.debug "Run #{execution.success? ? 'successful' : 'failed'}."
+      if execution.success?
+        @errors = 0
+        if @never_been_cleared
+          first_clear
+        else
+          clear_current_failure
+        end
+      else
+        @errors += 1
+        return unless failure = determine_error(execution)
+        set_current_failure(failure)
+      end
+    end
+
+    def set_current_failure(failure)
+      if failure > @current_failure
+        WatchtowerJob.log.debug "Set alarm to #{failure.name}"
+        clear_current_failure
+        @current_failure = failure
+        failure.send_start_event
+      end
+    end
+
+    def first_clear
+      WatchtowerJob.log.debug "Clearing All Possible Failures"
+      @current_failure = nil
+      @never_been_cleared = false
+      ExecutionFailure.clear_all_failures_possible(id)
+    end
+
+    def clear_current_failure
+      if @current_failure
+        WatchtowerJob.log.debug "Clearing #{@current_failure.name}"
+        failure = @current_failure
+        @current_failure = nil
+        failure.send_clear_event
+      end
+    end
+
+    def determine_error(execution)
+      #please make sure you update the wiki as you make changes here
+      if execution.error_messages_include?(/Failed during Getsend/, -2)
+        return ExecutionFailure.new_getsend_minus_two(execution)
+      elsif execution.error_messages_include?(/Could not start interface\. Error communicating with the gateway\./)
+        return ExecutionFailure.new_gateway_error_failure(execution)
+      elsif execution.message =~ /Incomplete previous run detected/ #&& execution.sherlock_feed?
+        return ExecutionFailure.new_incomplete_previous_run(execution)
+      elsif execution.error_messages_include?(/Failed during Getsend/, -7) && execution.imports_enabled?
+        return ExecutionFailure.new_getsend_minus_seven(execution)
+      elsif @errors >= MAESTRO_ERROR_THRESHOLD && execution.error_messages_include?(/Interface was killed automatically, run time exceeded/) && photo_feed?
+        return ExecutionFailure.new_maestro_max_runtime_exceeded(execution)
+      elsif execution.message =~ /Failed during Import/
+        return ExecutionFailure.new_failed_import(execution)
+      elsif @errors >= ERROR_THRESHOLD
+        return ExecutionFailure.new_repeated_errors(execution)
+      end
+    end
+
+    def photo_feed?
+      @is_photo_feed ||= @data_feed_attributes['data-types'] == ['Photo']
+    end
+
+    def retrieve_data_feed_details(id)
+      retries ||= 0
+      sleep(retries ** 2)
+      RestClient.get "#{WatchtowerJob.dfr2_api_url}/data_feeds/#{id}?include_config=false", :content_type => 'application/json'
+    rescue => e
+      retry if (retries += 1) <= 4
+      WatchtowerJob.log.error e.message
+      WatchtowerJob.log.error "error with GET from dfr2-api, retry #{retries}"
+      raise e
+    end
+  end
+end

data/lib/watchtower_job/event_collector.rb

@@ -0,0 +1,45 @@
+require 'einstein_aws_keys'
+
+module WatchtowerJob
+  class EventCollector
+    def initialize(collector_url)
+      uri = URI.parse(collector_url)
+      @collector_service = Net::HTTP.new(uri.host, uri.port)
+      @collector_service.use_ssl = (uri.scheme == 'https')
+      @component_password = Einstein::AwsKeys::AwsKeys.get_component_password
+    end
+
+    def post_event(event_body)
+      retries ||= 0
+      WatchtowerJob.log.debug "Event post to #{event_body[:config_item_type]}:#{event_body[:config_item_name]}, type: #{event_body[:event_type]}, value: #{event_body[:value]}"
+      post = Net::HTTP::Post.new("/events", {'Content-Type' =>'application/json'})
+      post.body = event_body.to_json
+      post.basic_auth('watchtower_job', '49adf188447b20daaa5f45a960ec1001')
+      sleep(retries ** 2)
+      response = @collector_service.start {|https| https.request(post)}
+      raise "Einstein Event failed to post! Status: #{response.code} Body: #{response.body}" unless response.code == "200"
+      response
+    rescue => e
+      retry if (retries += 1) <= 4
+      WatchtowerJob.log.error e.message
+      WatchtowerJob.log.error "error with POST to einstein events, retry #{retries}"
+      raise e
+    end
+
+    def event_body(config_item_type:,config_item_name:,event_type:,value:,start_time:Time.now,alternate_key:nil,info:{})
+      body = {
+        :source => 'watch_tower_job',
+        :config_item_type => config_item_type,
+        :config_item_name => config_item_name,
+        :event_type => event_type,
+        :start_ts => start_time,
+        :value => value,
+        :info => info
+      }
+      if alternate_key
+        body.merge!({:alternate_name_key => alternate_key})
+      end
+      body
+    end
+  end
+end

data/lib/watchtower_job/execution_failure.rb

@@ -0,0 +1,79 @@
+module WatchtowerJob
+  class ExecutionFailure
+    attr_reader :name, :data_feed_id, :failure_start, :priority, :start_value, :clear_value
+    def initialize(name,data_feed_id,failure_start,priority,start_value='set',clear_value='clear')
+      @name,@data_feed_id,@failure_start,@priority,@start_value,@clear_value = name,data_feed_id,failure_start,priority,start_value,clear_value
+    end
+
+    def send_start_event
+      send_to_einstein(start_value,@failure_start)
+    end
+
+    def send_clear_event(clear_time=Time.now)
+      send_to_einstein(clear_value, clear_time)
+    end
+
+    def <=>(other)
+      return 1 if other.nil?
+      @priority <=> other.priority
+    end
+
+    def >(other)
+      (self <=> other) == 1
+    end
+
+    def self.clear_all_failures_possible(data_feed_id)
+      ExecutionFailure.failure_helper_methods.each do |new_failure|
+        generic_failure = ExecutionFailure.send(new_failure, data_feed_id: data_feed_id, failure_start: nil)
+        generic_failure.send_clear_event
+      end
+    end
+
+    def self.failure_helper_methods
+      self.methods.select{|m| m.to_s =~ /^new_/}
+    end
+
+    def self.new_getsend_minus_two(interface_execution=nil,data_feed_id:interface_execution.data_feed_id,failure_start:interface_execution.end_time)
+      ExecutionFailure.new('Getsend -2', data_feed_id, failure_start, 0)
+    end
+
+    def self.new_gateway_error_failure(interface_execution=nil,data_feed_id:interface_execution.data_feed_id,failure_start:interface_execution.end_time)
+      ExecutionFailure.new('Gateway Error Alarm', data_feed_id, failure_start, 10)
+    end
+
+    def self.new_getsend_minus_seven(interface_execution=nil,data_feed_id:interface_execution.data_feed_id,failure_start:interface_execution.end_time)
+      ExecutionFailure.new('Getsend -7', data_feed_id, failure_start, 20)
+    end
+
+    def self.new_incomplete_previous_run(interface_execution=nil,data_feed_id:interface_execution.data_feed_id,failure_start:interface_execution.end_time)
+      ExecutionFailure.new('Incomplete Previous Run', data_feed_id, failure_start, 30)
+    end
+
+    def self.new_maestro_max_runtime_exceeded(interface_execution=nil,data_feed_id:interface_execution.data_feed_id,failure_start:interface_execution.end_time)
+      ExecutionFailure.new('Maestro Max Runtime Exceeded', data_feed_id, failure_start, 50)
+    end
+
+    def self.new_failed_import(interface_execution=nil,data_feed_id:interface_execution.data_feed_id,failure_start:interface_execution.end_time)
+      ExecutionFailure.new('Failed Import', data_feed_id, failure_start, 70)
+    end
+
+    def self.new_repeated_errors(interface_execution=nil,data_feed_id:interface_execution.data_feed_id,failure_start:interface_execution.end_time)
+      ExecutionFailure.new('Repeated Errors', data_feed_id, failure_start, 100)
+    end
+
+  private
+
+    def send_to_einstein(value,time)
+      data = WatchtowerJob.einstein_event_collector.event_body(
+        config_item_type:'Data Feed NT2',
+        config_item_name:@data_feed_id,
+        alternate_key:'data_feed_id',
+        event_type:@name.downcase,
+        value:value,
+        start_time:time
+      )
+      WatchtowerJob.einstein_event_collector.post_event(data)
+    end
+
+  end
+end

data/lib/watchtower_job/interface_execution.rb

@@ -0,0 +1,72 @@
+module WatchtowerJob
+  class InterfaceExecution
+    def self.required_values
+      %w(datafeedId runEndTime interfaceExecutionId interfaceId interfaceVersion site type message success)
+    end
+
+    attr_reader :end_time, :data_feed_id, :interface_version, :message, :workflow
+    def initialize(execution_message)
+      @data_feed_id = execution_message['datafeedId']
+      @interface_version = execution_message['interfaceVersion']
+      @message = execution_message['message']
+      @workflow = execution_message['type']
+      @success = execution_message['success']
+      @site, @agency = execution_message['site'], execution_message['agency']
+      @interface_id = execution_message['interfaceId']
+      @execution_id = execution_message['interfaceExecutionId']
+      @end_time = Time.at(execution_message['runEndTime']/1000)
+    end
+
+    def success?
+      @success
+    end
+
+    def errors
+      @errors ||= JSON.parse(retreive_errors)
+    end
+
+    def retreive_errors
+      retries ||= 0
+      sleep(retries ** 2)
+      response = WatchtowerJob.nt2_service.start do |https|
+        https.request(Net::HTTP::Get.new("/interface_execution/#{@execution_id}/errors"))
+      end
+      raise "GET of NT2 errors failed. Status: #{response.code} Body: #{response.body}" unless response.code == "200"
+      response.body
+    rescue => e
+      retry if (retries += 1) <= 4
+      WatchtowerJob.log.error e.message
+      WatchtowerJob.log.error "error with GET from NT2 errors, retry #{retries}"
+      raise e
+    end
+
+    def error_messages_include?(regex,error_code=nil)
+      errors.each do |error|
+        if error['errorDescription'] =~ regex
+          #if error_code is available, it must also match
+          if !error_code || error['errorCode'] == error_code
+            return true
+          end
+        end
+      end
+      return false
+    end
+
+    def sherlock_feed?
+      ['SHERLOCK', 'SHERLOCK_FILE'].include? @workflow
+    end
+
+    def imports_enabled?
+      site_operationals = WatchtowerJob.vine_site_service.get_details(@site)['operationals']['vine']
+      return false unless operationals_support_imports?(site_operationals)
+      return true unless @agency
+      agency_operationals = WatchtowerJob.vine_site_service.get_details(@site,@agency)['operationals']['vine']
+      operationals_support_imports?(agency_operationals)
+    end
+
+    def operationals_support_imports?(vine_operationals)
+      return false if vine_operationals['master']['disabled'] || !vine_operationals['master']['available']
+      vine_operationals['import']['available'] && !vine_operationals['import']['disabled']
+    end
+  end
+end

data/lib/watchtower_job/kinesis_reader.rb

@@ -0,0 +1,43 @@
+require 'aws-sdk'
+
+module WatchtowerJob
+  class KinesisReader
+    attr_reader :last_sequence_number
+    def initialize(config,last_sequence_number=nil)
+      @config = config
+      @last_sequence_number = last_sequence_number
+    end
+
+    def get_iterator
+      description = client.describe_stream({stream_name: stream_name})
+      start_shard = description.stream_description.shards.first
+      if @last_sequence_number
+        return client.get_shard_iterator(stream_name: stream_name, shard_id: start_shard.shard_id,shard_iterator_type:"AFTER_SEQUENCE_NUMBER",starting_sequence_number: @last_sequence_number).shard_iterator
+      end
+      client.get_shard_iterator(stream_name: stream_name, shard_id: start_shard.shard_id,shard_iterator_type:"TRIM_HORIZON").shard_iterator
+    end
+
+    def read(iterator=get_iterator,&block)
+      while (records = client.get_records(shard_iterator:iterator,limit:10)) do
+        iterator = records[:next_shard_iterator]
+        records[:records].each do |message|
+          WatchtowerJob.log.debug "Stream Message: #{message}"
+          yield message.dup
+          @last_sequence_number = message['sequence_number']
+        end
+        if records[:records].empty?
+          yield nil
+        end
+      end
+      return false #it's done, but it should never be done!
+    end
+
+    def client
+      return @client ||= Aws::Kinesis::Client.new(@config)
+    end
+
+    def stream_name
+      @stream_name ||= client.list_streams[:stream_names].find{|name| name =~ /#{WatchtowerJob.nt2_env}/}
+    end
+  end
+end

data/lib/watchtower_job/version.rb

@@ -0,0 +1,3 @@
+module WatchtowerJob
+  VERSION = "1.3.0"
+end

data/lib/watchtower_job/vine_site_services.rb

@@ -0,0 +1,43 @@
+require 'hawk'
+require 'rest-client'
+
+module WatchtowerJob
+  class VineSiteServices
+    SERVICES_ID = 'mock'
+    SERVICES_KEY = 'mock'
+    SERVICES_USER = 'watchtower'
+
+    def host
+      "site-#{WatchtowerJob.nt2_env}.vineserv.appriss.com"
+    end
+
+    def build_hawk_header(path)
+      time_in_millis = Time.now.to_f.round
+      Hawk::Client.build_authorization_header(
+        :credentials => { :id => SERVICES_ID, :key => SERVICES_KEY, :algorithm => 'sha256' },
+        :ts => time_in_millis, :method => 'GET', :request_uri => path,
+        :host => host, :port => 443, :ext => "userId=#{SERVICES_USER}"
+      )
+    end
+
+    def get_details(site,agency=nil)
+      path = "/sites/#{site}"
+      if agency
+        path += "/agencies/#{agency}"
+      end
+      hawk_auth = build_hawk_header(path)
+      JSON.parse retrieve_agency_details(path,hawk_auth)
+    end
+
+    def retrieve_agency_details(path,hawk_auth)
+      retries ||= 0
+      sleep(retries ** 2)
+      RestClient.get "https://#{host}#{path}", :content_type => 'application/json', :Authorization => hawk_auth
+    rescue => e
+      retry if (retries += 1) <= 4
+      WatchtowerJob.log.error e.message
+      WatchtowerJob.log.error "error with GET from vine services, retry #{retries}"
+      raise e
+    end
+  end
+end

data/lib/watchtower_job/workflow_counts.rb

@@ -0,0 +1,54 @@
+module WatchtowerJob
+  class WorkflowCounts
+    WORKFLOWS = %w(GETSEND FTPPROC SHERLOCK SHERLOCK_FILE) #these are the only ones to monitor
+    def initialize
+      @workflows = Hash.new
+      reset_workflows_hash
+    end
+    
+    def count(workflow_type)
+      unless WORKFLOWS.include?(workflow_type)
+        WatchtowerJob.log.warn("#{workflow_type} is unsupported workflow type!") unless WORKFLOWS.include?(workflow_type)
+        return nil
+      end
+      @workflows[lookup_ci_name(workflow_type)] += 1
+    end
+
+    def send_counts
+      @workflows.each do |config_item_name, count|
+        WatchtowerJob.einstein_event_collector.post_event(
+          WatchtowerJob.einstein_event_collector.event_body(
+            config_item_type: 'Service',
+            config_item_name: config_item_name,
+            event_type: 'successful runs',
+            value: count
+          )
+        )
+      end
+      reset_workflows_hash
+    end
+
+    def lookup_ci_name(workflow)
+      name =
+        case workflow
+        when 'GETSEND'
+          'NT2.GetsendWorkflow'
+        when 'FTPPROC'
+          'NT2.FTPProcWorkflow'
+        when 'SHERLOCK', 'SHERLOCK_FILE'
+          'NT2.SherlockWorkflow'
+        else
+          return nil
+        end
+      return "PREP.#{name}" if WatchtowerJob.nt2_env == 'prep'
+      name
+    end 
+
+  private
+    def reset_workflows_hash
+      WORKFLOWS.map{|wf| lookup_ci_name(wf)}.uniq.each do |ci|
+        @workflows[ci] = 0
+      end
+    end
+  end
+end

data/send_an_event_for_each_failure_type.rb

@@ -0,0 +1,30 @@
+require 'watchtower_job'
+
+WatchtowerJob.set_einstein_environment(ARGV.shift || 'dev')
+
+class WatchtowerJob::EventCollector
+  alias original_event_body event_body
+  def event_body(**args)
+    body = original_event_body(args)
+    body[:info]['einstein-metadata'] = {note: 'This was generated manually to get events into Einstein.  If you have questions, please contact pbecker.'}
+    body
+  end
+end
+
+fake_execution_run = {
+  'datafeedId' => 1700000002, #this is "Charles Test Sherlock"
+  'runEndTime' => (Time.now - 60).to_i * 1000,
+  'interfaceExecutionId' => 0,
+  'interfaceId' => 1700000002,
+  'interfaceVersion' => 0,
+  'site' => 'FAKE',
+  'type' => 'Watchtower Test',
+  'message' => 'Watchtower Test',
+  'success' => true
+}
+
+ie = WatchtowerJob::InterfaceExecution.new(fake_execution_run)
+methods_to_generate_failures = (WatchtowerJob::ExecutionFailure.methods - Object.methods).select{|sym| sym.to_s =~ /^new/}
+execution_failures = methods_to_generate_failures.map{|method_name| WatchtowerJob::ExecutionFailure.send(method_name, ie)}
+execution_failures.map(&:send_start_event)
+execution_failures.map(&:send_clear_event)

data/watchtower_job.gemspec

@@ -0,0 +1,39 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'watchtower_job/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "watchtower_job"
+  spec.version       = WatchtowerJob::VERSION
+  spec.authors       = ["Philip Becker"]
+  spec.email         = ["pbecker@appriss.com"]
+
+  spec.summary       = %q{WatchTower observes the health of Data Feed interface executions.}
+  spec.description   = %q{Reads the interface execution kinesis stream for NT2.  Then it uses that execution info combined with information from VINE Services to determine if this is an error state.  If it is, it sends a message to Einstein.  It also clears Einstein alarms related to tracked errors.  State is not tracked between runs, so if the job fails it won't manage previously created errors.}
+
+
+  spec.files         = `git ls-files -z`.split("\x0").reject do |f|
+    f.match(%r{^(test|spec|features)/})
+  end
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.14"
+  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "rspec", "~> 3.0"
+  spec.add_development_dependency "guard"
+  spec.add_development_dependency 'guard-rspec'
+  spec.add_development_dependency 'webmock'
+  spec.add_development_dependency 'byebug'
+  spec.add_development_dependency 'awesome_print'
+
+  spec.add_dependency 'aws-sdk', '~> 2.7.15'
+  spec.add_dependency 'hawk-auth'
+  spec.add_dependency 'rest-client'
+  spec.add_dependency 'einstein_aws_keys'
+  spec.add_dependency 'einstein_environment'
+  spec.add_dependency 'slop', '~> 3.6.0'
+  spec.add_dependency 'SyslogLogger', '~> 2.0'
+end

metadata

@@ -0,0 +1,282 @@
+--- !ruby/object:Gem::Specification
+name: watchtower_job
+version: !ruby/object:Gem::Version
+  version: 1.3.0
+platform: ruby
+authors:
+- Philip Becker
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2018-04-17 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.14'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.14'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: guard
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: guard-rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: webmock
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: byebug
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: awesome_print
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: aws-sdk
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.7.15
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.7.15
+- !ruby/object:Gem::Dependency
+  name: hawk-auth
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rest-client
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: einstein_aws_keys
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: einstein_environment
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: slop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.6.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.6.0
+- !ruby/object:Gem::Dependency
+  name: SyslogLogger
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+description: Reads the interface execution kinesis stream for NT2.  Then it uses that
+  execution info combined with information from VINE Services to determine if this
+  is an error state.  If it is, it sends a message to Einstein.  It also clears Einstein
+  alarms related to tracked errors.  State is not tracked between runs, so if the
+  job fails it won't manage previously created errors.
+email:
+- pbecker@appriss.com
+executables:
+- watchtower_job
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rspec"
+- ".travis.yml"
+- Gemfile
+- Guardfile
+- README.md
+- Rakefile
+- bin/console
+- bin/setup
+- exe/watchtower_job
+- initd_script_basic
+- initd_script_counts
+- lib/watchtower_job.rb
+- lib/watchtower_job/application.rb
+- lib/watchtower_job/data_feed.rb
+- lib/watchtower_job/event_collector.rb
+- lib/watchtower_job/execution_failure.rb
+- lib/watchtower_job/interface_execution.rb
+- lib/watchtower_job/kinesis_reader.rb
+- lib/watchtower_job/version.rb
+- lib/watchtower_job/vine_site_services.rb
+- lib/watchtower_job/workflow_counts.rb
+- send_an_event_for_each_failure_type.rb
+- watchtower_job.gemspec
+homepage: 
+licenses: []
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.5.1
+signing_key: 
+specification_version: 4
+summary: WatchTower observes the health of Data Feed interface executions.
+test_files: []