scout_agent 3.0.0

data/lib/scout_agent/database/mission_log.rb

@@ -0,0 +1,282 @@
+#!/usr/bin/env ruby -wKU
+
+module ScoutAgent
+  class Database
+    class MissionLog < Database
+      # 
+      # A default number of seconds a mission is allowed to run before it is
+      # halted.
+      # 
+      DEFAULT_TIMEOUT  = 60
+      # The default number of minutes before a mission is run again.
+      DEFAULT_INTERVAL = 3
+      # A size limit for the reports table to prevent data from building up.
+      REPORTS_LIMIT    = 3000
+      
+      def update_schema(version = schema_version)
+        case version
+        when 0
+          <<-END_INITIAL_SCHEMA.trim
+          CREATE TABLE plans (
+            last_modified REQUIRED_TEXT_TYPE COLLATE NOCASE
+          );
+          
+          CREATE TABLE missions (
+            plan_id     DEFAULT_INTEGER_TYPE 1,
+            id          INTEGER NOT NULL PRIMARY KEY,
+            name        TEXT COLLATE NOCASE,
+            code        REQUIRED_TEXT_TYPE,
+            options     TEXT,
+            memory      TEXT DEFAULT '{}',
+            timeout     POSITIVE_INTEGER_TYPE DEFAULT #{DEFAULT_TIMEOUT},
+            interval    POSITIVE_INTEGER_TYPE DEFAULT #{DEFAULT_INTERVAL},
+            last_run_at DATETIME_TYPE,
+            next_run_at REQUIRED_DATETIME_TYPE
+          );
+          FOREIGN_KEY_CHECK_TRIGGER missions plan_id plans ROWID
+          
+          CREATE TABLE reports (
+            mission_id INTEGER NOT NULL,
+            type       TEXT NOT NULL COLLATE NOCASE
+              CHECK(type IN ('report', 'hint', 'alert',  'error')),
+            fields     REQUIRED_TEXT_TYPE,
+            created_at DATETIME_TYPE
+          );
+          FOREIGN_KEY_CHECK_TRIGGER reports mission_id missions id
+          DEFAULT_LOCALTIME_TRIGGER reports created_at
+          LIMIT_TABLE_SIZE_TRIGGER  reports #{REPORTS_LIMIT}
+          END_INITIAL_SCHEMA
+        end
+      end
+      
+      def current_plan
+        plan = read_from_sqlite { |sqlite|
+          sqlite.first_row_from(<<-END_FIND_PLAN.trim)
+          SELECT   ROWID AS id, last_modified
+          FROM     plans
+          ORDER BY ROWID DESC
+          LIMIT    1
+          END_FIND_PLAN
+        }
+        plan.empty? ? nil : plan
+      rescue Amalgalite::SQLite3::Error => error  # failed to retrieve plan
+        log.error("Database current plan error:  #{error.message}.")
+        nil  # not found
+      end
+      
+      def update_plan(last_modified, missions)
+        write_to_sqlite do |sqlite|
+          begin
+            sqlite.execute(<<-END_INSERT_PLAN.trim, last_modified)
+            INSERT OR REPLACE INTO plans(ROWID, last_modified) VALUES(1, ?)
+            END_INSERT_PLAN
+
+            ids = missions.map { |m| m["id"] }
+            sqlite.execute(<<-END_DELETE_MISSIONS.trim, *ids)
+            DELETE FROM missions
+            WHERE       id NOT IN (#{(['?'] * ids.size).join(', ')})
+            END_DELETE_MISSIONS
+          rescue Amalgalite::SQLite3::Error => error  # failed to update plan
+            log.error("Database bad plan error:  #{error.message}.")
+            sqlite.rollback  # these changes are all or nothing
+            return           # try again to update plan later
+          end
+          missions.each do |mission|
+            params = [ mission["name"],
+                       mission["code"],
+                       mission["options"].to_json,
+                       mission["timeout"].to_s =~ /\A\d*[1-9]\z/  ?
+                         mission["timeout"].to_i                  :
+                         DEFAULT_TIMEOUT,
+                       mission["interval"].to_s =~ /\A\d*[1-9]\z/ ?
+                         mission["interval"].to_i                 :
+                         DEFAULT_INTERVAL,
+                       mission["id"],
+                       Time.now.to_db_s(:trim_seconds) ]
+            begin
+              if sqlite.first_value_from(
+                   "SELECT id FROM missions WHERE id = ? LIMIT 1",
+                   mission["id"]
+                 )
+                params.pop  # remove next_run_at
+                sqlite.execute(<<-END_UPDATE_MISSION.trim, *params)
+                UPDATE missions
+                SET    name     = ?, code = ?, options = ?, timeout = ?,
+                       interval = ?
+                WHERE  id = ?
+                END_UPDATE_MISSION
+              else
+                sqlite.execute(<<-END_INSERT_MISSION.trim, *params)
+                INSERT INTO
+                  missions( name,     code, options,    timeout,
+                            interval,   id, next_run_at )
+                  VALUES(      ?,        ?,           ?,      ?,
+                               ?,        ?,           ? )
+                END_INSERT_MISSION
+              end
+            rescue Amalgalite::SQLite3::Error => error  # failed to set mission
+              # do nothing:  skip bad mission and move on
+              log.error( "Database bad mission (#{mission['name']}) error:  " +
+                         "#{error.message}." )
+            end
+          end
+        end
+      rescue Amalgalite::SQLite3::Error => error  # failed to get a write lock
+        # try again to update plan later
+        log.error("Database mission update locking error:  #{error.message}.")
+      end
+      
+      def current_mission
+        mission = read_from_sqlite { |sqlite|
+          return nil unless plan = current_plan
+          params = [plan[:id], Time.now.to_db_s]
+          sqlite.first_row_from(<<-END_FIND_MISSION.trim, *params)
+          SELECT   id, timeout, interval, last_run_at, name,
+                   code, options, memory
+          FROM     missions
+          WHERE    plan_id = ? AND next_run_at <= ?
+          ORDER BY ROWID
+          LIMIT    1
+          END_FIND_MISSION
+        }
+        if mission.empty?
+          nil  # not found
+        else
+          mission[:last_run_at] = Time.from_db_s(mission[:last_run_at])
+          %w[options memory].each do |serialized|
+            begin
+              mission[serialized] = JSON.parse(mission[serialized].to_s)
+            rescue JSON::ParserError
+              log.warn("Mission #{serialized} could not be parsed.")
+              mission[serialized] = { }
+            end
+          end
+          mission
+        end
+      rescue Amalgalite::SQLite3::Error => error  # failed to retrieve mission
+        log.error("Database current mission error:  #{error.message}.")
+        nil  # not found
+      end
+      
+      def update_mission_memory(mission_id, memory)
+        write_to_sqlite do |sqlite|
+          sqlite.execute(<<-END_UPDATE_MISSION.trim, memory.to_json, mission_id)
+          UPDATE missions SET memory = ? WHERE id = ?
+          END_UPDATE_MISSION
+        end
+      rescue Amalgalite::SQLite3::Error => error  # failed to update memory
+        # do nothing:  mission will receive previous memory state
+        log.error("Database memory update error:  #{error.message}.")
+      end
+      
+      def complete_mission(mission)
+        write_to_sqlite do |sqlite|
+          run_time = Time.now
+          params   = [ run_time.to_db_s,
+                       ( run_time +
+                         mission[:interval] * 60 ).to_db_s(:trim_seconds),
+                       mission[:id] ]
+          sqlite.execute(<<-END_UPDATE_MISSION.trim, *params)
+          UPDATE missions SET last_run_at = ?, next_run_at = ? WHERE id = ?
+          END_UPDATE_MISSION
+        end
+        true  # it's safe to continue
+      rescue Amalgalite::SQLite3::Error => error  # failed to update mission
+        log.error("Database complete mission error:  #{error.message}.")
+        false  # warn the caller that the mission will still match
+      end
+      
+      def reset_missions(*ids)
+        write_to_sqlite do |sqlite|
+          sqlite.execute(<<-END_RESET_MISSIONS.trim, *ids)
+          UPDATE missions
+          SET    next_run_at = strftime('%Y-%m-%d %H:%M', 'now', 'localtime')
+          WHERE  id IN (#{(['?'] * ids.size).join(', ')})
+          END_RESET_MISSIONS
+        end
+      rescue Amalgalite::SQLite3::Error => error  # failed to reset missions
+        # do nothing:  missions will be run at their scheduled time
+        log.error("Database mission reset error:  #{error.message}.")
+      end
+      
+      def write_report(mission_id, type, fields)
+        write_to_sqlite do |sqlite|
+          params = [mission_id, type, fields.to_json]
+          sqlite.execute(<<-END_INSERT_REPORT.trim, *params)
+          INSERT INTO reports(mission_id, type, fields) VALUES(?, ?, ?)
+          END_INSERT_REPORT
+        end
+        true  # report successfully written
+      rescue Amalgalite::SQLite3::Error => error  # failed to create report
+        log.error("Database write report error:  #{error.message}.")
+        false  # couldn't be written
+      end
+      
+      def current_reports
+        write_to_sqlite { |sqlite|
+          begin
+            report_ids = Array.new
+            reports    = query(<<-END_FIND_REPORTS.trim) { |row|
+            SELECT     reports.ROWID AS id, reports.type, reports.fields,
+                       reports.created_at, missions.id AS plugin_id
+            FROM       reports
+            INNER JOIN missions ON reports.mission_id = missions.id
+            ORDER BY   created_at
+            LIMIT      500
+            END_FIND_REPORTS
+              begin
+                row[:fields] = JSON.parse(row[:fields].to_s)
+              rescue JSON::ParserError
+                # skip the transform since we can't parse it
+                log.warn("Report fields malformed.")
+              end
+              if created = Time.from_db_s(row[:created_at])
+                row[:created_at] = created.utc.to_db_s
+              else
+                log.warn("Report timestamp missing.")
+              end
+              report_ids << row.delete_at(:id)
+            }
+          rescue Amalgalite::SQLite3::Error => error  # failed to find reports
+            log.error("Database reports error:  #{error.message}.")
+            return Array.new  # return empty results
+          end
+          return reports if reports.empty?
+          begin
+              sqlite.execute(<<-END_DELETE_REPORTS.trim, *report_ids)
+              DELETE FROM reports
+              WHERE       ROWID IN (#{(['?'] * report_ids.size).join(', ')})
+              END_DELETE_REPORTS
+          rescue Amalgalite::SQLite3::Error => error  # failed to remove reports
+            # cancel sending this batch
+            log.error("Database delivered reports error:  #{error.message}.")
+            sqlite.rollback   # we can't submit unless we're sure they are gone
+            return Array.new  # return empty results
+          end
+          reports  # the reports ready for sending
+        }
+      rescue Amalgalite::SQLite3::Error => error  # failed to get a write lock
+        # try again to read reports later
+        log.error("Database reports locking error:  #{error.message}.")
+      end
+      
+      def seconds_to_next_mission
+        default     = DEFAULT_INTERVAL * 60
+        next_run_at = read_from_sqlite { |sqlite|
+          sqlite.first_value_from(<<-END_FIND_MISSION.trim)
+          SELECT next_run_at FROM missions ORDER BY next_run_at LIMIT 1
+          END_FIND_MISSION
+        }
+        if next_run = Time.from_db_s(next_run_at)
+          seconds = next_run - Time.now
+          seconds > 0 ? seconds : default
+        else
+          default
+        end
+      rescue Amalgalite::SQLite3::Error => error  # failed to locate last run
+        log.error("Database next mission error:  #{error.message}.")
+        default  # use default
+      end
+    end
+  end
+end

data/lib/scout_agent/database/queue.rb

@@ -0,0 +1,126 @@
+#!/usr/bin/env ruby -wKU
+
+module ScoutAgent
+  class Database
+    class Queue < Database
+      # A size limit for the queue to prevent data from building up.
+      QUEUE_LIMIT = 3000
+      
+      def update_schema(version = schema_version)
+        case version
+        when 0
+          <<-END_INITIAL_SCHEMA.trim
+          CREATE TABLE queue (
+            mission_id TEXT NOT NULL
+              CHECK( mission_id IN ('report', 'hint', 'alert', 'error') OR
+                     CAST(mission_id AS 'integer') > 0 ),
+            fields     REQUIRED_TEXT_TYPE,
+            created_at DATETIME_TYPE
+          );
+          DEFAULT_LOCALTIME_TRIGGER queue created_at
+          LIMIT_TABLE_SIZE_TRIGGER  queue #{QUEUE_LIMIT}
+          END_INITIAL_SCHEMA
+        end
+      end
+      
+      def enqueue(mission_id, fields)
+        write_to_sqlite do |sqlite|
+          sqlite.execute(<<-END_ENQUEUE.trim, mission_id, fields.to_json)
+          INSERT INTO queue(mission_id, fields) VALUES(?, ?)
+          END_ENQUEUE
+        end
+        true
+      rescue Amalgalite::SQLite3::Error => error  # failed to enqueue message
+        log.error("Database queuing error:  #{error.message}.")
+        false  # reject bad message
+      end
+      
+      def peek(mission_id)
+        queued = read_from_sqlite { |sqlite|
+          sqlite.first_row_from(<<-END_FIND_QUEUED.trim, mission_id.to_s)
+          SELECT ROWID AS id, fields, created_at FROM queue WHERE mission_id = ?
+          END_FIND_QUEUED
+        }
+        if queued.empty?
+          nil  # not found
+        else
+          begin
+            queued[:fields] = JSON.parse(queued[:fields].to_s)
+          rescue JSON::ParserError  # failed to parse
+            # leave for mission to decode it
+            log.warn("Queued fields malformed.")
+          end
+          if created = Time.from_db_s(queued[:created_at])
+            queued[:created_at] = created
+          else
+            log.warn("Queued timestamp missing.")
+          end
+          queued
+        end
+      rescue Amalgalite::SQLite3::Error => error  # failed to retrieve message
+        log.error("Database peeking error:  #{error.message}.")
+        nil  # not found
+      end
+      
+      def queued_reports
+        write_to_sqlite { |sqlite|
+          begin
+            report_ids = Array.new
+            reports    = query(<<-END_FIND_REPORTS.trim) { |row|
+            SELECT     ROWID AS id, mission_id AS type, fields, created_at
+            FROM       queue
+            WHERE      mission_id IN ('report', 'hint', 'alert', 'error')
+            ORDER BY   created_at
+            LIMIT      500
+            END_FIND_REPORTS
+              begin
+                row[:fields] = JSON.parse(row[:fields].to_s)
+                if row[:fields].include? "plugin_id"
+                  row[:plugin_id] = row[:fields].delete("plugin_id")
+                end
+              rescue JSON::ParserError  # failed to parse
+                # skip the transform since we can't parse it
+                log.warn("Queued fields malformed.")
+              end
+              if created = Time.from_db_s(row[:created_at])
+                row[:created_at] = created.utc.to_db_s
+              else
+                log.warn("Queued timestamp missing.")
+              end
+              report_ids << row.delete_at(:id)
+            }
+          rescue Amalgalite::SQLite3::Error => error  # failed to find reports
+            log.error("Database queued reports error:  #{error.message}.")
+            return Array.new  # return empty results
+          end
+          return reports if reports.empty?
+          unless dequeue(*report_ids)
+            # cancel sending this batch
+            sqlite.rollback   # we can't submit unless we're sure they are gone
+            return Array.new  # return empty results
+          end
+          reports  # the reports ready for sending
+        }
+      rescue Amalgalite::SQLite3::Error => error  # failed to get a write lock
+        # try again to read reports later
+        log.error("Database queued reports locking error:  #{error.message}.")
+      end
+      
+      def dequeue(*ids)
+        write_to_sqlite do |sqlite|
+          sqlite.execute(<<-END_DELETE_QUEUED.trim, *ids)
+          DELETE FROM queue WHERE ROWID IN (#{(['?'] * ids.size).join(', ')})
+          END_DELETE_QUEUED
+        end
+        true
+      rescue Amalgalite::SQLite3::Error => error  # failed to remove messages
+        # 
+        # do nothing:  messages will be delivered again,
+        #              mission can block duplicate
+        # 
+        log.error("Database dequeuing error:  #{error.message}.")
+        false
+      end
+    end
+  end
+end

data/lib/scout_agent/database/snapshots.rb

@@ -0,0 +1,187 @@
+#!/usr/bin/env ruby -wKU
+
+module ScoutAgent
+  class Database
+    class Snapshots < Database
+      # A maximum time in seconds after which a command run is halted.
+      DEFAULT_TIMEOUT  = 60
+      # 
+      # A minimum time in minutes that must pass before a command will be run
+      # again in the next snapshot.
+      # 
+      DEFAULT_INTERVAL = 10
+      # A size limit for the runs table to prevent data from building up.
+      RUNS_LIMIT       = 3000
+      
+      def update_schema(version = schema_version)
+        case version
+        when 0
+          <<-END_INITIAL_SCHEMA.trim
+          CREATE TABLE commands (
+            code        REQUIRED_TEXT_TYPE    PRIMARY KEY,
+            timeout     POSITIVE_INTEGER_TYPE DEFAULT #{DEFAULT_TIMEOUT},
+            interval    DEFAULT_INTEGER_TYPE  #{DEFAULT_INTERVAL},
+            last_run_at DATETIME_TYPE,
+            next_run_at DATETIME_TYPE
+          );
+          DEFAULT_LOCALTIME_TRIGGER commands next_run_at trim_seconds
+
+          CREATE TABLE runs (
+            code        REQUIRED_TEXT_TYPE,
+            output      TEXT,
+            exit_status INTEGER,
+            snapshot_at REQUIRED_DATETIME_TYPE,
+            run_time    ZERO_OR_POSITIVE_REAL_TYPE,
+            PRIMARY KEY(code, snapshot_at)
+          );
+          LIMIT_TABLE_SIZE_TRIGGER runs #{RUNS_LIMIT}
+          END_INITIAL_SCHEMA
+        end
+      end
+
+      def update_commands(commands)
+        write_to_sqlite do |sqlite|
+          codes = commands.map { |m| m["code"] }
+          begin
+            sqlite.execute(<<-END_DELETE_COMMANDS.trim, *codes)
+            DELETE FROM commands
+            WHERE       code NOT IN (#{(['?'] * codes.size).join(', ')})
+            END_DELETE_COMMANDS
+          rescue Amalgalite::SQLite3::Error => error  # failed to remove
+            log.error("Database command updating error:  #{error.message}.")
+            return  # try again to update commands later
+          end
+          commands.each do |command|
+            params = [ command["timeout"].to_s =~ /\A\d*[1-9]\z/ ?
+                         command["timeout"].to_i                 :
+                         DEFAULT_TIMEOUT,
+                       (command["interval"] || DEFAULT_INTERVAL).to_i,
+                       command["code"] ]
+            begin
+              if sqlite.first_value_from(
+                   "SELECT ROWID FROM commands WHERE code = ? LIMIT 1",
+                   command["code"]
+                 )
+                sqlite.execute(<<-END_UPDATE_COMMAND.trim, *params)
+                UPDATE commands SET timeout = ?, interval = ? WHERE code = ?
+                END_UPDATE_COMMAND
+              else
+                sqlite.execute(<<-END_INSERT_COMMAND.trim, *params)
+                INSERT INTO commands(timeout, interval, code) VALUES(?, ?, ?)
+                END_INSERT_COMMAND
+              end
+            rescue Amalgalite::SQLite3::Error => error  # failed to set command
+              # do nothing:  skip bad command and move on
+              log.error( "Database bad command (#{command['code']}) error:  " +
+                         "#{error.message}." )
+            end
+          end
+        end
+      rescue Amalgalite::SQLite3::Error => error  # failed to get a write lock
+        # try again to update commands later
+        log.error("Database command update locking error:  #{error.message}.")
+      end
+      
+      def current_commands
+        query(<<-END_FIND_COMMANDS.trim, Time.now.to_db_s) { |row|
+        SELECT   ROWID AS id, timeout, interval, last_run_at, code
+        FROM     commands
+        WHERE    next_run_at <= ?
+        END_FIND_COMMANDS
+          row[:last_run_at] = Time.from_db_s(row[:last_run_at])
+        }
+      rescue Amalgalite::SQLite3::Error => error  # failed to find commands
+        log.error("Database commands error:  #{error.message}.")
+        Array.new  # return empty results
+      end
+      
+      def have_commands?
+        read_from_sqlite { |sqlite|
+          !!sqlite.first_value_from("SELECT ROWID FROM commands LIMIT 1")
+        }
+      rescue Amalgalite::SQLite3::Error => error  # failed to find commands
+        log.error("Database command check error:  #{error.message}.")
+        nil  # commands not found
+      end
+      
+      def complete_run(command, output, exit_status, snapshot_at, run_time)
+        write_to_sqlite do |sqlite|
+          params = [ command[:code],
+                     output,
+                     exit_status,
+                     snapshot_at.to_db_s,
+                     run_time ]
+          begin
+            sqlite.execute(<<-END_INSERT_RUN.trim, *params)
+            INSERT INTO
+              runs(   code, output, exit_status, snapshot_at, run_time )
+              VALUES(    ?,      ?,           ?,           ?,        ? )
+            END_INSERT_RUN
+          rescue Amalgalite::SQLite3::Error => error  # failed to add run
+            # do nothing:  skip bad command and move on
+            log.error( "Database bad command run (#{command[:code]}) error:  " +
+                       "#{error.message}." )
+          end
+          run_time = Time.now
+          params   = [ run_time.to_db_s,
+                       ( run_time +
+                         command[:interval] * 60 ).to_db_s(:trim_seconds),
+                       command[:id] ]
+          begin
+            sqlite.execute(<<-END_UPDATE_COMMAND.trim, *params)
+            UPDATE commands SET last_run_at = ?, next_run_at = ? WHERE ROWID = ?
+            END_UPDATE_COMMAND
+          rescue Amalgalite::SQLite3::Error => error  # failed to update command
+            # do nothing:  command will be run again
+            log.error( "Database bad command (#{command[:code]}) update " +
+                       "error:  #{error.message}." )
+          end
+        end
+      rescue Amalgalite::SQLite3::Error => error  # failed to get a write lock
+        # try again to update commands later
+        log.error("Database complete command locking error:  #{error.message}.")
+      end
+      
+      def current_runs
+        write_to_sqlite { |sqlite|
+          begin
+            run_ids = Array.new
+            runs    = query(<<-END_FIND_RUNS.trim) { |row|
+            SELECT     ROWID AS id, code, output, exit_status,
+                       snapshot_at AS created_at, run_time
+            FROM       runs
+            ORDER BY   snapshot_at
+            LIMIT      500
+            END_FIND_RUNS
+              if created = Time.from_db_s(row[:created_at])
+                row[:created_at] = created.utc.to_db_s
+              else
+                log.warn("Run timestamp missing.")
+              end
+              run_ids << row.delete_at(:id)
+            }
+          rescue Amalgalite::SQLite3::Error => error  # failed to find runs
+            log.error("Database runs error:  #{error.message}.")
+            return Array.new  # return empty results
+          end
+          return runs if runs.empty?
+          begin
+            sqlite.execute(<<-END_DELETE_RUNS.trim, *run_ids)
+            DELETE FROM runs
+            WHERE       ROWID IN (#{(['?'] * run_ids.size).join(', ')})
+            END_DELETE_RUNS
+          rescue Amalgalite::SQLite3::Error => error  # failed to remove runs
+            # cancel sending this batch
+            log.error("Database delivered runs error:  #{error.message}.")
+            sqlite.rollback   # we can't submit unless we're sure they are gone
+            return Array.new  # return empty results
+          end
+          runs  # the runs ready for sending
+        }
+      rescue Amalgalite::SQLite3::Error => error  # failed to get a write lock
+        # try again to read runs later
+        log.error("Database runs locking error:  #{error.message}.")
+      end
+    end
+  end
+end