cosmos 4.1.1 → 4.2.0

data/lib/cosmos/dart/lib/dart_tcpip_server_interface.rb

@@ -0,0 +1,142 @@
+# encoding: ascii-8bit
+
+# Copyright 2018 Ball Aerospace & Technologies Corp.
+# All Rights Reserved.
+#
+# This program is free software; you can modify and/or redistribute it
+# under the terms of the GNU General Public License
+# as published by the Free Software Foundation; version 3 with
+# attribution addendums as found in the LICENSE.txt
+
+require 'dart_common'
+
+class DartTcpipServerInterface < Cosmos::TcpipServerInterface
+  include DartCommon
+
+  def initialize(write_timeout = 60, read_timeout = 5)
+    port = Cosmos::System.ports['DART_STREAM']
+    super(port, port, write_timeout, read_timeout, 'PREIDENTIFIED')
+    @listen_address = Cosmos::System.listen_hosts['DART_STREAM']
+    @plr_cache = {}
+  end
+
+  def connect
+    @cancel_threads = false
+    @read_queue.clear if @read_queue
+    start_listen_thread(@read_port, true, true)
+    @write_thread = nil
+    @connected = true
+  end
+
+  def read_thread_body(interface)
+    packet = interface.read
+    return if !packet || @cancel_threads
+
+    # Expect to receive a packet that is completed composed of a JSON object with the following fields
+    # start_time_sec, start_time_usec
+    # end_time_sec, end_time_usec
+    # cmd_tlm = CMD, or TLM
+    # packets [[target_name, packet_name], ...]
+    json_data = packet.buffer(false)
+    json_object = JSON.parse(json_data)
+    Cosmos::Logger.info("Received Request: #{json_data}")
+    start_time = Time.at(json_object['start_time_sec'], json_object['start_time_usec'])
+    end_time = Time.at(json_object['end_time_sec'], json_object['end_time_usec'])
+    cmd_tlm = json_object['cmd_tlm']
+    packets = json_object['packets']
+    meta_ids = json_object['meta_ids']
+    meta_filters = json_object['meta_filters']
+    if cmd_tlm and cmd_tlm.to_s.upcase == 'CMD'
+      is_tlm = false
+    else
+      is_tlm = true
+    end
+
+    if meta_filters and (!meta_ids or meta_ids.length <= 0)
+      meta_ids = process_meta_filters(meta_filters, is_tlm, end_time)
+    end
+
+    # Retrieve and stream out requested packets here
+    pairs = []
+    if packets
+      packets.each do |target_name, packet_name|
+        target = Target.where("name = ?", target_name.to_s.upcase).first
+        if target
+          packet = Packet.where("target_id = ? and name = ? and is_tlm = #{is_tlm}", target.id, packet_name.to_s.upcase).first
+          if packet
+            pairs << [target.id, packet.id]
+          end
+        end
+      end
+    end
+
+    if pairs.length > 0
+      where_clause = "("
+      index = 0
+      pairs.each do |target_id, packet_id|
+        where_clause << " or " if index != 0
+        where_clause << "(target_id = #{target_id} and packet_id = #{packet_id})"
+        index += 1
+      end
+      where_clause << ")"
+    else
+      where_clause = "is_tlm = #{is_tlm}"
+    end
+
+    meta_ple = nil
+    packet_count = 0
+    batch_size = 100
+    batch_count = 0
+    loop do
+      ples = PacketLogEntry.where(where_clause)
+      ples = ples.where(:meta_id => meta_ids) if meta_ids
+      if start_time <= end_time
+        ples = ples.where("time >= ?", start_time)
+        ples = ples.where("time <= ?", end_time)
+        ples = ples.order(time: :asc)
+      else
+        ples = ples.where("time >= ?", end_time)
+        ples = ples.where("time <= ?", start_time)
+        ples = ples.order(time: :desc)
+      end
+      ples = ples.limit(batch_size).offset(batch_count * batch_size)
+      break if ples.length <= 0
+
+      ples.each do |ple|
+        if !meta_ple or ple.meta_id != meta_ple.id
+          meta_ple = PacketLogEntry.find(ple.meta_id)
+          meta_packet = read_packet_from_ple(meta_ple)
+          if meta_packet
+            begin
+              interface.write(meta_packet)
+              packet_count += 1
+            rescue Exception
+              Cosmos::Logger.error("Request ended with meta packet write error")
+              break
+            end
+          else
+            Cosmos::Logger.error("No Meta Packet Read: #{meta_ple.inspect}")
+          end
+        end
+
+        if meta_ple.id != ple.id
+          packet = read_packet_from_ple(ple)
+          if packet
+            begin
+              interface.write(packet)
+              packet_count += 1
+            rescue Exception
+              Cosmos::Logger.error("Request ended with write error")
+              break
+            end
+          else
+            Cosmos::Logger.error("No Packet Read: #{ple.inspect}")
+          end
+        end
+      end
+
+      batch_count += 1
+    end
+    Cosmos::Logger.info("Request fully served #{packet_count} packets")
+  end
+end

data/lib/cosmos/dart/processes/dart.rb

@@ -0,0 +1,145 @@
+# encoding: ascii-8bit
+
+# Copyright 2018 Ball Aerospace & Technologies Corp.
+# All Rights Reserved.
+#
+# This program is free software; you can modify and/or redistribute it
+# under the terms of the GNU General Public License
+# as published by the Free Software Foundation; version 3 with
+# attribution addendums as found in the LICENSE.txt
+
+require File.expand_path('../../config/environment', __FILE__)
+require 'dart_common'
+require 'dart_logging'
+require 'childprocess'
+require 'dart_database_cleaner'
+
+class Dart
+  include DartCommon
+
+  @@force_cleanup = false
+  def self.force_cleanup=(value)
+    @@force_cleanup = value
+  end
+
+  # Start all the DART processes:
+  # 1. Ingester - writes all packets to the DART log file
+  # 2. Reducer - reduces decommutated data by minute, hour, and day
+  # 3. Stream Server - TCPIP server which handles requests for raw data
+  #      streamed from the packet log binary file
+  # 4. Decom Server - JSON DRB server which handles requests for decommutated
+  #      or reduced data from the database
+  # 5..n Worker - Decommutates data from the packet log binary file into the DB
+  def run
+    Cosmos::Logger.level = Cosmos::Logger::INFO
+    dart_logging = DartLogging.new('dart')
+
+    # Cleanup the database before starting processes
+    DartDatabaseCleaner.clean(@@force_cleanup)
+
+    ruby_process_name = ENV['DART_RUBY']
+    if RUBY_ENGINE != 'ruby'
+      ruby_process_name ||= 'jruby'
+    else
+      ruby_process_name ||= 'ruby'
+    end
+
+    num_workers = ENV['DART_NUM_WORKERS']
+    num_workers ||= 1
+    num_workers = num_workers.to_i
+
+    process_definitions = [
+      [ruby_process_name, File.join(__dir__, 'dart_ingester.rb')],
+      [ruby_process_name, File.join(__dir__, 'dart_reducer.rb')],
+      [ruby_process_name, File.join(__dir__, 'dart_stream_server.rb')],
+      [ruby_process_name, File.join(__dir__, 'dart_decom_server.rb')]
+    ]
+
+    num_workers.times do |index|
+      process_definitions << [ruby_process_name, File.join(__dir__, 'dart_worker.rb'), index.to_s, num_workers.to_s]
+    end
+
+    processes = []
+    p_mutex = Mutex.new
+
+    # Start all the processes.rb
+    Cosmos::Logger.info("Dart starting each process...")
+
+    process_definitions.each do |p|
+      Cosmos::Logger.info("Starting: #{p.join(' ')}")
+      processes << ChildProcess.build(*p)
+      processes[-1].start
+    end
+
+    # Setup signal handlers to shutdown cleanly
+    ["TERM", "INT"].each do |sig|
+      Signal.trap(sig) do
+        @shutdown = true
+        Thread.new do
+          p_mutex.synchronize do
+            Cosmos::Logger.info("Shutting down processes...")
+            processes.each_with_index do |p, index|
+              Thread.new do
+                Cosmos::Logger.info("Soft Shutting down process: #{process_definitions[index].join(' ')}")
+                Process.kill("SIGINT", p.pid)
+              end
+            end
+            sleep(2)
+            processes.each_with_index do |p, index|
+              unless p.exited?
+                Cosmos::Logger.info("Hard Shutting down process: #{process_definitions[index].join(' ')}")
+                p.stop
+              end
+            end
+            @shutdown_complete = true
+          end
+        end
+      end
+    end
+
+    # Monitor processes and respawn if died
+    @shutdown = false
+    @shutdown_complete = false
+    Cosmos::Logger.info("Dart Monitoring processes...")
+    loop do
+      p_mutex.synchronize do
+        processes.each_with_index do |p, index|
+          break if @shutdown
+          unless p.alive?
+            # Respawn process
+            Cosmos::Logger.error("Unexpected process died... respawning! #{process_definitions[index].join(' ')}")
+            processes[index] = ChildProcess.build(*process_definitions[index])
+            processes[index].leader = true
+            processes[index].start
+          end
+        end
+      end
+      break if @shutdown
+      sleep(1)
+      break if @shutdown
+    end
+
+    loop do
+      break if @shutdown_complete
+      sleep(1)
+    end
+
+  ensure
+    Cosmos::Logger.info("Dart shutdown complete")
+    shutdown_cmd_tlm()
+    dart_logging.stop
+  end
+
+  def self.run
+    Cosmos.catch_fatal_exception do
+      a = self.new
+      a.run
+    end
+  end
+end
+
+parser = DartCommon.handle_argv(false)
+parser.on("--force-cleanup", "Force database cleanup") do |arg|
+  Dart.force_cleanup = true
+end
+parser.parse!

data/lib/cosmos/dart/processes/dart_decom_server.rb

@@ -0,0 +1,39 @@
+# encoding: ascii-8bit
+
+# Copyright 2018 Ball Aerospace & Technologies Corp.
+# All Rights Reserved.
+#
+# This program is free software; you can modify and/or redistribute it
+# under the terms of the GNU General Public License
+# as published by the Free Software Foundation; version 3 with
+# attribution addendums as found in the LICENSE.txt
+
+require File.expand_path('../../config/environment', __FILE__)
+require 'dart_common'
+require 'dart_decom_query'
+
+Cosmos.catch_fatal_exception do
+  DartCommon.handle_argv
+
+  Cosmos::Logger.level = Cosmos::Logger::INFO
+  dart_logging = DartLogging.new('dart_decom_server')
+
+  json_drb = Cosmos::JsonDRb.new
+  json_drb.acl = Cosmos::System.acl if Cosmos::System.acl
+  begin
+    json_drb.method_whitelist = ['query', 'item_names']
+    begin
+      json_drb.start_service(Cosmos::System.listen_hosts['DART_DECOM'],
+        Cosmos::System.ports['DART_DECOM'], DartDecomQuery.new)
+    rescue Exception
+      raise Cosmos::FatalError.new("Error starting JsonDRb on port #{Cosmos::System.ports['DART_DECOM']}.\nPerhaps another DART Decom Server is already running?")
+    end
+    ["TERM", "INT"].each {|sig| Signal.trap(sig) {exit}}
+    Cosmos::Logger.info("Dart Decom Server Started...")
+    sleep(1) while true
+  rescue Interrupt
+    Cosmos::Logger.info("Dart Decom Server Closing...")
+    json_drb.stop_service
+    dart_logging.stop
+  end
+end

data/lib/cosmos/dart/processes/dart_import.rb

@@ -0,0 +1,63 @@
+# encoding: ascii-8bit
+
+# Copyright 2018 Ball Aerospace & Technologies Corp.
+# All Rights Reserved.
+#
+# This program is free software; you can modify and/or redistribute it
+# under the terms of the GNU General Public License
+# as published by the Free Software Foundation; version 3 with
+# attribution addendums as found in the LICENSE.txt
+
+# This code must be run on the database server
+# The file to be imported should be placed in its final storage location
+# Note that it is imported in place with algorithms that attempt to prevent
+# duplicate creation of Database entries
+
+require 'ostruct'
+require 'optparse'
+require 'cosmos/version'
+
+options = OpenStruct.new
+options.force = false
+
+parser = OptionParser.new do |option_parser|
+  option_parser.banner = "Usage: dart_import filename"
+  option_parser.separator("")
+
+  # Create the help option
+  option_parser.on("-h", "--help", "Show this message") do
+    puts option_parser
+    exit(0)
+  end
+
+  # Create the version option
+  option_parser.on("-v", "--version", "Show version") do
+    puts "COSMOS Version: #{COSMOS_VERSION}"
+    puts "User Version: #{USER_VERSION}" if defined? USER_VERSION
+    exit(0)
+  end
+
+  # Create the system option
+  option_parser.on("--system FILE", "Use an alternative system.txt file") do |arg|
+    System.instance(File.join(USERPATH, 'config', 'system', arg))
+  end
+
+  # Create the force option
+  option_parser.on("-f", "--force", "Force parsing entire file") do
+    options.force = true
+  end
+end
+
+parser.parse!(ARGV)
+unless ARGV[0]
+  puts parser
+  exit(1)
+end
+
+require File.expand_path('../../config/environment', __FILE__)
+require 'dart_importer'
+
+Cosmos.catch_fatal_exception do
+  code = DartImporter.new.import(File.expand_path(ARGV[0]), options.force)
+  exit(code)
+end

data/lib/cosmos/dart/processes/dart_ingester.rb

@@ -0,0 +1,92 @@
+# encoding: ascii-8bit
+
+# Copyright 2018 Ball Aerospace & Technologies Corp.
+# All Rights Reserved.
+#
+# This program is free software; you can modify and/or redistribute it
+# under the terms of the GNU General Public License
+# as published by the Free Software Foundation; version 3 with
+# attribution addendums as found in the LICENSE.txt
+
+require File.expand_path('../../config/environment', __FILE__)
+require 'dart_packet_log_writer'
+require 'dart_logging'
+
+# Handles packets by writing them to the dart log file. New SYSTEM META packets
+# cause a new log file to be started.
+class DartInterfaceThread < Cosmos::InterfaceThread
+  attr_writer :packet_log_writer
+  attr_writer :log_type
+
+  def handle_packet(packet)
+    if packet.target_name == 'SYSTEM'.freeze and packet.packet_name == 'META'.freeze
+      Cosmos::Logger.info("#{@log_type}: #{packet.target_name} #{packet.packet_name}")
+      # Update Current Value Table Used By Packet Log Writer
+      cvt_packet = Cosmos::System.telemetry.update!(packet.target_name, packet.packet_name, packet.buffer)
+      cvt_packet.received_time = packet.received_time
+      @packet_log_writer.start
+      @packet_log_writer.write(cvt_packet)
+    else
+      @packet_log_writer.write(packet)
+    end
+  end
+end
+
+Cosmos.catch_fatal_exception do
+  DartCommon.handle_argv
+
+  Cosmos::Logger.level = Cosmos::Logger::INFO
+  dart_logging = DartLogging.new('dart_ingester')
+
+  tlm_log_writer = DartPacketLogWriter.new(
+    :TLM,    # Log telemetry
+    'dart_', # Put dart_ in the log file name
+    true,    # Enable logging
+    nil,     # Don't cycle on time
+    2_000_000_000, # Cycle the log at 2GB
+    Cosmos::System.paths['DART_DATA']) # Log into the DART_DATA dir
+
+  tlm_interface = Cosmos::TcpipClientInterface.new(
+    Cosmos::System.connect_hosts['CTS_PREIDENTIFIED'], # Connect to the CTS machine
+    nil, # Don't write commands
+    Cosmos::System.ports['CTS_PREIDENTIFIED'], # Read telemetry from the CTS port
+    nil, # No write timeout
+    nil, # No read timeout
+    'PREIDENTIFIED') # PREIDENTIFIED protocol
+
+  tlm_thread = DartInterfaceThread.new(tlm_interface)
+  tlm_thread.packet_log_writer = tlm_log_writer
+  tlm_thread.log_type = :TLM
+
+  cmd_log_writer = DartPacketLogWriter.new(
+    :CMD,    # Log commands
+    'dart_', # Put dart_ in the log file name
+    true,    # Enable logging
+    nil,     # Don't cycle on time
+    2_000_000_000, # Cycle the log at 2GB
+    Cosmos::System.paths['DART_DATA']) # Log into the DART_DATA dir
+
+  cmd_interface = Cosmos::TcpipClientInterface.new(
+    Cosmos::System.connect_hosts['CTS_CMD_ROUTER'], # Connect to the CTS machine
+    nil, # Don't write commands
+    Cosmos::System.ports['CTS_CMD_ROUTER'], # Read commands from the CMD port
+    nil, # No write timeout
+    nil, # No read timeout
+    'PREIDENTIFIED') # PREIDENTIFIED protocol
+
+  cmd_thread = DartInterfaceThread.new(cmd_interface)
+  cmd_thread.packet_log_writer = cmd_log_writer
+  cmd_thread.log_type = :CMD
+
+  begin
+    tlm_thread.start
+    cmd_thread.start
+    sleep(1) while true
+  rescue Interrupt
+    tlm_thread.stop
+    cmd_thread.stop
+    tlm_log_writer.shutdown
+    cmd_log_writer.shutdown
+    dart_logging.stop
+  end
+end