cosmos 4.4.0 → 4.4.1

data/install/config/system/system.txt

@@ -14,6 +14,7 @@ LISTEN_HOST REPLAY_PREIDENTIFIED 0.0.0.0 # 127.0.0.1 is more secure if you don't
 LISTEN_HOST REPLAY_CMD_ROUTER 0.0.0.0 # 127.0.0.1 is more secure if you don't need external connections
 LISTEN_HOST DART_STREAM 0.0.0.0 # 127.0.0.1 is more secure if you don't need external connections
 LISTEN_HOST DART_DECOM 0.0.0.0 # 127.0.0.1 is more secure if you don't need external connections
+LISTEN_HOST DART_MASTER 0.0.0.0 # 127.0.0.1 is more secure if you don't need external connections
 
 # Connect Hosts - Ip addresses or hostnames to connect to when running the tools
 CONNECT_HOST CTS_API 127.0.0.1
@@ -25,6 +26,7 @@ CONNECT_HOST REPLAY_PREIDENTIFIED 127.0.0.1
 CONNECT_HOST REPLAY_CMD_ROUTER 127.0.0.1
 CONNECT_HOST DART_STREAM 127.0.0.1
 CONNECT_HOST DART_DECOM 127.0.0.1
+CONNECT_HOST DART_MASTER 127.0.0.1
 
 # Ethernet Ports
 PORT CTS_API 7777
@@ -36,6 +38,7 @@ PORT REPLAY_PREIDENTIFIED 7879
 PORT REPLAY_CMD_ROUTER 7880
 PORT DART_STREAM 8777
 PORT DART_DECOM 8779
+PORT DART_MASTER 8780
 
 # Default Packet Log Writer and Reader
 DEFAULT_PACKET_LOG_WRITER packet_log_writer.rb

data/lib/cosmos/dart/config/boot.rb

@@ -1,3 +1,3 @@
-ENV['BUNDLE_GEMFILE'] ||= File.expand_path('../Gemfile', __dir__)
+#ENV['BUNDLE_GEMFILE'] ||= File.expand_path('../Gemfile', __dir__)
 
 require 'bundler/setup' # Set up gems listed in the Gemfile.

data/lib/cosmos/dart/config/database.yml

@@ -1,6 +1,8 @@
 default: &default
   template: template0
   encoding: SQL_ASCII
+  collation: C 
+  ctype: C 
   adapter: postgresql
   pool: 50
   timeout: 5000

data/lib/cosmos/dart/lib/dart_common.rb

@@ -10,6 +10,7 @@
 
 require 'cosmos/script'
 require 'optparse'
+require 'dart_constants'
 
 # Autoload models here to remove problems loading within Cosmos namespace
 Target
@@ -20,6 +21,8 @@ PacketLogEntry
 # Implement methods common to DART (Data Archival Retrieval and Trending).
 # Most of these methods handle accessing the DART database.
 module DartCommon
+  include DartConstants
+
   # @return [Integer] Maximimum byte size of strings in the database
   MAX_STRING_BYTE_SIZE = 191 # Works well with mysql utf8mb4 if we want to support mysql in the future
   # @return [Integer] Maximimum bit size of strings in the database
@@ -184,7 +187,7 @@ module DartCommon
           end
         end
         t.index :time
-        t.index :reduced_state, :where => "reduced_state < 2"
+        t.index :reduced_state, :where => "reduced_state < #{REDUCED}"
       end
       create_reduction_table("t#{packet_config.id}_#{table_index}_h", table_data_types, table_index) # hour
       create_reduction_table("t#{packet_config.id}_#{table_index}_m", table_data_types, table_index) # month
@@ -288,7 +291,11 @@ module DartCommon
         reader.open(packet_log.filename)
         @plr_cache[packet_log.id] = reader
       end
-      return reader.read_at_offset(ple.data_offset)
+      packet = reader.read_at_offset(ple.data_offset) 
+      unless packet
+        Cosmos::Logger.error("Failed to read at offset #{ple.data_offset} (file offset: #{reader.bytes_read}) with file size #{reader.size}")
+      end
+      return packet 
     rescue Exception => error
       Cosmos::Logger.error("Error Reading Packet Log Entry:\n#{error.formatted}")
       return nil
@@ -319,7 +326,7 @@ module DartCommon
     reduction,
     reduction_modifier,
     item_name_modifier,
-    limit = 10000,
+    limit = MAX_DECOM_RESULTS,
     offset = 0,
     meta_ids = [])
 
@@ -687,7 +694,7 @@ module DartCommon
         end
       end
       t.index :start_time
-      t.index :reduced_state, :where => "reduced_state < 2"
+      t.index :reduced_state, :where => "reduced_state < #{REDUCED}"
     end
   end
 

data/lib/cosmos/dart/lib/dart_constants.rb

@@ -0,0 +1,15 @@
+# encoding: ascii-8bit
+
+# Copyright 2018 Ball Aerospace & Technologies Corp.
+# All Rights Reserved.
+#
+# This program is free software; you can modify and/or redistribute it
+# under the terms of the GNU General Public License
+# as published by the Free Software Foundation; version 3 with
+# attribution addendums as found in the LICENSE.txt
+
+# Implement constants common to DART (Data Archival Retrieval and Trending).
+module DartConstants
+  # @return [Integer] maximum number of results returned from a decommutation query
+  MAX_DECOM_RESULTS = 10000
+end

data/lib/cosmos/dart/lib/dart_decom_query.rb

@@ -43,10 +43,9 @@ class DartDecomQuery
   #     meta_ids => Optional IDs related to the meta data you want to filter by. This requires
   #       making a separate request for the particular meta data in question and recording
   #       the returned meta_ids for use in a subsequent request.
-  #     limit => Maximum number of data items to return, must be less than 10000
-  #     offset => Offset into the data stream. Since the maximum number of values allowed
-  #       is 10000, you can set the offset to 10000, then 20000, etc to get additional values.
-  #       By default the offset is 0.
+  #     limit => Maximum number of data items to return, must be less than DartCommon::MAX_DECOM_RESULTS
+  #     offset => Offset into the data stream. Use this to get more than the DartCommon::MAX_DECOM_RESULTS
+  #       by making multipe requests with multiples of the DartCommon::MAX_DECOM_RESULTS value.
   #     cmd_tlm => Whether the item is a command or telemetry. Default is telemetry.
   # @return [Array<Array<String, Integer, Integer, Integer, Integer>>] Array of arrays containing
   #   the item name, item seconds, item microseconds, samples (always 1 for NONE reduction, varies
@@ -143,7 +142,7 @@ class DartDecomQuery
       end
 
       limit = request['limit'].to_i
-      limit = 10000 if limit <= 0 or limit > 10000
+      limit = MAX_DECOM_RESULTS if limit <= 0 or limit > MAX_DECOM_RESULTS
 
       offset = request['offset'].to_i
       offset = 0 if offset < 0
@@ -168,7 +167,7 @@ class DartDecomQuery
   # @param is_tlm true or false
   # @return [Array<String>] Array of item names
   def item_names(target_name, packet_name, is_tlm = true)
-    Cosmos::Logger.info("#{time.formatted}: item_names")
+    Cosmos::Logger.info("#{Time.now.formatted}: item_names")
 
     target = Target.where("name = ?", target_name).first
     raise "Target #{target_name} not found" unless target

data/lib/cosmos/dart/lib/dart_decommutator.rb

@@ -11,6 +11,7 @@
 require 'dart_common'
 require 'dart_logging'
 require 'packet_log_entry'
+require 'cosmos/io/json_drb_object'
 
 class DartDecommutatorStatus
   attr_accessor :count
@@ -35,25 +36,41 @@ class DartDecommutator
 
   # Wait 60s before giving up on the PacketConfig becoming ready
   PACKET_CONFIG_READY_TIMEOUT = 60
-
+  
+  # Delay between updating the DART status packet.   Simply throttles this rarely viewed status
+  STATUS_UPDATE_PERIOD_SECONDS = 60.seconds
+  
   def initialize(worker_id = 0, num_workers = 1)
     sync_targets_and_packets()
     @worker_id = worker_id
     @num_workers = num_workers
     @status = DartDecommutatorStatus.new
+    @master = Cosmos::JsonDRbObject.new(Cosmos::System.connect_hosts['DART_MASTER'], Cosmos::System.ports['DART_MASTER'])
+  end
+
+  def timeit(message, &block)
+    time_start = Time.now
+    yield
+    Cosmos::Logger.info("#{Time.now - time_start}s #{message}")
   end
 
   # Run forever looking for data to decommutate
   def run
-    status_time = Time.now + 60.seconds
+    status_time = Time.now + STATUS_UPDATE_PERIOD_SECONDS
     while true
-      time_start = Time.now # Remember start time so we can throttle
-      # Get all entries that are ready and decommutation hasn't started
-      PacketLogEntry.where("decom_state = #{PacketLogEntry::NOT_STARTED} and ready = true").
-                     # Mod the ID to allow distribution of effort, in_batches processes 1000 at a time
-                     where("id % #{@num_workers} = #{@worker_id}").in_batches do |group|
-        group.each do |ple|
+      ple_id = nil
+      start_time = nil
+      end_time = nil
+      begin
+        ple_ids = @master.get_decom_ple_ids()
+      rescue DRb::DRbConnError
+        sleep(1)
+        next
+      end
+      if ple_ids and ple_ids.length > 0
+        ple_ids.each do |ple_id|
           begin
+            ple = PacketLogEntry.find(ple_id)
             meta_ple = get_meta_ple(ple)
             next unless meta_ple
             system_meta = get_system_meta(ple, meta_ple)
@@ -67,7 +84,7 @@ class DartDecommutator
             # If we timeout this code will simply exit the application
             wait_for_ready_packet_config(packet_config)
             decom_packet(ple, packet, packet_config)
-
+            
             # Update status
             if Time.now > status_time
               status_time = Time.now + 60.seconds
@@ -86,15 +103,14 @@ class DartDecommutator
           rescue => err
             handle_error("PLE:#{ple.id}:ERROR\n#{err.formatted}")
           end
-        end # each ple
-      end # batches
-
-      # Throttle to no faster than 1 Hz
-      delta = Time.now - time_start
-      sleep(1 - delta) if delta < 1 && delta > 0
+        end
+      else
+        sleep(1)
+      end
     end
   rescue Interrupt
     Cosmos::Logger.info("Dart Worker Closing From Signal...")
+    @master.shutdown
   end
 
   protected
@@ -238,51 +254,45 @@ class DartDecommutator
   end
 
   def decom_packet(ple, packet, packet_config)
-    # Update packet config times
-    if !packet_config.start_time or (packet.packet_time < packet_config.start_time)
-      packet_config.start_time = packet.packet_time
-      packet_config.save!
-    end
-    if !packet_config.end_time or (packet.packet_time > packet_config.end_time)
-      packet_config.end_time = packet.packet_time
-      packet_config.save!
-    end
+    ActiveRecord::Base.transaction do
+      # Update packet config times
+      if !packet_config.start_time or (packet.packet_time < packet_config.start_time)
+        packet_config.start_time = packet.packet_time
+        packet_config.save!
+      end
+      if !packet_config.end_time or (packet.packet_time > packet_config.end_time)
+        packet_config.end_time = packet.packet_time
+        packet_config.save!
+      end
 
-    # Mark the log entry IN_PROGRESS as we decommutate the data
-    ple.decom_state = PacketLogEntry::IN_PROGRESS
-    ple.save!
-    values = get_values(packet)
+      values = get_values(packet)
 
-    table_index = 0
-    rows = []
-    # Create rows in the decommutation table model
-    values.each_slice(MAX_COLUMNS_PER_TABLE) do |table_values|
-      model = get_decom_table_model(packet_config.id, table_index)
-      row = model.new
-      row.time = ple.time
-      row.ple_id = ple.id
-      row.packet_log_id = ple.packet_log_id
-      row.meta_id = ple.meta_id
-      row.reduced_state = INITIALIZING
-      table_values.each_with_index do |value, index|
-        item_index = (table_index * MAX_COLUMNS_PER_TABLE) + index
-        row.write_attribute("i#{item_index}", value)
+      table_index = 0
+      rows = []
+      # Create rows in the decommutation table model
+      values.each_slice(MAX_COLUMNS_PER_TABLE) do |table_values|
+        model = get_decom_table_model(packet_config.id, table_index)
+        row = model.new
+        row.time = ple.time
+        row.ple_id = ple.id
+        row.packet_log_id = ple.packet_log_id
+        row.meta_id = ple.meta_id
+        row.reduced_state = READY_TO_REDUCE
+        table_values.each_with_index do |value, index|
+          item_index = (table_index * MAX_COLUMNS_PER_TABLE) + index
+          row.write_attribute("i#{item_index}", value)
+        end
+        row.save!
+        rows << row
+        table_index += 1
       end
-      row.save!
-      rows << row
-      table_index += 1
-    end
-    # Mark ready to reduce
-    rows.each do |row|
-      row.reduced_state = READY_TO_REDUCE
-      row.save!
-    end
 
-    # The log entry has been decommutated, mark COMPLETE
-    ple.decom_state = PacketLogEntry::COMPLETE
-    ple.save!
-    @status.count += 1
-    Cosmos::Logger.debug("PLE:#{ple.id}:#{ple.decom_state_string}")
+      # The log entry has been decommutated, mark COMPLETE
+      ple.decom_state = PacketLogEntry::COMPLETE
+      ple.save!
+      @status.count += 1
+      Cosmos::Logger.debug("PLE:#{ple.id}:#{ple.decom_state_string}")
+    end
   end
 
   def handle_error(message)

data/lib/cosmos/dart/lib/dart_master_query.rb

@@ -0,0 +1,71 @@
+# encoding: ascii-8bit
+
+# Copyright 2018 Ball Aerospace & Technologies Corp.
+# All Rights Reserved.
+#
+# This program is free software; you can modify and/or redistribute it
+# under the terms of the GNU General Public License
+# as published by the Free Software Foundation; version 3 with
+# attribution addendums as found in the LICENSE.txt
+
+require 'dart_common'
+require 'dart_logging'
+require 'thread'
+
+# Rails Json screws up COSMOS handling of Nan, etc.
+require "active_support/core_ext/object/json"
+module ActiveSupport
+  module ToJsonWithActiveSupportEncoder # :nodoc:
+    def to_json(options = nil)
+      super(options)
+    end
+  end
+end
+
+# JsonDRb server which responds to queries for decommutated and reduced data
+# from the database.
+class DartMasterQuery
+  include DartCommon
+
+  def initialize(ples_per_request = 5)
+    # Keep a thread to make sure we have the current list of items to decom
+    @ples_per_request = ples_per_request
+    @mutex = Mutex.new
+    @decom_list = []
+    @thread = Thread.new do
+      loop do
+        # Get all entries that are ready and decommutation hasn't started
+        if @decom_list.length <= 0
+          @mutex.synchronize do
+            begin
+              @decom_list.replace(PacketLogEntry.where("decom_state = #{PacketLogEntry::NOT_STARTED} and ready = true").order("id ASC").limit(1000).pluck(:id))
+            rescue Exception => error
+              Cosmos::Logger.error("Error getting packets to decom\n#{error.formatted}")
+            end
+          end
+        else
+          sleep(1)
+        end
+      end
+    end
+  end
+
+  # Returns the id of a ple that needs to be decommed next
+  #
+  def get_decom_ple_ids()
+    begin
+      @mutex.synchronize do
+        result = []
+        @ples_per_request.times do
+          ple_id = @decom_list.shift
+          result << ple_id if ple_id
+        end
+        return result
+      end
+    rescue Exception => error
+      msg = "Master Error: #{error.message}"
+      raise $!, msg, $!.backtrace
+    end
+  end
+
+end

data/lib/cosmos/dart/lib/dart_reducer_worker_thread.rb

@@ -14,6 +14,11 @@ ItemToDecomTableMapping
 
 # Thread which performs data reduction in the DART database.
 class DartReducerWorkerThread
+  # This constant controls how much spread there must be in the data before doing a reduction.   Since our minimum
+  # reduction is 1 minute, we will wait until we have at least two minutes of spread.  Not as important for higher order 
+  # reductions but also ensures that there is a spread in the data points.
+  HOLD_OFF_TIME = 2.minutes
+  
   # Create a new thread and start it
   #
   # @param master_queue [Queue] Queue which the new thread will be added to
@@ -61,145 +66,44 @@ class DartReducerWorkerThread
 
       time_delta, base_model_time_column, time_method = job_attributes(job_type)
       rows = []
+      done = false
       # Find all the rows in the decommutation table which are ready to reduce
-      base_model.where("reduced_state = #{DartCommon::READY_TO_REDUCE}").order("meta_id ASC, #{base_model_time_column} ASC").find_each do |row|
-        rows << row
-        first_row_time = rows[0].send(base_model_time_column)
-        last_row_time = rows[-1].send(base_model_time_column)
-        # Ensure we have conditions to process the reduction data
-        next unless (last_row_time - first_row_time) > time_delta || # Enough samples or
-            # The time attribute (min, hour, day) has changed or
-            first_row_time.send(time_method) != last_row_time.send(time_method) ||
-            rows[0].meta_id != rows[-1].meta_id # New meta data
-
-        # Sample from the start to the second to last row because the last row
-        # is where we detected a change. The last row will be part of a new sample set.
-        sample_rows = rows[0..-2]
-        new_row = reduction_model.new
-        new_row.start_time = first_row_time
-        new_row.num_samples = sample_rows.length
-        new_row.meta_id = sample_rows[0].meta_id
-        new_row.packet_log_id = sample_rows[0].packet_log_id
-        # Process each of the ItemToDecomTableMapping to get the item to be reduced
-        mappings.each do |mapping|
-          item_name = "i#{mapping.item_index}"
-          min_item_name = "i#{mapping.item_index}min"
-          max_item_name = "i#{mapping.item_index}max"
-          avg_item_name = "i#{mapping.item_index}avg"
-          stddev_item_name = "i#{mapping.item_index}stddev"
-          min_value = nil
-          max_value = nil
-          total_samples = 0 # s0
-          avg_value = 0.0 # s1
-          s2 = 0.0
-          stddev_value = 0.0
-          min_nan_found = false
-          max_nan_found = false
-          avg_nan_found = false
-          stddev_nan_found = false
-          # Process each of the rows in the base model which is the decommutation table
-          # or a lesser reduction table (the minute or hour table).
-          sample_rows.each do |row_to_reduce|
-            # If we processing minute data we're reading from the base decommutation table
-            # thus there is only raw values to read
-            if job_type == :MINUTE
-              value = row_to_reduce.read_attribute(item_name)
-              min_sample = value
-              max_sample = value
-              avg_sample = value
-              if value.nil?
-                handle_error("#{item_name} is nil in #{row_to_reduce.class}:#{row_to_reduce.id}")
-                next
-              end
-            else # :HOUR or :DAY
-              # We're processing hour or day data so we're reducing previously reduced data
-              # thus there are min, max, and average values to read
-              min_sample = row_to_reduce.read_attribute(min_item_name)
-              max_sample = row_to_reduce.read_attribute(max_item_name)
-              avg_sample = row_to_reduce.read_attribute(avg_item_name)
-              stddev_sample = row_to_reduce.read_attribute(stddev_item_name)
-              if min_sample.nil?
-                handle_error("#{min_item_name} is nil in #{row_to_reduce.class}:#{row_to_reduce.id}")
-                next
-              end
-              if max_sample.nil?
-                handle_error("#{max_item_name} is nil in #{row_to_reduce.class}:#{row_to_reduce.id}")
-                next
-              end
-              if avg_sample.nil?
-                handle_error("#{avg_item_name} is nil in #{row_to_reduce.class}:#{row_to_reduce.id}")
-                next
-              end
-              if stddev_sample.nil?
-                handle_error("#{stddev_item_name} is nil in #{row_to_reduce.class}:#{row_to_reduce.id}")
-                next
-              end
-            end
-
-            if nan_value?(min_sample)
-              min_nan_found = true
-            else
-              if !min_value or min_sample < min_value
-                min_value = min_sample
-              end
-            end
-
-            if nan_value?(max_sample)
-              max_nan_found = true
-            else
-              if !max_value or max_sample > max_value
-                max_value = max_sample
-              end
-            end
-
-            if nan_value?(avg_sample)
-              avg_nan_found = true
-            else
-              # MINUTE data is reducing the decommutated values
-              if job_type == :MINUTE
-                total_samples += 1 # s0
-                avg_value += avg_sample # s1
-                s2 += (avg_sample * avg_sample)
-              else # :HOUR or :DAY
-                # Aggregated Stddev
-                # See https://math.stackexchange.com/questions/1547141/aggregating-standard-deviation-to-a-summary-point
-                total_samples += row_to_reduce.num_samples # s0
-                avg_value += (avg_sample * row_to_reduce.num_samples) # s1
-                s2 += row_to_reduce.num_samples * (avg_sample * avg_sample + stddev_sample * stddev_sample)
+      row_ids = base_model.where("reduced_state = #{DartCommon::READY_TO_REDUCE}").order("meta_id ASC, #{base_model_time_column} ASC").pluck(:id)
+      if row_ids.length > 0
+        first_row = base_model.find(row_ids[0])
+        last_row = base_model.find(row_ids[-1])
+        first_query_row_time = first_row.send(base_model_time_column)
+        last_query_row_time = last_row.send(base_model_time_column)
+        # Require at least a 2 minute spread to ensure a full minute of context is available
+        if (last_query_row_time - first_query_row_time) > HOLD_OFF_TIME
+          row_ids.in_groups_of(1000, false).each do |group_row_ids|
+            break if done
+            query_rows = base_model.order("meta_id ASC, #{base_model_time_column} ASC").where(id: group_row_ids)
+            query_rows.each do |row|
+              rows << row
+              first_row_time = rows[0].send(base_model_time_column)
+              last_row_time = rows[-1].send(base_model_time_column)
+              
+              # Break if we are near the end of a minute
+              if (last_query_row_time - last_row_time) < 1.minute
+                done = true
+                break
               end
+              
+              # Ensure we have conditions to process the reduction data
+              next unless (last_row_time - first_row_time) > time_delta || # Enough samples or
+                  # The time attribute (min, hour, day) has changed or
+                  first_row_time.send(time_method) != last_row_time.send(time_method) ||
+                  rows[0].meta_id != rows[-1].meta_id # New meta data
+              
+              # Sample from the start to the second to last row because the last row
+              # is where we detected a change. The last row will be part of a new sample set.
+              sample_rows = rows[0..-2]
+              create_reduced_row(sample_rows, base_model, reduction_model, base_model_time_column, mappings, job_type)
+              rows = rows[-1..-1] # Start a new sample with the last item in the previous sample
             end
           end
-          if total_samples != 0
-            # Aggregated Stddev
-            # See https://math.stackexchange.com/questions/1547141/aggregating-standard-deviation-to-a-summary-point
-            avg_value = avg_value.to_f / total_samples
-            # Note: For very large numbers with very small deviations this sqrt can fail.  If so then just set the stddev to 0.
-            begin
-              stddev_value = sqrt((s2 / total_samples) - (avg_value * avg_value))
-            rescue Exception
-              stddev_value = 0.0
-            end
-          end
-          min_value = Float::NAN if min_nan_found and !min_value
-          max_value = Float::NAN if max_nan_found and !max_value
-          if avg_nan_found and total_samples == 0
-            avg_value = Float::NAN
-            stddev_value = Float::NAN
-          end
-          new_row.write_attribute(min_item_name, min_value)
-          new_row.write_attribute(max_item_name, max_value)
-          new_row.write_attribute(avg_item_name, avg_value)
-          new_row.write_attribute(stddev_item_name, stddev_value)
         end
-        base_model.where(id: sample_rows.map(&:id)).update_all(:reduced_state => DartCommon::REDUCED)
-        new_row.save! # Create the reduced data row in the database
-        base_model.where(id: sample_rows.map(&:id)).update_all(:reduced_id => new_row.id)
-        new_row.reduced_state = DartCommon::READY_TO_REDUCE
-        new_row.save!
-        @status.count += 1
-
-        rows = rows[-1..-1] # Start a new sample with the last item in the previous sample
-        Cosmos::Logger.debug("Created #{new_row.class}:#{new_row.id} with #{mappings.length} items from #{new_row.num_samples} samples")
       end
       complete_job(job_type, packet_config_id, table_index)
     end # while @running
@@ -208,6 +112,133 @@ class DartReducerWorkerThread
     handle_error("Reducer Thread Unexpectedly Died: #{error.formatted}")
   end
 
+  def create_reduced_row(sample_rows, base_model, reduction_model, base_model_time_column, mappings, job_type)
+    new_row = reduction_model.new
+    new_row.start_time = sample_rows[0].send(base_model_time_column)
+    new_row.num_samples = sample_rows.length
+    new_row.meta_id = sample_rows[0].meta_id
+    new_row.packet_log_id = sample_rows[0].packet_log_id
+    # Process each of the ItemToDecomTableMapping to get the item to be reduced
+    mappings.each do |mapping|
+      item_name = "i#{mapping.item_index}"
+      min_item_name = "i#{mapping.item_index}min"
+      max_item_name = "i#{mapping.item_index}max"
+      avg_item_name = "i#{mapping.item_index}avg"
+      stddev_item_name = "i#{mapping.item_index}stddev"
+      min_value = nil
+      max_value = nil
+      total_samples = 0 # s0
+      avg_value = 0.0 # s1
+      s2 = 0.0
+      stddev_value = 0.0
+      min_nan_found = false
+      max_nan_found = false
+      avg_nan_found = false
+      stddev_nan_found = false
+      # Process each of the rows in the base model which is the decommutation table
+      # or a lesser reduction table (the minute or hour table).
+      sample_rows.each do |row_to_reduce|
+        # If we processing minute data we're reading from the base decommutation table
+        # thus there is only raw values to read
+        if job_type == :MINUTE
+          value = row_to_reduce.read_attribute(item_name)
+          min_sample = value
+          max_sample = value
+          avg_sample = value
+          if value.nil?
+            handle_error("#{item_name} is nil in #{row_to_reduce.class}:#{row_to_reduce.id}")
+            next
+          end
+        else # :HOUR or :DAY
+          # We're processing hour or day data so we're reducing previously reduced data
+          # thus there are min, max, and average values to read
+          min_sample = row_to_reduce.read_attribute(min_item_name)
+          max_sample = row_to_reduce.read_attribute(max_item_name)
+          avg_sample = row_to_reduce.read_attribute(avg_item_name)
+          stddev_sample = row_to_reduce.read_attribute(stddev_item_name)
+          if min_sample.nil?
+            handle_error("#{min_item_name} is nil in #{row_to_reduce.class}:#{row_to_reduce.id}")
+            next
+          end
+          if max_sample.nil?
+            handle_error("#{max_item_name} is nil in #{row_to_reduce.class}:#{row_to_reduce.id}")
+            next
+          end
+          if avg_sample.nil?
+            handle_error("#{avg_item_name} is nil in #{row_to_reduce.class}:#{row_to_reduce.id}")
+            next
+          end
+          if stddev_sample.nil?
+            handle_error("#{stddev_item_name} is nil in #{row_to_reduce.class}:#{row_to_reduce.id}")
+            next
+          end
+        end
+
+        if nan_value?(min_sample)
+          min_nan_found = true
+        else
+          if !min_value or min_sample < min_value
+            min_value = min_sample
+          end
+        end
+
+        if nan_value?(max_sample)
+          max_nan_found = true
+        else
+          if !max_value or max_sample > max_value
+            max_value = max_sample
+          end
+        end
+
+        if nan_value?(avg_sample)
+          avg_nan_found = true
+        else
+          # MINUTE data is reducing the decommutated values
+          if job_type == :MINUTE
+            total_samples += 1 # s0
+            avg_value += avg_sample # s1
+            s2 += (avg_sample * avg_sample)
+          else # :HOUR or :DAY
+            # Aggregated Stddev
+            # See https://math.stackexchange.com/questions/1547141/aggregating-standard-deviation-to-a-summary-point
+            total_samples += row_to_reduce.num_samples # s0
+            avg_value += (avg_sample * row_to_reduce.num_samples) # s1
+            s2 += row_to_reduce.num_samples * (avg_sample * avg_sample + stddev_sample * stddev_sample)
+          end
+        end
+      end
+      if total_samples != 0
+        # Aggregated Stddev
+        # See https://math.stackexchange.com/questions/1547141/aggregating-standard-deviation-to-a-summary-point
+        avg_value = avg_value.to_f / total_samples
+        # Note: For very large numbers with very small deviations this sqrt can fail.  If so then just set the stddev to 0.
+        begin
+          stddev_value = sqrt((s2 / total_samples) - (avg_value * avg_value))
+        rescue Exception
+          stddev_value = 0.0
+        end
+      end
+      min_value = Float::NAN if min_nan_found and !min_value
+      max_value = Float::NAN if max_nan_found and !max_value
+      if avg_nan_found and total_samples == 0
+        avg_value = Float::NAN
+        stddev_value = Float::NAN
+      end
+      new_row.write_attribute(min_item_name, min_value)
+      new_row.write_attribute(max_item_name, max_value)
+      new_row.write_attribute(avg_item_name, avg_value)
+      new_row.write_attribute(stddev_item_name, stddev_value)
+    end
+    base_model.where(id: sample_rows.map(&:id)).update_all(:reduced_state => DartCommon::REDUCED)
+    new_row.save! # Create the reduced data row in the database
+    base_model.where(id: sample_rows.map(&:id)).update_all(:reduced_id => new_row.id)
+    new_row.reduced_state = DartCommon::READY_TO_REDUCE
+    new_row.save!
+    @status.count += 1
+
+    Cosmos::Logger.debug("Created #{new_row.class}:#{new_row.id} with #{mappings.length} items from #{new_row.num_samples} samples")    
+  end
+  
   # Shutdown the worker thread
   def shutdown
     @running = false