cosmos 4.2.4 → 4.5.0

data/lib/cosmos/dart/lib/dart_decommutator.rb

@@ -11,6 +11,7 @@
 require 'dart_common'
 require 'dart_logging'
 require 'packet_log_entry'
+require 'cosmos/io/json_drb_object'
 
 class DartDecommutatorStatus
   attr_accessor :count
@@ -36,24 +37,40 @@ class DartDecommutator
   # Wait 60s before giving up on the PacketConfig becoming ready
   PACKET_CONFIG_READY_TIMEOUT = 60
 
+  # Delay between updating the DART status packet.   Simply throttles this rarely viewed status
+  STATUS_UPDATE_PERIOD_SECONDS = 60.seconds
+
   def initialize(worker_id = 0, num_workers = 1)
     sync_targets_and_packets()
     @worker_id = worker_id
     @num_workers = num_workers
     @status = DartDecommutatorStatus.new
+    @master = Cosmos::JsonDRbObject.new(Cosmos::System.connect_hosts['DART_MASTER'], Cosmos::System.ports['DART_MASTER'], 1.0, Cosmos::System.x_csrf_token)
+  end
+
+  def timeit(message, &block)
+    time_start = Time.now
+    yield
+    Cosmos::Logger.info("#{Time.now - time_start}s #{message}")
   end
 
   # Run forever looking for data to decommutate
   def run
-    status_time = Time.now + 60.seconds
+    status_time = Time.now + STATUS_UPDATE_PERIOD_SECONDS
     while true
-      time_start = Time.now # Remember start time so we can throttle
-      # Get all entries that are ready and decommutation hasn't started
-      PacketLogEntry.where("decom_state = #{PacketLogEntry::NOT_STARTED} and ready = true").
-                     # Mod the ID to allow distribution of effort, in_batches processes 1000 at a time
-                     where("id % #{@num_workers} = #{@worker_id}").in_batches do |group|
-        group.each do |ple|
+      ple_id = nil
+      start_time = nil
+      end_time = nil
+      begin
+        ple_ids = @master.get_decom_ple_ids()
+      rescue DRb::DRbConnError
+        sleep(1)
+        next
+      end
+      if ple_ids and ple_ids.length > 0
+        ple_ids.each do |ple_id|
           begin
+            ple = PacketLogEntry.find(ple_id)
             meta_ple = get_meta_ple(ple)
             next unless meta_ple
             system_meta = get_system_meta(ple, meta_ple)
@@ -86,15 +103,14 @@ class DartDecommutator
           rescue => err
             handle_error("PLE:#{ple.id}:ERROR\n#{err.formatted}")
           end
-        end # each ple
-      end # batches
-
-      # Throttle to no faster than 1 Hz
-      delta = Time.now - time_start
-      sleep(1 - delta) if delta < 1 && delta > 0
+        end
+      else
+        sleep(1)
+      end
     end
   rescue Interrupt
     Cosmos::Logger.info("Dart Worker Closing From Signal...")
+    @master.shutdown
   end
 
   protected
@@ -238,51 +254,45 @@ class DartDecommutator
   end
 
   def decom_packet(ple, packet, packet_config)
-    # Update packet config times
-    if !packet_config.start_time or (packet.received_time < packet_config.start_time)
-      packet_config.start_time = packet.received_time
-      packet_config.save!
-    end
-    if !packet_config.end_time or (packet.received_time > packet_config.end_time)
-      packet_config.end_time = packet.received_time
-      packet_config.save!
-    end
+    ActiveRecord::Base.transaction do
+      # Update packet config times
+      if !packet_config.start_time or (packet.packet_time < packet_config.start_time)
+        packet_config.start_time = packet.packet_time
+        packet_config.save!
+      end
+      if !packet_config.end_time or (packet.packet_time > packet_config.end_time)
+        packet_config.end_time = packet.packet_time
+        packet_config.save!
+      end
 
-    # Mark the log entry IN_PROGRESS as we decommutate the data
-    ple.decom_state = PacketLogEntry::IN_PROGRESS
-    ple.save!
-    values = get_values(packet)
+      values = get_values(packet)
 
-    table_index = 0
-    rows = []
-    # Create rows in the decommutation table model
-    values.each_slice(MAX_COLUMNS_PER_TABLE) do |table_values|
-      model = get_decom_table_model(packet_config.id, table_index)
-      row = model.new
-      row.time = ple.time
-      row.ple_id = ple.id
-      row.packet_log_id = ple.packet_log_id
-      row.meta_id = ple.meta_id
-      row.reduced_state = INITIALIZING
-      table_values.each_with_index do |value, index|
-        item_index = (table_index * MAX_COLUMNS_PER_TABLE) + index
-        row.write_attribute("i#{item_index}", value)
+      table_index = 0
+      rows = []
+      # Create rows in the decommutation table model
+      values.each_slice(MAX_COLUMNS_PER_TABLE) do |table_values|
+        model = get_decom_table_model(packet_config.id, table_index)
+        row = model.new
+        row.time = ple.time
+        row.ple_id = ple.id
+        row.packet_log_id = ple.packet_log_id
+        row.meta_id = ple.meta_id
+        row.reduced_state = READY_TO_REDUCE
+        table_values.each_with_index do |value, index|
+          item_index = (table_index * MAX_COLUMNS_PER_TABLE) + index
+          row.write_attribute("i#{item_index}", value)
+        end
+        row.save!
+        rows << row
+        table_index += 1
       end
-      row.save!
-      rows << row
-      table_index += 1
-    end
-    # Mark ready to reduce
-    rows.each do |row|
-      row.reduced_state = READY_TO_REDUCE
-      row.save!
-    end
 
-    # The log entry has been decommutated, mark COMPLETE
-    ple.decom_state = PacketLogEntry::COMPLETE
-    ple.save!
-    @status.count += 1
-    Cosmos::Logger.debug("PLE:#{ple.id}:#{ple.decom_state_string}")
+      # The log entry has been decommutated, mark COMPLETE
+      ple.decom_state = PacketLogEntry::COMPLETE
+      ple.save!
+      @status.count += 1
+      Cosmos::Logger.debug("PLE:#{ple.id}:#{ple.decom_state_string}")
+    end
   end
 
   def handle_error(message)

data/lib/cosmos/dart/lib/dart_importer.rb

@@ -80,7 +80,7 @@ class DartImporter
       Cosmos::Logger.info("First and Last Packet in File not in database")
 
       # Check if time range of packets is not present in database
-      ple = PacketLogEntry.where("time >= ? and time <= ?", first_packet.received_time, last_packet.received_time).first
+      ple = PacketLogEntry.where("time >= ? and time <= ?", first_packet.packet_time, last_packet.packet_time).first
       if !ple # Can go fast if not present at all
         Cosmos::Logger.info("  Fast Import Enabled...")
         fast = true
@@ -134,7 +134,7 @@ class DartImporter
           ple = PacketLogEntry.new
           ple.target_id = target_id
           ple.packet_id = packet_id
-          ple.time = packet.received_time
+          ple.time = packet.packet_time
           ple.packet_log_id = packet_log.id
           ple.data_offset = data_offset
           ple.meta_id = meta_id
@@ -149,7 +149,7 @@ class DartImporter
           ple.save!(validate: false)
         else
           ple_data << "," if ple_data.length > 0
-          ple_data << "(#{target_id},#{packet_id},'#{packet.received_time.dup.utc.iso8601(6)}',#{packet_log.id},#{data_offset},#{meta_id},#{is_tlm},true)"
+          ple_data << "(#{target_id},#{packet_id},'#{packet.packet_time.dup.utc.iso8601(6)}',#{packet_log.id},#{data_offset},#{meta_id},#{is_tlm},true)"
           ple_data_count += 1
         end
 

data/lib/cosmos/dart/lib/dart_master_query.rb

@@ -0,0 +1,71 @@
+# encoding: ascii-8bit
+
+# Copyright 2018 Ball Aerospace & Technologies Corp.
+# All Rights Reserved.
+#
+# This program is free software; you can modify and/or redistribute it
+# under the terms of the GNU General Public License
+# as published by the Free Software Foundation; version 3 with
+# attribution addendums as found in the LICENSE.txt
+
+require 'dart_common'
+require 'dart_logging'
+require 'thread'
+
+# Rails Json screws up COSMOS handling of Nan, etc.
+require "active_support/core_ext/object/json"
+module ActiveSupport
+  module ToJsonWithActiveSupportEncoder # :nodoc:
+    def to_json(options = nil)
+      super(options)
+    end
+  end
+end
+
+# JsonDRb server which responds to queries for decommutated and reduced data
+# from the database.
+class DartMasterQuery
+  include DartCommon
+
+  def initialize(ples_per_request = 5)
+    # Keep a thread to make sure we have the current list of items to decom
+    @ples_per_request = ples_per_request
+    @mutex = Mutex.new
+    @decom_list = []
+    @thread = Thread.new do
+      loop do
+        # Get all entries that are ready and decommutation hasn't started
+        if @decom_list.length <= 0
+          @mutex.synchronize do
+            begin
+              @decom_list.replace(PacketLogEntry.where("decom_state = #{PacketLogEntry::NOT_STARTED} and ready = true").order("id ASC").limit(1000).pluck(:id))
+            rescue Exception => error
+              Cosmos::Logger.error("Error getting packets to decom\n#{error.formatted}")
+            end
+          end
+        else
+          sleep(1)
+        end
+      end
+    end
+  end
+
+  # Returns the id of a ple that needs to be decommed next
+  #
+  def get_decom_ple_ids()
+    begin
+      @mutex.synchronize do
+        result = []
+        @ples_per_request.times do
+          ple_id = @decom_list.shift
+          result << ple_id if ple_id
+        end
+        return result
+      end
+    rescue Exception => error
+      msg = "Master Error: #{error.message}"
+      raise $!, msg, $!.backtrace
+    end
+  end
+
+end

data/lib/cosmos/dart/lib/dart_packet_log_writer.rb

@@ -114,7 +114,7 @@ class DartPacketLogWriter < Cosmos::PacketLogWriter
       ple = PacketLogEntry.new
       ple.target_id = target_id
       ple.packet_id = packet_id
-      ple.time = packet.received_time
+      ple.time = packet.packet_time
       ple.packet_log_id = @packet_log_id
       ple.data_offset = @file_size
       ple.meta_id = @meta_id
@@ -134,7 +134,7 @@ class DartPacketLogWriter < Cosmos::PacketLogWriter
         @sync_count = 0
       end
       @ple_data << "," if @ple_data.length > 0
-      @ple_data << "(#{target_id},#{packet_id},'#{packet.received_time.dup.utc.iso8601(6)}',#{@packet_log_id},#{@file_size},#{@meta_id},#{@is_tlm},true)"
+      @ple_data << "(#{target_id},#{packet_id},'#{packet.packet_time.dup.utc.iso8601(6)}',#{@packet_log_id},#{@file_size},#{@meta_id},#{@is_tlm},true)"
     end
   end
 

data/lib/cosmos/dart/lib/dart_reducer_worker_thread.rb

@@ -14,6 +14,11 @@ ItemToDecomTableMapping
 
 # Thread which performs data reduction in the DART database.
 class DartReducerWorkerThread
+  # This constant controls how much spread there must be in the data before doing a reduction.   Since our minimum
+  # reduction is 1 minute, we will wait until we have at least two minutes of spread.  Not as important for higher order 
+  # reductions but also ensures that there is a spread in the data points.
+  HOLD_OFF_TIME = 2.minutes
+  
   # Create a new thread and start it
   #
   # @param master_queue [Queue] Queue which the new thread will be added to
@@ -61,145 +66,44 @@ class DartReducerWorkerThread
 
       time_delta, base_model_time_column, time_method = job_attributes(job_type)
       rows = []
+      done = false
       # Find all the rows in the decommutation table which are ready to reduce
-      base_model.where("reduced_state = #{DartCommon::READY_TO_REDUCE}").order("meta_id ASC, #{base_model_time_column} ASC").find_each do |row|
-        rows << row
-        first_row_time = rows[0].send(base_model_time_column)
-        last_row_time = rows[-1].send(base_model_time_column)
-        # Ensure we have conditions to process the reduction data
-        next unless (last_row_time - first_row_time) > time_delta || # Enough samples or
-            # The time attribute (min, hour, day) has changed or
-            first_row_time.send(time_method) != last_row_time.send(time_method) ||
-            rows[0].meta_id != rows[-1].meta_id # New meta data
-
-        # Sample from the start to the second to last row because the last row
-        # is where we detected a change. The last row will be part of a new sample set.
-        sample_rows = rows[0..-2]
-        new_row = reduction_model.new
-        new_row.start_time = first_row_time
-        new_row.num_samples = sample_rows.length
-        new_row.meta_id = sample_rows[0].meta_id
-        new_row.packet_log_id = sample_rows[0].packet_log_id
-        # Process each of the ItemToDecomTableMapping to get the item to be reduced
-        mappings.each do |mapping|
-          item_name = "i#{mapping.item_index}"
-          min_item_name = "i#{mapping.item_index}min"
-          max_item_name = "i#{mapping.item_index}max"
-          avg_item_name = "i#{mapping.item_index}avg"
-          stddev_item_name = "i#{mapping.item_index}stddev"
-          min_value = nil
-          max_value = nil
-          total_samples = 0 # s0
-          avg_value = 0.0 # s1
-          s2 = 0.0
-          stddev_value = 0.0
-          min_nan_found = false
-          max_nan_found = false
-          avg_nan_found = false
-          stddev_nan_found = false
-          # Process each of the rows in the base model which is the decommutation table
-          # or a lesser reduction table (the minute or hour table).
-          sample_rows.each do |row_to_reduce|
-            # If we processing minute data we're reading from the base decommutation table
-            # thus there is only raw values to read
-            if job_type == :MINUTE
-              value = row_to_reduce.read_attribute(item_name)
-              min_sample = value
-              max_sample = value
-              avg_sample = value
-              if value.nil?
-                handle_error("#{item_name} is nil in #{row_to_reduce.class}:#{row_to_reduce.id}")
-                next
-              end
-            else # :HOUR or :DAY
-              # We're processing hour or day data so we're reducing previously reduced data
-              # thus there are min, max, and average values to read
-              min_sample = row_to_reduce.read_attribute(min_item_name)
-              max_sample = row_to_reduce.read_attribute(max_item_name)
-              avg_sample = row_to_reduce.read_attribute(avg_item_name)
-              stddev_sample = row_to_reduce.read_attribute(stddev_item_name)
-              if min_sample.nil?
-                handle_error("#{min_item_name} is nil in #{row_to_reduce.class}:#{row_to_reduce.id}")
-                next
-              end
-              if max_sample.nil?
-                handle_error("#{max_item_name} is nil in #{row_to_reduce.class}:#{row_to_reduce.id}")
-                next
-              end
-              if avg_sample.nil?
-                handle_error("#{avg_item_name} is nil in #{row_to_reduce.class}:#{row_to_reduce.id}")
-                next
-              end
-              if stddev_sample.nil?
-                handle_error("#{stddev_item_name} is nil in #{row_to_reduce.class}:#{row_to_reduce.id}")
-                next
-              end
-            end
-
-            if nan_value?(min_sample)
-              min_nan_found = true
-            else
-              if !min_value or min_sample < min_value
-                min_value = min_sample
-              end
-            end
-
-            if nan_value?(max_sample)
-              max_nan_found = true
-            else
-              if !max_value or max_sample > max_value
-                max_value = max_sample
-              end
-            end
-
-            if nan_value?(avg_sample)
-              avg_nan_found = true
-            else
-              # MINUTE data is reducing the decommutated values
-              if job_type == :MINUTE
-                total_samples += 1 # s0
-                avg_value += avg_sample # s1
-                s2 += (avg_sample * avg_sample)
-              else # :HOUR or :DAY
-                # Aggregated Stddev
-                # See https://math.stackexchange.com/questions/1547141/aggregating-standard-deviation-to-a-summary-point
-                total_samples += row_to_reduce.num_samples # s0
-                avg_value += (avg_sample * row_to_reduce.num_samples) # s1
-                s2 += row_to_reduce.num_samples * (avg_sample * avg_sample + stddev_sample * stddev_sample)
+      row_ids = base_model.where("reduced_state = #{DartCommon::READY_TO_REDUCE}").order("meta_id ASC, #{base_model_time_column} ASC").pluck(:id)
+      if row_ids.length > 0
+        first_row = base_model.find(row_ids[0])
+        last_row = base_model.find(row_ids[-1])
+        first_query_row_time = first_row.send(base_model_time_column)
+        last_query_row_time = last_row.send(base_model_time_column)
+        # Require at least a 2 minute spread to ensure a full minute of context is available
+        if (last_query_row_time - first_query_row_time) > HOLD_OFF_TIME
+          row_ids.in_groups_of(1000, false).each do |group_row_ids|
+            break if done
+            query_rows = base_model.order("meta_id ASC, #{base_model_time_column} ASC").where(id: group_row_ids)
+            query_rows.each do |row|
+              rows << row
+              first_row_time = rows[0].send(base_model_time_column)
+              last_row_time = rows[-1].send(base_model_time_column)
+              
+              # Break if we are near the end of a minute
+              if (last_query_row_time - last_row_time) < 1.minute
+                done = true
+                break
               end
+              
+              # Ensure we have conditions to process the reduction data
+              next unless (last_row_time - first_row_time) > time_delta || # Enough samples or
+                  # The time attribute (min, hour, day) has changed or
+                  first_row_time.send(time_method) != last_row_time.send(time_method) ||
+                  rows[0].meta_id != rows[-1].meta_id # New meta data
+              
+              # Sample from the start to the second to last row because the last row
+              # is where we detected a change. The last row will be part of a new sample set.
+              sample_rows = rows[0..-2]
+              create_reduced_row(sample_rows, base_model, reduction_model, base_model_time_column, mappings, job_type)
+              rows = rows[-1..-1] # Start a new sample with the last item in the previous sample
             end
           end
-          if total_samples != 0
-            # Aggregated Stddev
-            # See https://math.stackexchange.com/questions/1547141/aggregating-standard-deviation-to-a-summary-point
-            avg_value = avg_value.to_f / total_samples
-            # Note: For very large numbers with very small deviations this sqrt can fail.  If so then just set the stddev to 0.
-            begin
-              stddev_value = sqrt((s2 / total_samples) - (avg_value * avg_value))
-            rescue Exception
-              stddev_value = 0.0
-            end
-          end
-          min_value = Float::NAN if min_nan_found and !min_value
-          max_value = Float::NAN if max_nan_found and !max_value
-          if avg_nan_found and total_samples == 0
-            avg_value = Float::NAN
-            stddev_value = Float::NAN
-          end
-          new_row.write_attribute(min_item_name, min_value)
-          new_row.write_attribute(max_item_name, max_value)
-          new_row.write_attribute(avg_item_name, avg_value)
-          new_row.write_attribute(stddev_item_name, stddev_value)
         end
-        base_model.where(id: sample_rows.map(&:id)).update_all(:reduced_state => DartCommon::REDUCED)
-        new_row.save! # Create the reduced data row in the database
-        base_model.where(id: sample_rows.map(&:id)).update_all(:reduced_id => new_row.id)
-        new_row.reduced_state = DartCommon::READY_TO_REDUCE
-        new_row.save!
-        @status.count += 1
-
-        rows = rows[-1..-1] # Start a new sample with the last item in the previous sample
-        Cosmos::Logger.debug("Created #{new_row.class}:#{new_row.id} with #{mappings.length} items from #{new_row.num_samples} samples")
       end
       complete_job(job_type, packet_config_id, table_index)
     end # while @running
@@ -208,6 +112,133 @@ class DartReducerWorkerThread
     handle_error("Reducer Thread Unexpectedly Died: #{error.formatted}")
   end
 
+  def create_reduced_row(sample_rows, base_model, reduction_model, base_model_time_column, mappings, job_type)
+    new_row = reduction_model.new
+    new_row.start_time = sample_rows[0].send(base_model_time_column)
+    new_row.num_samples = sample_rows.length
+    new_row.meta_id = sample_rows[0].meta_id
+    new_row.packet_log_id = sample_rows[0].packet_log_id
+    # Process each of the ItemToDecomTableMapping to get the item to be reduced
+    mappings.each do |mapping|
+      item_name = "i#{mapping.item_index}"
+      min_item_name = "i#{mapping.item_index}min"
+      max_item_name = "i#{mapping.item_index}max"
+      avg_item_name = "i#{mapping.item_index}avg"
+      stddev_item_name = "i#{mapping.item_index}stddev"
+      min_value = nil
+      max_value = nil
+      total_samples = 0 # s0
+      avg_value = 0.0 # s1
+      s2 = 0.0
+      stddev_value = 0.0
+      min_nan_found = false
+      max_nan_found = false
+      avg_nan_found = false
+      stddev_nan_found = false
+      # Process each of the rows in the base model which is the decommutation table
+      # or a lesser reduction table (the minute or hour table).
+      sample_rows.each do |row_to_reduce|
+        # If we processing minute data we're reading from the base decommutation table
+        # thus there is only raw values to read
+        if job_type == :MINUTE
+          value = row_to_reduce.read_attribute(item_name)
+          min_sample = value
+          max_sample = value
+          avg_sample = value
+          if value.nil?
+            handle_error("#{item_name} is nil in #{row_to_reduce.class}:#{row_to_reduce.id}")
+            next
+          end
+        else # :HOUR or :DAY
+          # We're processing hour or day data so we're reducing previously reduced data
+          # thus there are min, max, and average values to read
+          min_sample = row_to_reduce.read_attribute(min_item_name)
+          max_sample = row_to_reduce.read_attribute(max_item_name)
+          avg_sample = row_to_reduce.read_attribute(avg_item_name)
+          stddev_sample = row_to_reduce.read_attribute(stddev_item_name)
+          if min_sample.nil?
+            handle_error("#{min_item_name} is nil in #{row_to_reduce.class}:#{row_to_reduce.id}")
+            next
+          end
+          if max_sample.nil?
+            handle_error("#{max_item_name} is nil in #{row_to_reduce.class}:#{row_to_reduce.id}")
+            next
+          end
+          if avg_sample.nil?
+            handle_error("#{avg_item_name} is nil in #{row_to_reduce.class}:#{row_to_reduce.id}")
+            next
+          end
+          if stddev_sample.nil?
+            handle_error("#{stddev_item_name} is nil in #{row_to_reduce.class}:#{row_to_reduce.id}")
+            next
+          end
+        end
+
+        if nan_value?(min_sample)
+          min_nan_found = true
+        else
+          if !min_value or min_sample < min_value
+            min_value = min_sample
+          end
+        end
+
+        if nan_value?(max_sample)
+          max_nan_found = true
+        else
+          if !max_value or max_sample > max_value
+            max_value = max_sample
+          end
+        end
+
+        if nan_value?(avg_sample)
+          avg_nan_found = true
+        else
+          # MINUTE data is reducing the decommutated values
+          if job_type == :MINUTE
+            total_samples += 1 # s0
+            avg_value += avg_sample # s1
+            s2 += (avg_sample * avg_sample)
+          else # :HOUR or :DAY
+            # Aggregated Stddev
+            # See https://math.stackexchange.com/questions/1547141/aggregating-standard-deviation-to-a-summary-point
+            total_samples += row_to_reduce.num_samples # s0
+            avg_value += (avg_sample * row_to_reduce.num_samples) # s1
+            s2 += row_to_reduce.num_samples * (avg_sample * avg_sample + stddev_sample * stddev_sample)
+          end
+        end
+      end
+      if total_samples != 0
+        # Aggregated Stddev
+        # See https://math.stackexchange.com/questions/1547141/aggregating-standard-deviation-to-a-summary-point
+        avg_value = avg_value.to_f / total_samples
+        # Note: For very large numbers with very small deviations this sqrt can fail.  If so then just set the stddev to 0.
+        begin
+          stddev_value = sqrt((s2 / total_samples) - (avg_value * avg_value))
+        rescue Exception
+          stddev_value = 0.0
+        end
+      end
+      min_value = Float::NAN if min_nan_found and !min_value
+      max_value = Float::NAN if max_nan_found and !max_value
+      if avg_nan_found and total_samples == 0
+        avg_value = Float::NAN
+        stddev_value = Float::NAN
+      end
+      new_row.write_attribute(min_item_name, min_value)
+      new_row.write_attribute(max_item_name, max_value)
+      new_row.write_attribute(avg_item_name, avg_value)
+      new_row.write_attribute(stddev_item_name, stddev_value)
+    end
+    base_model.where(id: sample_rows.map(&:id)).update_all(:reduced_state => DartCommon::REDUCED)
+    new_row.save! # Create the reduced data row in the database
+    base_model.where(id: sample_rows.map(&:id)).update_all(:reduced_id => new_row.id)
+    new_row.reduced_state = DartCommon::READY_TO_REDUCE
+    new_row.save!
+    @status.count += 1
+
+    Cosmos::Logger.debug("Created #{new_row.class}:#{new_row.id} with #{mappings.length} items from #{new_row.num_samples} samples")    
+  end
+  
   # Shutdown the worker thread
   def shutdown
     @running = false