openc3 7.0.0.pre.rc3 → 7.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 920a681652723ab469d3311e6f28332ea202068300c3a506d3314c552f5c0f30
-  data.tar.gz: 2f35277fa1e25eb30646a27e8f45365cb2fd39d7d3b24758baabf004e9fe54b5
+  metadata.gz: 0a06901f01bc9d8e7b6f0b123296da67475c08ea9cf50ef788a86b6da1308fcb
+  data.tar.gz: 34732b912cea1b4677f3e94ac0ba9f437e99217ccf54f9250816176ef5b63db1
 SHA512:
-  metadata.gz: 1cd5cb567a743e90742e76db3b4ab8498f34d8b6ba6dc0032e351bb6e3fdfbc4f8db826cc907d500f4670ef461bd49193be74e8264648b6a0f5908d090b7e6cf
-  data.tar.gz: 9af5921a88413af21113b00c0f99dcc023e70c21b4dfc9e7f0f8840fbe226765f86d80e82a7b874458468c45cea590a3e6b0ab2d702d29613818e19e48b473be
+  metadata.gz: 5cda606ccb010606ad4b432685f53428cbf10b216f1cd06f2f55baac28fb723514dc3f8dfeadafc0958734237d3a7faf3f42439695481145914434e2a4bb1a58
+  data.tar.gz: 98532ea424c421554d45a6d155fadfc27e67ddf969a96be7653daebb30d5530dbb7601e13b5d3a3afa7dfd56718c161c060e81d67a5ee967a9248225554e1a03

data/data/config/interface_modifiers.yaml

@@ -53,7 +53,7 @@ RECONNECT_DELAY:
   parameters:
     - name: Delay
       required: true
-      description: Delay in seconds between reconnect attempts. The default is 15 seconds.
+      description: Delay in seconds between reconnect attempts. The default is 5 seconds.
       values: ([0-9]*[.])?[0-9]+
 DISABLE_DISCONNECT:
   summary: Disable the Disconnect button on the Interfaces tab in the Server

data/data/config/item_modifiers.yaml

@@ -72,7 +72,8 @@ GENERIC_READ_CONVERSION_START:
     class (Note, referencing the packet as 'myself' is still supported for backwards
     compatibility). The last line of code should return the converted
     value. The GENERIC_READ_CONVERSION_END keyword specifies that all lines of
-    code for the conversion have been given.
+    code for the conversion have been given. To specify the bit size, type, and array size of the converted data,
+    use the CONVERTED_DATA keyword.
   warning: Generic conversions are not a good long term solution. Consider creating
     a conversion class and using READ_CONVERSION instead. READ_CONVERSION is easier
     to debug and has higher performance.
@@ -81,22 +82,33 @@ GENERIC_READ_CONVERSION_START:
       GENERIC_READ_CONVERSION_START
         (value * 1.5).to_i # Convert the value by a scale factor
       GENERIC_READ_CONVERSION_END
+      CONVERTED_DATA 32 UINT
   python_example: |
     APPEND_ITEM ITEM1 32 UINT
       GENERIC_READ_CONVERSION_START
         int(value * 1.5) # Convert the value by a scale factor
       GENERIC_READ_CONVERSION_END
+      CONVERTED_DATA 32 UINT
+GENERIC_READ_CONVERSION_END:
+  summary: Complete a generic read conversion
+CONVERTED_DATA:
+  summary: Defines the bit size, type, and array size of the converted data for a read conversion
+  description: This keyword is used in conjunction with DERIVED items to specify the bit size, type, and array size of the converted data.
+    If this keyword is not used, DERIVED items are stored as strings in the decommutated data.
+  since: 7.0.0
   parameters:
+    - name: Converted Bit Size
+      required: true
+      description: Bit size of converted value
+      values: \d+
     - name: Converted Type
-      required: false
+      required: true
       description: Type of the converted value
       values: <%= %w(INT UINT FLOAT STRING BLOCK) %>
-    - name: Converted Bit Size
+    - name: Converted Array Size
       required: false
-      description: Bit size of converted value
+      description: Bit size of the total array if the converted value is an array. Only specified if the converted type is an array.
       values: \d+
-GENERIC_READ_CONVERSION_END:
-  summary: Complete a generic read conversion
 LIMITS:
   summary: Defines a set of limits for a telemetry item
   description: If limits are violated a message is printed in the Command and Telemetry Server

data/data/config/telemetry.yaml

@@ -10,7 +10,7 @@ TELEMETRY:
       required: true
       description: Name of the target this telemetry packet is associated with
       values: .+
-    - name: Command
+    - name: Packet
       required: true
       description:
         Name of this telemetry packet. Also referred to as its mnemonic.

data/lib/openc3/accessors/json_accessor.rb

@@ -20,7 +20,7 @@ require 'openc3/accessors/accessor'
 OpenC3.disable_warnings do
   class JsonPath
     def self.process_object(obj_or_str, opts = {})
-      obj_or_str.is_a?(String) ? MultiJson.load(obj_or_str, max_nesting: opts[:max_nesting], create_additions: true, allow_nan: true) : obj_or_str
+      obj_or_str.is_a?(String) ? JSON.parse(obj_or_str, max_nesting: opts[:max_nesting], create_additions: true, allow_nan: true) : obj_or_str
     end
   end
 end

data/lib/openc3/api/tlm_api.rb

@@ -23,7 +23,7 @@
 
 require 'openc3/models/target_model'
 require 'openc3/models/cvt_model'
-require 'openc3/packets/packet'
+# require 'openc3/packets/packet' # Circular require
 require 'openc3/topics/telemetry_topic'
 require 'openc3/topics/interface_topic'
 require 'openc3/topics/decom_interface_topic'
@@ -281,7 +281,7 @@ module OpenC3
 
           case value_type
           when 'FORMATTED', 'WITH_UNITS'
-            if item['format_string']
+            if item['format_string'] or item['units']
               results << [target_name, orig_packet_name, item_name, 'FORMATTED'].join('__')
             # This logic must match the logic in Packet#decom
             elsif item['states'] or (item['read_conversion'] and item['data_type'] != 'DERIVED')
@@ -304,7 +304,7 @@ module OpenC3
           if item['limits']['DEFAULT']
             results[-1] += '__LIMITS'
           end
-        rescue RuntimeError => e
+        rescue RuntimeError
           results << nil
         end
       end

data/lib/openc3/config/config_parser.rb

@@ -15,10 +15,11 @@
 # This file may also be used under the terms of a commercial license
 # if purchased from OpenC3, Inc.
 
-require 'openc3/top_level'
+# require 'openc3/top_level' # Circular require
 require 'openc3/ext/config_parser' if RUBY_ENGINE == 'ruby' and !ENV['OPENC3_NO_EXT']
 require 'erb'
 require 'fileutils'
+require 'tempfile'
 
 module OpenC3
   # Reads OpenC3 style configuration data which consists of keywords followed
@@ -218,6 +219,7 @@ module OpenC3
                    &)
       ensure
         file.close unless file.closed?
+        file.unlink
       end
     end
 
@@ -417,9 +419,7 @@ module OpenC3
       elsif copy.include?(':') # Check for Windows drive letter
         copy = copy.split(':')[1]
       end
-      parsed_filename = File.join(Dir.tmpdir, 'openc3', 'tmp', copy)
-      FileUtils.mkdir_p(File.dirname(parsed_filename)) # Create the path
-      file = File.open(parsed_filename, 'w+')
+      file = Tempfile.new(copy)
       file.puts output
       file.rewind # Rewind so the file is ready to read
       file

data/lib/openc3/conversions/conversion.rb

@@ -20,11 +20,11 @@ module OpenC3
   class Conversion
     # @return [Symbol] The converted data type. Must be one of
     #   {OpenC3::StructureItem#data_type}
-    attr_reader :converted_type
+    attr_accessor :converted_type
     # @return [Integer] The size in bits of the converted value
-    attr_reader :converted_bit_size
+    attr_accessor :converted_bit_size
     # @return [Integer] The size in bits of the converted array value
-    attr_reader :converted_array_size
+    attr_accessor :converted_array_size
     # @return [Array] The arguments passed to the conversion
     attr_reader :params
 

data/lib/openc3/core_ext/faraday.rb

@@ -1,4 +1,8 @@
+# Remove warnings in CGI
+saved_verbose = $VERBOSE
+$VERBOSE = false
 require 'faraday'
+$VERBOSE = saved_verbose
 
 module Faraday
   class Response

data/lib/openc3/logs/log_writer.rb

@@ -126,6 +126,7 @@ module OpenC3
       @cleanup_times = []
       @previous_time_nsec_since_epoch = nil
       @tmp_dir = Dir.mktmpdir
+      @wait_threads = []
 
       # This is an optimization to avoid creating a new entry object
       # each time we create an entry which we do a LOT!
@@ -154,9 +155,8 @@ module OpenC3
 
     # Stops all logging and closes the current log file.
     def stop
-      threads = nil
-      @mutex.synchronize { threads = close_file(false); @logging_enabled = false; }
-      return threads
+      @mutex.synchronize { close_file(false); @logging_enabled = false; }
+      return @wait_threads
     end
 
     # Stop all logging, close the current log file, and kill the logging threads.
@@ -173,6 +173,13 @@ module OpenC3
       return threads
     end
 
+    def cleanup
+      if @tmp_dir
+        FileUtils.remove_entry_secure(@tmp_dir, true)
+        @tmp_dir = nil
+      end
+    end
+
     def graceful_kill
       @cancel_threads = true
     end
@@ -307,8 +314,19 @@ module OpenC3
     # to keep a full file's worth of data in the stream. This is what prevents continuous stream growth.
     # Returns thread that moves log to bucket
     def close_file(take_mutex = true)
-      threads = []
       @mutex.lock if take_mutex
+
+      # Remove old wait_threads
+      to_remove = []
+      @wait_threads.each do |thread|
+        unless thread.alive?
+          to_remove << thread
+        end
+      end
+      to_remove.each do |thread|
+        @wait_threads.delete(thread)
+      end
+
       begin
         if @file
           begin
@@ -322,7 +340,7 @@ module OpenC3
               # Cleanup timestamps here so they are unset for the next file
               @first_time = nil
               @last_time = nil
-              threads << BucketUtilities.move_log_file_to_bucket(@filename, bucket_key)
+              @wait_threads << BucketUtilities.move_log_file_to_bucket(@filename, bucket_key)
               # Now that the file is in storage, trim the Redis stream after a delay
               @cleanup_offsets << {}
               @last_offsets.each do |redis_topic, last_offset|
@@ -342,7 +360,7 @@ module OpenC3
       ensure
         @mutex.unlock if take_mutex
       end
-      return threads
+      return @wait_threads
     end
 
     def bucket_filename

data/lib/openc3/logs/packet_log_writer.rb

@@ -137,7 +137,6 @@ module OpenC3
     # Closing a log file isn't critical so we just log an error
     # Returns threads that moves log to bucket
     def close_file(take_mutex = true)
-      threads = []
       @mutex.lock if take_mutex
       begin
         # Need to write the OFFSET_MARKER for each packet
@@ -145,12 +144,10 @@ module OpenC3
           write_entry(:OFFSET_MARKER, nil, nil, nil, nil, nil, last_offset + ',' + redis_topic, nil) if @file
         end
 
-        threads.concat(super(false))
-
+        return super(false)
       ensure
         @mutex.unlock if take_mutex
       end
-      return threads
     end
 
     def get_packet_index(cmd_or_tlm, target_name, packet_name, entry_type, data)

data/lib/openc3/logs/stream_log_pair.rb

@@ -43,13 +43,20 @@ module OpenC3
 
     # Close any open stream log files
     def stop
-      @read_log.stop
-      @write_log.stop
+      threads = @read_log.stop
+      threads.concat(@write_log.stop)
+      return threads
     end
 
     def shutdown
-      @read_log.shutdown
-      @write_log.shutdown
+      threads = @read_log.shutdown
+      threads.concat(@write_log.shutdown)
+      return threads
+    end
+
+    def cleanup
+      @read_log.cleanup
+      @write_log.cleanup
     end
 
     # Clone the stream log pair

data/lib/openc3/logs/text_log_writer.rb

@@ -68,7 +68,6 @@ module OpenC3
     # Closing a log file isn't critical so we just log an error
     # Returns threads that moves log to bucket
     def close_file(take_mutex = true)
-      threads = []
       @mutex.lock if take_mutex
       begin
         # Need to write the OFFSET_MARKER for each packet
@@ -79,12 +78,10 @@ module OpenC3
           write_entry(time.to_nsec_from_epoch, data.as_json(allow_nan: true).to_json(allow_nan: true)) if @file
         end
 
-        threads.concat(super(false))
-
+        return super(false)
       ensure
         @mutex.unlock if take_mutex
       end
-      return threads
     end
 
     def extension

data/lib/openc3/microservices/interface_microservice.rb

@@ -779,7 +779,6 @@ module OpenC3
         else
           @logger.error "#{@interface.name}: #{connect_error.formatted}"
           unless @connection_failed_messages.include?(connect_error.message)
-            OpenC3.write_exception_file(connect_error)
             @connection_failed_messages << connect_error.message
           end
         end
@@ -800,7 +799,6 @@ module OpenC3
         else
           @logger.error "#{@interface.name}: #{err.formatted}"
           unless @connection_lost_messages.include?(err.message)
-            OpenC3.write_exception_file(err)
             @connection_lost_messages << err.message
           end
         end
@@ -888,6 +886,14 @@ module OpenC3
     def shutdown(_sig = nil)
       @logger.info "#{@interface ? @interface.name : @name}: shutdown requested"
       stop()
+      if @interface and @interface.stream_log_pair
+        threads = @interface.stream_log_pair.shutdown
+        # Wait for all the logging threads to move files to buckets
+        threads.flatten.compact.each do |thread|
+          thread.join
+        end
+        @interface.stream_log_pair.cleanup
+      end
       super()
     end
 

data/lib/openc3/microservices/log_microservice.rb

@@ -125,8 +125,8 @@ module OpenC3
     def shutdown
       # Make sure all the existing logs are properly closed down
       threads = []
-      @plws.each do |target_name, plw_hash|
-        plw_hash.each do |type, plw|
+      @plws.each do |_target_name, plw_hash|
+        plw_hash.each do |_type, plw|
           threads.concat(plw.shutdown)
         end
       end
@@ -134,6 +134,11 @@ module OpenC3
       threads.flatten.compact.each do |thread|
         thread.join
       end
+      @plws.each do |_target_name, plw_hash|
+        plw_hash.each do |_type, plw|
+          plw.cleanup
+        end
+      end
       super()
     end
   end

data/lib/openc3/microservices/microservice.rb

@@ -88,6 +88,7 @@ module OpenC3
       @name = name
       split_name = name.split("__")
       raise "Name #{name} doesn't match convention of SCOPE__TYPE__NAME" if split_name.length != 3
+      microservice_type = split_name[1].to_s.upcase
 
       @scope = split_name[0]
       $openc3_scope = @scope
@@ -102,8 +103,14 @@ module OpenC3
 
       OpenC3.setup_open_telemetry(@name, false)
 
+      @temp_dir = OpenC3.sanitize_path(File.join(Dir.tmpdir, @name))
+
       # Create temp folder for this microservice
-      @temp_dir = Dir.mktmpdir
+      # This will already have been setup by plugin_microservice.rb if USER
+      if is_plugin or microservice_type != 'USER'
+        FileUtils.remove_entry_secure(@temp_dir, true)
+        Dir.mkdir(@temp_dir)
+      end
 
       # Get microservice configuration from Redis
       @config = MicroserviceModel.get(name: @name, scope: @scope)
@@ -142,14 +149,13 @@ module OpenC3
         cmd_array = @config["cmd"]
 
         # Get Microservice files from bucket storage
-        temp_dir = Dir.mktmpdir
         bucket = ENV['OPENC3_CONFIG_BUCKET']
         client = Bucket.getClient()
 
         prefix = "#{@scope}/microservices/#{@name}/"
         file_count = 0
         client.list_objects(bucket: bucket, prefix: prefix).each do |object|
-          response_target = File.join(temp_dir, object.key.split(prefix)[-1])
+          response_target = OpenC3.sanitize_path(File.join(@temp_dir, object.key.split(prefix)[-1]))
           FileUtils.mkdir_p(File.dirname(response_target))
           client.get_object(bucket: bucket, key: object.key, path: response_target)
           file_count += 1
@@ -157,7 +163,7 @@ module OpenC3
 
         # Adjust @work_dir to microservice files downloaded if files and a relative path
         if file_count > 0 and @work_dir[0] != '/'
-          @work_dir = File.join(temp_dir, @work_dir)
+          @work_dir = OpenC3.sanitize_path(File.join(@temp_dir, @work_dir))
         end
 
         # Check Syntax on any ruby files

data/lib/openc3/microservices/queue_microservice.rb

@@ -18,6 +18,8 @@ require 'openc3/utilities/authentication'
 require 'openc3/api/api'
 
 module OpenC3
+  saved_verbose = $VERBOSE
+  $VERBOSE = false
   module Script
     private
     # Override the prompt_for_hazardous method to always return true since there is no user to prompt
@@ -25,6 +27,7 @@ module OpenC3
       return true
     end
   end
+  $VERBOSE = saved_verbose
 
   # The queue processor runs in a single thread and processes commands via cmd_api.
   class QueueProcessor

data/lib/openc3/microservices/scope_cleanup_microservice.rb

@@ -16,6 +16,120 @@ require 'openc3/microservices/cleanup_microservice'
 
 module OpenC3
   class ScopeCleanupMicroservice < CleanupMicroservice
+    TSDB_HEALTH_QUERY =
+"SELECT
+    table_name,
+    table_row_count,
+    wal_pending_row_count,
+    CASE
+        WHEN table_suspended THEN 'SUSPENDED'
+        WHEN table_memory_pressure_level = 2 THEN 'BACKOFF'
+        WHEN table_memory_pressure_level = 1 THEN 'PRESSURE'
+        ELSE 'OK'
+    END AS status,
+    wal_txn - table_txn AS lag_txns,
+    table_write_amp_p50 AS write_amp,
+    table_merge_rate_p99 AS slowest_merge
+FROM tables()
+WHERE walEnabled
+ORDER BY
+    table_suspended DESC,
+    table_memory_pressure_level DESC,
+    wal_pending_row_count DESC;"
+
+    GROWTH_NUM_SAMPLE_PERIODS = 4
+
+    def initialize(*args)
+      super(*args)
+      @run_time = nil
+      @cleanup_poll_time = nil
+      @delta_time = 0.0
+      @wal_pending_row_count = {}
+      @lag_txns = {}
+    end
+
+    def cleanup(areas, bucket)
+      current_time = Time.now
+      if @run_time
+        delta = current_time - @run_time
+        if delta > 0.0
+          @delta_time += delta
+        end
+      end
+      @run_time = current_time
+      if @delta_time > @cleanup_poll_time
+        @delta_time = 0.0
+        super(areas, bucket)
+      end
+
+      # Always check TSDB health
+      if @scope == 'DEFAULT'
+        begin
+          conn = OpenC3::QuestDBClient.connection
+          result = conn.exec(TSDB_HEALTH_QUERY)
+          columns = result.fields
+          rows = result.values
+
+          table_name_column = columns.index("table_name")
+          wal_pending_row_count_column = columns.index("wal_pending_row_count")
+          status_column = columns.index("status")
+          lag_txns_column = columns.index("lag_txns")
+
+          rows.each do |values|
+            table_name = values[table_name_column]
+            wal_pending_row_count = values[wal_pending_row_count_column].to_i
+            status = values[status_column]
+            lag_txns = values[lag_txns_column].to_i
+
+            if status != 'OK'
+              @logger.error("QuestDB: #{table_name} in bad state: #{status}")
+
+              if status == 'SUSPENDED'
+                # Try to automatically unsuspend
+                @logger.info("QuestDB: Attempting to unsuspend: #{table_name}")
+                conn.exec("ALTER TABLE #{table_name} RESUME WAL;")
+              end
+            end
+
+            @wal_pending_row_count[table_name] ||= []
+            @wal_pending_row_count[table_name] << wal_pending_row_count
+            @lag_txns[table_name] ||= []
+            @lag_txns[table_name] << lag_txns
+
+            if @wal_pending_row_count[table_name].length > GROWTH_NUM_SAMPLE_PERIODS
+              if detect_growth(@wal_pending_row_count[table_name], GROWTH_NUM_SAMPLE_PERIODS)
+                # Crossed threshold of sample periods of growth
+                @logger.error("QuestDB: #{table_name} has growing wal_pending_row_count: #{wal_pending_row_count}")
+              end
+
+              # Leave the last GROWTH_NUM_SAMPLE_PERIODS samples
+              @wal_pending_row_count[table_name] = @wal_pending_row_count[table_name][-GROWTH_NUM_SAMPLE_PERIODS..-1]
+            end
+
+            if @lag_txns[table_name].length > GROWTH_NUM_SAMPLE_PERIODS
+              if detect_growth(@lag_txns[table_name], GROWTH_NUM_SAMPLE_PERIODS)
+                # Crossed threshold of sample periods of growth
+                @logger.error("QuestDB: #{table_name} has growing lag_txns: #{lag_txns}")
+              end
+
+              # Leave the last GROWTH_NUM_SAMPLE_PERIODS samples
+              @lag_txns[table_name] = @lag_txns[table_name][-GROWTH_NUM_SAMPLE_PERIODS..-1]
+            end
+          end
+        rescue => e
+          OpenC3::QuestDBClient.disconnect
+          @logger.error("QuestDB Error: #{e.formatted}")
+        end
+      end
+    end
+
+    def detect_growth(array, num_samples)
+      num_samples.times do |index|
+        return false if array[index + 1] <= array[index]
+      end
+      return true
+    end
+
     def get_areas_and_poll_time
       scope = ScopeModel.get_model(name: @scope)
       areas = [
@@ -28,7 +142,8 @@ module OpenC3
         areas << ["NOSCOPE/tool_logs/sr", scope.tool_log_retain_time]
       end
 
-      return areas, scope.cleanup_poll_time
+      @cleanup_poll_time = scope.cleanup_poll_time
+      return areas, 60 # Run every 1 minute for TSDB checks
     end
   end
 end

data/lib/openc3/microservices/text_log_microservice.rb

@@ -98,13 +98,16 @@ module OpenC3
     def shutdown
       # Make sure all the existing logs are properly closed down
       threads = []
-      @tlws.each do |topic, tlw|
+      @tlws.each do |_topic, tlw|
         threads.concat(tlw.shutdown)
       end
       # Wait for all the logging threads to move files to buckets
       threads.flatten.compact.each do |thread|
         thread.join
       end
+      @tlws.each do |_topic, tlw|
+        tlw.cleanup
+      end
       super()
     end
   end

data/lib/openc3/migrations/20260204000000_remove_decom_reducer.rb

@@ -23,6 +23,8 @@ module OpenC3
         target_models = TargetModel.all(scope: scope)
         target_models.each do |name, target_model|
           # Remove deprecated decom log settings from target model
+          target_model.delete("cmd_unique_id_mode")
+          target_model.delete("tlm_unique_id_mode")
           target_model.delete("cmd_decom_log_cycle_time")
           target_model.delete("cmd_decom_log_cycle_size")
           target_model.delete("cmd_decom_log_retain_time")

data/lib/openc3/models/activity_model.rb

@@ -18,6 +18,7 @@
 # https://www.rubydoc.info/gems/redis/Redis/Commands/SortedSets
 
 require 'openc3/models/model'
+require 'openc3/models/timeline_model'
 require 'openc3/topics/timeline_topic'
 require 'securerandom'
 
@@ -28,6 +29,11 @@ module OpenC3
 
   class ActivityModel < Model
     MAX_DURATION = Time::SEC_PER_DAY
+    # Grace window (in seconds) to allow creating activities slightly in the past.
+    # This handles race conditions where real-time activity notifications arrive
+    # after the start time has already passed (e.g. from external systems).
+    # This is consistent with the -15 second window in the timeline microservice.
+    START_GRACE_SECONDS = 15
     PRIMARY_KEY = '__openc3_timelines'.freeze # MUST be equal to `TimelineModel::PRIMARY_KEY` minus the leading __
     # See run_activity(activity) in openc3/lib/openc3/microservices/timeline_microservice.rb
     VALID_KINDS = %w(command script reserve expire)
@@ -212,7 +218,7 @@ module OpenC3
     end
 
     # validate the input to the rules we have created for timelines.
-    # - A task's start MUST NOT be in the past.
+    # - A task's start MUST NOT be more than START_GRACE_SECONDS in the past.
     # - A task's start MUST be before the stop.
     # - A task CAN NOT be longer than MAX_DURATION (86400) in seconds.
     # - A task MUST have a kind.
@@ -230,8 +236,8 @@ module OpenC3
       rescue NoMethodError
         raise ActivityInputError.new "start and stop must be seconds: #{start}, #{stop}"
       end
-      if now_f >= start and kind != 'expire'
-        raise ActivityInputError.new "activity must be in the future, current_time: #{now_f} vs #{start}"
+      if now_f >= start + START_GRACE_SECONDS and kind != 'expire'
+        raise ActivityInputError.new "activity must not be more than #{START_GRACE_SECONDS} seconds in the past, current_time: #{now_f} vs #{start}"
       elsif duration > MAX_DURATION and kind != 'expire'
         raise ActivityInputError.new "activity can not be longer than #{MAX_DURATION} seconds"
       elsif duration <= 0
@@ -262,6 +268,12 @@ module OpenC3
     # Update the Redis hash at primary_key and set the score equal to the start Epoch time
     # the member is set to the JSON generated via calling as_json
     def create(overlap: true, username: nil)
+      # Validate that the timeline exists in this scope before creating activities.
+      # Activities must be attached to an existing timeline within the same scope.
+      unless TimelineModel.get(name: @name, scope: @scope)
+        raise ActivityError.new "timeline '#{@name}' does not exist in scope '#{@scope}'"
+      end
+
       if @recurring['end'] and @recurring['frequency'] and @recurring['span']
         # First validate the initial recurring activity ... all others are just offsets
         validate_input(start: @start, stop: @stop, kind: @kind, data: @data)