@fleetbase/fleetops-engine 0.6.66 → 0.6.68

package/server/src/Jobs/PollTelematicTelemetry.php

@@ -23,12 +23,21 @@ class PollTelematicTelemetry implements ShouldQueue, ShouldBeUnique
     use InteractsWithQueue;
     use Queueable;
 
-    public int $tries     = 5;
-    public int $timeout   = 120;
-    public int $uniqueFor = 3600;
+    // Reservations released while earlier batches drain are not provider failures.
+    public int $tries         = 0;
+    public int $maxExceptions = 5;
+    // Keep every attempt below Laravel's baseline Redis retry_after of 90 seconds.
+    public int $timeout          = 80;
+    public int $uniqueFor        = 3600;
+    public ?string $manualJobId  = null;
+    public array $requestOptions = [];
+    public ?int $retryDeadline   = null;
 
-    public function __construct(public string $telematicUuid)
+    public function __construct(public string $telematicUuid, ?string $manualJobId = null, array $requestOptions = [])
     {
+        $this->manualJobId    = $manualJobId;
+        $this->requestOptions = $requestOptions;
+        $this->retryDeadline  = now()->addMinutes(15)->getTimestamp();
     }
 
     public function uniqueId(): string
@@ -38,11 +47,35 @@ class PollTelematicTelemetry implements ShouldQueue, ShouldBeUnique
 
     public function backoff(): array
     {
-        return [15, 60, 180, 300];
+        // A scheduled retry holds the uniqueness lease that suppresses minute ticks,
+        // so it must not wait longer than the polling interval after a transient failure.
+        return $this->manualJobId ? [15, 60, 180, 300] : [15, 30, 60, 60];
+    }
+
+    public function retryUntil(): \Illuminate\Support\Carbon
+    {
+        if ($this->retryDeadline === null) {
+            // Older serialized jobs have no deadline property. Anchor their first
+            // observed attempt once, rather than extending it on each redelivery.
+            $key                 = 'telemetry:poll:retry-deadline:' . ($this->job?->uuid() ?? $this->manualJobId ?? $this->telematicUuid);
+            $this->retryDeadline = Cache::remember($key, 86400, fn () => now()->addMinutes(15)->getTimestamp());
+        }
+
+        return \Illuminate\Support\Carbon::createFromTimestampUTC($this->retryDeadline);
     }
 
     public function handle(TelematicProviderRegistry $registry, Inbox $inbox): void
     {
+        if (now()->gte($this->retryUntil())) {
+            $error = new \Illuminate\Queue\MaxAttemptsExceededException('Telemetry polling retry deadline exceeded.');
+            if ($this->job) {
+                $this->fail($error);
+            } else {
+                $this->failed($error);
+            }
+
+            return;
+        }
         $telematic = Telematic::withoutGlobalScopes()->where('uuid', $this->telematicUuid)->first();
         if (!$telematic || !Inbox::enabled($telematic)) {
             return;
@@ -57,25 +90,48 @@ class PollTelematicTelemetry implements ShouldQueue, ShouldBeUnique
         }
         // Coalesce ticks while a previous sweep is still being ingested.
         if (DB::table('telematic_deliveries')->where('telematic_uuid', $this->telematicUuid)->where('source', 'poll')->whereIn('status', ['pending', 'retry', 'processing'])->exists()) {
+            if ($this->manualJobId && $this->job) {
+                $this->manualProgress(['last_sync_result' => 'retrying', 'last_sync_phase' => 'waiting_for_ingestion']);
+                $this->release(15);
+            }
+
             return;
         }
-        $lock = Cache::lock('telemetry:poll:' . $this->telematicUuid, 150);
+        $lock = Cache::lock('telemetry:poll:' . $this->telematicUuid, 85);
         if (!$lock->get()) {
+            if ($this->manualJobId && $this->job) {
+                $this->release(15);
+            }
+
             return;
         }
         $run = (string) Str::uuid();
-        DB::table('telematic_sync_runs')->insert(['uuid' => $run, 'telematic_uuid' => $telematic->uuid, 'status' => 'fetching', 'created_at' => now(), 'updated_at' => now()]);
         try {
+            $started = microtime(true);
+            DB::table('telematic_sync_runs')->insert(['uuid' => $run, 'telematic_uuid' => $telematic->uuid, 'status' => 'fetching', 'created_at' => now(), 'updated_at' => now()]);
+            $this->adoptPendingRequest($run);
+            $this->manualProgress([
+                'last_sync_run_uuid'      => $run, 'last_sync_run_job_id' => $this->manualJobId,
+                'last_sync_result'        => 'running', 'last_sync_phase' => 'fetching_inventory',
+                'last_sync_fetched_total' => 0, 'last_sync_page_count' => 0,
+                'last_sync_linked_total'  => 0, 'last_sync_failed_total' => 0, 'last_sync_error' => null,
+            ]);
             $provider->connect($telematic);
             $cursor  = null;
             $cursors = [];
             $seen    = [];
-            $started = microtime(true);
             for ($page = 0; $page < ($options['max_pages'] ?? 100); $page++) {
-                if (microtime(true) - $started > 90) {
-                    throw new \RuntimeException('Polling time budget exceeded; sweep incomplete.');
-                }
-                $response = $provider->fetchDevices(['limit' => $options['page_size'] ?? 1000, 'cursor' => $cursor, 'timeout' => 20, 'connect_timeout' => 5]);
+                $remaining = $this->remainingBudget($started);
+                // Large fleet responses can exceed 20 seconds. Allow a configured
+                // request timeout without extending the bounded sweep deadline.
+                $timeout  = min(max(1, (int) ($options['request_timeout_seconds'] ?? 45)), $remaining);
+                $response = $provider->fetchDevices([
+                    'limit'             => min(max(1, (int) ($this->requestOptions['limit'] ?? $options['page_size'] ?? 1000)), 1000),
+                    'cursor'            => $cursor, 'filters' => $this->requestOptions['filters'] ?? [],
+                    'refresh_inventory' => $this->manualJobId !== null && $cursor === null,
+                    'timeout'           => $timeout, 'connect_timeout' => min(5, $timeout),
+                ]);
+                $this->remainingBudget($started);
                 $received = now()->utc();
                 $units    = $response['devices'];
                 $ids      = array_map(fn ($unit) => $provider->normalizeDevice($unit)['device_id'] ?? null, $units);
@@ -86,11 +142,15 @@ class PollTelematicTelemetry implements ShouldQueue, ShouldBeUnique
                     $seen[(string) $id] = true;
                 }
                 foreach (array_chunk($units, max(1, (int) ($options['batch_size'] ?? 100))) as $batch) {
+                    $this->remainingBudget($started);
                     $inbox->accept($telematic, $batch, 'poll', $run, $received->toDateTimeString());
                 }
+                $this->remainingBudget($started);
                 DB::table('telematic_sync_runs')->where('uuid', $run)->update(['pages' => $page + 1, 'units' => count($seen), 'updated_at' => now()]);
+                $this->manualProgress(['last_sync_fetched_total' => count($seen), 'last_sync_page_count' => $page + 1, 'last_sync_phase' => 'fetching_inventory']);
                 if (!$response['has_more']) {
                     DB::table('telematic_sync_runs')->where('uuid', $run)->update(['status' => 'ingesting', 'updated_at' => now()]);
+                    $this->manualProgress(['last_sync_phase' => 'ingesting', 'last_sync_inventory_total' => count($seen)]);
                     Inbox::finishRun($run);
 
                     return;
@@ -105,6 +165,7 @@ class PollTelematicTelemetry implements ShouldQueue, ShouldBeUnique
             throw new \RuntimeException('Maximum page count reached; sweep incomplete.');
         } catch (\Throwable $e) {
             DB::table('telematic_sync_runs')->where('uuid', $run)->update(['status' => 'incomplete', 'error' => $e instanceof TelematicRateLimitExceededException ? 'Rate limited; retry scheduled.' : 'Polling failed; retry scheduled. ' . class_basename($e), 'updated_at' => now()]);
+            $this->manualProgress(['last_sync_result' => 'retrying', 'last_sync_phase' => 'retrying', 'last_sync_error' => 'Polling failed; retry scheduled. ' . class_basename($e)]);
             if ($e instanceof TelematicRateLimitExceededException && $this->job) {
                 $this->release($e->context()['retry_after'] ?? 60);
 
@@ -115,4 +176,94 @@ class PollTelematicTelemetry implements ShouldQueue, ShouldBeUnique
             $lock->release();
         }
     }
+
+    private function remainingBudget(float $started): int
+    {
+        $remaining = (int) floor(60 - (microtime(true) - $started));
+        if ($remaining < 1) {
+            throw new \RuntimeException('Polling time budget exceeded; sweep incomplete.');
+        }
+
+        return $remaining;
+    }
+
+    /**
+     * Bind a manual request that has no active run to this scheduled sweep.
+     *
+     * Every poll job for a connection shares one unique lease, so while a scheduled sweep
+     * runs, the request's own job is either not queued (the request was recorded while
+     * the lease was held) or no longer exists (lost before or after an interrupted run).
+     */
+    private function adoptPendingRequest(string $run): void
+    {
+        if ($this->manualJobId) {
+            return;
+        }
+        DB::transaction(function () use ($run) {
+            $connection = Telematic::withoutGlobalScopes()->where('uuid', $this->telematicUuid)->lockForUpdate()->first();
+            $meta       = $connection?->meta ?? [];
+            $request    = data_get($meta, 'last_sync_job_id');
+            if (!$request || !in_array(data_get($meta, 'last_sync_result'), Inbox::PENDING_REQUEST_RESULTS, true)) {
+                return;
+            }
+            $bound = data_get($meta, 'last_sync_run_uuid');
+            if ($bound && DB::table('telematic_sync_runs')->where('uuid', $bound)->whereIn('status', ['fetching', 'ingesting'])->exists()) {
+                return;
+            }
+            $connection->status = 'synchronizing';
+            $connection->meta   = array_merge($meta, [
+                'last_sync_run_uuid'      => $run, 'last_sync_run_job_id' => $request,
+                'last_sync_result'        => 'running', 'last_sync_phase' => 'fetching_inventory',
+                'last_sync_fetched_total' => 0, 'last_sync_page_count' => 0,
+                'last_sync_linked_total'  => 0, 'last_sync_failed_total' => 0, 'last_sync_error' => null,
+                'last_sync_progress_at'   => now()->toDateTimeString(),
+            ]);
+            $connection->save();
+        });
+    }
+
+    private function manualProgress(array $attributes): void
+    {
+        if (!$this->manualJobId) {
+            return;
+        }
+        DB::transaction(function () use ($attributes) {
+            $connection = Telematic::withoutGlobalScopes()->where('uuid', $this->telematicUuid)->lockForUpdate()->first();
+            if (!$connection || !Inbox::enabled($connection) || data_get($connection->meta, 'last_sync_job_id') !== $this->manualJobId) {
+                return;
+            }
+            $connection->status = 'synchronizing';
+            $connection->meta   = array_merge($connection->meta ?? [], $attributes, ['last_sync_progress_at' => now()->toDateTimeString()]);
+            $connection->save();
+        });
+    }
+
+    public function failed(\Throwable $error): void
+    {
+        if (!$this->manualJobId) {
+            return;
+        }
+        $run = null;
+        DB::transaction(function () use ($error, &$run) {
+            $connection = Telematic::withoutGlobalScopes()->where('uuid', $this->telematicUuid)->lockForUpdate()->first();
+            if (!$connection || !Inbox::enabled($connection) || data_get($connection->meta, 'last_sync_job_id') !== $this->manualJobId) {
+                return;
+            }
+            if (data_get($connection->meta, 'last_sync_run_job_id') === $this->manualJobId) {
+                $run = data_get($connection->meta, 'last_sync_run_uuid');
+            }
+            $connection->status = 'error';
+            $connection->meta   = array_merge($connection->meta ?? [], [
+                'last_sync_result'     => 'failed', 'last_sync_phase' => 'failed',
+                'last_sync_error'      => 'Polling retries exhausted. ' . class_basename($error),
+                'last_sync_error_type' => class_basename($error), 'last_sync_failed_at' => now()->toDateTimeString(),
+            ]);
+            $connection->save();
+        });
+        // Do not lock run rows while holding the connection lock: finishRun takes
+        // those locks in the opposite order when the final delivery completes.
+        if ($run) {
+            DB::table('telematic_sync_runs')->where('uuid', $run)->where('status', 'fetching')->update(['status' => 'incomplete', 'error' => 'Polling attempt interrupted.', 'updated_at' => now()]);
+        }
+    }
 }

package/server/src/Jobs/ProcessTelematicDelivery.php

@@ -38,7 +38,8 @@ class ProcessTelematicDelivery implements ShouldQueue, ShouldBeUnique
 
     public function handle(Ingestor $ingestor, TelematicService $service): void
     {
-        $lock = Cache::lock('telemetry:delivery:' . $this->deliveryUuid, 90);
+        $started = microtime(true);
+        $lock    = Cache::lock('telemetry:delivery:' . $this->deliveryUuid, 90);
         if (!$lock->get()) {
             return;
         }
@@ -66,21 +67,48 @@ class ProcessTelematicDelivery implements ShouldQueue, ShouldBeUnique
                 return;
             }
             $this->update(['status' => 'processing', 'attempts' => $row->attempts + 1, 'available_at' => now()->addSeconds(120)]);
+            $failed       = [];
+            $failureTypes = [];
             try {
-                $units = $provider->telemetryUnits(json_decode(Crypt::decryptString($row->retry_payload ?? $row->payload), true, 512, JSON_THROW_ON_ERROR));
+                $payload = json_decode(Crypt::decryptString($row->retry_payload ?? $row->payload), true, 512, JSON_THROW_ON_ERROR);
+                if ($row->retry_payload && ($payload['checkpoint_version'] ?? null) === 1) {
+                    // This envelope is internal and encrypted; the original webhook
+                    // contract was already validated before creating the checkpoint.
+                    $units        = $payload['remaining'] ?? null;
+                    $failed       = $payload['failed'] ?? null;
+                    $failureTypes = $payload['failure_types'] ?? null;
+                    foreach ([$units, $failed, $failureTypes] as $items) {
+                        if (!is_array($items) || !array_is_list($items)) {
+                            throw new \InvalidArgumentException('Invalid delivery checkpoint.');
+                        }
+                    }
+                } elseif ($row->source === 'poll') {
+                    // Polling persists the provider's inventory records, including units
+                    // that have never reported a position. Validate this internal envelope
+                    // here and let ingestion report invalid positions per unit below.
+                    if (!is_array($payload) || !array_is_list($payload) || $payload === []) {
+                        throw new \InvalidArgumentException('Expected a non-empty polling unit list.');
+                    }
+                    $units = $payload;
+                } else {
+                    // External deliveries must still pass the provider's position contract.
+                    $units = $provider->telemetryUnits($payload);
+                }
             } catch (\Throwable) {
                 $this->update(['status' => 'quarantined', 'failed' => 1, 'error' => 'Unsupported or unreadable position payload; inspect and replay after adapter correction.']);
                 Inbox::finishRun($row->run_uuid);
 
                 return;
             }
-            $failed       = [];
-            $failureTypes = [];
             $applied      = 0;
             $invalid      = (int) $row->invalid_count;
-            $sourceDelay  = null;
+            // Polled inventory legitimately includes units without a valid fix (for example,
+            // never reported). They stay counted, but only pushed samples are failures to replay.
+            $pushed       = $row->source !== 'poll';
+            $sourceDelay  = $row->source_delay_seconds;
             $queueDelay   = max(0, now()->timestamp - \Illuminate\Support\Carbon::parse($row->received_at, 'UTC')->timestamp);
-            foreach ($units as $unit) {
+            $checkpointAt = $started;
+            foreach ($units as $index => $unit) {
                 try {
                     $result      = $ingestor->ingest($telematic, $provider, $unit, $service, $row->received_at, $row->source);
                     $sourceDelay = max($sourceDelay ?? 0, data_get($result['device']->meta, 'telemetry.source_delay_seconds', 0));
@@ -93,19 +121,45 @@ class ProcessTelematicDelivery implements ShouldQueue, ShouldBeUnique
                     $failed[]       = $unit;
                     $failureTypes[] = class_basename($e);
                 }
+                $elapsed = microtime(true);
+                $yield   = $elapsed - $started >= 40 && $index + 1 < count($units);
+                if ($yield || ($index + 1) % 10 === 0 || $elapsed - $checkpointAt >= 2) {
+                    // Persist progress independently of the worker reservation. A
+                    // restart resumes the remaining tail rather than replaying a
+                    // growing prefix until the delivery exhausts its retry budget.
+                    $this->update([
+                        'retry_payload' => Crypt::encryptString(json_encode([
+                            'checkpoint_version' => 1, 'remaining' => array_slice($units, $index + 1),
+                            'failed'             => $failed, 'failure_types' => array_values(array_unique($failureTypes)),
+                        ], JSON_THROW_ON_ERROR)),
+                        'applied'             => $row->applied + $applied, 'failed' => count($failed) + ($pushed ? $invalid : 0), 'invalid_count' => $invalid,
+                        'queue_delay_seconds' => $queueDelay, 'source_delay_seconds' => $sourceDelay,
+                        'status'              => $yield ? 'retry' : 'processing',
+                        // Cooperative continuation is not a failed attempt. Actual
+                        // crashes keep the increment persisted at the start above.
+                        'attempts'     => $yield ? $row->attempts : $row->attempts + 1,
+                        'available_at' => $yield ? now() : now()->addSeconds(120),
+                    ]);
+                    $checkpointAt = $elapsed;
+                }
+                if ($yield) {
+                    // The normal inbox drain queues the continuation after this
+                    // job releases its uniqueness lease; it uses the same queue.
+                    return;
+                }
             }
             $this->update(['queue_delay_seconds' => $queueDelay, 'source_delay_seconds' => $sourceDelay]);
             if ($failed && $row->attempts < 4) {
                 $this->update([
                     'status'  => 'retry', 'retry_payload' => Crypt::encryptString(json_encode($failed, JSON_THROW_ON_ERROR)),
-                    'applied' => $row->applied + $applied, 'failed' => count($failed) + $invalid, 'invalid_count' => $invalid,
+                    'applied' => $row->applied + $applied, 'failed' => count($failed) + ($pushed ? $invalid : 0), 'invalid_count' => $invalid,
                     'error'   => 'Some units failed ingestion; retry scheduled. ' . implode(', ', array_unique($failureTypes)), 'available_at' => now()->addSeconds(15 * (2 ** $row->attempts)),
                 ]);
             } else {
                 $this->update([
-                    'status'       => ($failed || $invalid) ? 'quarantined' : 'processed',
-                    'applied'      => $row->applied + $applied, 'failed' => count($failed) + $invalid, 'invalid_count' => $invalid,
-                    'error'        => ($failed || $invalid) ? 'Invalid positions or exhausted ingestion retries; inspect and replay.' : null,
+                    'status'       => ($failed || ($pushed ? $invalid : 0)) ? 'quarantined' : 'processed',
+                    'applied'      => $row->applied + $applied, 'failed' => count($failed) + ($pushed ? $invalid : 0), 'invalid_count' => $invalid,
+                    'error'        => ($failed || ($pushed ? $invalid : 0)) ? 'Invalid positions or exhausted ingestion retries; inspect and replay.' : null,
                     'processed_at' => now(),
                 ]);
             }

package/server/src/Jobs/SyncTelematicDevicesJob.php

@@ -40,6 +40,7 @@ class SyncTelematicDevicesJob implements ShouldQueue
         $this->telematic = $telematic;
         $this->options   = $options;
         $this->jobId     = $jobId ?? \Illuminate\Support\Str::uuid()->toString();
+        $this->onQueue(\Fleetbase\FleetOps\Support\Telematics\Telemetry\Configuration::pollQueue());
     }
 
     /**
@@ -47,6 +48,24 @@ class SyncTelematicDevicesJob implements ShouldQueue
      */
     public function handle(TelematicProviderRegistry $registry, TelematicService $service): void
     {
+        // Jobs queued before a provider opts into batching must use the same bounded
+        // path as new manual requests, without acquiring the legacy hour-long lock.
+        $provider = $registry->resolve($this->telematic->provider);
+        if ($provider instanceof \Fleetbase\FleetOps\Contracts\TelemetryProviderInterface
+            && (\Fleetbase\FleetOps\Support\Telematics\Telemetry\Configuration::options($provider)['manual_batch_sync'] ?? false)) {
+            $this->telematic->refresh();
+            if (\Fleetbase\FleetOps\Support\Telematics\Telemetry\Inbox::enabled($this->telematic)) {
+                try {
+                    $service->queueTelemetrySync($this->telematic, $this->options, $this->jobId);
+                } catch (\Illuminate\Validation\ValidationException) {
+                    // Polling was paused after this job was queued. Failing here would mark the
+                    // connection as errored and detach any active request's progress tracking.
+                    Log::info('Skipped legacy telematics sync redirect.', ['telematic_uuid' => $this->telematic->uuid]);
+                }
+            }
+
+            return;
+        }
         $correlationId = \Illuminate\Support\Str::uuid()->toString();
         $lockKey       = 'fleetops:sync-telematic-devices:' . $this->telematic->uuid;
         $lock          = Cache::lock($lockKey, $this->timeout + 60);
@@ -96,12 +115,54 @@ class SyncTelematicDevicesJob implements ShouldQueue
             $totalEnrichment            = 0;
             $totalEnrichmentCompleted   = 0;
             $totalEnrichmentFailures    = 0;
+            $itemsSinceCheckpoint       = 0;
+            $lastCheckpointAt           = now();
+
+            $checkpoint = function (string $phase, bool $force = false) use (
+                &$itemsSinceCheckpoint, &$lastCheckpointAt, &$providerSyncMeta, &$totalFetched,
+                &$totalLinked, &$totalLinkAttempts, &$totalEvents, &$totalSensors, &$totalSkipped,
+                &$pageCount, &$lastProviderAllCount, &$lastProviderFiltersCount,
+                &$inventoryFetched, &$inventoryLinked, &$inventorySkipped,
+                &$totalEnrichment, &$totalEnrichmentCompleted, &$totalEnrichmentFailures
+            ): void {
+                if (!$force && $itemsSinceCheckpoint < 25 && $lastCheckpointAt->diffInSeconds(now(), true) < 5) {
+                    return;
+                }
+                $this->telematic->status = 'synchronizing';
+                $this->telematic->meta   = array_merge($this->telematic->meta ?? [], $providerSyncMeta, [
+                    'last_sync_job_id'                  => $this->jobId,
+                    'last_sync_result'                  => 'running',
+                    'last_sync_phase'                   => $phase,
+                    'last_sync_progress_at'             => now()->toDateTimeString(),
+                    'last_sync_fetched_total'           => $totalFetched,
+                    'last_sync_linked_total'            => $totalLinked,
+                    'last_sync_link_attempts_total'     => $totalLinkAttempts,
+                    'last_sync_events_total'            => $totalEvents,
+                    'last_sync_sensors_total'           => $totalSensors,
+                    'last_sync_skipped_total'           => $totalSkipped,
+                    'last_sync_page_count'              => $pageCount,
+                    'last_sync_provider_total'          => $lastProviderFiltersCount ?? $lastProviderAllCount,
+                    'last_sync_provider_all_count'      => $lastProviderAllCount,
+                    'last_sync_provider_filters_count'  => $lastProviderFiltersCount,
+                    'last_sync_inventory_total'         => $inventoryFetched,
+                    'last_sync_inventory_linked_total'  => $inventoryLinked,
+                    'last_sync_inventory_skipped_total' => $inventorySkipped,
+                    'last_sync_enrichment_total'        => $totalEnrichment,
+                    'last_sync_enrichment_completed'    => $totalEnrichmentCompleted,
+                    'last_sync_enrichment_failures'     => $totalEnrichmentFailures,
+                ]);
+                $this->telematic->save();
+                $itemsSinceCheckpoint = 0;
+                $lastCheckpointAt     = now();
+            };
 
             try {
+                $checkpoint('connecting', true);
                 $provider = $registry->resolve($this->telematic->provider);
                 $provider->connect($this->telematic);
 
                 do {
+                    $checkpoint('fetching_inventory', true);
                     $response = $provider->fetchDevices([
                         'limit'   => $this->options['limit'] ?? null,
                         'cursor'  => $cursor,
@@ -137,6 +198,9 @@ class SyncTelematicDevicesJob implements ShouldQueue
                         'has_more'       => $response['has_more'] ?? false,
                     ]);
 
+                    // Persist retrieval before any potentially slow per-device database work.
+                    $inventoryFetched = $totalFetched;
+                    $checkpoint('ingesting_inventory', true);
                     foreach ($devices as $devicePayload) {
                         $normalizedDevice = $provider->normalizeDevice($devicePayload);
                         try {
@@ -162,7 +226,12 @@ class SyncTelematicDevicesJob implements ShouldQueue
                                 'imei'             => $devicePayload['imei'] ?? null,
                             ]);
                         }
+                        $inventoryLinked  = $totalLinked;
+                        $inventorySkipped = $totalSkipped;
+                        $itemsSinceCheckpoint++;
+                        $checkpoint('ingesting_inventory');
                     }
+                    $checkpoint('ingesting_inventory', true);
 
                     $cursor = $response['next_cursor'] ?? null;
 
@@ -221,6 +290,7 @@ class SyncTelematicDevicesJob implements ShouldQueue
                         'device_count'   => count($inventoryPayloads),
                     ]);
 
+                    $checkpoint('fetching_enrichment', true);
                     $enrichmentResponse = $provider->fetchDeviceTelemetrySnapshots($inventoryPayloads, [
                         'limit'   => $this->options['limit'] ?? null,
                         'filters' => $this->options['filters'] ?? [],
@@ -229,6 +299,7 @@ class SyncTelematicDevicesJob implements ShouldQueue
                     $enrichedDevices  = $enrichmentResponse['devices'] ?? [];
                     $totalEnrichment += count($enrichedDevices);
 
+                    $checkpoint('ingesting_enrichment', true);
                     foreach ($enrichedDevices as $devicePayload) {
                         $normalizedDevice = $provider->normalizeDevice($devicePayload);
                         try {
@@ -256,7 +327,10 @@ class SyncTelematicDevicesJob implements ShouldQueue
                                 'imei'             => $devicePayload['imei'] ?? null,
                             ]);
                         }
+                        $itemsSinceCheckpoint++;
+                        $checkpoint('ingesting_enrichment');
                     }
+                    $checkpoint('ingesting_enrichment', true);
 
                     Log::info($this->telematic->provider === 'safee' ? 'Safee telemetry enrichment completed' : 'Telematics telemetry enrichment completed', [
                         'correlation_id' => $correlationId,
@@ -289,6 +363,8 @@ class SyncTelematicDevicesJob implements ShouldQueue
                     'last_sync_job_id'                  => $this->jobId,
                     'last_sync_completed_at'            => now()->toDateTimeString(),
                     'last_sync_result'                  => 'success',
+                    'last_sync_phase'                   => 'completed',
+                    'last_sync_progress_at'             => now()->toDateTimeString(),
                     'last_sync_total'                   => $totalLinked,
                     'last_sync_fetched_total'           => $totalFetched,
                     'last_sync_linked_total'            => $totalLinked,
@@ -310,7 +386,7 @@ class SyncTelematicDevicesJob implements ShouldQueue
                     'last_sync_error_context'           => null,
                 ]);
                 $this->telematic->save();
-            } catch (\Exception $e) {
+            } catch (\Throwable $e) {
                 $failureContext = method_exists($e, 'context') ? $e->context() : [];
                 $failureMessage = $this->safeSyncErrorMessage($e);
 

package/server/src/Jobs/TestTelematicConnectionJob.php

@@ -37,6 +37,7 @@ class TestTelematicConnectionJob implements ShouldQueue
     {
         $this->telematic = $telematic;
         $this->jobId     = $jobId ?? Str::uuid()->toString();
+        $this->onQueue(\Fleetbase\FleetOps\Support\Telematics\Telemetry\Configuration::pollQueue());
     }
 
     /**

package/server/src/Providers/FleetOpsServiceProvider.php

@@ -149,8 +149,10 @@ class FleetOpsServiceProvider extends CoreServiceProvider
             $schedule->command('fleetops:process-maintenance-triggers')->daily()->withoutOverlapping()->storeOutputInDb();
             $schedule->command('fleetops:send-maintenance-reminders')->daily()->withoutOverlapping()->storeOutputInDb();
             $schedule->command('fleetops:process-operational-alerts')->everyMinute()->withoutOverlapping()->storeOutputInDb();
-            $schedule->command('fleetops:sync-telematics')->everyMinute()->withoutOverlapping()->storeOutputInDb();
-            $schedule->command('fleetops:drain-telematic-inbox')->everyMinute()->withoutOverlapping();
+            // These commands dispatch bounded jobs. A scheduler restart must not
+            // leave the default 24-hour overlap lease blocking telemetry recovery.
+            $schedule->command('fleetops:sync-telematics')->everyMinute()->withoutOverlapping(2)->storeOutputInDb();
+            $schedule->command('fleetops:drain-telematic-inbox')->everyMinute()->withoutOverlapping(2);
         });
         $this->registerNotifications();
         $this->registerAiCapabilities();
@@ -196,6 +198,7 @@ class FleetOpsServiceProvider extends CoreServiceProvider
         $this->mergeConfigFrom(__DIR__ . '/../../config/fleetops.php', 'fleetops');
         $this->mergeConfigFrom(__DIR__ . '/../../config/telematics.php', 'telematics');
         $this->mergeConfigFrom(__DIR__ . '/../../config/afaqy.php', 'telematics.afaqy');
+        $this->mergeConfigFrom(__DIR__ . '/../../config/safee.php', 'telematics.safee');
         $this->mergeConfigFrom(__DIR__ . '/../../config/telemetry.php', 'telematics.telemetry');
         $this->mergeConfigFrom(__DIR__ . '/../../config/fuel-providers.php', 'fuel-providers');
         $this->mergeConfigFrom(__DIR__ . '/../../config/api.php', 'api');