@fleetbase/fleetops-engine 0.6.66 → 0.6.67

package/server/src/Jobs/ProcessTelematicDelivery.php

@@ -38,7 +38,8 @@ class ProcessTelematicDelivery implements ShouldQueue, ShouldBeUnique
 
     public function handle(Ingestor $ingestor, TelematicService $service): void
     {
-        $lock = Cache::lock('telemetry:delivery:' . $this->deliveryUuid, 90);
+        $started = microtime(true);
+        $lock    = Cache::lock('telemetry:delivery:' . $this->deliveryUuid, 90);
         if (!$lock->get()) {
             return;
         }
@@ -66,21 +67,48 @@ class ProcessTelematicDelivery implements ShouldQueue, ShouldBeUnique
                 return;
             }
             $this->update(['status' => 'processing', 'attempts' => $row->attempts + 1, 'available_at' => now()->addSeconds(120)]);
+            $failed       = [];
+            $failureTypes = [];
             try {
-                $units = $provider->telemetryUnits(json_decode(Crypt::decryptString($row->retry_payload ?? $row->payload), true, 512, JSON_THROW_ON_ERROR));
+                $payload = json_decode(Crypt::decryptString($row->retry_payload ?? $row->payload), true, 512, JSON_THROW_ON_ERROR);
+                if ($row->retry_payload && ($payload['checkpoint_version'] ?? null) === 1) {
+                    // This envelope is internal and encrypted; the original webhook
+                    // contract was already validated before creating the checkpoint.
+                    $units        = $payload['remaining'] ?? null;
+                    $failed       = $payload['failed'] ?? null;
+                    $failureTypes = $payload['failure_types'] ?? null;
+                    foreach ([$units, $failed, $failureTypes] as $items) {
+                        if (!is_array($items) || !array_is_list($items)) {
+                            throw new \InvalidArgumentException('Invalid delivery checkpoint.');
+                        }
+                    }
+                } elseif ($row->source === 'poll') {
+                    // Polling persists the provider's inventory records, including units
+                    // that have never reported a position. Validate this internal envelope
+                    // here and let ingestion report invalid positions per unit below.
+                    if (!is_array($payload) || !array_is_list($payload) || $payload === []) {
+                        throw new \InvalidArgumentException('Expected a non-empty polling unit list.');
+                    }
+                    $units = $payload;
+                } else {
+                    // External deliveries must still pass the provider's position contract.
+                    $units = $provider->telemetryUnits($payload);
+                }
             } catch (\Throwable) {
                 $this->update(['status' => 'quarantined', 'failed' => 1, 'error' => 'Unsupported or unreadable position payload; inspect and replay after adapter correction.']);
                 Inbox::finishRun($row->run_uuid);
 
                 return;
             }
-            $failed       = [];
-            $failureTypes = [];
             $applied      = 0;
             $invalid      = (int) $row->invalid_count;
-            $sourceDelay  = null;
+            // Polled inventory legitimately includes units without a valid fix (for example,
+            // never reported). They stay counted, but only pushed samples are failures to replay.
+            $pushed       = $row->source !== 'poll';
+            $sourceDelay  = $row->source_delay_seconds;
             $queueDelay   = max(0, now()->timestamp - \Illuminate\Support\Carbon::parse($row->received_at, 'UTC')->timestamp);
-            foreach ($units as $unit) {
+            $checkpointAt = $started;
+            foreach ($units as $index => $unit) {
                 try {
                     $result      = $ingestor->ingest($telematic, $provider, $unit, $service, $row->received_at, $row->source);
                     $sourceDelay = max($sourceDelay ?? 0, data_get($result['device']->meta, 'telemetry.source_delay_seconds', 0));
@@ -93,19 +121,45 @@ class ProcessTelematicDelivery implements ShouldQueue, ShouldBeUnique
                     $failed[]       = $unit;
                     $failureTypes[] = class_basename($e);
                 }
+                $elapsed = microtime(true);
+                $yield   = $elapsed - $started >= 40 && $index + 1 < count($units);
+                if ($yield || ($index + 1) % 10 === 0 || $elapsed - $checkpointAt >= 2) {
+                    // Persist progress independently of the worker reservation. A
+                    // restart resumes the remaining tail rather than replaying a
+                    // growing prefix until the delivery exhausts its retry budget.
+                    $this->update([
+                        'retry_payload' => Crypt::encryptString(json_encode([
+                            'checkpoint_version' => 1, 'remaining' => array_slice($units, $index + 1),
+                            'failed'             => $failed, 'failure_types' => array_values(array_unique($failureTypes)),
+                        ], JSON_THROW_ON_ERROR)),
+                        'applied'             => $row->applied + $applied, 'failed' => count($failed) + ($pushed ? $invalid : 0), 'invalid_count' => $invalid,
+                        'queue_delay_seconds' => $queueDelay, 'source_delay_seconds' => $sourceDelay,
+                        'status'              => $yield ? 'retry' : 'processing',
+                        // Cooperative continuation is not a failed attempt. Actual
+                        // crashes keep the increment persisted at the start above.
+                        'attempts'     => $yield ? $row->attempts : $row->attempts + 1,
+                        'available_at' => $yield ? now() : now()->addSeconds(120),
+                    ]);
+                    $checkpointAt = $elapsed;
+                }
+                if ($yield) {
+                    // The normal inbox drain queues the continuation after this
+                    // job releases its uniqueness lease; it uses the same queue.
+                    return;
+                }
             }
             $this->update(['queue_delay_seconds' => $queueDelay, 'source_delay_seconds' => $sourceDelay]);
             if ($failed && $row->attempts < 4) {
                 $this->update([
                     'status'  => 'retry', 'retry_payload' => Crypt::encryptString(json_encode($failed, JSON_THROW_ON_ERROR)),
-                    'applied' => $row->applied + $applied, 'failed' => count($failed) + $invalid, 'invalid_count' => $invalid,
+                    'applied' => $row->applied + $applied, 'failed' => count($failed) + ($pushed ? $invalid : 0), 'invalid_count' => $invalid,
                     'error'   => 'Some units failed ingestion; retry scheduled. ' . implode(', ', array_unique($failureTypes)), 'available_at' => now()->addSeconds(15 * (2 ** $row->attempts)),
                 ]);
             } else {
                 $this->update([
-                    'status'       => ($failed || $invalid) ? 'quarantined' : 'processed',
-                    'applied'      => $row->applied + $applied, 'failed' => count($failed) + $invalid, 'invalid_count' => $invalid,
-                    'error'        => ($failed || $invalid) ? 'Invalid positions or exhausted ingestion retries; inspect and replay.' : null,
+                    'status'       => ($failed || ($pushed ? $invalid : 0)) ? 'quarantined' : 'processed',
+                    'applied'      => $row->applied + $applied, 'failed' => count($failed) + ($pushed ? $invalid : 0), 'invalid_count' => $invalid,
+                    'error'        => ($failed || ($pushed ? $invalid : 0)) ? 'Invalid positions or exhausted ingestion retries; inspect and replay.' : null,
                     'processed_at' => now(),
                 ]);
             }

package/server/src/Jobs/SyncTelematicDevicesJob.php

@@ -40,6 +40,7 @@ class SyncTelematicDevicesJob implements ShouldQueue
         $this->telematic = $telematic;
         $this->options   = $options;
         $this->jobId     = $jobId ?? \Illuminate\Support\Str::uuid()->toString();
+        $this->onQueue(\Fleetbase\FleetOps\Support\Telematics\Telemetry\Configuration::pollQueue());
     }
 
     /**
@@ -47,6 +48,24 @@ class SyncTelematicDevicesJob implements ShouldQueue
      */
     public function handle(TelematicProviderRegistry $registry, TelematicService $service): void
     {
+        // Jobs queued before a provider opts into batching must use the same bounded
+        // path as new manual requests, without acquiring the legacy hour-long lock.
+        $provider = $registry->resolve($this->telematic->provider);
+        if ($provider instanceof \Fleetbase\FleetOps\Contracts\TelemetryProviderInterface
+            && (\Fleetbase\FleetOps\Support\Telematics\Telemetry\Configuration::options($provider)['manual_batch_sync'] ?? false)) {
+            $this->telematic->refresh();
+            if (\Fleetbase\FleetOps\Support\Telematics\Telemetry\Inbox::enabled($this->telematic)) {
+                try {
+                    $service->queueTelemetrySync($this->telematic, $this->options, $this->jobId);
+                } catch (\Illuminate\Validation\ValidationException) {
+                    // Paused or already queued/running. Failing here would mark the connection
+                    // as errored and detach the active request's progress tracking.
+                    Log::info('Skipped legacy telematics sync redirect.', ['telematic_uuid' => $this->telematic->uuid]);
+                }
+            }
+
+            return;
+        }
         $correlationId = \Illuminate\Support\Str::uuid()->toString();
         $lockKey       = 'fleetops:sync-telematic-devices:' . $this->telematic->uuid;
         $lock          = Cache::lock($lockKey, $this->timeout + 60);
@@ -96,12 +115,54 @@ class SyncTelematicDevicesJob implements ShouldQueue
             $totalEnrichment            = 0;
             $totalEnrichmentCompleted   = 0;
             $totalEnrichmentFailures    = 0;
+            $itemsSinceCheckpoint       = 0;
+            $lastCheckpointAt           = now();
+
+            $checkpoint = function (string $phase, bool $force = false) use (
+                &$itemsSinceCheckpoint, &$lastCheckpointAt, &$providerSyncMeta, &$totalFetched,
+                &$totalLinked, &$totalLinkAttempts, &$totalEvents, &$totalSensors, &$totalSkipped,
+                &$pageCount, &$lastProviderAllCount, &$lastProviderFiltersCount,
+                &$inventoryFetched, &$inventoryLinked, &$inventorySkipped,
+                &$totalEnrichment, &$totalEnrichmentCompleted, &$totalEnrichmentFailures
+            ): void {
+                if (!$force && $itemsSinceCheckpoint < 25 && $lastCheckpointAt->diffInSeconds(now(), true) < 5) {
+                    return;
+                }
+                $this->telematic->status = 'synchronizing';
+                $this->telematic->meta   = array_merge($this->telematic->meta ?? [], $providerSyncMeta, [
+                    'last_sync_job_id'                  => $this->jobId,
+                    'last_sync_result'                  => 'running',
+                    'last_sync_phase'                   => $phase,
+                    'last_sync_progress_at'             => now()->toDateTimeString(),
+                    'last_sync_fetched_total'           => $totalFetched,
+                    'last_sync_linked_total'            => $totalLinked,
+                    'last_sync_link_attempts_total'     => $totalLinkAttempts,
+                    'last_sync_events_total'            => $totalEvents,
+                    'last_sync_sensors_total'           => $totalSensors,
+                    'last_sync_skipped_total'           => $totalSkipped,
+                    'last_sync_page_count'              => $pageCount,
+                    'last_sync_provider_total'          => $lastProviderFiltersCount ?? $lastProviderAllCount,
+                    'last_sync_provider_all_count'      => $lastProviderAllCount,
+                    'last_sync_provider_filters_count'  => $lastProviderFiltersCount,
+                    'last_sync_inventory_total'         => $inventoryFetched,
+                    'last_sync_inventory_linked_total'  => $inventoryLinked,
+                    'last_sync_inventory_skipped_total' => $inventorySkipped,
+                    'last_sync_enrichment_total'        => $totalEnrichment,
+                    'last_sync_enrichment_completed'    => $totalEnrichmentCompleted,
+                    'last_sync_enrichment_failures'     => $totalEnrichmentFailures,
+                ]);
+                $this->telematic->save();
+                $itemsSinceCheckpoint = 0;
+                $lastCheckpointAt     = now();
+            };
 
             try {
+                $checkpoint('connecting', true);
                 $provider = $registry->resolve($this->telematic->provider);
                 $provider->connect($this->telematic);
 
                 do {
+                    $checkpoint('fetching_inventory', true);
                     $response = $provider->fetchDevices([
                         'limit'   => $this->options['limit'] ?? null,
                         'cursor'  => $cursor,
@@ -137,6 +198,9 @@ class SyncTelematicDevicesJob implements ShouldQueue
                         'has_more'       => $response['has_more'] ?? false,
                     ]);
 
+                    // Persist retrieval before any potentially slow per-device database work.
+                    $inventoryFetched = $totalFetched;
+                    $checkpoint('ingesting_inventory', true);
                     foreach ($devices as $devicePayload) {
                         $normalizedDevice = $provider->normalizeDevice($devicePayload);
                         try {
@@ -162,7 +226,12 @@ class SyncTelematicDevicesJob implements ShouldQueue
                                 'imei'             => $devicePayload['imei'] ?? null,
                             ]);
                         }
+                        $inventoryLinked  = $totalLinked;
+                        $inventorySkipped = $totalSkipped;
+                        $itemsSinceCheckpoint++;
+                        $checkpoint('ingesting_inventory');
                     }
+                    $checkpoint('ingesting_inventory', true);
 
                     $cursor = $response['next_cursor'] ?? null;
 
@@ -221,6 +290,7 @@ class SyncTelematicDevicesJob implements ShouldQueue
                         'device_count'   => count($inventoryPayloads),
                     ]);
 
+                    $checkpoint('fetching_enrichment', true);
                     $enrichmentResponse = $provider->fetchDeviceTelemetrySnapshots($inventoryPayloads, [
                         'limit'   => $this->options['limit'] ?? null,
                         'filters' => $this->options['filters'] ?? [],
@@ -229,6 +299,7 @@ class SyncTelematicDevicesJob implements ShouldQueue
                     $enrichedDevices  = $enrichmentResponse['devices'] ?? [];
                     $totalEnrichment += count($enrichedDevices);
 
+                    $checkpoint('ingesting_enrichment', true);
                     foreach ($enrichedDevices as $devicePayload) {
                         $normalizedDevice = $provider->normalizeDevice($devicePayload);
                         try {
@@ -256,7 +327,10 @@ class SyncTelematicDevicesJob implements ShouldQueue
                                 'imei'             => $devicePayload['imei'] ?? null,
                             ]);
                         }
+                        $itemsSinceCheckpoint++;
+                        $checkpoint('ingesting_enrichment');
                     }
+                    $checkpoint('ingesting_enrichment', true);
 
                     Log::info($this->telematic->provider === 'safee' ? 'Safee telemetry enrichment completed' : 'Telematics telemetry enrichment completed', [
                         'correlation_id' => $correlationId,
@@ -289,6 +363,8 @@ class SyncTelematicDevicesJob implements ShouldQueue
                     'last_sync_job_id'                  => $this->jobId,
                     'last_sync_completed_at'            => now()->toDateTimeString(),
                     'last_sync_result'                  => 'success',
+                    'last_sync_phase'                   => 'completed',
+                    'last_sync_progress_at'             => now()->toDateTimeString(),
                     'last_sync_total'                   => $totalLinked,
                     'last_sync_fetched_total'           => $totalFetched,
                     'last_sync_linked_total'            => $totalLinked,
@@ -310,7 +386,7 @@ class SyncTelematicDevicesJob implements ShouldQueue
                     'last_sync_error_context'           => null,
                 ]);
                 $this->telematic->save();
-            } catch (\Exception $e) {
+            } catch (\Throwable $e) {
                 $failureContext = method_exists($e, 'context') ? $e->context() : [];
                 $failureMessage = $this->safeSyncErrorMessage($e);
 

package/server/src/Jobs/TestTelematicConnectionJob.php

@@ -37,6 +37,7 @@ class TestTelematicConnectionJob implements ShouldQueue
     {
         $this->telematic = $telematic;
         $this->jobId     = $jobId ?? Str::uuid()->toString();
+        $this->onQueue(\Fleetbase\FleetOps\Support\Telematics\Telemetry\Configuration::pollQueue());
     }
 
     /**

package/server/src/Providers/FleetOpsServiceProvider.php

@@ -149,8 +149,10 @@ class FleetOpsServiceProvider extends CoreServiceProvider
             $schedule->command('fleetops:process-maintenance-triggers')->daily()->withoutOverlapping()->storeOutputInDb();
             $schedule->command('fleetops:send-maintenance-reminders')->daily()->withoutOverlapping()->storeOutputInDb();
             $schedule->command('fleetops:process-operational-alerts')->everyMinute()->withoutOverlapping()->storeOutputInDb();
-            $schedule->command('fleetops:sync-telematics')->everyMinute()->withoutOverlapping()->storeOutputInDb();
-            $schedule->command('fleetops:drain-telematic-inbox')->everyMinute()->withoutOverlapping();
+            // These commands dispatch bounded jobs. A scheduler restart must not
+            // leave the default 24-hour overlap lease blocking telemetry recovery.
+            $schedule->command('fleetops:sync-telematics')->everyMinute()->withoutOverlapping(2)->storeOutputInDb();
+            $schedule->command('fleetops:drain-telematic-inbox')->everyMinute()->withoutOverlapping(2);
         });
         $this->registerNotifications();
         $this->registerAiCapabilities();
@@ -196,6 +198,7 @@ class FleetOpsServiceProvider extends CoreServiceProvider
         $this->mergeConfigFrom(__DIR__ . '/../../config/fleetops.php', 'fleetops');
         $this->mergeConfigFrom(__DIR__ . '/../../config/telematics.php', 'telematics');
         $this->mergeConfigFrom(__DIR__ . '/../../config/afaqy.php', 'telematics.afaqy');
+        $this->mergeConfigFrom(__DIR__ . '/../../config/safee.php', 'telematics.safee');
         $this->mergeConfigFrom(__DIR__ . '/../../config/telemetry.php', 'telematics.telemetry');
         $this->mergeConfigFrom(__DIR__ . '/../../config/fuel-providers.php', 'fuel-providers');
         $this->mergeConfigFrom(__DIR__ . '/../../config/api.php', 'api');

package/server/src/Support/Telematics/Providers/AfaqyProvider.php

@@ -5,6 +5,7 @@ namespace Fleetbase\FleetOps\Support\Telematics\Providers;
 use Fleetbase\FleetOps\Exceptions\TelematicProviderException;
 use Fleetbase\FleetOps\Exceptions\TelematicRateLimitExceededException;
 use Fleetbase\FleetOps\Support\Telematics\Afaqy\Payload;
+use Illuminate\Contracts\Encryption\DecryptException;
 use Illuminate\Http\Client\ConnectionException;
 use Illuminate\Http\Client\Response;
 use Illuminate\Support\Facades\Cache;
@@ -20,12 +21,13 @@ use Illuminate\Support\Facades\Log;
  */
 class AfaqyProvider extends AbstractProvider implements \Fleetbase\FleetOps\Contracts\TelemetryProviderInterface
 {
-    protected string $baseUrl                   = 'https://api.afaqy.sa';
-    protected int $requestsPerMinute            = 60;
-    protected int $dataTimeout                  = 120;
-    protected int $connectTimeout               = 15;
-    protected int $connectionTestTimeout        = 30;
-    protected int $connectionTestConnectTimeout = 10;
+    protected string $baseUrl                    = 'https://api.afaqy.sa';
+    protected int $requestsPerMinute             = 60;
+    protected int $dataTimeout                   = 120;
+    protected int $connectTimeout                = 15;
+    protected int $connectionTestTimeout         = 30;
+    protected int $connectionTestConnectTimeout  = 10;
+    protected ?float $requestDeadline            = null;
 
     protected function prepareAuthentication(): void
     {
@@ -343,9 +345,11 @@ class AfaqyProvider extends AbstractProvider implements \Fleetbase\FleetOps\Cont
         }
 
         $this->reserveRequest();
+        $timeout  = $this->remainingRequestTimeout(30);
         $response = Http::asJson()
             ->acceptJson()
-            ->timeout(30)
+            ->timeout($timeout)
+            ->connectTimeout(min($this->connectTimeout, $timeout))
             ->post($this->baseUrl . '/auth/login', [
                 'data' => [
                     'username' => $this->credentials['username'],
@@ -373,6 +377,20 @@ class AfaqyProvider extends AbstractProvider implements \Fleetbase\FleetOps\Cont
     }
 
     protected function authenticatedPost(string $endpoint, array $payload = [], bool $tokenInQuery = false, bool $allowRetry = true, ?int $timeout = null, ?int $connectTimeout = null): array
+    {
+        $previousDeadline = $this->requestDeadline;
+        $this->requestDeadline ??= $this->requestTime() + max(1, $timeout ?? $this->dataTimeout);
+
+        try {
+            return $this->postWithinDeadline($endpoint, $payload, $tokenInQuery, $allowRetry, $timeout, $connectTimeout);
+        } finally {
+            // Providers may be reused. A completed or failed request must not
+            // leave its deadline on the next independent request.
+            $this->requestDeadline = $previousDeadline;
+        }
+    }
+
+    protected function postWithinDeadline(string $endpoint, array $payload, bool $tokenInQuery, bool $allowRetry, ?int $timeout, ?int $connectTimeout): array
     {
         [$url, $body] = $this->buildAuthenticatedRequest($endpoint, $payload, $tokenInQuery);
 
@@ -381,6 +399,8 @@ class AfaqyProvider extends AbstractProvider implements \Fleetbase\FleetOps\Cont
         $connectTimeout ??= $this->connectTimeout;
 
         $this->reserveRequest();
+        $timeout        = $this->remainingRequestTimeout($timeout);
+        $connectTimeout = min(max(1, $connectTimeout), $timeout);
         try {
             $response = Http::withHeaders($this->headers)
                 ->timeout($timeout)
@@ -468,9 +488,22 @@ class AfaqyProvider extends AbstractProvider implements \Fleetbase\FleetOps\Cont
     {
         $key = 'afaqy:token:' . $this->accountKey() . ':' . hash('sha256', $this->credentials['password'] ?? '');
 
-        return Cache::lock($key . ':lock', 40)->block(5, function () use ($key, $rejected) {
+        return Cache::lock($key . ':lock', 40)->block($this->remainingRequestTimeout(5), function () use ($key, $rejected) {
             $stored = Cache::get($key);
-            $token  = $stored ? Crypt::decryptString($stored) : null;
+            $token  = null;
+            if ($stored) {
+                try {
+                    $token = Crypt::decryptString($stored);
+                } catch (DecryptException) {
+                    // A cached token is disposable. Evict only this entry while holding
+                    // the existing account lock, then authenticate with resolved credentials.
+                    Cache::forget($key);
+                    Log::warning('AFAQY cached token could not be decrypted; re-authenticating.', [
+                        'telematic_uuid' => $this->telematic?->uuid,
+                        'reason'         => 'cached_token_decryption_failed',
+                    ]);
+                }
+            }
             if (!$token || $token === $rejected) {
                 $token = $this->authenticate();
                 Cache::put($key, Crypt::encryptString($token), 29 * 86400);
@@ -483,7 +516,8 @@ class AfaqyProvider extends AbstractProvider implements \Fleetbase\FleetOps\Cont
     protected function reserveRequest(): void
     {
         $key = 'afaqy:rate:' . $this->accountKey();
-        Cache::lock($key . ':lock', 5)->block(2, function () use ($key) {
+        Cache::lock($key . ':lock', 5)->block($this->remainingRequestTimeout(2), function () use ($key) {
+            $this->remainingRequestTimeout(1);
             $now      = microtime(true);
             $until    = (float) Cache::get($key . ':blocked', 0);
             $requests = array_values(array_filter(Cache::get($key, []), fn ($at) => $at > $now - 60));
@@ -495,6 +529,27 @@ class AfaqyProvider extends AbstractProvider implements \Fleetbase\FleetOps\Cont
         });
     }
 
+    /** Monotonic time keeps the request budget independent of wall-clock corrections. */
+    protected function requestTime(): float
+    {
+        return hrtime(true) / 1_000_000_000;
+    }
+
+    protected function remainingRequestTimeout(int $maximum): int
+    {
+        $maximum = max(1, $maximum);
+        if ($this->requestDeadline === null) {
+            return $maximum;
+        }
+
+        $remaining = (int) floor($this->requestDeadline - $this->requestTime());
+        if ($remaining < 1) {
+            throw new TelematicProviderException('AFAQY request time budget exhausted before another operation could start.');
+        }
+
+        return min($maximum, $remaining);
+    }
+
     protected function checkThrottle(Response $response): void
     {
         if ($response->status() !== 429) {