@andrian.yablonskyy/thub-coordinator 1.0.26 → 1.0.27

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@andrian.yablonskyy/thub-coordinator",
-  "version": "1.0.26",
+  "version": "1.0.27",
   "description": "TestHub Coordinator — job queue, resource registry, scheduler, heartbeat monitor, log/artifact store and web dashboard",
   "main": "src/server.js",
   "engines": {

package/public/css/thub.css

@@ -1,6 +1,6 @@
 #log-viewer {
   font-family: ui-monospace, SFMono-Regular, Menlo, Consolas, monospace;
-  font-size: 0.8125rem;
+  font-size: 0.75rem;
   white-space: pre-wrap;
   word-break: break-all;
 }
@@ -39,19 +39,12 @@
   text-align: left;
 }
 
-/* Resource card (views/mixins/resource.pug): 30% wider than Bootstrap's
-   default 500px modal. Only applies from 576px up, like Bootstrap's own
-   width — below that the modal fills the screen as before. */
-.modal.thub-resource-card {
-  --bs-modal-width: calc(500px * 1.3);
-}
-
 /* A status badge inside a page heading (e.g. the job page's state): badges
    are 0.75em of their parent, which in an h4 makes them oversized — keep
    it close to the badges in tables, centered on the heading text. */
 .thub-heading-badge {
-  font-size: 0.55em;
-  vertical-align: middle;
+  font-size: 0.6rem;
+  vertical-align: super;
 }
 
 /* Toast text (layout.pug): "\n" in a flash message is a line break, and a
@@ -61,3 +54,7 @@
   overflow-wrap: anywhere;
   min-width: 0;
 }
+
+.resource-capabilities {
+  font-size: 0.75rem;
+}

package/src/api/admin.js

@@ -51,6 +51,17 @@ function createAdminRouter({ services }){
     res.status(204).end();
   });
 
+  // Scheduled host reboot: { cron } (5-field, host local time; "" clears).
+  router.post('/resources/:id/reboot-schedule', (req, res, next) => {
+    try {
+      const r = services.registry.setRebootSchedule(req.params.id, req.body?.cron ?? '');
+      res.json({ id: r.id, rebootSchedule: r.reboot_schedule, applied: r.reboot_schedule_applied });
+    }
+    catch (err){
+      next(err);
+    }
+  });
+
   router.post('/resources/:id/maintenance', (req, res, next) => {
     try {
       const resource = services.registry.setMaintenance(req.params.id, !!req.body.enabled);

package/src/api/resource.js

@@ -84,7 +84,7 @@ function createResourceRouter({ services, config }){
 
   router.post('/resources/:id/heartbeat', auth, requireOwnResource, (req, res, next) => {
     try {
-      const { state, activeJobId, localLock, metrics, addresses, hostUptimeSec, activity } = req.body;
+      const { state, activeJobId, localLock, metrics, addresses, hostUptimeSec, activity, rebootSchedule } = req.body;
       services.registry.heartbeat(req.params.id, {
         state,
         activeJobId,
@@ -94,7 +94,8 @@ function createResourceRouter({ services, config }){
         remoteAddr: req.ip,
         clientVersion: appVersion(req, 'client'),
         hostUptimeSec,
-        activity
+        activity,
+        rebootSchedule
       });
       const commands = services.commands.drain(req.params.id),
         // Repeated on every heartbeat until the Client reports the new
@@ -103,6 +104,11 @@ function createResourceRouter({ services, config }){
       if (updateTo){
         commands.push({ command: 'self-update', version: updateTo });
       }
+      // Also repeated until the Client reports applying it (README §10).
+      const reboot = services.registry.pendingRebootSchedule(req.params.id);
+      if (reboot){
+        commands.push({ command: 'set-reboot-schedule', cron: reboot.cron });
+      }
       // Removal pending (jobs.removeResource): once the Client reports its
       // job stopped, finish it. This response still goes out; the next
       // heartbeat gets 401, the token being gone with the resource.

package/src/db/migrations/017_add_reboot_schedule.sql

@@ -0,0 +1,7 @@
+-- Scheduled host reboot per Client (README §10): a 5-field cron expression
+-- in the Client host's local time, set on the resource card. `reboot_schedule`
+-- is what an admin saved; `reboot_schedule_applied` what the Client last
+-- reported applying — while they differ, every heartbeat reply carries a
+-- set-reboot-schedule command (api/resource.js). NULL = no scheduled reboot.
+ALTER TABLE resources ADD COLUMN reboot_schedule TEXT;
+ALTER TABLE resources ADD COLUMN reboot_schedule_applied TEXT;

package/src/dev/virtual.js

@@ -0,0 +1,286 @@
+/**
+ * @file        packages/coordinator/src/dev/virtual.js
+ * @description DEV mode only (DEV_MODE=1 with THUB_BOOTSTRAP_ADMIN_PASSWORD): a virtual agent, virtual Clients that
+ *              heartbeat and run assigned jobs in-process, and a seeded job history — for working on the dashboard
+ *              without real hardware (README §13.1)
+ *
+ * @author      Andrian Yablonskyy
+ * @copyright   Copyright (c) 2026 Andrian Yablonskyy. All rights reserved.
+ *
+ * This file is part of TestHub and is proprietary and confidential.
+ * Unauthorized copying, modification, distribution, or use of this file,
+ * via any medium, is strictly prohibited without prior written permission
+ * from AdSystem.PRO.
+ */
+
+'use strict';
+
+const { JOB_STATES, ACTIVE_JOB_STATES } = require('@andrian.yablonskyy/thub-common'),
+  { hashToken } = require('../services/tokens'),
+  { version } = require('../../package.json');
+
+// Fixed, so `thub --token <it>` works against a dev Coordinator without
+// copying a fresh token each time. Only ever created in DEV mode.
+const DEV_AGENT_NAME = 'virtual-agent',
+  DEV_AGENT_TOKEN = 'agt_dev-virtual-agent-token',
+  VIRTUAL_CLIENTS = [
+    {
+      clientId: 'virtual-client-sw-01',
+      name: 'virtual-sw-01',
+      type: 'sw',
+      labels: ['virtual', 'emulator'],
+      capabilities: {
+        sw: { image: 'dut-emulator:dev', registry: null, allowDockerHub: true, allowJobImages: true, cpus: 2, memory: '2g' },
+        allowJobCommands: true
+      },
+      address: '10.0.0.41'
+    },
+    {
+      clientId: 'virtual-client-hw-01',
+      name: 'virtual-hw-01',
+      type: 'hw',
+      labels: ['virtual', 'board:nucleo-f401re', 'uart', 'stlink'],
+      capabilities: {
+        hw: {
+          stlinks: [{ path: '/dev/thub/dut1-stlink', index: 1, serial: '066DFF485457725187092834', present: true }],
+          uarts: [{ path: '/dev/thub/dut1-uart', index: 1, baudRate: 115200, present: true }],
+          usbs: [{ path: '/dev/thub/dut1-usb', index: 1, present: false }],
+          relays: [{ channel: 0, baseUrl: 'http://localhost:3000' }],
+          power: { method: 'relay' }
+        },
+        allowJobCommands: false
+      },
+      address: '10.0.0.42'
+    }
+  ],
+  // Seeded history: [daysAgo, source, target type, final state, duration s, user].
+  HISTORY = [
+    [6.8, 'ci', 'hw', 'PASSED', 412, null],
+    [6.1, 'cli', 'sw', 'FAILED', 95, 'alice'],
+    [5.4, 'ci', 'sw', 'PASSED', 188, null],
+    [4.9, 'ci', 'hw', 'ERROR', 31, null],
+    [4.2, 'cli', 'hw', 'PASSED', 640, 'bob'],
+    [3.6, 'ci', 'sw', 'TIMEOUT', 1800, null],
+    [2.7, 'cli', 'sw', 'CANCELED', 44, 'alice'],
+    [2.1, 'ci', 'hw', 'PASSED', 377, null],
+    [1.3, 'ci', 'sw', 'FAILED', 152, null],
+    [0.8, 'cli', 'hw', 'LOST', 260, 'bob'],
+    [0.4, 'ci', 'sw', 'PASSED', 201, null],
+    [0.1, 'cli', 'sw', 'PASSED', 73, 'alice']
+  ],
+  LOG_SCRIPT = [
+    ['runner', 'downloading firmware (virtual)'],
+    ['runner', 'downloading tests (virtual)'],
+    ['flash', 'st-flash write app.bin 0x08000000 ... done (virtual)'],
+    ['uart', '[BOOT] app v1.4.0-dev'],
+    ['uart', '[INIT] peripherals ok'],
+    ['runner', 'running: run-tests.sh --suite default'],
+    ['runner', 'test_boot ............ ok'],
+    ['runner', 'test_uart_echo ....... ok'],
+    ['runner', 'test_flash_crc ....... ok'],
+    ['runner', 'test_power_cycle ..... ok']
+  ],
+
+  spec = (type, user, extra = {}) => ({
+    target: { type, labels: ['virtual'] },
+    firmware: type === 'sw' ? { image: 'alpine:3.20' } : { url: 'https://artifactory.example.com/fw-local/app/1.4.0-dev/app.bin' },
+    tests: { url: 'https://artifactory.example.com/fw-local/app/1.4.0-dev/tests.tar.gz', suite: 'default' },
+    ...(user ? { user } : {}),
+    meta: { repo: 'yourorg/firmware', branch: 'main', virtual: true },
+    ...extra
+  });
+
+function isDevMode(env = process.env){
+  return env.DEV_MODE === '1' && Boolean(env.THUB_BOOTSTRAP_ADMIN_PASSWORD);
+}
+
+// The virtual agent, with its fixed token (re-applied if revoked or changed).
+function ensureAgent(services){
+  const { db } = services,
+    existing = db.prepare('SELECT * FROM agents WHERE name = ?').get(DEV_AGENT_NAME);
+  if (existing){
+    db.prepare('UPDATE agents SET token_hash = ?, revoked_at = NULL WHERE id = ?').run(hashToken(DEV_AGENT_TOKEN), existing.id);
+    return existing.id;
+  }
+  const id = 'agt_dev-virtual-agent';
+  db.prepare('INSERT INTO agents (id, name, kind, token_hash, created_at) VALUES (?, ?, ?, ?, ?)').run(
+    id, DEV_AGENT_NAME, 'cli', hashToken(DEV_AGENT_TOKEN), new Date().toISOString()
+  );
+  services.agents.touchLastUsed(id, version);
+  return id;
+}
+
+// A week of finished jobs in every final state, on the virtual Clients —
+// only into an empty jobs table, so restarts don't pile them up.
+function seedHistory(services, agentId, resourceIds){
+  const { db, jobs, logs } = services;
+  if (db.prepare('SELECT COUNT(*) AS n FROM jobs').get().n > 0){
+    return 0;
+  }
+  for (const [daysAgo, source, type, state, durationSec, user]of HISTORY){
+    const job = jobs.create({ agentId, source, spec: spec(type, user) }),
+      created = Date.now() - daysAgo * 86400000,
+      started = created + 20000,
+      finished = started + durationSec * 1000,
+      ran = state !== 'CANCELED' || durationSec > 0,
+      failed = state === 'FAILED' ? 2 : 0;
+    db.prepare(
+      `UPDATE jobs SET state = ?, resource_id = ?, created_at = ?, assigned_at = ?, started_at = ?, finished_at = ?,
+         duration_sec = ?, exit_code = ?, summary = ?, message = ? WHERE id = ?`
+    ).run(
+      state,
+      resourceIds[type],
+      new Date(created).toISOString(),
+      new Date(created + 5000).toISOString(),
+      ran ? new Date(started).toISOString() : null,
+      new Date(finished).toISOString(),
+      ran ? durationSec : null,
+      { PASSED: 0, FAILED: 1 }[state] ?? null,
+      JSON.stringify(['PASSED', 'FAILED'].includes(state)
+        ? { total: 12, passed: 12 - failed, failed, skipped: 0 }
+        : { error: `virtual ${state.toLowerCase()}` }),
+      { TIMEOUT: 'Job exceeded its timeout', LOST: 'Resource stopped heartbeating', ERROR: 'Flashing failed (virtual)' }[state] || null,
+      job.id
+    );
+    logs.appendBatch(job.id, LOG_SCRIPT.slice(0, state === 'PASSED' ? LOG_SCRIPT.length : 6).map(([stream, line], i) => ({
+      ts: new Date(started + i * 1000).toISOString(), stream, line
+    })));
+  }
+  return HISTORY.length;
+}
+
+// Virtual Clients, run in-process: they heartbeat like real ones (so they
+// stay IDLE instead of going OUT_OF_SERVICE) and play out any job the
+// scheduler assigns them — accept, PREPARING, RUNNING with streamed log
+// lines, then a PASSED (mostly) or FAILED result — through the same
+// services the Client API uses, so SSE, durations and artifacts all work.
+function startVirtualClients(services, config){
+  const { registry, jobs, logs, bus } = services,
+    intervalMs = config.heartbeat.intervalSec * 1000,
+    clients = new Map(); // resourceId -> { def, activity, running }
+
+  for (const def of VIRTUAL_CLIENTS){
+    const { resourceId } = registry.registerAuto({
+      clientId: def.clientId,
+      name: def.name,
+      type: def.type,
+      labels: def.labels,
+      hostInfo: { hostname: `${def.name}.virtual`, platform: 'linux', addresses: [{ iface: 'eth0', address: def.address, family: 'IPv4' }], timeZone: 'UTC' },
+      capabilities: { ...def.capabilities, rebootSupported: false },
+      remoteAddr: '127.0.0.1',
+      clientVersion: version
+    });
+    clients.set(resourceId, { def, activity: { state: 'idle', jobId: null, since: Date.now() }, running: null });
+  }
+
+  const bootedAt = Date.now() - 3 * 86400000;
+  function heartbeat(resourceId){
+    const c = clients.get(resourceId),
+      busy = Boolean(c.running);
+    registry.heartbeat(resourceId, {
+      state: busy ? 'busy' : 'idle',
+      activeJobId: c.running,
+      localLock: false,
+      addresses: [{ iface: 'eth0', address: c.def.address, family: 'IPv4' }],
+      remoteAddr: '127.0.0.1',
+      clientVersion: version,
+      hostUptimeSec: Math.round((Date.now() - bootedAt) / 1000),
+      activity: { state: c.activity.state, jobId: c.activity.jobId, durationSec: Math.round((Date.now() - c.activity.since) / 1000) },
+      rebootSchedule: registry.get(resourceId)?.reboot_schedule || null // "applies" whatever is saved
+    });
+    services.commands.drain(resourceId); // nothing real to act on
+  }
+
+  const beat = () => {
+      for (const resourceId of clients.keys()){
+        try {
+          heartbeat(resourceId);
+        }
+        catch (err){
+          console.error(`DEV virtual client heartbeat: ${err.message}`);
+        }
+      }
+    },
+    timer = setInterval(beat, intervalMs);
+  timer.unref?.();
+  beat();
+
+  const sleep = (ms) => new Promise((r) => setTimeout(r, ms)),
+    stillActive = (jobId) => ACTIVE_JOB_STATES.has(jobs.get(jobId)?.state);
+
+  async function run(resourceId, jobId){
+    const c = clients.get(resourceId);
+    c.running = jobId;
+    c.activity = { state: 'job', jobId, since: Date.now() };
+    try {
+      await sleep(800);
+      if (!stillActive(jobId)){
+        return;
+      }
+      jobs.setState(jobId, JOB_STATES.PREPARING);
+      logs.appendBatch(jobId, [{ stream: 'runner', line: `(virtual client ${c.def.name}) preparing` }]);
+      await sleep(1500);
+      if (!stillActive(jobId)){
+        return;
+      }
+      jobs.setState(jobId, JOB_STATES.RUNNING);
+      for (const [stream, line]of LOG_SCRIPT){
+        await sleep(600 + Math.random() * 900);
+        if (!stillActive(jobId)){
+          return; // canceled meanwhile
+        }
+        logs.appendBatch(jobId, [{ stream, line }]);
+      }
+      const failed = Math.random() < 0.2 ? 1 : 0;
+      jobs.applyResult(jobId, resourceId, {
+        state: failed ? JOB_STATES.FAILED : JOB_STATES.PASSED,
+        exitCode: failed,
+        summary: { total: 4, passed: 4 - failed, failed, skipped: 0 }
+      });
+    }
+    catch (err){
+      console.error(`DEV virtual client ${c.def.name}: ${err.message}`);
+    }
+    finally {
+      c.running = null;
+      c.activity = { state: 'idle', jobId: null, since: Date.now() };
+      heartbeat(resourceId);
+    }
+  }
+
+  bus.on('job.assigned', ({ jobId, resourceId }) => {
+    if (clients.has(resourceId)){
+      run(resourceId, jobId);
+    }
+  });
+
+  return {
+    resourceIds: Object.fromEntries([...clients].map(([id, c]) => [c.def.type, id])),
+    stop: () => clearInterval(timer)
+  };
+}
+
+// Entry point from server.js. Returns null outside DEV mode.
+function startDevMode(services, config, env = process.env){
+  if (!isDevMode(env)){
+    return null;
+  }
+  const agentId = ensureAgent(services),
+    virtual = startVirtualClients(services, config),
+    seeded = seedHistory(services, agentId, virtual.resourceIds);
+  // A few live ones for the virtual Clients to pick up right away.
+  if (seeded){
+    for (const [type, user]of [['sw', 'alice'], ['hw', null], ['sw', 'bob']]){
+      services.jobs.create({ agentId, source: 'cli', spec: spec(type, user) });
+    }
+  }
+  console.warn(
+    '*** DEV MODE (DEV_MODE=1): virtual Clients virtual-sw-01 / virtual-hw-01 are running in-process' +
+      `${seeded ? `, and ${seeded} past jobs + 3 live ones were seeded` : ''}.\n` +
+      `*** Virtual agent token: ${DEV_AGENT_TOKEN}\n` +
+      `***   e.g. thub --url ${config.publicUrl} --token ${DEV_AGENT_TOKEN} run --type sw --image alpine --tests https://x/t.tgz`
+  );
+  return virtual;
+}
+
+module.exports = { startDevMode, isDevMode, DEV_AGENT_TOKEN };

package/src/server.js

@@ -35,6 +35,7 @@ const path = require('node:path'),
   { createCommandsService } = require('./services/commands'),
   { createRetentionService } = require('./services/retention'),
   { createCleanupService } = require('./services/cleanup'),
+  { startDevMode } = require('./dev/virtual'),
   { createUpdatesService } = require('./services/updates'),
 
   { createAgentRouter } = require('./api/agent'),
@@ -154,6 +155,9 @@ function start(configPath){
     app = createApp(config, services);
 
   ensureBootstrapAdmin(services);
+  // DEV_MODE=1 plus THUB_BOOTSTRAP_ADMIN_PASSWORD only: virtual agent,
+  // Clients and jobs to work on the dashboard with (src/dev/virtual.js).
+  services.dev = startDevMode(services, config);
 
   const server = app.listen(config.port, config.host, () => {
     console.log(`Config: ${config.configPath || '(built-in defaults)'}; data: ${config.dataDir}`);

package/src/services/registry.js

@@ -14,7 +14,7 @@
 'use strict';
 
 const { v4: uuid } = require('uuid'),
-  { RESOURCE_STATES, BUSY_SOURCES, ACTIVE_JOB_STATES, compareVersions } = require('@andrian.yablonskyy/thub-common'),
+  { RESOURCE_STATES, BUSY_SOURCES, ACTIVE_JOB_STATES, compareVersions, parseCron } = require('@andrian.yablonskyy/thub-common'),
   { generateToken, hashToken } = require('./tokens');
 
 function rowToResource(row){
@@ -73,7 +73,15 @@ function sanitizeCapabilities(caps){
     return null;
   }
   const typed = sanitizeTypedCapabilities(caps);
-  return typed ? { ...typed, allowJobCommands: caps.allowJobCommands === true } : null;
+  return typed
+    ? {
+      ...typed,
+      allowJobCommands: caps.allowJobCommands === true,
+      // Whether the host has the root reboot helper (thub-client-reboot.path)
+      // a scheduled reboot needs; unknown (null) for older Clients.
+      rebootSupported: typeof caps.rebootSupported === 'boolean' ? caps.rebootSupported : null
+    }
+    : null;
 }
 
 function sanitizeTypedCapabilities(caps){
@@ -109,7 +117,7 @@ function sanitizeTypedCapabilities(caps){
 // Heartbeat durations (README §10) arrive as seconds relative to "now" on
 // the Client and are anchored to the Coordinator's clock here. Anything
 // negative, non-numeric or over ~10 years is ignored.
-const ACTIVITY_STATES = new Set(['idle', 'job', 'locked', 'update-hold']),
+const ACTIVITY_STATES = new Set(['idle', 'job', 'locked', 'update-hold', 'reboot-hold']),
   MAX_DURATION_SEC = 10 * 365 * 86400;
 
 function secondsAgo(now, sec){
@@ -184,7 +192,12 @@ function createRegistryService(db, { bus, events }){
   function registerAuto({ clientId, name, type, labels = [], groups = [], hostInfo, capabilities, remoteAddr, clientVersion = null }){
     const resourceToken = generateToken('res'),
       remote = normalizeRemoteAddr(remoteAddr);
-    hostInfo = { ...(hostInfo || {}), addresses: sanitizeAddresses(hostInfo?.addresses) || [] };
+    hostInfo = {
+      ...(hostInfo || {}),
+      addresses: sanitizeAddresses(hostInfo?.addresses) || [],
+      // The host's own IANA zone — a scheduled reboot's cron runs in it.
+      timeZone: typeof hostInfo?.timeZone === 'string' && /^[A-Za-z0-9_+/-]{1,64}$/.test(hostInfo.timeZone) ? hostInfo.timeZone : null
+    };
     const caps = sanitizeCapabilities(capabilities),
       capsJson = caps ? JSON.stringify(caps) : null;
     let existing = getByClientId(clientId);
@@ -296,7 +309,7 @@ function createRegistryService(db, { bus, events }){
   // Reconcile status from a heartbeat's self-reported state (§5.1).
   function heartbeat(
     resourceId,
-    { state, activeJobId, localLock, metrics, addresses, remoteAddr, clientVersion = null, hostUptimeSec, activity } = {}
+    { state, activeJobId, localLock, metrics, addresses, remoteAddr, clientVersion = null, hostUptimeSec, activity, rebootSchedule } = {}
   ){
     const resource = get(resourceId);
     if (!resource){
@@ -329,6 +342,15 @@ function createRegistryService(db, { bus, events }){
       busyReason = null;
     }
 
+    // The reboot schedule the Client applies (null: none); older Clients
+    // don't report one at all — then leave what's stored.
+    if (rebootSchedule !== undefined){
+      db.prepare('UPDATE resources SET reboot_schedule_applied = ? WHERE id = ?').run(
+        typeof rebootSchedule === 'string' && rebootSchedule ? rebootSchedule.slice(0, 200) : null,
+        resourceId
+      );
+    }
+
     const now = Date.now(),
       cleanActivity = sanitizeActivity(now, activity),
 
@@ -521,6 +543,37 @@ function createRegistryService(db, { bus, events }){
     return db.prepare('SELECT * FROM resources WHERE remove_requested_at IS NOT NULL').all().map(rowToResource);
   }
 
+  // Scheduled host reboot (resource card): a cron expression, validated
+  // here; '' or null clears it. Delivered on the Client's next heartbeat.
+  function setRebootSchedule(resourceId, cron, { by } = {}){
+    const r = get(resourceId);
+    if (!r){
+      throw Object.assign(new Error('Unknown resource'), { status: 404 });
+    }
+    const text = typeof cron === 'string' ? cron.trim().replace(/\s+/g, ' ') : '';
+    if (text){
+      try {
+        parseCron(text);
+      }
+      catch (err){
+        throw Object.assign(new Error(`Invalid reboot schedule "${text}": ${err.message}`), { status: 400 });
+      }
+    }
+    db.prepare('UPDATE resources SET reboot_schedule = ? WHERE id = ?').run(text || null, resourceId);
+    events.record('resource', resourceId, 'resource.reboot_schedule', { cron: text || null, by });
+    return get(resourceId);
+  }
+
+  // What a heartbeat reply must tell the Client, if its applied schedule
+  // isn't the saved one: { cron } (null = clear). Repeated until it reports it.
+  function pendingRebootSchedule(resourceId){
+    const r = get(resourceId);
+    if (!r || (r.reboot_schedule || null) === (r.reboot_schedule_applied || null)){
+      return null;
+    }
+    return { cron: r.reboot_schedule || null };
+  }
+
   function findIdleCandidates(type, labels, groupId, resourceId, needs){
     const rows = db
       .prepare('SELECT * FROM resources WHERE status = ? AND type = ? AND remove_requested_at IS NULL')
@@ -552,6 +605,8 @@ function createRegistryService(db, { bus, events }){
     pendingUpdate,
     assignToJob,
     remove,
+    setRebootSchedule,
+    pendingRebootSchedule,
     requestRemoval,
     pendingRemovals,
     findIdleCandidates,

package/src/web/routes.js

@@ -21,7 +21,8 @@ const fs = require('node:fs'),
   { requireAdminSession, requireAdminRole } = require('../auth'),
   { attachJobStream } = require('../api/sse'),
   {
-    RESOURCE_STATES, JOB_STATES, ACTIVE_JOB_STATES, TERMINAL_JOB_STATES, isNewer, compareVersions, formatDateTime, parseDateTime
+    RESOURCE_STATES, JOB_STATES, ACTIVE_JOB_STATES, TERMINAL_JOB_STATES, isNewer, compareVersions, formatDateTime, parseDateTime,
+    nextCronRun
   } = require('@andrian.yablonskyy/thub-common'),
   { retentionCutoff, JOB_RETENTION } = require('../services/cleanup'),
   listPrefs = require('../services/list-prefs');
@@ -152,6 +153,8 @@ function createWebRouter({ services, config }){
       ];
     }
     res.locals.isNewer = isNewer;
+    res.locals.nextCronRun = nextCronRun;
+    res.locals.formatDateTime = formatDateTime;
     res.locals.currentPath = req.originalUrl;
     if (req.session){
       req.session.flash = [];
@@ -365,6 +368,10 @@ function createWebRouter({ services, config }){
         services.commands.push(r.id, { command: 'unlock' });
         flash(req, 'warning', `Releasing the local lock on ${r.name} (on its next heartbeat).`);
       }
+      else if (activity === 'reboot-hold'){
+        services.commands.push(r.id, { command: 'cancel-reboot' });
+        flash(req, 'warning', `Canceling the scheduled reboot of ${r.name}'s host (on its next heartbeat).`);
+      }
       else if (activity === 'update-hold'){
         services.commands.push(r.id, { command: 'cancel-update' });
         if (r.update_to){
@@ -375,7 +382,7 @@ function createWebRouter({ services, config }){
       else {
         flash(req, 'info', `${r.name} isn't doing anything to cancel.`);
       }
-      if (!job && ['locked', 'update-hold'].includes(activity)){
+      if (!job && ['locked', 'update-hold', 'reboot-hold'].includes(activity)){
         services.events.record('resource', r.id, 'resource.activity_canceled', { activity, by: req.session.user.username });
       }
     }
@@ -405,6 +412,22 @@ function createWebRouter({ services, config }){
     res.redirect(returnTo(req, '/resources'));
   });
 
+  // Scheduled host reboot (resource card): save a cron expression, or clear
+  // it; the Client applies it on its next heartbeat.
+  router.post('/resources/:id/reboot-schedule', requireAdminRole, (req, res) => {
+    try {
+      const cron = req.body.clear === '1' ? '' : req.body.cron,
+        r = services.registry.setRebootSchedule(req.params.id, cron, { by: req.session.user.username });
+      flash(req, 'success', r.reboot_schedule
+        ? `Reboot schedule for ${r.name} saved: ${r.reboot_schedule}.\nThe Client applies it on its next heartbeat.`
+        : `Scheduled reboot for ${r.name} cleared.\nThe Client applies it on its next heartbeat.`);
+    }
+    catch (err){
+      flash(req, 'danger', err.message);
+    }
+    res.redirect(returnTo(req, '/resources'));
+  });
+
   router.post('/resources/:id/rotate-token', requireAdminRole, (req, res) => {
     const { generateToken, hashToken } = require('../services/tokens');
     const token = generateToken('res');

package/test/dev-mode.test.js

@@ -0,0 +1,63 @@
+/**
+ * @file        packages/coordinator/test/dev-mode.test.js
+ * @description Tests: DEV mode (virtual agent, Clients and jobs) — only with DEV_MODE=1 and a bootstrap password
+ *
+ * @author      Andrian Yablonskyy
+ * @copyright   Copyright (c) 2026 Andrian Yablonskyy. All rights reserved.
+ *
+ * This file is part of TestHub and is proprietary and confidential.
+ * Unauthorized copying, modification, distribution, or use of this file,
+ * via any medium, is strictly prohibited without prior written permission
+ * from AdSystem.PRO.
+ */
+
+'use strict';
+
+const test = require('node:test'),
+  assert = require('node:assert/strict'),
+  fs = require('node:fs'),
+  os = require('node:os'),
+  path = require('node:path'),
+  { loadConfig } = require('../src/config'),
+  { buildServices } = require('../src/server'),
+  { startDevMode, isDevMode, DEV_AGENT_TOKEN } = require('../src/dev/virtual'),
+  { hashToken } = require('../src/services/tokens');
+
+test('DEV mode needs both DEV_MODE=1 and THUB_BOOTSTRAP_ADMIN_PASSWORD', () => {
+  assert.equal(isDevMode({ DEV_MODE: '1', THUB_BOOTSTRAP_ADMIN_PASSWORD: 'pw' }), true);
+  assert.equal(isDevMode({ DEV_MODE: '1' }), false);
+  assert.equal(isDevMode({ THUB_BOOTSTRAP_ADMIN_PASSWORD: 'pw' }), false);
+  assert.equal(isDevMode({ DEV_MODE: 'true', THUB_BOOTSTRAP_ADMIN_PASSWORD: 'pw' }), false);
+});
+
+test('seeds a virtual agent, virtual Clients and jobs once; nothing outside DEV mode', (t) => {
+  const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'thub-dev-')),
+    file = path.join(dir, 'coordinator.json');
+  fs.writeFileSync(file, JSON.stringify({ dataDir: path.join(dir, 'data'), updates: { checkIntervalMin: 0 } }));
+  const config = loadConfig(file),
+    services = buildServices(config),
+    count = (table) => services.db.prepare(`SELECT COUNT(*) AS n FROM ${table}`).get().n,
+    warn = t.mock.method(console, 'warn', () => {});
+
+  assert.equal(startDevMode(services, config, {}), null);
+  assert.equal(count('agents') + count('resources') + count('jobs'), 0);
+
+  const env = { DEV_MODE: '1', THUB_BOOTSTRAP_ADMIN_PASSWORD: 'pw' },
+    first = startDevMode(services, config, env);
+  t.after(() => first.stop());
+  assert.equal(services.agents.getByTokenHash(hashToken(DEV_AGENT_TOKEN)).name, 'virtual-agent');
+  assert.deepEqual(services.registry.list().map((r) => r.name).sort(), ['virtual-hw-01', 'virtual-sw-01']);
+  const jobs = count('jobs'),
+    states = new Set(services.db.prepare('SELECT state FROM jobs').all().map((r) => r.state));
+  assert.equal(jobs, 15);
+  for (const s of ['PASSED', 'FAILED', 'ERROR', 'TIMEOUT', 'CANCELED', 'LOST', 'QUEUED']){
+    assert.ok(states.has(s), s);
+  }
+
+  const second = startDevMode(services, config, env); // a restart
+  t.after(() => second.stop());
+  assert.equal(count('jobs'), jobs);
+  assert.equal(count('agents'), 1);
+  assert.equal(count('resources'), 2);
+  assert.equal(warn.mock.callCount(), 2);
+});

package/test/scheduler.test.js

@@ -565,6 +565,26 @@ test('a job-supplied command only goes to Clients with allowJobCommands; otherwi
   assert.equal(jobs.get(job.id).resource_id, resourceId);
 });
 
+test('reboot schedule: validated, then resent on heartbeats until the Client reports applying it', () => {
+  const { registry } = buildTestServices(),
+    r = registerResource(registry, { name: 'lab-reboot', type: 'sw' });
+
+  assert.throws(() => registry.setRebootSchedule(r.id, '61 3 * * *'), /Invalid reboot schedule.*minute: 61/);
+  assert.equal(registry.pendingRebootSchedule(r.id), null); // nothing set, nothing applied
+
+  registry.setRebootSchedule(r.id, '  30   3 * * sun ');
+  assert.deepEqual(registry.pendingRebootSchedule(r.id), { cron: '30 3 * * sun' });
+  registry.heartbeat(r.id, { state: 'idle' }); // an older Client: reports nothing
+  assert.deepEqual(registry.pendingRebootSchedule(r.id), { cron: '30 3 * * sun' });
+  registry.heartbeat(r.id, { state: 'idle', rebootSchedule: '30 3 * * sun' });
+  assert.equal(registry.pendingRebootSchedule(r.id), null);
+
+  registry.setRebootSchedule(r.id, '');
+  assert.deepEqual(registry.pendingRebootSchedule(r.id), { cron: null }); // tell it to clear
+  registry.heartbeat(r.id, { state: 'idle', rebootSchedule: null });
+  assert.equal(registry.pendingRebootSchedule(r.id), null);
+});
+
 test('a resource can belong to several groups at once', () => {
   const { registry, groups } = buildTestServices(),
     g1 = groups.create({ name: 'g1' }),

package/views/jobs/show.pug

@@ -41,9 +41,6 @@ block content
       dt.col-sm-2 Message
       dd.col-sm-10= job.message
 
-  h2.h6.mt-4 Log
-  +logViewer(job.id, jobActive)
-
   h2.h6.mt-4#artifacts Artifacts
   if artifacts.length
     ul.list-group
@@ -53,3 +50,6 @@ block content
           a(href=a.url) download
   else
     p.text-body-secondary No artifacts yet.
+
+  h2.h6.mt-4 Log
+  +logViewer(job.id, jobActive)

package/views/mixins/resource.pug

@@ -180,7 +180,7 @@ mixin capabilitiesList(caps)
 mixin uptimeRows(r, returnTo)
   - const stale = r.status === 'OUT_OF_SERVICE' && r.last_heartbeat_at
   - const ref = stale ? new Date(r.last_heartbeat_at).getTime() : Date.now()
-  - const activityLabel = { idle: 'Idle', job: 'Running job', locked: 'Locked locally', 'update-hold': 'Holding for self-update' }
+  - const activityLabel = { idle: 'Idle', job: 'Running job', locked: 'Locked locally', 'update-hold': 'Holding for self-update', 'reboot-hold': 'Holding for scheduled reboot' }
   dt.col-sm-4 Host uptime
   dd.col-sm-8
     if r.host_booted_at
@@ -205,12 +205,80 @@ mixin uptimeRows(r, returnTo)
     if user.role === 'admin'
       +cancelActivityButton(r, returnTo)
 
+//- Scheduled host reboot (cron, in the Client host's local time): what's
+//- saved, when it next fires, whether the Client has applied it, and — for
+//- admins — an editor. Saved here, applied on the Client's next heartbeat.
+mixin rebootScheduleRow(r, returnTo)
+  - const tz = r.host_info && r.host_info.timeZone
+  - const caps = r.capabilities || {}
+  - const pending = (r.reboot_schedule || null) !== (r.reboot_schedule_applied || null)
+  - let next = null
+  - try { next = r.reboot_schedule && tz ? nextCronRun(r.reboot_schedule, new Date(), { timeZone: tz }) : null } catch (e) { next = null }
+  dt.col-sm-4 Scheduled reboot
+  dd.col-sm-8
+    if r.reboot_schedule
+      code= r.reboot_schedule
+      if next
+        span.text-body-secondary.small.ms-2(
+          data-bs-toggle="tooltip"
+          data-bs-title=`${formatDateTime(next, { timeZone: tz })} host time (${tz}); shown here in your time zone`
+        )= `next ${fmtDate(next)}`
+      else if !tz
+        span.text-body-secondary.small.ms-2(
+          data-bs-toggle="tooltip"
+          data-bs-title="The Client runs it in its host's local time; this Client is too old to report its time zone"
+        ) host local time
+    else
+      span.text-body-secondary none
+    if pending
+      span.badge.text-bg-warning.ms-2(
+        data-bs-toggle="tooltip"
+        data-bs-title="Saved — the Client applies it on its next heartbeat"
+      ) not applied yet
+    if caps.rebootSupported === false
+      span.badge.text-bg-danger.ms-2(
+        data-bs-toggle="tooltip"
+        data-bs-title="This host has no reboot helper (thub-client-reboot.path), so the schedule can't reboot it — reinstall the Client as root: sudo npm i -g @andrian.yablonskyy/thub-client"
+      ) can't reboot
+    if user.role === 'admin'
+      form.d-flex.gap-1.mt-1(
+        method="post"
+        action=`/resources/${r.id}/reboot-schedule`
+        data-confirm=`Reboot the host of ${r.name} on this schedule? It reboots the whole host — every Client instance on it. Running jobs finish first: new ones wait, for up to 2 h, and otherwise that time is skipped.`
+        data-confirm-title="Schedule host reboots?"
+        data-confirm-ok="Save schedule"
+        data-confirm-tone="warning"
+      )
+        input(type="hidden" name="returnTo" value=returnTo)
+        input.form-control.form-control-sm.font-monospace(
+          type="text"
+          name="cron"
+          value=r.reboot_schedule || ''
+          placeholder="30 3 * * sun"
+          aria-label=`Reboot schedule for ${r.name} (cron)`
+          required
+        )
+        button.btn.btn-sm.btn-outline-primary(type="submit") Save
+      if r.reboot_schedule
+        form.d-inline(method="post" action=`/resources/${r.id}/reboot-schedule`)
+          input(type="hidden" name="returnTo" value=returnTo)
+          input(type="hidden" name="clear" value="1")
+          button.btn.btn-sm.btn-link.p-0.small(type="submit") Clear schedule
+      .form-text
+        | cron: minute hour day month weekday, in the host's local time#{tz ? ` (${tz})` : ''} — e.g.
+        |
+        code 30 3 * * sun
+        |  (Sundays 03:30) or
+        |
+        code 0 4 * * *
+        | . Reboots the whole host.
+
 //- Cancel the Client's current job or local lock — icon only, on the card's
 //- Task / status line, and only while there's one. A self-update hold is
 //- canceled from the version line instead (+versionWithUpdate).
 mixin cancelActivityButton(r, returnTo)
   - const state = r.activity && r.activity.state
-  - const what = r.activeJob ? `job ${r.activeJob.id}` : state === 'locked' ? 'the local lock' : null
+  - const what = r.activeJob ? `job ${r.activeJob.id}` : state === 'locked' ? 'the local lock' : state === 'reboot-hold' ? 'the scheduled reboot' : null
   if what
     form.d-inline.ms-2(
       method="post"
@@ -234,60 +302,92 @@ mixin resourceCard(r, groupsById, returnTo)
   - const local = ((r.host_info && r.host_info.addresses) || []).filter((a) => !isLoopback(a.address))
   - const external = r.remote_addr && !isLoopback(r.remote_addr) ? r.remote_addr : null
   .modal.fade.thub-resource-card(id=`resource-card-${r.id}` tabindex="-1" aria-labelledby=`resource-card-${r.id}-title` aria-hidden="true")
-    .modal-dialog.modal-dialog-centered
+    .modal-dialog.modal-dialog-centered.modal-lg
       .modal-content
         .modal-header
           h5.modal-title(id=`resource-card-${r.id}-title`)= r.name
           button.btn-close(type="button" data-bs-dismiss="modal" aria-label="Close")
-        .modal-body
-          dl.row.mb-0
-            dt.col-sm-4 Name
-            dd.col-sm-8= r.name
-            dt.col-sm-4 Type
-            dd.col-sm-8.text-uppercase= r.type
-            dt.col-sm-4 Status
-            dd.col-sm-8: +resourceBadge(r.status, r.remove_requested_at)
-            +uptimeRows(r, returnTo)
-            dt.col-sm-4 Client version
-            dd.col-sm-8: +versionWithUpdate(r.client_version, updates.latest.client, r.update_to, `/resources/${r.id}/update`, returnTo, true, r.activity && r.activity.state === 'update-hold')
-            dt.col-sm-4 IP addresses
-            dd.col-sm-8
-              if !local.length && !external
-                span.text-body-secondary —
-              ul.list-unstyled.mb-0
-                each a in local
-                  li
-                    code= a.address
-                    span.text-body-secondary.small.ms-2= a.iface ? `${a.iface} · ${a.family}` : a.family
-                if external
-                  li
-                    code= external
-                    span.text-body-secondary.small.ms-2(
-                      data-bs-toggle="tooltip"
-                      data-bs-title="The address the Coordinator last saw this Client connect from"
-                    ) external
-            dt.col-sm-4 Capabilities
-            dd.col-sm-8
-              +capabilitiesList(r.capabilities)
-            dt.col-sm-4 Labels
-            dd.col-sm-8
-              if r.labels.length
-                each l in r.labels
-                  span.badge.text-bg-light.border.me-1= l
-              else
-                span.text-body-secondary —
-            dt.col-sm-4 Groups
-            dd.col-sm-8
-              if r.group_ids.length
-                each gid in r.group_ids
-                  span.badge.text-bg-info.me-1(data-bs-toggle="tooltip" data-bs-title=gid)= groupsById[gid] ? groupsById[gid].name : `(unknown: ${gid})`
-              else
-                span.text-body-secondary —
-            dt.col-sm-4 Busy source / reason
-            dd.col-sm-8= r.busy_source ? `${r.busy_source}${r.busy_reason ? ' — ' + r.busy_reason : ''}` : '—'
-            dt.col-sm-4 Last heartbeat
-            dd.col-sm-8= fmtDate(r.last_heartbeat_at, 'never')
-            if user.role === 'admin'
-              dt.col-sm-4 Actions
-              dd.col-sm-8.mb-0
-                +resourceActions(r, returnTo)
+        .modal-body.pt-2
+          //- Details: what the Client is and reports. Config: settings sent
+          //- to it (today the scheduled reboot); read-only for viewers.
+          ul.nav.nav-tabs.mb-3(role="tablist")
+            li.nav-item(role="presentation")
+              button.nav-link.active(
+                id=`rc-${r.id}-details-tab`
+                type="button"
+                role="tab"
+                data-bs-toggle="tab"
+                data-bs-target=`#rc-${r.id}-details`
+                aria-controls=`rc-${r.id}-details`
+                aria-selected="true"
+              ) Details
+            li.nav-item(role="presentation")
+              button.nav-link(
+                id=`rc-${r.id}-config-tab`
+                type="button"
+                role="tab"
+                data-bs-toggle="tab"
+                data-bs-target=`#rc-${r.id}-config`
+                aria-controls=`rc-${r.id}-config`
+                aria-selected="false"
+              ) Config
+          .tab-content
+            .tab-pane.fade.show.active(id=`rc-${r.id}-details` role="tabpanel" aria-labelledby=`rc-${r.id}-details-tab` tabindex="0")
+              +resourceDetails(r, groupsById, returnTo, local, external)
+            .tab-pane.fade(id=`rc-${r.id}-config` role="tabpanel" aria-labelledby=`rc-${r.id}-config-tab` tabindex="0")
+              dl.row.mb-0
+                +rebootScheduleRow(r, returnTo)
+
+//- The resource card's Details tab.
+mixin resourceDetails(r, groupsById, returnTo, local, external)
+  dl.row.mb-0
+    dt.col-sm-4 Name
+    dd.col-sm-8= r.name
+    dt.col-sm-4 Type
+    dd.col-sm-8.text-uppercase= r.type
+    dt.col-sm-4 Status
+    dd.col-sm-8: +resourceBadge(r.status, r.remove_requested_at)
+    +uptimeRows(r, returnTo)
+    dt.col-sm-4 Client version
+    dd.col-sm-8: +versionWithUpdate(r.client_version, updates.latest.client, r.update_to, `/resources/${r.id}/update`, returnTo, true, r.activity && r.activity.state === 'update-hold')
+    dt.col-sm-4 IP addresses
+    dd.col-sm-8
+      if !local.length && !external
+        span.text-body-secondary —
+      ul.list-unstyled.mb-0
+        each a in local
+          li
+            code= a.address
+            span.text-body-secondary.small.ms-2= a.iface ? `${a.iface} · ${a.family}` : a.family
+        if external
+          li
+            code= external
+            span.text-body-secondary.small.ms-2(
+              data-bs-toggle="tooltip"
+              data-bs-title="The address the Coordinator last saw this Client connect from"
+            ) external
+    dt.col-sm-4 Capabilities
+    dd.col-sm-8.resource-capabilities
+      +capabilitiesList(r.capabilities)
+    dt.col-sm-4 Labels
+    dd.col-sm-8
+      if r.labels.length
+        each l in r.labels
+          span.badge.text-bg-light.border.me-1= l
+      else
+        span.text-body-secondary —
+    dt.col-sm-4 Groups
+    dd.col-sm-8
+      if r.group_ids.length
+        each gid in r.group_ids
+          span.badge.text-bg-info.me-1(data-bs-toggle="tooltip" data-bs-title=gid)= groupsById[gid] ? groupsById[gid].name : `(unknown: ${gid})`
+      else
+        span.text-body-secondary —
+    dt.col-sm-4 Busy source / reason
+    dd.col-sm-8= r.busy_source ? `${r.busy_source}${r.busy_reason ? ' — ' + r.busy_reason : ''}` : '—'
+    dt.col-sm-4 Last heartbeat
+    dd.col-sm-8= fmtDate(r.last_heartbeat_at, 'never')
+    if user.role === 'admin'
+      dt.col-sm-4 Actions
+      dd.col-sm-8.mb-0
+        +resourceActions(r, returnTo)