@andrian.yablonskyy/thub-coordinator 1.0.27 → 1.0.28

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@andrian.yablonskyy/thub-coordinator",
-  "version": "1.0.27",
+  "version": "1.0.28",
   "description": "TestHub Coordinator — job queue, resource registry, scheduler, heartbeat monitor, log/artifact store and web dashboard",
   "main": "src/server.js",
   "engines": {

package/public/js/confirm.js

@@ -47,7 +47,16 @@
     field('ok').className = `btn btn-${tone}`;
     field('ok').textContent = form.dataset.confirmOk || 'Confirm';
     field('dismiss').textContent = form.dataset.confirmDismiss || 'Cancel';
-    modal.show();
+    // Asked from inside another modal (e.g. the resource card): close that
+    // one first — Bootstrap doesn't stack modals.
+    const host = form.closest('.modal.show');
+    if (host && host !== modalEl){
+      host.addEventListener('hidden.bs.modal', () => modal.show(), { once: true });
+      bootstrap.Modal.getInstance(host)?.hide();
+    }
+    else {
+      modal.show();
+    }
   });
 
   field('ok').addEventListener('click', () => {

package/src/api/admin.js

@@ -51,6 +51,17 @@ function createAdminRouter({ services }){
     res.status(204).end();
   });
 
+  // Reboot a Client's host now (its running job is canceled first).
+  router.post('/resources/:id/reboot', (req, res, next) => {
+    try {
+      const { resource, canceledJob } = services.jobs.requestReboot(req.params.id);
+      res.json({ id: resource.id, rebooting: true, canceledJob });
+    }
+    catch (err){
+      next(err);
+    }
+  });
+
   // Scheduled host reboot: { cron } (5-field, host local time; "" clears).
   router.post('/resources/:id/reboot-schedule', (req, res, next) => {
     try {

package/src/dev/virtual.js

@@ -30,8 +30,7 @@ const DEV_AGENT_NAME = 'virtual-agent',
       type: 'sw',
       labels: ['virtual', 'emulator'],
       capabilities: {
-        sw: { image: 'dut-emulator:dev', registry: null, allowDockerHub: true, allowJobImages: true, cpus: 2, memory: '2g' },
-        allowJobCommands: true
+        sw: { image: 'dut-emulator:dev', registry: null, allowDockerHub: true, allowJobImages: true, cpus: 2, memory: '2g' }
       },
       address: '10.0.0.41'
     },
@@ -47,8 +46,7 @@ const DEV_AGENT_NAME = 'virtual-agent',
           usbs: [{ path: '/dev/thub/dut1-usb', index: 1, present: false }],
           relays: [{ channel: 0, baseUrl: 'http://localhost:3000' }],
           power: { method: 'relay' }
-        },
-        allowJobCommands: false
+        }
       },
       address: '10.0.0.42'
     }
@@ -69,12 +67,12 @@ const DEV_AGENT_NAME = 'virtual-agent',
     [0.1, 'cli', 'sw', 'PASSED', 73, 'alice']
   ],
   LOG_SCRIPT = [
-    ['runner', 'downloading firmware (virtual)'],
-    ['runner', 'downloading tests (virtual)'],
+    ['runner', 'downloading app.bin (virtual)'],
+    ['runner', 'cloning yourorg/firmware-tests at main (virtual)'],
     ['flash', 'st-flash write app.bin 0x08000000 ... done (virtual)'],
     ['uart', '[BOOT] app v1.4.0-dev'],
     ['uart', '[INIT] peripherals ok'],
-    ['runner', 'running: run-tests.sh --suite default'],
+    ['runner', 'running: sh -c "./ci/run.sh"'],
     ['runner', 'test_boot ............ ok'],
     ['runner', 'test_uart_echo ....... ok'],
     ['runner', 'test_flash_crc ....... ok'],
@@ -83,8 +81,10 @@ const DEV_AGENT_NAME = 'virtual-agent',
 
   spec = (type, user, extra = {}) => ({
     target: { type, labels: ['virtual'] },
-    firmware: type === 'sw' ? { image: 'alpine:3.20' } : { url: 'https://artifactory.example.com/fw-local/app/1.4.0-dev/app.bin' },
-    tests: { url: 'https://artifactory.example.com/fw-local/app/1.4.0-dev/tests.tar.gz', suite: 'default' },
+    command: './ci/run.sh',
+    downloads: [{ url: 'https://artifactory.example.com/fw-local/app/1.4.0-dev/app.bin' }],
+    ...(type === 'sw' ? { image: 'alpine:3.20' } : {}),
+    git: { url: 'https://github.com/yourorg/firmware-tests.git', ref: 'main', depth: 1 },
     ...(user ? { user } : {}),
     meta: { repo: 'yourorg/firmware', branch: 'main', virtual: true },
     ...extra
@@ -278,7 +278,7 @@ function startDevMode(services, config, env = process.env){
     '*** DEV MODE (DEV_MODE=1): virtual Clients virtual-sw-01 / virtual-hw-01 are running in-process' +
       `${seeded ? `, and ${seeded} past jobs + 3 live ones were seeded` : ''}.\n` +
       `*** Virtual agent token: ${DEV_AGENT_TOKEN}\n` +
-      `***   e.g. thub --url ${config.publicUrl} --token ${DEV_AGENT_TOKEN} run --type sw --image alpine --tests https://x/t.tgz`
+      `***   e.g. thub --url ${config.publicUrl} --token ${DEV_AGENT_TOKEN} run --type sw --docker-image alpine --command 'uname -a'`
   );
   return virtual;
 }

package/src/services/jobs.js

@@ -13,7 +13,7 @@
 
 'use strict';
 
-const { validateJobSpec, JOB_STATES, ACTIVE_JOB_STATES, TERMINAL_JOB_STATES } = require('@andrian.yablonskyy/thub-common'),
+const { validateJobSpec, JOB_STATES, ACTIVE_JOB_STATES, TERMINAL_JOB_STATES, RESOURCE_STATES } = require('@andrian.yablonskyy/thub-common'),
   { paginate } = require('./list-prefs');
 
 function rowToJob(row){
@@ -145,22 +145,15 @@ function createJobsService(db, { bus, events, registry, artifacts, config }){
       spec.target.client = resource.id;
     }
 
-    // A job-supplied Docker image or command needs a Client that opted in —
-    // say which, rather than the generic "no resource can satisfy".
-    const needs = { jobImage: Boolean(spec.firmware.image), jobCommand: Boolean(spec.tests.command) },
+    // A job-supplied Docker image needs an SW Client that opted in — say so,
+    // rather than the generic "no resource can satisfy".
+    const needs = { jobImage: Boolean(spec.image) },
       { type, labels, group, client } = spec.target;
-    if ((needs.jobImage || needs.jobCommand) && registry.everSatisfiable(type, labels, group, client) &&
-      !registry.everSatisfiable(type, labels, group, client, needs)){
-      const missing = [
-        needs.jobImage && !registry.everSatisfiable(type, labels, group, client, { jobImage: true }) &&
-          `job-supplied Docker images (${spec.firmware.image}) — "sw": { "allowJobImages": true }`,
-        needs.jobCommand && !registry.everSatisfiable(type, labels, group, client, { jobCommand: true }) &&
-          'job-supplied commands (--run) — "allowJobCommands": true'
-      ].filter(Boolean);
+    if (needs.jobImage && registry.everSatisfiable(type, labels, group, client) && !registry.everSatisfiable(type, labels, group, client, needs)){
       throw Object.assign(
         new Error(
-          `No matching Client runs ${missing.length ? missing.join(', nor ') : 'both a job-supplied Docker image and command together'}` +
-            ' — enable it in a Client\'s config'
+          `No matching SW Client runs job-supplied Docker images (${spec.image}) — enable it on a Client ` +
+            'with "sw": { "allowJobImages": true } in its config'
         ),
         { status: 422 }
       );
@@ -263,7 +256,8 @@ function createJobsService(db, { bus, events, registry, artifacts, config }){
   }
 
   // Agent: POST /jobs/:id/cancel
-  function cancel(id, { agentId, isAdmin }){
+  // `reason`: shown as the job's message (e.g. "Canceled: user reboot request").
+  function cancel(id, { agentId, isAdmin, reason }){
     const job = get(id);
     if (!job){
       throw Object.assign(new Error('Unknown job'), { status: 404 });
@@ -278,7 +272,35 @@ function createJobsService(db, { bus, events, registry, artifacts, config }){
     if (job.resource_id){
       bus.emit('command', { resourceId: job.resource_id, command: 'cancel-job', jobId: id });
     }
-    return setState(id, JOB_STATES.CANCELED);
+    return setState(id, JOB_STATES.CANCELED, reason ? { message: reason } : {});
+  }
+
+  // Admin "Reboot" on a Client (resource card): its running job is canceled
+  // first (the host is going down anyway), then a `reboot` command goes out
+  // with its next heartbeat — the Client reboots the host through its root
+  // reboot helper once nothing on it is busy. Refused for a Client that's
+  // offline (it would reboot whenever it next reconnects) or can't reboot.
+  function requestReboot(resourceId, { by } = {}){
+    const r = registry.get(resourceId);
+    if (!r){
+      throw Object.assign(new Error('Unknown resource'), { status: 404 });
+    }
+    if (r.status === RESOURCE_STATES.OUT_OF_SERVICE || !r.last_heartbeat_at){
+      throw Object.assign(new Error(`${r.name} is offline — it can only be rebooted while it's connected`), { status: 409 });
+    }
+    if (r.capabilities?.rebootSupported === false){
+      throw Object.assign(
+        new Error(`${r.name}'s host has no reboot helper — reinstall the Client as root (sudo npm i -g @andrian.yablonskyy/thub-client)`),
+        { status: 409 }
+      );
+    }
+    const job = activeForResource(resourceId);
+    if (job){
+      cancel(job.id, { isAdmin: true, reason: 'Canceled: user reboot request' });
+    }
+    bus.emit('command', { resourceId, command: 'reboot', reason: `user reboot request${by ? ` by ${by}` : ''}` });
+    events.record('resource', resourceId, 'resource.reboot_requested', { by, canceledJob: job?.id || null });
+    return { resource: r, canceledJob: job?.id || null };
   }
 
   // Queued jobs pinned to a resource (`target.client`) can never run once it's
@@ -493,6 +515,7 @@ function createJobsService(db, { bus, events, registry, artifacts, config }){
     setState,
     cancel,
     cancelPinnedTo,
+    requestReboot,
     removeResource,
     completeRemoval,
     activeForResource,

package/src/services/registry.js

@@ -67,7 +67,6 @@ const MAX_DEVICES = 8,
     present: typeof d?.present === 'boolean' ? d.present : null
   });
 
-// Plus `allowJobCommands` (either type): runs a job's own `tests.command`.
 function sanitizeCapabilities(caps){
   if (!caps || typeof caps !== 'object'){
     return null;
@@ -76,7 +75,6 @@ function sanitizeCapabilities(caps){
   return typed
     ? {
       ...typed,
-      allowJobCommands: caps.allowJobCommands === true,
       // Whether the host has the root reboot helper (thub-client-reboot.path)
       // a scheduled reboot needs; unknown (null) for older Clients.
       rebootSupported: typeof caps.rebootSupported === 'boolean' ? caps.rebootSupported : null
@@ -498,14 +496,12 @@ function createRegistryService(db, { bus, events }){
   }
 
   // `needs`: what a job brings that a Client must have opted in to run —
-  // jobImage (firmware.image; sw.allowJobImages) and jobCommand
-  // (tests.command; allowJobCommands).
+  // jobImage (the job's own Docker image; sw.allowJobImages).
   function matchesTarget(r, labels, groupId, resourceId, needs = {}){
     return labels.every((l) => r.labels.includes(l)) &&
       (!groupId || r.group_ids.includes(groupId)) &&
       (!resourceId || r.id === resourceId) &&
-      (!needs.jobImage || r.capabilities?.sw?.allowJobImages === true) &&
-      (!needs.jobCommand || r.capabilities?.allowJobCommands === true);
+      (!needs.jobImage || r.capabilities?.sw?.allowJobImages === true);
   }
 
   // Admin: remove a resource from the registry (dashboard "Remove"). Refused

package/src/services/scheduler.js

@@ -17,7 +17,7 @@ const { JOB_STATES, BUSY_SOURCES } = require('@andrian.yablonskyy/thub-common');
 
 // What a job needs a Client to have opted in to (registry matchesTarget).
 function jobNeeds(spec){
-  return { jobImage: Boolean(spec.firmware?.image), jobCommand: Boolean(spec.tests?.command) };
+  return { jobImage: Boolean(spec.image) };
 }
 
 function rowToJob(row){

package/src/web/routes.js

@@ -412,6 +412,19 @@ function createWebRouter({ services, config }){
     res.redirect(returnTo(req, '/resources'));
   });
 
+  // Reboot now (resource card footer): cancels a running job, then the
+  // Client reboots its host on its next heartbeat (jobs.requestReboot).
+  router.post('/resources/:id/reboot', requireAdminRole, (req, res) => {
+    try {
+      const { resource: r, canceledJob } = services.jobs.requestReboot(req.params.id, { by: req.session.user.username });
+      flash(req, 'warning', `Rebooting ${r.name}'s host on its next heartbeat.` + (canceledJob ? `\nCanceled its running job ${canceledJob}.` : ''));
+    }
+    catch (err){
+      flash(req, 'danger', err.message);
+    }
+    res.redirect(returnTo(req, '/resources'));
+  });
+
   // Scheduled host reboot (resource card): save a cron expression, or clear
   // it; the Client applies it on its next heartbeat.
   router.post('/resources/:id/reboot-schedule', requireAdminRole, (req, res) => {

package/test/cleanup.test.js

@@ -55,7 +55,7 @@ function setup(jobRetention = 'forever'){
 
   // A job that finished at `finishedAt` (or is still queued, if null), with a log line.
   function job(finishedAt){
-    const spec = { target: { type: 'sw' }, firmware: { url: 'https://x/a' }, tests: { url: 'https://x/t' } },
+    const spec = { target: { type: 'sw' }, command: './run.sh' },
       j = jobs.create({ agentId: agent.id, source: 'cli', spec });
     db.prepare('INSERT INTO job_logs (job_id, seq, ts, stream, line) VALUES (?, 1, ?, \'runner\', ?)').run(j.id, NOW.toISOString(), 'x'.repeat(2000));
     if (finishedAt){

package/test/list-prefs.test.js

@@ -74,7 +74,7 @@ test('jobs.page sorts and pages in SQL, with empty values last', () => {
       jobs.create({
         agentId: agent.id,
         source: 'cli',
-        spec: { target: { type: 'sw' }, firmware: { url: 'https://x/a' }, tests: { url: 'https://x/t' }, ...(user ? { user } : {}) }
+        spec: { target: { type: 'sw' }, command: './run.sh', ...(user ? { user } : {}) }
       }).id),
 
     byUser = (dir) => jobs.page({ sort: 'user', dir, size: 'all' }).rows.map((j) => j.spec.user ?? null);

package/test/scheduler.test.js

@@ -57,7 +57,7 @@ function buildTestServices(overrides = {}){
     jobs = createJobsService(db, { bus, events, registry, artifacts, config }),
     scheduler = createScheduler(db, { bus, events, registry, config }),
     heartbeatMonitor = createHeartbeatMonitor(db, { bus, events, registry, jobs, config });
-  return { db, registry, agents, groups, artifacts, jobs, scheduler, heartbeatMonitor, artifactsDir };
+  return { db, bus, registry, agents, groups, artifacts, jobs, scheduler, heartbeatMonitor, artifactsDir };
 }
 
 function registerResource(registry, { name, type, labels = [], groups = [], clientId } = {}){
@@ -68,8 +68,7 @@ function registerResource(registry, { name, type, labels = [], groups = [], clie
 function makeSpec(overrides = {}){
   return {
     target: { type: 'sw', labels: [] },
-    firmware: { url: 'https://x/app.bin' },
-    tests: { url: 'https://x/tests.tar.gz' },
+    command: './run.sh',
     ...overrides
   };
 }
@@ -528,7 +527,7 @@ test('a removed Client re-registers as a fresh resource under the same name', ()
 test('a job-supplied Docker image only goes to SW Clients that allow it; otherwise 422', () => {
   const { registry, agents, jobs, scheduler } = buildTestServices(),
     { agent } = agents.create({ name: 'dev', kind: 'cli' }),
-    imageJob = { target: { type: 'sw', labels: [] }, firmware: { image: 'alpine' }, tests: { url: 'https://x/t.tar.gz' } },
+    imageJob = { target: { type: 'sw', labels: [] }, image: 'alpine', command: './run.sh' },
     strict = registerResource(registry, { name: 'sw-strict', type: 'sw' });
   registry.heartbeat(strict.id, { state: 'idle' });
 
@@ -544,25 +543,25 @@ test('a job-supplied Docker image only goes to SW Clients that allow it; otherwi
 
   const fileJob = jobs.create({ agentId: agent.id, source: 'cli', spec: makeSpec() });
   scheduler.runPass();
-  assert.equal(jobs.get(fileJob.id).resource_id, strict.id); // firmware URLs still go anywhere
+  assert.equal(jobs.get(fileJob.id).resource_id, strict.id); // jobs without an image still go anywhere
 });
 
-test('a job-supplied command only goes to Clients with allowJobCommands; otherwise 422 naming it', () => {
+test('every job carries its command and runs on any matching Client; an old-Agent spec is refused', () => {
   const { registry, agents, jobs, scheduler } = buildTestServices(),
     { agent } = agents.create({ name: 'dev', kind: 'cli' }),
-    commandJob = makeSpec({ tests: { url: 'https://x/t.tar.gz', command: 'make test' } }),
-    plain = registerResource(registry, { name: 'sw-plain', type: 'sw' });
-  registry.heartbeat(plain.id, { state: 'idle' });
-
-  assert.throws(() => jobs.create({ agentId: agent.id, source: 'cli', spec: commandJob }), /allowJobCommands/);
+    r = registerResource(registry, { name: 'sw-any', type: 'sw' });
+  registry.heartbeat(r.id, { state: 'idle' });
 
-  const { resourceId } = registry.registerAuto({
-    clientId: 'c-cmd', name: 'sw-cmd', type: 'sw', labels: [], capabilities: { sw: { image: 'emu' }, allowJobCommands: true }
-  });
-  registry.heartbeat(resourceId, { state: 'idle' });
-  const job = jobs.create({ agentId: agent.id, source: 'cli', spec: commandJob });
+  const job = jobs.create({ agentId: agent.id, source: 'cli', spec: makeSpec({ command: 'make test', args: ['-j4'] }) });
   scheduler.runPass();
-  assert.equal(jobs.get(job.id).resource_id, resourceId);
+  assert.equal(jobs.get(job.id).resource_id, r.id);
+  assert.equal(jobs.get(job.id).spec.command, 'make test');
+
+  assert.throws(
+    () => jobs.create({ agentId: agent.id, source: 'cli', spec: { target: { type: 'sw' }, firmware: { url: 'https://x/a' }, tests: { url: 'https://x/t' } } }),
+    /older Agent.*--command/
+  );
+  assert.throws(() => jobs.create({ agentId: agent.id, source: 'cli', spec: { target: { type: 'sw' } } }), /command/);
 });
 
 test('reboot schedule: validated, then resent on heartbeats until the Client reports applying it', () => {
@@ -585,6 +584,40 @@ test('reboot schedule: validated, then resent on heartbeats until the Client rep
   assert.equal(registry.pendingRebootSchedule(r.id), null);
 });
 
+test('reboot now: refused offline or without a helper; cancels a running job first, then sends reboot', () => {
+  const { bus, registry, agents, jobs, scheduler } = buildTestServices(),
+    { agent } = agents.create({ name: 'dev', kind: 'cli' }),
+    reg = (name, rebootSupported) => registry.get(registry.registerAuto({
+      clientId: `c-${name}`, name, type: 'sw', labels: [], capabilities: { sw: { image: 'emu' }, rebootSupported }
+    }).resourceId),
+    offline = reg('rb-offline', true),
+    noHelper = reg('rb-nohelper', false),
+    idle = reg('rb-idle', true),
+    busy = reg('rb-busy', true),
+    commands = [];
+  for (const r of [noHelper, idle, busy]){
+    registry.heartbeat(r.id, { state: 'idle' });
+  }
+  bus.on('command', (c) => commands.push(c));
+
+  assert.throws(() => jobs.requestReboot(offline.id), /offline/);
+  assert.throws(() => jobs.requestReboot(noHelper.id), /no reboot helper/);
+
+  assert.equal(jobs.requestReboot(idle.id).canceledJob, null);
+  assert.deepEqual(commands.map((c) => [c.resourceId, c.command]), [[idle.id, 'reboot']]);
+  commands.length = 0;
+
+  const job = jobs.create({ agentId: agent.id, source: 'cli', spec: makeSpec({ target: { type: 'sw', client: busy.id } }) });
+  scheduler.runPass();
+  jobs.setState(job.id, JOB_STATES.RUNNING);
+  assert.equal(jobs.requestReboot(busy.id, { by: 'admin' }).canceledJob, job.id);
+  assert.deepEqual(commands.map((c) => c.command), ['cancel-job', 'reboot']); // same heartbeat reply, in order
+  assert.equal(commands[1].reason, 'user reboot request by admin');
+  const after = jobs.get(job.id);
+  assert.equal(after.state, JOB_STATES.CANCELED);
+  assert.equal(after.message, 'Canceled: user reboot request');
+});
+
 test('a resource can belong to several groups at once', () => {
   const { registry, groups } = buildTestServices(),
     g1 = groups.create({ name: 'g1' }),

package/views/mixins/resource.pug

@@ -337,6 +337,37 @@ mixin resourceCard(r, groupsById, returnTo)
             .tab-pane.fade(id=`rc-${r.id}-config` role="tabpanel" aria-labelledby=`rc-${r.id}-config-tab` tabindex="0")
               dl.row.mb-0
                 +rebootScheduleRow(r, returnTo)
+        if user.role === 'admin'
+          .modal-footer
+            +rebootButton(r, returnTo)
+
+//- Reboot the Client's host now (card footer): confirmed first — naming the
+//- job that gets canceled, if one is running — then sent with the Client's
+//- next heartbeat. Disabled while it's offline or has no reboot helper.
+mixin rebootButton(r, returnTo)
+  - const offline = r.status === 'OUT_OF_SERVICE' || !r.last_heartbeat_at
+  - const unsupported = r.capabilities && r.capabilities.rebootSupported === false
+  - const why = offline ? 'Offline — a Client can only be rebooted while connected' : unsupported ? "This host has no reboot helper — reinstall the Client as root" : null
+  if why
+    span.d-inline-block(tabindex="0" data-bs-toggle="tooltip" data-bs-title=why)
+      button.btn.btn-outline-danger(type="button" disabled style="pointer-events: none;")
+        i.bi.bi-bootstrap-reboot.me-1
+        | Reboot
+  else
+    form(
+      method="post"
+      action=`/resources/${r.id}/reboot`
+      data-confirm=r.activeJob
+        ? `${r.name} is running job ${r.activeJob.id}. Rebooting cancels that job (reason: user reboot request), then reboots the host on the Client's next heartbeat. Every Client instance on that host goes down with it.`
+        : `Reboot the host of ${r.name}? It reboots on the Client's next heartbeat, once nothing on the host is busy. Every Client instance on that host goes down with it.`
+      data-confirm-title=r.activeJob ? `Cancel job ${r.activeJob.id} and reboot?` : `Reboot ${r.name}?`
+      data-confirm-ok=r.activeJob ? 'Cancel job and reboot' : 'Reboot'
+      data-confirm-dismiss="Don't reboot"
+    )
+      input(type="hidden" name="returnTo" value=returnTo)
+      button.btn.btn-outline-danger(type="submit")
+        i.bi.bi-bootstrap-reboot.me-1
+        | Reboot
 
 //- The resource card's Details tab.
 mixin resourceDetails(r, groupsById, returnTo, local, external)