@andrian.yablonskyy/thub-coordinator 1.0.38 → 1.0.40

package/bin/thub-admin.js

@@ -30,10 +30,10 @@ const { loadConfig } = require('../src/config'),
   { createAgentsService } = require('../src/services/agents'),
   { createGroupsService } = require('../src/services/groups'),
   { createAdminUsersService } = require('../src/services/admin-users'),
-  { createArtifactsService } = require('../src/services/artifacts'),
   { createJobsService } = require('../src/services/jobs'),
   { createCleanupService } = require('../src/services/cleanup'),
   { generateToken } = require('../src/services/tokens'),
+  { createSettingsService } = require('../src/services/settings'),
   { PACKAGES, fetchLatestVersion, isNewer, isValidVersion, npmBin, formatDateTime, parseDateTime } = require('@andrian.yablonskyy/thub-common'),
   { spawnSync } = require('node:child_process'),
   { version: installedVersion } = require('../package.json');
@@ -46,12 +46,17 @@ function usage(){
   thub-admin resource maintenance <resourceId> --on|--off
   thub-admin jobs reset --yes     Cancel every queued/assigned/preparing/running job
   thub-admin jobs clean --yes [--before "dd/mm/yyyy HH:MM:SS"]
-                                  Permanently delete finished jobs (with logs and artifacts) and
+                                  Permanently delete finished jobs (with their logs) and
                                   history from before then (local time; default: now), then
                                   compact the database file
   thub-admin group add <name> [--comment <text>]
   thub-admin group list
   thub-admin group remove <groupId>
+  thub-admin settings list        Settings changed from the dashboard (Settings page)
+  thub-admin settings reset [<key>...]
+                                  Drop them (all, or the named ones) — back to the config
+                                  file's values at the next restart; the way back if a
+                                  setting locked you out of the dashboard
   thub-admin check-update         Compare this Coordinator with the latest published version
   thub-admin self-update [--to X.Y.Z]
                                   Update this Coordinator (sudo npm i -g; restarts the service)
@@ -134,10 +139,26 @@ function main(){
     registry = createRegistryService(db, { bus, events }),
     agents = createAgentsService(db, { events }),
     adminUsers = createAdminUsersService(db),
-    artifacts = createArtifactsService(db, { config }),
-    jobs = createJobsService(db, { bus, events, registry, artifacts, config }),
+    jobs = createJobsService(db, { bus, events, registry, config }),
     groups = createGroupsService(db, { events, registry });
 
+  if (cmd === 'settings' && (sub === 'list' || sub === 'reset')){
+    const settings = createSettingsService(db, { config });
+    if (sub === 'reset'){
+      const n = settings.reset(rest);
+      console.log(`Removed ${n} dashboard setting(s). Restart the Coordinator to use the config file's values: sudo systemctl restart thub-coordinator`);
+      return;
+    }
+    const changed = settings.list().filter((s) => s.source === 'dashboard');
+    if (!changed.length){
+      console.log('No settings changed from the dashboard — everything comes from the config file (or the environment).');
+    }
+    for (const s of changed){
+      console.log(`${s.key} = ${s.type === 'secret' ? '(secret)' : JSON.stringify(s.value)}   (${s.updatedBy || '?'}, ${s.updatedAt})`);
+    }
+    return;
+  }
+
   if (cmd === 'create-admin'){
     const { positional, flags } = parseFlags([sub, ...rest].filter(Boolean)),
       [username, password] = positional;
@@ -193,7 +214,7 @@ function main(){
   if (cmd === 'jobs' && sub === 'clean'){
     const { flags } = parseFlags(rest);
     if (!flags.yes){
-      console.error('This permanently deletes finished jobs and their logs/artifacts. Re-run with --yes to confirm.');
+      console.error('This permanently deletes finished jobs and their logs. Re-run with --yes to confirm.');
       process.exit(4);
     }
     const before = flags.before ? parseDateTime(flags.before) : new Date();

package/config.json

@@ -19,12 +19,11 @@
     "defaultTimeoutSec": 1800,
     "maxTimeoutSec": 14400
   },
-  "retention": {
-    "logRetentionDays": 14,
-    "artifactRetentionDays": 30
+  "rateLimit": {
+    "requestsPerMinute": 500,
+    "loginPerMinute": 10
   },
-  "artifacts": {
-    "maxUploadMb": 512,
-    "linkTtlHours": 168
+  "retention": {
+    "jobRetention": "forever"
   }
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@andrian.yablonskyy/thub-coordinator",
-  "version": "1.0.38",
+  "version": "1.0.40",
   "description": "TestHub Coordinator — job queue, resource registry, scheduler, heartbeat monitor, log/artifact store and web dashboard",
   "main": "src/server.js",
   "engines": {
@@ -19,7 +19,7 @@
     "lint:fix": "eslint . --fix"
   },
   "dependencies": {
-    "@andrian.yablonskyy/thub-common": "^1.0.32",
+    "@andrian.yablonskyy/thub-common": "^1.0.33",
     "better-sqlite3": "^12.4.1",
     "express": "^5.1.0",
     "express-session": "^1.18.2",

package/public/js/actions.js

@@ -0,0 +1,58 @@
+/**
+ * @file        packages/coordinator/public/js/actions.js
+ * @description Dashboard: small declarative behaviours (data-* attributes) that used to be inline on*= handlers
+ *
+ * @author      Andrian Yablonskyy
+ * @copyright   Copyright (c) 2026 Andrian Yablonskyy. All rights reserved.
+ *
+ * This file is part of TestHub and is proprietary and confidential.
+ * Unauthorized copying, modification, distribution, or use of this file,
+ * via any medium, is strictly prohibited without prior written permission
+ * from AdSystem.PRO.
+ */
+
+'use strict';
+
+// The Content-Security-Policy (server.js) allows no inline script, so no
+// onclick=/onchange=/onsubmit= in the templates: they declare what they
+// want with these attributes instead. Delegated, so it also works for
+// content swapped in by live updates (live.js).
+(function (){
+  // <select data-navigate>: its option values are URLs to go to.
+  // <select data-autosubmit>: submit its form on change.
+  document.addEventListener('change', (e) => {
+    const el = e.target;
+    if (el.matches?.('select[data-navigate]')){
+      window.location.href = el.value;
+    }
+    else if (el.matches?.('[data-autosubmit]') && el.form){
+      el.form.submit();
+    }
+  });
+
+  // <button data-fill="#field" data-value="…">: put the value into #field.
+  document.addEventListener('click', (e) => {
+    const btn = e.target.closest('[data-fill]'),
+      target = btn && document.querySelector(btn.dataset.fill);
+    if (target){
+      target.value = btn.dataset.value;
+    }
+  });
+
+  // <form data-submit-busy>: disable its button and show a spinner while
+  // the request runs (e.g. the navbar's check for updates).
+  document.addEventListener('submit', (e) => {
+    const form = e.target;
+    if (!form.matches?.('form[data-submit-busy]') || e.defaultPrevented){
+      return;
+    }
+    const button = form.querySelector('button'),
+      icon = button?.querySelector('i');
+    if (button){
+      button.disabled = true;
+    }
+    if (icon){
+      icon.className = 'spinner-border spinner-border-sm';
+    }
+  });
+})();

package/public/js/log-viewer.js

@@ -1,6 +1,6 @@
 /**
  * @file        packages/coordinator/public/js/log-viewer.js
- * @description Dashboard: streams or fetches a job's log output into the log viewer pane
+ * @description Dashboard: a job's log in the log viewer pane — loaded from its end, earlier lines as you scroll up, live lines streamed
  *
  * @author      Andrian Yablonskyy
  * @copyright   Copyright (c) 2026 Andrian Yablonskyy. All rights reserved.
@@ -21,7 +21,11 @@
   const jobId = pre.dataset.jobId,
     jobActive = pre.dataset.jobActive === 'true',
     filter = document.getElementById('stream-filter'),
-    lines = [],
+    info = document.getElementById('log-info'),
+    loadAllButton = document.getElementById('log-load-all'),
+    PAGE = 1000, // lines per page (routes.js LOG_PAGE)
+    PAGE_ALL = 5000, // per request when loading everything
+    NEAR_TOP_PX = 150,
     // Same dd/mm/yyyy HH:MM:SS, 24-hour format and profile time zone as
     // every server-rendered date (thub-common formatDateTime).
     timeFormat = new Intl.DateTimeFormat('en-GB', {
@@ -35,6 +39,16 @@
       hourCycle: 'h23'
     });
 
+  // What's loaded, oldest first: a window ending at the log's end. `firstSeq`
+  // is where the next earlier page ends; `total` counts every line the job
+  // has (loaded or not), live lines included.
+  let lines = [],
+    firstSeq = null,
+    hasMore = false,
+    total = 0,
+    lastSeq = 0, // newest log line we have (for the live stream to follow on)
+    loading = false;
+
   function fmtTime(iso){
     const d = new Date(iso);
     if (Number.isNaN(d.getTime())){
@@ -44,8 +58,28 @@
     return `${p.day}/${p.month}/${p.year} ${p.hour}:${p.minute}:${p.second}`;
   }
 
-  // Search (views/mixins/log-viewer.pug): case-insensitive, literal text.
-  // Matches are <mark>ed; `current` is the one the arrows step to.
+  const number = (n) => n.toLocaleString('en-GB');
+
+  function updateInfo(){
+    const logLines = lines.filter((l) => l.seq !== undefined).length;
+    if (loading){
+      info.textContent = 'Loading earlier lines…';
+    }
+    else if (!total){
+      info.textContent = jobActive ? 'No output yet.' : 'This job produced no output.';
+    }
+    else if (hasMore){
+      info.textContent = `Showing the last ${number(logLines)} of ${number(total)} lines — scroll up for earlier ones.`;
+    }
+    else {
+      info.textContent = `All ${number(total)} lines.`;
+    }
+    loadAllButton.hidden = !hasMore || loading;
+  }
+
+  // Search (views/mixins/log-viewer.pug): case-insensitive, literal text,
+  // over the loaded lines. Matches are <mark>ed; `current` is the one the
+  // arrows step to.
   const search = document.getElementById('log-search'),
     onlyMatching = document.getElementById('log-search-only'),
     count = document.getElementById('log-search-count'),
@@ -62,7 +96,7 @@
   function showCurrent(scroll){
     hits.forEach((el, i) => el.classList.toggle('current', i === current));
     if (current >= 0){
-      count.textContent = `${current + 1} / ${hits.length}`;
+      count.textContent = `${current + 1} / ${hits.length}${hasMore ? ' loaded' : ''}`;
       if (scroll){
         const el = hits[current];
         pre.scrollTop = el.offsetTop - pre.clientHeight / 2;
@@ -78,22 +112,42 @@
     showCurrent(true);
   }
 
-  // `live`: re-rendered for a newly streamed line — keep the reader's place
-  // while searching; without a search, follow the end of the log as before.
-  function render({ live = false } = {}){
+  // `mode` says what changed, and so where the view should stay:
+  //   reset   — first load, a new search or filter: the end (or first match)
+  //   live    — a streamed line: follow the end only if the reader is there
+  //   prepend — earlier lines loaded above: keep the same lines in view
+  function render(mode = 'reset'){
     const wanted = filter.value,
       query = search.value.trim(),
+      atBottom = pre.scrollHeight - pre.scrollTop - pre.clientHeight < 30,
+      fromBottom = pre.scrollHeight - pre.scrollTop,
+      scrollTop = pre.scrollTop,
+      hitsBefore = hits.length,
       shown = lines
         .filter((l) => !wanted || l.stream === wanted)
         .map((l) => `[${fmtTime(l.ts)}] [${l.stream}] ${l.line}`);
 
+    function place(){
+      if (mode === 'prepend'){
+        pre.scrollTop = pre.scrollHeight - fromBottom;
+      }
+      else if (mode === 'reset' || atBottom){
+        pre.scrollTop = pre.scrollHeight;
+      }
+      else {
+        pre.scrollTop = scrollTop;
+      }
+    }
+
+    updateInfo();
     if (!query){
       pre.textContent = shown.join('\n');
       hits = [];
       current = -1;
       count.hidden = true;
       prev.disabled = next.disabled = true;
-      pre.scrollTop = pre.scrollHeight;
+      place();
+      fillIfShort();
       return;
     }
 
@@ -112,7 +166,6 @@
         out.push(html + escapeHtml(text.slice(last)));
       }
     }
-    const scrollTop = pre.scrollTop;
     pre.innerHTML = out.join('\n');
     hits = [...pre.querySelectorAll('mark.thub-log-hit')];
     count.hidden = false;
@@ -120,18 +173,23 @@
     prev.disabled = next.disabled = hits.length < 2;
     if (!hits.length){
       current = -1;
-      count.textContent = 'No matches';
+      count.textContent = hasMore ? 'No matches in loaded lines' : 'No matches';
+      place();
+      fillIfShort();
       return;
     }
-    if (live){
-      current = Math.min(Math.max(current, 0), hits.length - 1);
-      showCurrent(false);
-      pre.scrollTop = scrollTop;
-    }
-    else {
+    if (mode === 'reset'){
       current = 0;
       showCurrent(true);
     }
+    else {
+      // Earlier lines add their matches in front: the same match moves down.
+      current = mode === 'prepend' ? current + (hits.length - hitsBefore) : current;
+      current = Math.min(Math.max(current, 0), hits.length - 1);
+      showCurrent(false);
+      place();
+    }
+    fillIfShort();
   }
 
   // Streamed lines can arrive in bursts: render at most once per frame.
@@ -142,10 +200,75 @@
     queued = true;
     requestAnimationFrame(() => {
       queued = false;
-      render({ live: true });
+      render('live');
     });
   }
 
+  // --- loading pages ------------------------------------------------------
+  async function fetchPage(before, limit){
+    const qs = new URLSearchParams({ limit: String(limit) });
+    if (before){
+      qs.set('before', String(before));
+    }
+    const res = await fetch(`/jobs/${encodeURIComponent(jobId)}/logs?${qs}`, { cache: 'no-store' });
+    if (!res.ok){
+      throw new Error(`HTTP ${res.status}`);
+    }
+    return res.json();
+  }
+
+  function absorb(page, where){
+    const fetched = page.lines;
+    if (fetched.length){
+      firstSeq = fetched[0].seq;
+    }
+    lines = where === 'prepend' ? [...fetched, ...lines] : fetched;
+    if (where !== 'prepend' && fetched.length){
+      lastSeq = fetched.at(-1).seq;
+    }
+    hasMore = page.hasMore;
+    total = Math.max(total, page.total);
+  }
+
+  async function loadEarlier(limit = PAGE){
+    if (loading || !hasMore){
+      return;
+    }
+    loading = true;
+    updateInfo();
+    try {
+      absorb(await fetchPage(firstSeq, limit), 'prepend');
+    }
+    catch {
+      info.textContent = 'Couldn’t load earlier lines — scroll up to try again.';
+      loading = false;
+      return;
+    }
+    loading = false;
+    render('prepend');
+  }
+
+  async function loadAll(){
+    while (hasMore && !loading){
+      await loadEarlier(PAGE_ALL);
+    }
+  }
+
+  // A filter or "only matching lines" can leave too little to scroll: keep
+  // loading earlier pages until there's something to scroll, or nothing left.
+  function fillIfShort(){
+    if (hasMore && !loading && pre.scrollHeight <= pre.clientHeight + NEAR_TOP_PX){
+      loadEarlier();
+    }
+  }
+
+  pre.addEventListener('scroll', () => {
+    if (pre.scrollTop < NEAR_TOP_PX){
+      loadEarlier();
+    }
+  }, { passive: true });
+  loadAllButton.addEventListener('click', loadAll);
+
   let typing;
   search.addEventListener('input', () => {
     clearTimeout(typing);
@@ -167,43 +290,52 @@
   onlyMatching.addEventListener('change', () => render());
   filter.addEventListener('change', () => render());
 
-  // Already finished when the page loaded: the output will never change
-  // again, so just fetch it once — no EventSource, no live connection, no
-  // reload. (Opening a stream here used to trigger an auto-reload once it
-  // immediately got the job's "end" event, which reloaded the page, which
-  // re-opened the stream, which reloaded again — an infinite loop for
-  // anyone viewing a finished job.)
-  if (!jobActive){
-    fetch(`/jobs/${jobId}/logs`)
-      .then((r) => r.json())
-      .then(({ lines: fetched }) => {
-        lines.push(...fetched);
-        render();
-      })
-      .catch(() => {});
-    return;
-  }
+  // --- start: the end of the log, then (active job) live lines after it ----
+  (async () => {
+    try {
+      absorb(await fetchPage(null, PAGE), 'replace');
+    }
+    catch {
+      info.textContent = 'Couldn’t load the log — reload the page to try again.';
+      return;
+    }
+    render();
 
-  const source = new EventSource(`/jobs/${jobId}/stream`);
-  source.addEventListener('log', (e) => {
-    lines.push(JSON.parse(e.data));
-    renderLive();
-  });
-  source.addEventListener('state', (e) => {
-    const { state, resource } = JSON.parse(e.data);
-    lines.push({ ts: new Date().toISOString(), stream: 'state', line: `-> ${state}${resource ? ' on ' + resource : ''}` });
-    renderLive();
-  });
-  source.addEventListener('end', (e) => {
-    const { state } = JSON.parse(e.data);
-    lines.push({ ts: new Date().toISOString(), stream: 'state', line: `job finished: ${state}` });
-    renderLive();
-    source.close();
-    // The job was active when we opened this page and just finished —
-    // reload once to pick up the now-final badge/artifacts list.
-    setTimeout(() => window.location.reload(), 1500);
-  });
-  source.onerror = () => {
-    // EventSource auto-reconnects; nothing to do here.
-  };
+    // Already finished when the page loaded: the output will never change
+    // again — no EventSource, no live connection. (Opening a stream here used
+    // to reload the page on its immediate "end" event, forever.)
+    if (!jobActive){
+      return;
+    }
+    // Live lines start right after what's loaded; on a reconnect the
+    // browser resumes from the last one it got (Last-Event-ID).
+    const source = new EventSource(`/jobs/${encodeURIComponent(jobId)}/stream?after=${lastSeq}`);
+    source.addEventListener('log', (e) => {
+      const seq = Number(e.lastEventId) || undefined;
+      if (seq && seq <= lastSeq){
+        return; // already have it
+      }
+      lastSeq = seq ?? lastSeq;
+      lines.push({ ...JSON.parse(e.data), seq });
+      total += 1;
+      renderLive();
+    });
+    source.addEventListener('state', (e) => {
+      const { state, resource } = JSON.parse(e.data);
+      lines.push({ ts: new Date().toISOString(), stream: 'state', line: `-> ${state}${resource ? ' on ' + resource : ''}` });
+      renderLive();
+    });
+    source.addEventListener('end', (e) => {
+      const { state } = JSON.parse(e.data);
+      lines.push({ ts: new Date().toISOString(), stream: 'state', line: `job finished: ${state}` });
+      renderLive();
+      source.close();
+      // The job was active when we opened this page and just finished —
+      // reload once to pick up the now-final badge and details.
+      setTimeout(() => window.location.reload(), 1500);
+    });
+    source.onerror = () => {
+      // EventSource auto-reconnects; nothing to do here.
+    };
+  })();
 })();

package/public/js/settings.js

@@ -0,0 +1,66 @@
+/**
+ * @file        packages/coordinator/public/js/settings.js
+ * @description Dashboard Settings page: generate/show the join key, and wait out a restart
+ *
+ * @author      Andrian Yablonskyy
+ * @copyright   Copyright (c) 2026 Andrian Yablonskyy. All rights reserved.
+ *
+ * This file is part of TestHub and is proprietary and confidential.
+ * Unauthorized copying, modification, distribution, or use of this file,
+ * via any medium, is strictly prohibited without prior written permission
+ * from AdSystem.PRO.
+ */
+
+'use strict';
+
+(function (){
+  // Generate: a random key, shown so it can be copied into the Clients'
+  // configs (joinKey) before saving.
+  document.querySelectorAll('[data-secret-generate]').forEach((btn) => {
+    btn.addEventListener('click', () => {
+      const input = document.querySelector(btn.dataset.secretGenerate),
+        bytes = crypto.getRandomValues(new Uint8Array(32)),
+        b64 = btoa(String.fromCharCode(...bytes)).replace(/\+/g, '-').replace(/\//g, '_').replace(/=+$/, '');
+      input.value = `jk_${b64}`;
+      input.type = 'text';
+      input.select();
+    });
+  });
+  document.querySelectorAll('[data-secret-show]').forEach((btn) => {
+    btn.addEventListener('click', () => {
+      const input = document.querySelector(btn.dataset.secretShow);
+      input.type = input.type === 'password' ? 'text' : 'password';
+    });
+  });
+
+  // Restarting: poll until a process with a different start time answers,
+  // then reload (riding out the seconds it's down).
+  const banner = document.querySelector('[data-restarting]');
+  if (banner){
+    const before = banner.dataset.restarting,
+      started = Date.now(),
+      poll = async () => {
+        try {
+          const res = await fetch('/admin/settings/status', { cache: 'no-store', redirect: 'manual' });
+          if (res.type === 'opaqueredirect'){
+            window.location.href = '/login'; // the session didn't survive? sign in again
+            return;
+          }
+          if (res.ok && (await res.json()).startedAt !== before){
+            window.location.href = '/admin/settings';
+            return;
+          }
+        }
+        catch {
+          // down for now
+        }
+        if (Date.now() - started > 120_000){
+          banner.lastElementChild.textContent = 'The Coordinator hasn\'t come back after 2 minutes — check journalctl -u thub-coordinator on its host.';
+          banner.querySelector('.spinner-border')?.remove();
+          return;
+        }
+        setTimeout(poll, 1500);
+      };
+    setTimeout(poll, 1500);
+  }
+})();

package/scripts/install-default-config.js

@@ -1,7 +1,7 @@
 /**
  * @file        scripts/install-default-config.js
  * @description npm postinstall: on a real global install, creates the Coordinator's directory layout
- *              (~/.config/thub, ~/.thub and its artifacts/avatars/work subdirs) and, if it
+ *              (~/.config/thub, ~/.thub and its avatars/work subdirs) and, if it
  *              doesn't already exist, ~/.config/thub/coordinator.json with every default option and
  *              a freshly generated sessionSecret (README §12, §13)
  *

package/scripts/install-target.js

@@ -52,7 +52,7 @@ function resolveTargetUser(){
 }
 
 // Default layout (README §13): config under ~/.config/thub, runtime data
-// (SQLite DB, uploaded artifacts, avatars, work files) under ~/.thub.
+// (SQLite DB, avatars, work files) under ~/.thub.
 // An existing coordinator.json's own dataDir wins over the default, so a
 // re-install prepares (and the systemd unit allows writes to) the
 // directory the Coordinator will really use.
@@ -69,7 +69,7 @@ function targetPaths(user){
     // Written by the dashboard's "Update app" button, watched by
     // thub-coordinator-update.path (README §10.2).
     updateRequestFile: path.join(dataDir, 'update-request.json'),
-    dataSubdirs: ['artifacts', 'avatars', 'work'].map((d) => path.join(dataDir, d))
+    dataSubdirs: ['avatars', 'work'].map((d) => path.join(dataDir, d))
   };
 }
 

package/src/api/admin.js

@@ -134,7 +134,7 @@ function createAdminRouter({ services }){
     res.json({ canceled });
   });
 
-  // Permanently deletes finished jobs and their logs/artifacts (§13.1).
+  // Permanently deletes finished jobs and their logs (§13.1).
   // Database cleanup (§9): optional `before` (ISO 8601; default now) —
   // finished jobs that ended before it, older history, then VACUUM.
   router.post('/jobs/clean-history', (req, res) => {