@andrian.yablonskyy/thub-coordinator 1.0.37 → 1.0.39

package/src/services/live.js

@@ -0,0 +1,124 @@
+/**
+ * @file        packages/coordinator/src/services/live.js
+ * @description Dashboard live updates: detects when what a list page shows (resources, agents, jobs) has changed and notifies subscribers
+ *
+ * @author      Andrian Yablonskyy
+ * @copyright   Copyright (c) 2026 Andrian Yablonskyy. All rights reserved.
+ *
+ * This file is part of TestHub and is proprietary and confidential.
+ * Unauthorized copying, modification, distribution, or use of this file,
+ * via any medium, is strictly prohibited without prior written permission
+ * from AdSystem.PRO.
+ */
+
+'use strict';
+
+const crypto = require('node:crypto'),
+  { ACTIVE_JOB_STATES } = require('@andrian.yablonskyy/thub-common');
+
+// How the dashboard stays current (README §10): the page holds one
+// GET /live stream that only says *which* topics changed; it then re-fetches
+// itself and swaps the parts that differ (public/js/live.js). Rather than
+// every place that changes a resource or agent having to announce it, a
+// fingerprint of each topic is recomputed and compared — right away on bus
+// events (a job starting or finishing shows up at once) and every
+// `intervalMs` as the catch-all for everything else (heartbeats, the
+// sweeper, maintenance, renames, update requests, …).
+const TOPICS = ['resources', 'agents', 'jobs'],
+  BUS_EVENTS = ['job.queued', 'job.assigned', 'job.state', 'job.finished', 'resource.idle', 'resource.reregistered'];
+
+function createLiveService(db, { bus, updates, intervalMs = 5000, debounceMs = 250 } = {}){
+  const subscribers = new Set(),
+    activeStates = [...ACTIVE_JOB_STATES],
+    hash = (value) => crypto.createHash('sha1').update(JSON.stringify(value)).digest('base64'),
+    // One cheap query per topic covering everything its pages display.
+    // Resources: every column (tens of rows) — last_heartbeat_at included,
+    // since the pages show it. Jobs: counts and latest times of the whole
+    // table (new, finished and deleted jobs) plus every active job's state.
+    queries = {
+      resources: () => [db.prepare('SELECT * FROM resources ORDER BY id').all(), updates?.status().latest],
+      agents: () => [
+        db.prepare('SELECT id, name, kind, version, update_to, last_used_at, revoked_at FROM agents ORDER BY id').all(),
+        updates?.status().latest
+      ],
+      jobs: () => [
+        db.prepare('SELECT COUNT(*) AS n, MAX(created_at) AS created, MAX(finished_at) AS finished FROM jobs').get(),
+        db.prepare(
+          `SELECT id, state, resource_id, message FROM jobs WHERE state IN (${activeStates.map(() => '?').join(',')}) ORDER BY id`
+        ).all(...activeStates)
+      ]
+    };
+  let fingerprints = null,
+    timer = null,
+    pending = null;
+
+  function compute(){
+    return Object.fromEntries(TOPICS.map((t) => [t, hash(queries[t]())]));
+  }
+
+  // Compare with the last fingerprints; tell every subscriber what changed.
+  function check(){
+    pending = null;
+    if (!subscribers.size){
+      return [];
+    }
+    const next = compute(),
+      changed = fingerprints ? TOPICS.filter((t) => next[t] !== fingerprints[t]) : [];
+    fingerprints = next;
+    if (changed.length){
+      for (const fn of subscribers){
+        try {
+          fn(changed);
+        }
+        catch {
+          // one broken subscriber (a closed stream) mustn't stop the others
+        }
+      }
+    }
+    return changed;
+  }
+
+  // Bus events arrive in bursts (state + log + finished): check once.
+  function soon(){
+    if (subscribers.size && !pending){
+      pending = setTimeout(check, debounceMs);
+      pending.unref?.();
+    }
+  }
+
+  function start(){
+    fingerprints = compute();
+    timer = setInterval(check, intervalMs);
+    timer.unref?.();
+    for (const e of BUS_EVENTS){
+      bus.on(e, soon);
+    }
+  }
+
+  function stop(){
+    clearInterval(timer);
+    clearTimeout(pending);
+    timer = pending = null;
+    fingerprints = null;
+    for (const e of BUS_EVENTS){
+      bus.off(e, soon);
+    }
+  }
+
+  // Nothing runs while no dashboard is open.
+  function subscribe(fn){
+    subscribers.add(fn);
+    if (subscribers.size === 1){
+      start();
+    }
+    return () => {
+      if (subscribers.delete(fn) && !subscribers.size){
+        stop();
+      }
+    };
+  }
+
+  return { subscribe, check, subscriberCount: () => subscribers.size, TOPICS };
+}
+
+module.exports = { createLiveService, TOPICS };

package/src/services/logs.js

@@ -15,7 +15,7 @@
 
 // §3.1 Log service: accepts batched log lines, assigns a monotonically
 // increasing seq per job, stores them, and fans them out over the bus for
-// SSE subscribers (§6.4). Retention is handled separately (§9).
+// SSE subscribers (§6.4). Lines are deleted with their job (§9).
 function createLogsService(db, { bus }){
   const insert = db.prepare(
       'INSERT INTO job_logs (job_id, seq, ts, stream, line) VALUES (?, ?, ?, ?, ?)'
@@ -47,24 +47,22 @@ function createLogsService(db, { bus }){
       .all(jobId, afterSeq, limit);
   }
 
-  // Used when a job finishes: fold job_logs into a flat console.log artifact.
-  function renderConsoleLog(jobId){
+  // The dashboard log viewer's pages (views/mixins/log-viewer.pug): the
+  // `limit` lines just before `beforeSeq` (the end of the log when omitted),
+  // oldest first, and whether there are earlier ones to scroll up to.
+  function pageBefore(jobId, beforeSeq, limit){
     const rows = db
-      .prepare('SELECT * FROM job_logs WHERE job_id = ? ORDER BY seq ASC')
-      .all(jobId);
-    return rows.map((r) => `[${r.ts}] [${r.stream}] ${r.line}`).join('\n') + (rows.length ? '\n' : '');
+        .prepare('SELECT seq, ts, stream, line FROM job_logs WHERE job_id = ? AND seq < ? ORDER BY seq DESC LIMIT ?')
+        .all(jobId, beforeSeq ?? Number.MAX_SAFE_INTEGER, limit + 1),
+      hasMore = rows.length > limit;
+    return { lines: rows.slice(0, limit).reverse(), hasMore };
   }
 
-  function purgeOlderThan(days){
-    const cutoff = new Date(Date.now() - days * 86400 * 1000).toISOString();
-    db.prepare(
-      `DELETE FROM job_logs WHERE job_id IN (
-         SELECT id FROM jobs WHERE finished_at IS NOT NULL AND finished_at <= ?
-       )`
-    ).run(cutoff);
+  function count(jobId){
+    return db.prepare('SELECT COUNT(*) AS n FROM job_logs WHERE job_id = ?').get(jobId).n;
   }
 
-  return { appendBatch, listSince, renderConsoleLog, purgeOlderThan };
+  return { appendBatch, listSince, pageBefore, count };
 }
 
 module.exports = { createLogsService };

package/src/services/retention.js

@@ -1,6 +1,6 @@
 /**
  * @file        packages/coordinator/src/services/retention.js
- * @description Deletes finished jobs' logs/artifacts past their retention window (README §9)
+ * @description Nightly consistent database backup (VACUUM INTO, README §9)
  *
  * @author      Andrian Yablonskyy
  * @copyright   Copyright (c) 2026 Andrian Yablonskyy. All rights reserved.
@@ -17,20 +17,11 @@ const path = require('node:path');
 
 const DAY_MS = 86400 * 1000;
 
-// §9: fold job_logs into a console.log artifact when a job finishes, purge
-// old logs/artifacts on the configured retention windows, and take a nightly
-// consistent backup via VACUUM INTO.
-function createRetentionService(db, { bus, logs, artifacts, config }){
-  bus.on('job.finished', ({ jobId }) => {
-    const text = logs.renderConsoleLog(jobId);
-    if (text){
-      artifacts.storeGenerated(jobId, 'console.log', Buffer.from(text, 'utf8'), 'text/plain');
-    }
-  });
-
+// §9: a nightly consistent backup via VACUUM INTO. (A job's log lines live
+// as long as the job itself — retention.jobRetention / Clean up database,
+// services/cleanup.js — and there are no artifacts to purge any more.)
+function createRetentionService(db, { config }){
   function runDaily(){
-    logs.purgeOlderThan(config.retention.logRetentionDays);
-    artifacts.purgeOlderThan(config.retention.artifactRetentionDays);
     const backupPath = path.join(config.dataDir, `backup-${new Date().toISOString().slice(0, 10)}.db`);
     db.exec(`VACUUM INTO '${backupPath.replace(/'/g, '\'\'')}'`);
   }

package/src/services/search.js

@@ -0,0 +1,59 @@
+/**
+ * @file        packages/coordinator/src/services/search.js
+ * @description Dashboard search (?q=): splits the query into words; a record matches when every word occurs in one of its fields
+ *
+ * @author      Andrian Yablonskyy
+ * @copyright   Copyright (c) 2026 Andrian Yablonskyy. All rights reserved.
+ *
+ * This file is part of TestHub and is proprietary and confidential.
+ * Unauthorized copying, modification, distribution, or use of this file,
+ * via any medium, is strictly prohibited without prior written permission
+ * from AdSystem.PRO.
+ */
+
+'use strict';
+
+const MAX_QUERY = 200,
+  MAX_TERMS = 10;
+
+// "  Lab-HW  nucleo " -> ['lab-hw', 'nucleo']. Case-insensitive, words in
+// any order; capped so a pasted wall of text can't build a huge query.
+function parseTerms(q){
+  if (typeof q !== 'string'){
+    return [];
+  }
+  return [...new Set(q.slice(0, MAX_QUERY).toLowerCase().split(/\s+/).filter(Boolean))].slice(0, MAX_TERMS);
+}
+
+// The query as shown back in the search box (trimmed, capped).
+function normalizeQuery(q){
+  return typeof q === 'string' ? q.slice(0, MAX_QUERY).trim() : '';
+}
+
+// In-memory lists (resources, groups, agents): `fields` are the record's
+// searchable values — strings, numbers, arrays of them, or null.
+function matches(fields, terms){
+  if (!terms.length){
+    return true;
+  }
+  const text = fields.flat(Infinity).filter((v) => v !== null && v !== undefined && v !== '').join('\n').toLowerCase();
+  return terms.every((t) => text.includes(t));
+}
+
+// SQL lists (jobs): every term must match at least one of `expressions`
+// (SQL expressions yielding text). LIKE is case-insensitive for ASCII in
+// SQLite; % and _ in the query are matched literally.
+function likeClause(expressions, terms){
+  if (!terms.length){
+    return { sql: '', params: [] };
+  }
+  const one = `(${expressions.map((e) => `COALESCE(${e}, '') LIKE ? ESCAPE '\\'`).join(' OR ')})`,
+    params = [];
+  for (const t of terms){
+    const pattern = `%${t.replace(/[\\%_]/g, (c) => `\\${c}`)}%`;
+    params.push(...expressions.map(() => pattern));
+  }
+  return { sql: terms.map(() => one).join(' AND '), params };
+}
+
+module.exports = { parseTerms, normalizeQuery, matches, likeClause, MAX_QUERY };

package/src/services/session-store.js

@@ -0,0 +1,127 @@
+/**
+ * @file        packages/coordinator/src/services/session-store.js
+ * @description Dashboard session store in the Coordinator's SQLite database: survives restarts, prunes expired sessions
+ *
+ * @author      Andrian Yablonskyy
+ * @copyright   Copyright (c) 2026 Andrian Yablonskyy. All rights reserved.
+ *
+ * This file is part of TestHub and is proprietary and confidential.
+ * Unauthorized copying, modification, distribution, or use of this file,
+ * via any medium, is strictly prohibited without prior written permission
+ * from AdSystem.PRO.
+ */
+
+'use strict';
+
+const session = require('express-session');
+
+// express-session's default MemoryStore loses every session on a restart,
+// never drops expired ones (memory only grows) and warns against itself in
+// production. This one keeps them in the `sessions` table (migration 022),
+// so signing in survives a Coordinator restart or self-update, and an
+// expired session is deleted when read and by a periodic sweep.
+const PRUNE_INTERVAL_MS = 15 * 60 * 1000,
+  // A session without a cookie expiry (never the case here: routes.js sets
+  // each user's idle timeout) is kept this long.
+  FALLBACK_TTL_MS = 24 * 60 * 60 * 1000;
+
+class SqliteSessionStore extends session.Store{
+  constructor(db, { pruneIntervalMs = PRUNE_INTERVAL_MS, now = Date.now } = {}){
+    super();
+    this.now = now;
+    this.stmts = {
+      get: db.prepare('SELECT sess, expires FROM sessions WHERE sid = ?'),
+      set: db.prepare(
+        `INSERT INTO sessions (sid, sess, expires) VALUES (?, ?, ?)
+         ON CONFLICT(sid) DO UPDATE SET sess = excluded.sess, expires = excluded.expires`
+      ),
+      touch: db.prepare('UPDATE sessions SET sess = ?, expires = ? WHERE sid = ?'),
+      destroy: db.prepare('DELETE FROM sessions WHERE sid = ?'),
+      prune: db.prepare('DELETE FROM sessions WHERE expires <= ?'),
+      length: db.prepare('SELECT COUNT(*) AS n FROM sessions WHERE expires > ?'),
+      clear: db.prepare('DELETE FROM sessions')
+    };
+    if (pruneIntervalMs > 0){
+      this.pruneTimer = setInterval(() => this.prune(), pruneIntervalMs);
+      this.pruneTimer.unref?.();
+    }
+  }
+
+  expiresOf(sess){
+    const at = sess?.cookie?.expires ? new Date(sess.cookie.expires).getTime() : NaN;
+    return Number.isFinite(at) ? at : this.now() + FALLBACK_TTL_MS;
+  }
+
+  // Each method answers through `cb`, as express-session expects; errors
+  // (a locked or broken database) go there too instead of throwing.
+  run(cb, fn){
+    let result;
+    try {
+      result = fn();
+    }
+    catch (err){
+      return cb?.(err);
+    }
+    cb?.(null, result);
+  }
+
+  get(sid, cb){
+    this.run(cb, () => {
+      const row = this.stmts.get.get(sid);
+      if (!row){
+        return null;
+      }
+      if (row.expires <= this.now()){
+        this.stmts.destroy.run(sid);
+        return null;
+      }
+      return JSON.parse(row.sess);
+    });
+  }
+
+  set(sid, sess, cb){
+    this.run(cb, () => {
+      this.stmts.set.run(sid, JSON.stringify(sess), this.expiresOf(sess));
+    });
+  }
+
+  // Called for a request that didn't change the session: moves its expiry
+  // (rolling idle timeout, §10.1). A passive live-update request leaves the
+  // cookie's expiry as it was (server.js), so this writes the same value.
+  touch(sid, sess, cb){
+    this.run(cb, () => {
+      this.stmts.touch.run(JSON.stringify(sess), this.expiresOf(sess), sid);
+    });
+  }
+
+  destroy(sid, cb){
+    this.run(cb, () => {
+      this.stmts.destroy.run(sid);
+    });
+  }
+
+  length(cb){
+    this.run(cb, () => this.stmts.length.get(this.now()).n);
+  }
+
+  clear(cb){
+    this.run(cb, () => {
+      this.stmts.clear.run();
+    });
+  }
+
+  prune(){
+    try {
+      return this.stmts.prune.run(this.now()).changes;
+    }
+    catch {
+      return 0; // the next sweep tries again
+    }
+  }
+
+  close(){
+    clearInterval(this.pruneTimer);
+  }
+}
+
+module.exports = { SqliteSessionStore };