navi-hey 0.5.2 → 1.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (72) hide show
  1. package/README.md +4 -0
  2. package/bin/navi.js +3 -3
  3. package/lib/{factories → background}/JobFactory.js +2 -2
  4. package/lib/{registry → background}/JobRegistry.js +18 -1
  5. package/lib/{registry → background}/JobRegistryInstance.js +51 -6
  6. package/lib/{models → background}/Worker.js +5 -3
  7. package/lib/{factories → background}/WorkerFactory.js +2 -2
  8. package/lib/{registry → background}/WorkersRegistryInstance.js +1 -1
  9. package/lib/{models → enqueuers}/ActionEnqueuer.js +4 -1
  10. package/lib/{models → enqueuers}/AssetRequestEnqueuer.js +6 -2
  11. package/lib/exceptions/ConflictError.js +14 -0
  12. package/lib/{models → jobs}/ActionProcessingJob.js +4 -4
  13. package/lib/{models → jobs}/AssetDownloadJob.js +6 -6
  14. package/lib/{models → jobs}/HtmlParseJob.js +6 -6
  15. package/lib/{models → jobs}/ResourceRequestJob.js +11 -11
  16. package/lib/models/Config.js +4 -2
  17. package/lib/models/FailureConfig.js +36 -0
  18. package/lib/models/ResourceRequest.js +5 -2
  19. package/lib/models/ResourceRequestAction.js +6 -3
  20. package/lib/models/WebConfig.js +5 -1
  21. package/lib/registry/ClientRegistry.js +60 -2
  22. package/lib/registry/LogRegistry.js +146 -0
  23. package/lib/registry/LogRegistryInstance.js +156 -0
  24. package/lib/serializers/BaseUrlsSerializer.js +18 -0
  25. package/lib/serializers/JobIndexSerializer.js +10 -3
  26. package/lib/serializers/LogSerializer.js +26 -0
  27. package/lib/server/BaseUrlsRequestHandler.js +33 -0
  28. package/lib/server/EngineContinueRequestHandler.js +34 -0
  29. package/lib/server/EnginePauseRequestHandler.js +34 -0
  30. package/lib/server/EngineRestartRequestHandler.js +34 -0
  31. package/lib/server/EngineShutdownRequestHandler.js +29 -0
  32. package/lib/server/EngineStartRequestHandler.js +34 -0
  33. package/lib/server/EngineStatusRequestHandler.js +27 -0
  34. package/lib/server/EngineStopRequestHandler.js +34 -0
  35. package/lib/server/JobLogsRequestHandler.js +39 -0
  36. package/lib/server/JobRequestHandler.js +1 -1
  37. package/lib/server/JobRetryRequestHandler.js +40 -0
  38. package/lib/server/JobsFilter.js +55 -0
  39. package/lib/server/JobsRequestHandler.js +5 -2
  40. package/lib/server/LogsRequestHandler.js +55 -0
  41. package/lib/server/RouteRegister.js +40 -7
  42. package/lib/server/Router.js +44 -20
  43. package/lib/server/SettingsRequestHandler.js +34 -0
  44. package/lib/server/StatsRequestHandler.js +2 -2
  45. package/lib/server/WebServer.js +28 -8
  46. package/lib/services/Application.js +135 -81
  47. package/lib/services/ApplicationInstance.js +272 -0
  48. package/lib/services/Client.js +27 -22
  49. package/lib/services/ConfigLoader.js +2 -1
  50. package/lib/services/ConfigParser.js +13 -2
  51. package/lib/services/Engine.js +51 -6
  52. package/lib/services/EngineEvents.js +42 -0
  53. package/lib/services/FailureChecker.js +78 -0
  54. package/lib/services/WorkersAllocator.js +2 -2
  55. package/lib/utils/EnvResolver.js +22 -0
  56. package/lib/utils/HtmlElementParser.js +5 -4
  57. package/lib/utils/HtmlParser.js +4 -4
  58. package/lib/utils/PromiseAggregator.js +11 -5
  59. package/lib/utils/logging/BufferedLogger.js +8 -0
  60. package/lib/utils/logging/LogBuffer.js +25 -5
  61. package/lib/utils/logging/LogBufferCollection.js +56 -0
  62. package/lib/utils/logging/LogContext.js +51 -0
  63. package/lib/utils/logging/LogFilter.js +34 -0
  64. package/package.json +1 -1
  65. package/static/assets/index-BfggoMvd.js +11 -0
  66. package/static/assets/{index-DBkz-poX.css → index-CuWjW0sT.css} +1 -1
  67. package/static/index.html +2 -2
  68. package/static/assets/index-BF8_ow13.js +0 -11
  69. /package/lib/{models → background}/Job.js +0 -0
  70. /package/lib/{registry → background}/WorkersRegistry.js +0 -0
  71. /package/lib/{models → enqueuers}/ActionsEnqueuer.js +0 -0
  72. /package/lib/{factories → factory}/Factory.js +0 -0
@@ -1,6 +1,6 @@
1
1
  import { RequestHandler } from './RequestHandler.js';
2
+ import { JobRegistry } from '../background/JobRegistry.js';
2
3
  import { NotFoundError } from '../exceptions/NotFoundError.js';
3
- import { JobRegistry } from '../registry/JobRegistry.js';
4
4
  import { JobSerializer } from '../serializers/JobSerializer.js';
5
5
 
6
6
  /**
@@ -0,0 +1,40 @@
1
+ import { RequestHandler } from './RequestHandler.js';
2
+ import { JobRegistry } from '../background/JobRegistry.js';
3
+ import { ConflictError } from '../exceptions/ConflictError.js';
4
+ import { NotFoundError } from '../exceptions/NotFoundError.js';
5
+
6
+ const RETRYABLE_STATUSES = new Set(['failed', 'dead']);
7
+
8
+ /**
9
+ * Handles PATCH /jobs/:id/retry requests.
10
+ * Moves a failed or dead job directly to the retry queue.
11
+ * Returns 404 if the job does not exist, 409 if it is not in a retryable state.
12
+ * @author darthjee
13
+ */
14
+ class JobRetryRequestHandler extends RequestHandler {
15
+ /**
16
+ * Creates a new JobRetryRequestHandler instance.
17
+ */
18
+ constructor() {
19
+ super();
20
+ }
21
+
22
+ /**
23
+ * Retries the specified job by moving it to the retry queue.
24
+ * @param {object} req - The Express request object.
25
+ * @param {object} res - The Express response object.
26
+ * @returns {void}
27
+ */
28
+ handle(req, res) {
29
+ const { id } = req.params;
30
+ const result = JobRegistry.jobById(id);
31
+
32
+ if (!result) throw new NotFoundError('Job not found');
33
+ if (!RETRYABLE_STATUSES.has(result.status)) throw new ConflictError();
34
+
35
+ JobRegistry.retryJob(id);
36
+ res.json({ status: 'enqueued' });
37
+ }
38
+ }
39
+
40
+ export { JobRetryRequestHandler };
@@ -0,0 +1,55 @@
1
+ /**
2
+ * Filters a job collection by one or more job classes.
3
+ * @author darthjee
4
+ */
5
+ class JobsFilter {
6
+ /**
7
+ * Creates a new JobsFilter instance.
8
+ * @param {object[]} jobs - The collection of jobs to filter.
9
+ * @param {object} [filters={}] - The filters object (e.g. `{ class: ['ResourceRequestJob'] }`).
10
+ */
11
+ constructor(jobs, filters = {}) {
12
+ this._jobs = jobs;
13
+ this._filters = filters;
14
+ }
15
+
16
+ /**
17
+ * Returns the filtered job list.
18
+ * When no class filter is provided, all jobs are returned unchanged.
19
+ * @returns {object[]} The filtered jobs.
20
+ */
21
+ filter() {
22
+ const classes = this.#normalizeClasses();
23
+
24
+ if (classes.length === 0) {
25
+ return this._jobs;
26
+ }
27
+
28
+ return this._jobs.filter((job) => this.#matchesClass(job, classes));
29
+ }
30
+
31
+ /**
32
+ * Returns true when the job's class is listed in the requested classes.
33
+ * @param {object} job - A job instance.
34
+ * @param {string[]} classes - The list of accepted class names.
35
+ * @returns {boolean}
36
+ */
37
+ #matchesClass(job, classes) {
38
+ return classes.includes(job.constructor.name);
39
+ }
40
+
41
+ /**
42
+ * Normalises the class filter value to an array.
43
+ * Express's qs parser collapses a single `filters[class][]=Foo` to a plain string.
44
+ * @returns {string[]} The normalised array of class names.
45
+ */
46
+ #normalizeClasses() {
47
+ const value = this._filters.class;
48
+
49
+ if (!value) return [];
50
+ if (Array.isArray(value)) return value;
51
+ return [value];
52
+ }
53
+ }
54
+
55
+ export { JobsFilter };
@@ -1,5 +1,6 @@
1
+ import { JobsFilter } from './JobsFilter.js';
1
2
  import { RequestHandler } from './RequestHandler.js';
2
- import { JobRegistry } from '../registry/JobRegistry.js';
3
+ import { JobRegistry } from '../background/JobRegistry.js';
3
4
  import { JobSerializer } from '../serializers/JobSerializer.js';
4
5
 
5
6
  /**
@@ -23,7 +24,9 @@ class JobsRequestHandler extends RequestHandler {
23
24
  handle(req, res) {
24
25
  const { status } = req.params;
25
26
  const jobs = JobRegistry.jobsByStatus(status);
26
- res.json(JobSerializer.serialize(jobs, { status, view: 'index' }));
27
+ const filters = req.query.filters || {};
28
+ const filteredJobs = new JobsFilter(jobs, filters).filter();
29
+ res.json(JobSerializer.serialize(filteredJobs, { status, view: 'index' }));
27
30
  }
28
31
  }
29
32
 
@@ -0,0 +1,55 @@
1
+ import { RequestHandler } from './RequestHandler.js';
2
+ import { LogRegistry } from '../registry/LogRegistry.js';
3
+ import { LogSerializer } from '../serializers/LogSerializer.js';
4
+
5
+ /**
6
+ * Handles GET /logs.json requests.
7
+ * Routes to per-job or per-worker buffers via optional `jobId` / `workerId` query params.
8
+ * Returns 400 when both params are present simultaneously.
9
+ * @author darthjee
10
+ */
11
+ class LogsRequestHandler extends RequestHandler {
12
+ #pageSize;
13
+
14
+ /**
15
+ * Creates a new LogsRequestHandler.
16
+ * @param {object} [options={}]
17
+ * @param {number} [options.pageSize=20] - Maximum number of log entries to return.
18
+ */
19
+ constructor({ pageSize = 20 } = {}) {
20
+ super();
21
+ this.#pageSize = pageSize;
22
+ }
23
+
24
+ /**
25
+ * Responds with a paginated JSON array of log entries.
26
+ * @param {object} req - The Express request object.
27
+ * @param {object} res - The Express response object.
28
+ * @returns {void}
29
+ */
30
+ handle(req, res) {
31
+ const { last_id: lastId, jobId, workerId } = req.query;
32
+
33
+ if (jobId && workerId) {
34
+ res.status(400).json({ error: 'Cannot specify both jobId and workerId' });
35
+ return;
36
+ }
37
+
38
+ if (jobId) {
39
+ const logs = LogRegistry.getLogsByJobId(jobId);
40
+ res.json(LogSerializer.serialize(logs.slice(0, this.#pageSize)));
41
+ return;
42
+ }
43
+
44
+ if (workerId) {
45
+ const logs = LogRegistry.getLogsByWorkerId(workerId);
46
+ res.json(LogSerializer.serialize(logs.slice(0, this.#pageSize)));
47
+ return;
48
+ }
49
+
50
+ const logs = LogRegistry.getLogs({ lastId });
51
+ res.json(LogSerializer.serialize(logs.slice(0, this.#pageSize)));
52
+ }
53
+ }
54
+
55
+ export { LogsRequestHandler };
@@ -1,5 +1,7 @@
1
+ import { ConflictError } from '../exceptions/ConflictError.js';
1
2
  import { ForbiddenError } from '../exceptions/ForbiddenError.js';
2
3
  import { NotFoundError } from '../exceptions/NotFoundError.js';
4
+ import { Logger } from '../utils/logging/Logger.js';
3
5
 
4
6
  /**
5
7
  * Registers a route on an Express router by binding the handler's handle method.
@@ -27,26 +29,57 @@ class RouteRegister {
27
29
  this.#router.get(route, (req, res) => {
28
30
  try {
29
31
  handler.handle(req, res);
32
+ Logger.debug(`${req.method} ${req.path} ${res.statusCode}`);
30
33
  } catch (e) {
31
- this.#handleError(e, res);
34
+ this.#handleError(e, req, res);
32
35
  }
33
36
  });
34
37
  }
35
38
 
36
39
  /**
37
- * Maps a caught exception to an HTTP error response.
40
+ * Registers a PATCH route on the router.
41
+ * Catches ForbiddenError → 403, NotFoundError → 404, and any other error → 500.
42
+ * @param {object} params - Options for registering a route.
43
+ * @param {string} params.route - The route path (e.g. '/engine/pause').
44
+ * @param {object} params.handler - The handler whose handle method is called.
45
+ * @returns {void}
46
+ */
47
+ registerPatch({ route, handler }) {
48
+ this.#router.patch(route, async (req, res) => {
49
+ try {
50
+ await handler.handle(req, res);
51
+ Logger.debug(`${req.method} ${req.path} ${res.statusCode}`);
52
+ } catch (e) {
53
+ this.#handleError(e, req, res);
54
+ }
55
+ });
56
+ }
57
+
58
+ /**
59
+ * Maps a caught exception to an HTTP error response and logs the access entry.
38
60
  * @param {Error} e - The caught exception.
61
+ * @param {object} req - The Express request object.
39
62
  * @param {object} res - The Express response object.
40
63
  * @returns {void}
41
64
  */
42
- #handleError(e, res) {
43
- if (e instanceof ForbiddenError) {
44
- res.status(403).json({ error: 'Forbidden' });
65
+ #handleError(e, req, res) {
66
+ let statusCode;
67
+ let message;
68
+ if (e instanceof ConflictError) {
69
+ statusCode = 409;
70
+ message = 'Conflict';
71
+ } else if (e instanceof ForbiddenError) {
72
+ statusCode = 403;
73
+ message = 'Forbidden';
45
74
  } else if (e instanceof NotFoundError) {
46
- res.status(404).json({ error: e.message });
75
+ statusCode = 404;
76
+ message = e.message;
47
77
  } else {
48
- res.status(500).json({ error: 'Internal Server Error' });
78
+ statusCode = 500;
79
+ message = 'Internal Server Error';
49
80
  }
81
+ res.status(statusCode).json({ error: message });
82
+ Logger.debug(`${req.method} ${req.path} ${statusCode}`);
50
83
  }
51
84
  }
52
85
 
@@ -2,10 +2,22 @@ import path from 'path';
2
2
  import { fileURLToPath } from 'url';
3
3
  import express from 'express';
4
4
  import { AssetsRequestHandler } from './AssetsRequestHandler.js';
5
+ import { BaseUrlsRequestHandler } from './BaseUrlsRequestHandler.js';
6
+ import { EngineContinueRequestHandler } from './EngineContinueRequestHandler.js';
7
+ import { EnginePauseRequestHandler } from './EnginePauseRequestHandler.js';
8
+ import { EngineRestartRequestHandler } from './EngineRestartRequestHandler.js';
9
+ import { EngineShutdownRequestHandler } from './EngineShutdownRequestHandler.js';
10
+ import { EngineStartRequestHandler } from './EngineStartRequestHandler.js';
11
+ import { EngineStatusRequestHandler } from './EngineStatusRequestHandler.js';
12
+ import { EngineStopRequestHandler } from './EngineStopRequestHandler.js';
5
13
  import { IndexRequestHandler } from './IndexRequestHandler.js';
14
+ import { JobLogsRequestHandler } from './JobLogsRequestHandler.js';
6
15
  import { JobRequestHandler } from './JobRequestHandler.js';
16
+ import { JobRetryRequestHandler } from './JobRetryRequestHandler.js';
7
17
  import { JobsRequestHandler } from './JobsRequestHandler.js';
18
+ import { LogsRequestHandler } from './LogsRequestHandler.js';
8
19
  import { RouteRegister } from './RouteRegister.js';
20
+ import { SettingsRequestHandler } from './SettingsRequestHandler.js';
9
21
  import { StatsRequestHandler } from './StatsRequestHandler.js';
10
22
 
11
23
  const { Router: ExpressRouter } = express;
@@ -17,10 +29,16 @@ const staticDir = path.join(__dirname, '../../static');
17
29
  * @author darthjee
18
30
  */
19
31
  class Router {
32
+ #webConfig;
33
+
20
34
  /**
21
35
  * Creates a new Router instance.
36
+ * @param {object} [options={}]
37
+ * @param {object} [options.webConfig={}] - Web configuration, used by handlers that need it.
22
38
  */
23
- constructor() {}
39
+ constructor({ webConfig = {} } = {}) {
40
+ this.#webConfig = webConfig;
41
+ }
24
42
 
25
43
  /**
26
44
  * Creates and returns an Express Router with all routes registered.
@@ -30,29 +48,35 @@ class Router {
30
48
  const router = ExpressRouter();
31
49
  const register = new RouteRegister(router);
32
50
 
33
- register.register({
34
- route: '/stats.json',
35
- handler: new StatsRequestHandler(),
36
- });
37
-
38
- register.register({
39
- route: '/jobs/:status.json',
40
- handler: new JobsRequestHandler(),
41
- });
51
+ const GET_ROUTES = {
52
+ '/settings.json': new SettingsRequestHandler({ enableShutdown: this.#webConfig.enableShutdown }),
53
+ '/stats.json': new StatsRequestHandler(),
54
+ '/clients/base_urls.json': new BaseUrlsRequestHandler(),
55
+ '/jobs/:status.json': new JobsRequestHandler(),
56
+ '/jobs/:job_id/logs.json': new JobLogsRequestHandler({ pageSize: this.#webConfig.logsPageSize }),
57
+ '/job/:id.json': new JobRequestHandler(),
58
+ '/engine/status': new EngineStatusRequestHandler(),
59
+ '/logs.json': new LogsRequestHandler({ pageSize: this.#webConfig.logsPageSize }),
60
+ '/': new IndexRequestHandler(),
61
+ '/assets/*path': new AssetsRequestHandler(),
62
+ };
42
63
 
43
- register.register({
44
- route: '/job/:id.json',
45
- handler: new JobRequestHandler(),
46
- });
64
+ const PATCH_ROUTES = {
65
+ '/jobs/:id/retry': new JobRetryRequestHandler(),
66
+ '/engine/pause': new EnginePauseRequestHandler(),
67
+ '/engine/stop': new EngineStopRequestHandler(),
68
+ '/engine/continue': new EngineContinueRequestHandler(),
69
+ '/engine/start': new EngineStartRequestHandler(),
70
+ '/engine/restart': new EngineRestartRequestHandler(),
71
+ '/engine/shutdown': new EngineShutdownRequestHandler(),
72
+ };
47
73
 
48
- register.register({
49
- route: '/',
50
- handler: new IndexRequestHandler(),
74
+ Object.entries(GET_ROUTES).forEach(([route, handler]) => {
75
+ register.register({ route, handler });
51
76
  });
52
77
 
53
- register.register({
54
- route: '/assets/*path',
55
- handler: new AssetsRequestHandler(),
78
+ Object.entries(PATCH_ROUTES).forEach(([route, handler]) => {
79
+ register.registerPatch({ route, handler });
56
80
  });
57
81
 
58
82
  router.use(express.static(staticDir));
@@ -0,0 +1,34 @@
1
+ import { RequestHandler } from './RequestHandler.js';
2
+ import { ForbiddenError } from '../exceptions/ForbiddenError.js';
3
+
4
+ /**
5
+ * Handles GET /settings.json requests.
6
+ * Returns application settings visible to the frontend.
7
+ * Throws ForbiddenError when shutdown is disabled.
8
+ * @author darthjee
9
+ */
10
+ class SettingsRequestHandler extends RequestHandler {
11
+ #enableShutdown;
12
+
13
+ /**
14
+ * @param {object} [options={}]
15
+ * @param {boolean} [options.enableShutdown=true] - Whether shutdown is permitted.
16
+ */
17
+ constructor({ enableShutdown = true } = {}) {
18
+ super();
19
+ this.#enableShutdown = enableShutdown;
20
+ }
21
+
22
+ /**
23
+ * Responds with the application settings JSON.
24
+ * @param {object} _req - The Express request object.
25
+ * @param {object} res - The Express response object.
26
+ * @returns {void}
27
+ */
28
+ handle(_req, res) {
29
+ if (!this.#enableShutdown) throw new ForbiddenError();
30
+ res.json({ enable_shutdown: true });
31
+ }
32
+ }
33
+
34
+ export { SettingsRequestHandler };
@@ -1,6 +1,6 @@
1
1
  import { RequestHandler } from './RequestHandler.js';
2
- import { JobRegistry } from '../registry/JobRegistry.js';
3
- import { WorkersRegistry } from '../registry/WorkersRegistry.js';
2
+ import { JobRegistry } from '../background/JobRegistry.js';
3
+ import { WorkersRegistry } from '../background/WorkersRegistry.js';
4
4
 
5
5
  /**
6
6
  * Handles GET /stats.json requests.
@@ -9,25 +9,45 @@ import { Logger } from '../utils/logging/Logger.js';
9
9
  class WebServer {
10
10
  #port;
11
11
  #app;
12
+ #httpServer;
13
+ #startPromise;
12
14
 
13
15
  /**
14
16
  * @param {object} params - Options for initializing the WebServer.
15
- * @param {number} params.port - Port to listen on.
17
+ * @param {object} params.webConfig - The web configuration object.
16
18
  */
17
- constructor({ port }) {
18
- this.#port = port;
19
+ constructor({ webConfig }) {
20
+ this.#port = webConfig.port;
19
21
  this.#app = express();
20
- this.#app.use(new Router().build());
22
+ this.#app.use(new Router({ webConfig }).build());
21
23
  }
22
24
 
23
25
  /**
24
26
  * Starts the Express server on the configured port.
25
- * The returned http.Server can be used to close the server (e.g. in tests).
26
- * @returns {http.Server} The http.Server instance.
27
+ * Returns a Promise that resolves when the HTTP server fires its 'close' event,
28
+ * or rejects if the server fails to start.
29
+ * @returns {Promise<void>} A Promise that resolves when the server closes.
27
30
  */
28
31
  start() {
29
32
  Logger.info(`Listening to port ${this.#port}`);
30
- return this.#app.listen(this.#port);
33
+ this.#startPromise = new Promise((resolve, reject) => {
34
+ this.#httpServer = this.#app.listen(this.#port);
35
+ this.#httpServer.on('close', resolve);
36
+ this.#httpServer.on('error', reject);
37
+ });
38
+ return this.#startPromise;
39
+ }
40
+
41
+ /**
42
+ * Closes the HTTP server. Stops accepting new connections and forcibly closes
43
+ * any remaining keep-alive connections on the next event-loop tick so that
44
+ * the start() promise can resolve.
45
+ * @returns {Promise<void>|undefined} The promise from start(), or undefined if not started.
46
+ */
47
+ shutdown() {
48
+ this.#httpServer?.close();
49
+ setImmediate(() => this.#httpServer?.closeAllConnections());
50
+ return this.#startPromise;
31
51
  }
32
52
 
33
53
  /**
@@ -38,7 +58,7 @@ class WebServer {
38
58
  */
39
59
  static build({ webConfig }) {
40
60
  if (!webConfig) return null;
41
- return new WebServer({ port: webConfig.port });
61
+ return new WebServer({ webConfig });
42
62
  }
43
63
  }
44
64