navi-hey 0.4.0 → 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (40) hide show
  1. package/README.md +167 -0
  2. package/lib/exceptions/ForbiddenError.js +14 -0
  3. package/lib/exceptions/InvalidHtmlResponseBody.js +19 -0
  4. package/lib/exceptions/NotFoundError.js +13 -0
  5. package/lib/models/ActionProcessingJob.js +18 -9
  6. package/lib/models/AssetDownloadJob.js +58 -0
  7. package/lib/models/AssetRequest.js +39 -0
  8. package/lib/models/AssetRequestEnqueuer.js +68 -0
  9. package/lib/models/Config.js +4 -1
  10. package/lib/models/HtmlParseJob.js +71 -0
  11. package/lib/models/Job.js +12 -2
  12. package/lib/models/LogConfig.js +35 -0
  13. package/lib/models/ResourceRequest.js +24 -1
  14. package/lib/models/ResourceRequestJob.js +35 -3
  15. package/lib/models/WorkersConfig.js +5 -1
  16. package/lib/registry/JobRegistry.js +18 -0
  17. package/lib/registry/JobRegistryInstance.js +55 -2
  18. package/lib/serializers/JobIndexSerializer.js +26 -0
  19. package/lib/serializers/JobSerializer.js +26 -0
  20. package/lib/serializers/JobShowSerializer.js +29 -0
  21. package/lib/serializers/Serializer.js +36 -0
  22. package/lib/server/AssetsRequestHandler.js +50 -0
  23. package/lib/server/IndexRequestHandler.js +31 -0
  24. package/lib/server/JobRequestHandler.js +33 -0
  25. package/lib/server/JobsRequestHandler.js +30 -0
  26. package/lib/server/PathValidator.js +41 -0
  27. package/lib/server/RouteRegister.js +27 -1
  28. package/lib/server/Router.js +27 -3
  29. package/lib/services/Application.js +19 -2
  30. package/lib/services/Client.js +45 -6
  31. package/lib/services/ConfigParser.js +13 -2
  32. package/lib/utils/HtmlElementParser.js +39 -0
  33. package/lib/utils/HtmlParser.js +49 -0
  34. package/lib/utils/collections/Collection.js +9 -0
  35. package/lib/utils/collections/IdentifyableCollection.js +9 -0
  36. package/lib/utils/collections/Queue.js +8 -0
  37. package/package.json +10 -4
  38. package/static/assets/index-DBkz-poX.css +1 -0
  39. package/static/assets/index-sYKRvQgv.js +11 -0
  40. package/static/index.html +13 -0
@@ -1,5 +1,6 @@
1
1
  import { Job } from './Job.js';
2
2
  import { ResponseWrapper } from './ResponseWrapper.js';
3
+ import { JobRegistry } from '../registry/JobRegistry.js';
3
4
  import { Logger } from '../utils/logging/Logger.js';
4
5
 
5
6
  /**
@@ -27,6 +28,14 @@ class ResourceRequestJob extends Job {
27
28
  this.#clients = clients;
28
29
  }
29
30
 
31
+ /**
32
+ * Returns the job-specific arguments for serialization.
33
+ * @returns {{ url: string, parameters: object }} The job arguments.
34
+ */
35
+ get arguments() {
36
+ return { url: this.#resourceRequest.url, parameters: this.#parameters };
37
+ }
38
+
30
39
  /**
31
40
  * Performs the HTTP request for the resource request.
32
41
  * @returns {Promise} A promise that resolves with the HTTP response.
@@ -36,15 +45,37 @@ class ResourceRequestJob extends Job {
36
45
  try {
37
46
  this.lastError = undefined;
38
47
  const response = await this.#getClient().perform(this.#resourceRequest, this.#parameters);
39
- const wrapper = new ResponseWrapper(response, this.#parameters);
40
- this.#resourceRequest.enqueueActions(wrapper);
41
- return response;
48
+ return this.#handleResponse(response);
42
49
  } catch (error) {
43
50
  Logger.error(`Job #${this.id} failed: ${error}`);
44
51
  this._fail(error);
45
52
  }
46
53
  }
47
54
 
55
+ /**
56
+ * Handles the successful HTTP response: enqueues asset jobs and action jobs.
57
+ * @param {object} response The HTTP response object.
58
+ * @returns {object} The same response object.
59
+ * @private
60
+ */
61
+ #handleResponse(response) {
62
+ this.#enqueueAssets(response);
63
+ const wrapper = new ResponseWrapper(response, this.#parameters);
64
+ this.#resourceRequest.enqueueActions(wrapper);
65
+ return response;
66
+ }
67
+
68
+ /**
69
+ * Enqueues asset download jobs when the resource request declares asset rules.
70
+ * @param {object} response The HTTP response object.
71
+ * @private
72
+ */
73
+ #enqueueAssets(response) {
74
+ if (this.#resourceRequest.hasAssets()) {
75
+ this.#resourceRequest.enqueueAssets(response.data, JobRegistry, this.#clients);
76
+ }
77
+ }
78
+
48
79
  /**
49
80
  * Gets the client associated with this job's resource request.
50
81
  * @returns {Client} The client associated with this job's resource request.
@@ -59,3 +90,4 @@ class ResourceRequestJob extends Job {
59
90
  }
60
91
 
61
92
  export { ResourceRequestJob };
93
+
@@ -8,10 +8,14 @@ class WorkersConfig {
8
8
  * @param {object} [config={}] - The configuration object.
9
9
  * @param {number} [config.quantity=1] - The number of worker threads.
10
10
  * @param {number} [config.retry_cooldown=2000] - The cooldown after a job has failed before retry.
11
+ * @param {number} [config.sleep=500] - Milliseconds the engine sleeps between allocation ticks.
12
+ * @param {number} [config.'max-retries'=3] - Maximum number of times a job is retried before being marked dead.
11
13
  */
12
- constructor({ quantity = 1, retry_cooldown: retryCooldown = 2000 } = {}) {
14
+ constructor({ quantity = 1, retry_cooldown: retryCooldown = 2000, sleep = 500, 'max-retries': maxRetries = 3 } = {}) {
13
15
  this.quantity = quantity;
14
16
  this.retryCooldown = retryCooldown;
17
+ this.sleep = sleep;
18
+ this.maxRetries = maxRetries;
15
19
  }
16
20
  }
17
21
 
@@ -101,6 +101,24 @@ class JobRegistry {
101
101
  return JobRegistry.#getInstance().hasReadyJob();
102
102
  }
103
103
 
104
+ /**
105
+ * Returns all jobs in the given status collection.
106
+ * @param {string} status - The status name to filter by.
107
+ * @returns {object[]} Array of raw job instances, or an empty array for unknown status.
108
+ */
109
+ static jobsByStatus(status) {
110
+ return JobRegistry.#getInstance().jobsByStatus(status);
111
+ }
112
+
113
+ /**
114
+ * Searches all collections and returns the job and its status for the given ID.
115
+ * @param {string} id - The ID of the job to look up.
116
+ * @returns {{ job: object, status: string } | null} The raw job and its status, or null if not found.
117
+ */
118
+ static jobById(id) {
119
+ return JobRegistry.#getInstance().jobById(id);
120
+ }
121
+
104
122
  /**
105
123
  * Returns counts of jobs in each state.
106
124
  * @returns {{ enqueued: number, processing: number, failed: number, retryQueue: number, finished: number, dead: number }} Job counts per state.
@@ -18,6 +18,7 @@ class JobRegistryInstance {
18
18
  #dead;
19
19
  #processing;
20
20
  #cooldown;
21
+ #maxRetries;
21
22
 
22
23
  /**
23
24
  * @param {object} [options={}] - Construction options.
@@ -28,8 +29,9 @@ class JobRegistryInstance {
28
29
  * @param {IdentifyableCollection} [options.dead] - Initial dead jobs collection.
29
30
  * @param {IdentifyableCollection} [options.processing] - Initial processing jobs collection.
30
31
  * @param {number} [options.cooldown=5000] - Cooldown in milliseconds before a failed job is retried.
32
+ * @param {number} [options.maxRetries=3] - Maximum number of retries before a job is marked dead.
31
33
  */
32
- constructor({ queue, failed, retryQueue, finished, dead, processing, cooldown = 5000 } = {}) {
34
+ constructor({ queue, failed, retryQueue, finished, dead, processing, cooldown = 5000, maxRetries = 3 } = {}) {
33
35
  this.#enqueued = queue || new Queue();
34
36
  this.#failed = failed || new SortedCollection([], { sortBy: FAILED_SORT_BY });
35
37
  this.#retryQueue = retryQueue || new Queue();
@@ -37,6 +39,7 @@ class JobRegistryInstance {
37
39
  this.#dead = dead || new IdentifyableCollection();
38
40
  this.#processing = processing || new IdentifyableCollection();
39
41
  this.#cooldown = cooldown;
42
+ this.#maxRetries = maxRetries;
40
43
  }
41
44
 
42
45
  /**
@@ -59,7 +62,7 @@ class JobRegistryInstance {
59
62
  fail(job) {
60
63
  if (!job) return;
61
64
  this.#processing.remove(job.id);
62
- if (job.exhausted()) {
65
+ if (job.exhausted(this.#maxRetries)) {
63
66
  this.#dead.push(job);
64
67
  } else {
65
68
  job.applyCooldown(this.#cooldown);
@@ -130,6 +133,32 @@ class JobRegistryInstance {
130
133
  return this.#enqueued.hasAny() || this.#retryQueue.hasAny();
131
134
  }
132
135
 
136
+ /**
137
+ * Returns all jobs in the collection that corresponds to the given status.
138
+ * @param {string} status - The status name ('enqueued', 'processing', 'failed', 'retryQueue', 'finished', 'dead').
139
+ * @returns {object[]} Array of raw job instances, or an empty array for unknown status.
140
+ */
141
+ jobsByStatus(status) {
142
+ const collection = this.#collectionByStatus(status);
143
+ if (!collection) return [];
144
+ return collection.list();
145
+ }
146
+
147
+ /**
148
+ * Searches all collections and returns the job and its status for the given ID.
149
+ * @param {string} id - The ID of the job to look up.
150
+ * @returns {{ job: object, status: string } | null} The raw job and its status, or null if not found.
151
+ */
152
+ jobById(id) {
153
+ for (const status of Object.keys(this.#collectionsByStatus())) {
154
+ const job = this.#collectionByStatus(status).findById(id);
155
+ if (job) {
156
+ return { job, status };
157
+ }
158
+ }
159
+ return null;
160
+ }
161
+
133
162
  /**
134
163
  * Returns counts of jobs in each state.
135
164
  * @returns {{ enqueued: number, processing: number, failed: number, retryQueue: number, finished: number, dead: number }} Counts of jobs in each state.
@@ -144,6 +173,30 @@ class JobRegistryInstance {
144
173
  dead: this.#dead.size(),
145
174
  };
146
175
  }
176
+
177
+ /**
178
+ * Returns the collection corresponding to the given status, or undefined.
179
+ * @param {string} status - The status name.
180
+ * @returns {object|undefined} The collection for the given status, or undefined if unknown.
181
+ */
182
+ #collectionByStatus(status) {
183
+ return this.#collectionsByStatus()[status];
184
+ }
185
+
186
+ /**
187
+ * Returns an object mapping each status name to its corresponding collection.
188
+ * @returns {object} A map of status names to collection instances.
189
+ */
190
+ #collectionsByStatus() {
191
+ return {
192
+ enqueued: this.#enqueued,
193
+ processing: this.#processing,
194
+ failed: this.#failed,
195
+ retryQueue: this.#retryQueue,
196
+ finished: this.#finished,
197
+ dead: this.#dead,
198
+ };
199
+ }
147
200
  }
148
201
 
149
202
  export { JobRegistryInstance };
@@ -0,0 +1,26 @@
1
+ import { Serializer } from './Serializer.js';
2
+
3
+ /**
4
+ * Serializes one or more job instances into index-view plain data objects.
5
+ * @augments Serializer
6
+ */
7
+ class JobIndexSerializer extends Serializer {
8
+ /**
9
+ * Serializes a single job instance into a `{ id, status, attempts, jobClass }` object.
10
+ *
11
+ * @param {object} job - A job instance.
12
+ * @param {object} options - Serialization options.
13
+ * @param {string} options.status - The status label to embed in the serialized object.
14
+ * @returns {{ id: string, status: string, attempts: number, jobClass: string }} The serialized job data.
15
+ */
16
+ static _serializeObject(job, { status }) {
17
+ return {
18
+ id: job.id,
19
+ status,
20
+ attempts: job._attempts,
21
+ jobClass: job.constructor.name,
22
+ };
23
+ }
24
+ }
25
+
26
+ export { JobIndexSerializer };
@@ -0,0 +1,26 @@
1
+ import { JobIndexSerializer } from './JobIndexSerializer.js';
2
+ import { JobShowSerializer } from './JobShowSerializer.js';
3
+
4
+ /**
5
+ * Dispatcher serializer for job instances.
6
+ * Delegates to {@link JobIndexSerializer} or {@link JobShowSerializer} based on the
7
+ * `view` option.
8
+ */
9
+ class JobSerializer {
10
+ /**
11
+ * Serializes a job or array of jobs using the appropriate serializer for the given view.
12
+ *
13
+ * @param {object|object[]} jobOrList - A single job instance or array of job instances.
14
+ * @param {object} options - Serialization options.
15
+ * @param {string} options.status - The status label to embed in the serialized object.
16
+ * @param {'index'|'show'} [options.view='index'] - The view type to serialize for.
17
+ * @returns {object|object[]} The serialized job data.
18
+ */
19
+ static serialize(jobOrList, { status, view = 'index' } = {}) {
20
+ const serializer = view === 'show' ? JobShowSerializer : JobIndexSerializer;
21
+ return serializer.serialize(jobOrList, { status });
22
+ }
23
+ }
24
+
25
+ export { JobSerializer };
26
+
@@ -0,0 +1,29 @@
1
+ import { Serializer } from './Serializer.js';
2
+
3
+ /**
4
+ * Serializes one or more job instances into show-view plain data objects.
5
+ * @augments Serializer
6
+ */
7
+ class JobShowSerializer extends Serializer {
8
+ /**
9
+ * Serializes a single job instance into a full details object for the show view.
10
+ *
11
+ * @param {object} job - A job instance.
12
+ * @param {object} options - Serialization options.
13
+ * @param {string} options.status - The status label to embed in the serialized object.
14
+ * @returns {{ id: string, status: string, attempts: number, jobClass: string, arguments: object, remainingAttempts: number, readyInMs: number }} The serialized job data.
15
+ */
16
+ static _serializeObject(job, { status }) {
17
+ return {
18
+ id: job.id,
19
+ status,
20
+ attempts: job._attempts,
21
+ jobClass: job.constructor.name,
22
+ arguments: job.arguments,
23
+ remainingAttempts: Math.max(0, job.maxRetries - job._attempts),
24
+ readyInMs: Math.max(0, job.readyBy - Date.now()),
25
+ };
26
+ }
27
+ }
28
+
29
+ export { JobShowSerializer };
@@ -0,0 +1,36 @@
1
+ /**
2
+ * Abstract base class for serializers.
3
+ * Subclasses must implement the `_serializeObject` static method.
4
+ */
5
+ class Serializer {
6
+ /**
7
+ * Serializes an item or a list of items using `_serializeObject`.
8
+ *
9
+ * When `itemOrList` is an array, each element is serialized individually.
10
+ * When it is a single object, delegates to `_serializeObject`.
11
+ *
12
+ * @param {object|object[]} itemOrList - A single item or array of items.
13
+ * @param {object} options - Options forwarded to `_serializeObject`.
14
+ * @returns {object|object[]} The serialized item, or an array of serialized items.
15
+ */
16
+ static serialize(itemOrList, options) {
17
+ if (Array.isArray(itemOrList)) {
18
+ return itemOrList.map(item => this.serialize(item, options));
19
+ }
20
+ return this._serializeObject(itemOrList, options);
21
+ }
22
+
23
+ /**
24
+ * Serializes a single item into a plain object.
25
+ * Must be overridden by subclasses.
26
+ * @param {object} _item - The item to serialize.
27
+ * @param {object} _options - Serialization options.
28
+ * @throws {Error} Always throws — subclasses must implement this method.
29
+ * @returns {object} The serialized plain object.
30
+ */
31
+ static _serializeObject(_item, _options) {
32
+ throw new Error('Subclasses must implement _serializeObject');
33
+ }
34
+ }
35
+
36
+ export { Serializer };
@@ -0,0 +1,50 @@
1
+ import path from 'path';
2
+ import { fileURLToPath } from 'url';
3
+ import { PathValidator } from './PathValidator.js';
4
+ import { RequestHandler } from './RequestHandler.js';
5
+
6
+ const __dirname = path.dirname(fileURLToPath(import.meta.url));
7
+
8
+ /**
9
+ * Handles GET /assets/* requests by serving static asset files.
10
+ * Rejects path traversal attempts with 403 Forbidden.
11
+ * @author darthjee
12
+ */
13
+ class AssetsRequestHandler extends RequestHandler {
14
+ /**
15
+ * Creates a new AssetsRequestHandler instance.
16
+ */
17
+ constructor() {
18
+ super();
19
+ this.assetsDir = path.resolve(__dirname, '../../static/assets');
20
+ this.validator = new PathValidator(this.assetsDir);
21
+ }
22
+
23
+ /**
24
+ * Serves the requested asset file, rejecting any path traversal attempt.
25
+ * @param {object} req - The Express request object.
26
+ * @param {object} res - The Express response object.
27
+ * @returns {void}
28
+ */
29
+ handle(req, res) {
30
+ const assetPath = [].concat(req.params.path).join(path.sep);
31
+ const resolved = this.#resolveAssetPath(assetPath);
32
+ res.sendFile(resolved);
33
+ }
34
+
35
+ /**
36
+ * Resolves and validates the asset path, throwing if a traversal attempt is detected.
37
+ * @param {string} assetPath - The raw asset path from the request.
38
+ * @returns {string} The resolved absolute path.
39
+ * @throws {ForbiddenError} If the path attempts to escape the assets directory.
40
+ */
41
+ #resolveAssetPath(assetPath) {
42
+ const resolved = path.resolve(this.assetsDir, assetPath);
43
+
44
+ this.validator.validate(resolved);
45
+
46
+ return resolved;
47
+ }
48
+ }
49
+
50
+ export { AssetsRequestHandler };
@@ -0,0 +1,31 @@
1
+ import path from 'path';
2
+ import { fileURLToPath } from 'url';
3
+ import { RequestHandler } from './RequestHandler.js';
4
+
5
+ const __dirname = path.dirname(fileURLToPath(import.meta.url));
6
+ const staticDir = path.resolve(__dirname, '../../static');
7
+
8
+ /**
9
+ * Handles GET / and SPA catch-all requests by serving index.html.
10
+ * @author darthjee
11
+ */
12
+ class IndexRequestHandler extends RequestHandler {
13
+ /**
14
+ * Creates a new IndexRequestHandler instance.
15
+ */
16
+ constructor() {
17
+ super();
18
+ }
19
+
20
+ /**
21
+ * Responds by sending source/static/index.html.
22
+ * @param {object} _req - The Express request object.
23
+ * @param {object} res - The Express response object.
24
+ * @returns {void}
25
+ */
26
+ handle(_req, res) {
27
+ res.sendFile(path.join(staticDir, 'index.html'));
28
+ }
29
+ }
30
+
31
+ export { IndexRequestHandler };
@@ -0,0 +1,33 @@
1
+ import { RequestHandler } from './RequestHandler.js';
2
+ import { NotFoundError } from '../exceptions/NotFoundError.js';
3
+ import { JobRegistry } from '../registry/JobRegistry.js';
4
+ import { JobSerializer } from '../serializers/JobSerializer.js';
5
+
6
+ /**
7
+ * Handles GET /job/:id.json requests.
8
+ * @author darthjee
9
+ */
10
+ class JobRequestHandler extends RequestHandler {
11
+ /**
12
+ * Creates a new JobRequestHandler instance.
13
+ */
14
+ constructor() {
15
+ super();
16
+ }
17
+
18
+ /**
19
+ * Responds with the job details for the given ID, or 404 if not found.
20
+ * @param {object} req - The Express request object.
21
+ * @param {object} res - The Express response object.
22
+ * @returns {void}
23
+ */
24
+ handle(req, res) {
25
+ const result = JobRegistry.jobById(req.params.id);
26
+ if (!result) {
27
+ throw new NotFoundError('Job not found');
28
+ }
29
+ res.json(JobSerializer.serialize(result.job, { status: result.status, view: 'show' }));
30
+ }
31
+ }
32
+
33
+ export { JobRequestHandler };
@@ -0,0 +1,30 @@
1
+ import { RequestHandler } from './RequestHandler.js';
2
+ import { JobRegistry } from '../registry/JobRegistry.js';
3
+ import { JobSerializer } from '../serializers/JobSerializer.js';
4
+
5
+ /**
6
+ * Handles GET /jobs/:status.json requests.
7
+ * @author darthjee
8
+ */
9
+ class JobsRequestHandler extends RequestHandler {
10
+ /**
11
+ * Creates a new JobsRequestHandler instance.
12
+ */
13
+ constructor() {
14
+ super();
15
+ }
16
+
17
+ /**
18
+ * Responds with the list of jobs in the given status queue.
19
+ * @param {object} req - The Express request object.
20
+ * @param {object} res - The Express response object.
21
+ * @returns {void}
22
+ */
23
+ handle(req, res) {
24
+ const { status } = req.params;
25
+ const jobs = JobRegistry.jobsByStatus(status);
26
+ res.json(JobSerializer.serialize(jobs, { status, view: 'index' }));
27
+ }
28
+ }
29
+
30
+ export { JobsRequestHandler };
@@ -0,0 +1,41 @@
1
+ import path from 'path';
2
+ import { ForbiddenError } from '../exceptions/ForbiddenError.js';
3
+
4
+ /**
5
+ * Validates that a resolved file path stays within an allowed base directory,
6
+ * preventing path traversal attacks.
7
+ * @author darthjee
8
+ */
9
+ class PathValidator {
10
+ #baseDir;
11
+
12
+ /**
13
+ * @param {string} baseDir - The absolute path of the allowed base directory.
14
+ */
15
+ constructor(baseDir) {
16
+ this.#baseDir = baseDir;
17
+ }
18
+
19
+ /**
20
+ * Returns true if the given resolved path is safely inside the base directory.
21
+ * @param {string} resolvedPath - The already-resolved absolute path to validate.
22
+ * @returns {boolean} Whether the path is within the allowed base directory.
23
+ */
24
+ isValid(resolvedPath) {
25
+ return resolvedPath.startsWith(this.#baseDir + path.sep);
26
+ }
27
+
28
+ /**
29
+ * Throws a ForbiddenError if the given resolved path escapes the base directory.
30
+ * @param {string} resolvedPath - The already-resolved absolute path to validate.
31
+ * @returns {void}
32
+ * @throws {ForbiddenError} If the path attempts to escape the base directory.
33
+ */
34
+ validate(resolvedPath) {
35
+ if (!this.isValid(resolvedPath)) {
36
+ throw new ForbiddenError();
37
+ }
38
+ }
39
+ }
40
+
41
+ export { PathValidator };
@@ -1,3 +1,6 @@
1
+ import { ForbiddenError } from '../exceptions/ForbiddenError.js';
2
+ import { NotFoundError } from '../exceptions/NotFoundError.js';
3
+
1
4
  /**
2
5
  * Registers a route on an Express router by binding the handler's handle method.
3
6
  * @author darthjee
@@ -14,13 +17,36 @@ class RouteRegister {
14
17
 
15
18
  /**
16
19
  * Registers a GET route on the router.
20
+ * Catches ForbiddenError → 403, NotFoundError → 404, and any other error → 500.
17
21
  * @param {object} params - Options for registering a route.
18
22
  * @param {string} params.route - The route path (e.g. '/stats.json').
19
23
  * @param {object} params.handler - The handler whose handle method is called.
20
24
  * @returns {void}
21
25
  */
22
26
  register({ route, handler }) {
23
- this.#router.get(route, (req, res) => handler.handle(req, res));
27
+ this.#router.get(route, (req, res) => {
28
+ try {
29
+ handler.handle(req, res);
30
+ } catch (e) {
31
+ this.#handleError(e, res);
32
+ }
33
+ });
34
+ }
35
+
36
+ /**
37
+ * Maps a caught exception to an HTTP error response.
38
+ * @param {Error} e - The caught exception.
39
+ * @param {object} res - The Express response object.
40
+ * @returns {void}
41
+ */
42
+ #handleError(e, res) {
43
+ if (e instanceof ForbiddenError) {
44
+ res.status(403).json({ error: 'Forbidden' });
45
+ } else if (e instanceof NotFoundError) {
46
+ res.status(404).json({ error: e.message });
47
+ } else {
48
+ res.status(500).json({ error: 'Internal Server Error' });
49
+ }
24
50
  }
25
51
  }
26
52
 
@@ -1,12 +1,16 @@
1
1
  import path from 'path';
2
2
  import { fileURLToPath } from 'url';
3
3
  import express from 'express';
4
+ import { AssetsRequestHandler } from './AssetsRequestHandler.js';
5
+ import { IndexRequestHandler } from './IndexRequestHandler.js';
6
+ import { JobRequestHandler } from './JobRequestHandler.js';
7
+ import { JobsRequestHandler } from './JobsRequestHandler.js';
4
8
  import { RouteRegister } from './RouteRegister.js';
5
9
  import { StatsRequestHandler } from './StatsRequestHandler.js';
6
10
 
7
11
  const { Router: ExpressRouter } = express;
8
12
  const __dirname = path.dirname(fileURLToPath(import.meta.url));
9
- const publicDir = path.join(__dirname, '../../public');
13
+ const staticDir = path.join(__dirname, '../../static');
10
14
 
11
15
  /**
12
16
  * Builds the Express router with all application routes.
@@ -31,10 +35,30 @@ class Router {
31
35
  handler: new StatsRequestHandler(),
32
36
  });
33
37
 
34
- router.use(express.static(publicDir));
38
+ register.register({
39
+ route: '/jobs/:status.json',
40
+ handler: new JobsRequestHandler(),
41
+ });
42
+
43
+ register.register({
44
+ route: '/job/:id.json',
45
+ handler: new JobRequestHandler(),
46
+ });
47
+
48
+ register.register({
49
+ route: '/',
50
+ handler: new IndexRequestHandler(),
51
+ });
52
+
53
+ register.register({
54
+ route: '/assets/*path',
55
+ handler: new AssetsRequestHandler(),
56
+ });
57
+
58
+ router.use(express.static(staticDir));
35
59
 
36
60
  router.use((_req, res) => {
37
- res.sendFile(path.join(publicDir, 'index.html'));
61
+ new IndexRequestHandler().handle(_req, res);
38
62
  });
39
63
 
40
64
  return router;
@@ -2,10 +2,14 @@ import { Engine } from './Engine.js';
2
2
  import { ConfigurationFileNotProvided } from '../exceptions/ConfigurationFileNotProvided.js';
3
3
  import { JobFactory } from '../factories/JobFactory.js';
4
4
  import { ActionProcessingJob } from '../models/ActionProcessingJob.js';
5
+ import { AssetDownloadJob } from '../models/AssetDownloadJob.js';
5
6
  import { Config } from '../models/Config.js';
7
+ import { HtmlParseJob } from '../models/HtmlParseJob.js';
6
8
  import { JobRegistry } from '../registry/JobRegistry.js';
7
9
  import { WorkersRegistry } from '../registry/WorkersRegistry.js';
8
10
  import { WebServer } from '../server/WebServer.js';
11
+ import { BufferedLogger } from '../utils/logging/BufferedLogger.js';
12
+ import { Logger } from '../utils/logging/Logger.js';
9
13
  import { PromiseAggregator } from '../utils/PromiseAggregator.js';
10
14
  import { ResourceRequestCollector } from '../utils/ResourceRequestCollector.js';
11
15
 
@@ -16,6 +20,7 @@ import { ResourceRequestCollector } from '../utils/ResourceRequestCollector.js';
16
20
  */
17
21
  class Application {
18
22
  #workers;
23
+ #bufferedLogger;
19
24
 
20
25
  /**
21
26
  * Creates a new Application instance.
@@ -40,6 +45,8 @@ class Application {
40
45
 
41
46
  // Load the configuration from the specified path.
42
47
  this.config = Config.fromFile(configPath);
48
+ this.#bufferedLogger = new BufferedLogger(undefined, this.config.logConfig.size);
49
+ Logger.addLogger(this.#bufferedLogger);
43
50
  this.#initRegistries();
44
51
  }
45
52
 
@@ -65,7 +72,7 @@ class Application {
65
72
  * @returns {Engine} The created Engine instance.
66
73
  */
67
74
  buildEngine() {
68
- return new Engine();
75
+ return new Engine({ sleepMs: this.config.workersConfig.sleep });
69
76
  }
70
77
 
71
78
  /**
@@ -78,6 +85,14 @@ class Application {
78
85
  });
79
86
  }
80
87
 
88
+ /**
89
+ * Gets the buffered logger instance created during config loading.
90
+ * @returns {BufferedLogger} The buffered logger instance.
91
+ */
92
+ get bufferedLogger() {
93
+ return this.#bufferedLogger;
94
+ }
95
+
81
96
  /**
82
97
  * Enqueues all parameter-free ResourceRequests into the job registry.
83
98
  * These are requests whose URLs contain no {:placeholder} tokens and can be
@@ -97,8 +112,10 @@ class Application {
97
112
  #initRegistries() {
98
113
  JobFactory.build('ResourceRequestJob', { attributes: { clients: this.config.clientRegistry } });
99
114
  JobFactory.build('Action', { klass: ActionProcessingJob });
115
+ JobFactory.build('HtmlParse', { klass: HtmlParseJob, attributes: { jobRegistry: JobRegistry, clientRegistry: this.config.clientRegistry } });
116
+ JobFactory.build('AssetDownload', { klass: AssetDownloadJob, attributes: { clientRegistry: this.config.clientRegistry } });
100
117
 
101
- JobRegistry.build({ cooldown: this.config.workersConfig.retryCooldown });
118
+ JobRegistry.build({ cooldown: this.config.workersConfig.retryCooldown, maxRetries: this.config.workersConfig.maxRetries });
102
119
 
103
120
  WorkersRegistry.build({
104
121
  workers: this.#workers,