navi-hey 0.5.2 → 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -0
- package/bin/navi.js +3 -3
- package/lib/{factories → background}/JobFactory.js +2 -2
- package/lib/{registry → background}/JobRegistry.js +18 -1
- package/lib/{registry → background}/JobRegistryInstance.js +51 -6
- package/lib/{models → background}/Worker.js +5 -3
- package/lib/{factories → background}/WorkerFactory.js +2 -2
- package/lib/{registry → background}/WorkersRegistryInstance.js +1 -1
- package/lib/{models → enqueuers}/ActionEnqueuer.js +4 -1
- package/lib/{models → enqueuers}/AssetRequestEnqueuer.js +6 -2
- package/lib/exceptions/ConflictError.js +14 -0
- package/lib/{models → jobs}/ActionProcessingJob.js +4 -4
- package/lib/{models → jobs}/AssetDownloadJob.js +6 -6
- package/lib/{models → jobs}/HtmlParseJob.js +6 -6
- package/lib/{models → jobs}/ResourceRequestJob.js +11 -11
- package/lib/models/Config.js +4 -2
- package/lib/models/FailureConfig.js +36 -0
- package/lib/models/ResourceRequest.js +5 -2
- package/lib/models/ResourceRequestAction.js +6 -3
- package/lib/models/WebConfig.js +5 -1
- package/lib/registry/ClientRegistry.js +60 -2
- package/lib/registry/LogRegistry.js +146 -0
- package/lib/registry/LogRegistryInstance.js +156 -0
- package/lib/serializers/BaseUrlsSerializer.js +18 -0
- package/lib/serializers/JobIndexSerializer.js +10 -3
- package/lib/serializers/LogSerializer.js +26 -0
- package/lib/server/BaseUrlsRequestHandler.js +33 -0
- package/lib/server/EngineContinueRequestHandler.js +34 -0
- package/lib/server/EnginePauseRequestHandler.js +34 -0
- package/lib/server/EngineRestartRequestHandler.js +34 -0
- package/lib/server/EngineShutdownRequestHandler.js +29 -0
- package/lib/server/EngineStartRequestHandler.js +34 -0
- package/lib/server/EngineStatusRequestHandler.js +27 -0
- package/lib/server/EngineStopRequestHandler.js +34 -0
- package/lib/server/JobLogsRequestHandler.js +39 -0
- package/lib/server/JobRequestHandler.js +1 -1
- package/lib/server/JobRetryRequestHandler.js +40 -0
- package/lib/server/JobsFilter.js +55 -0
- package/lib/server/JobsRequestHandler.js +5 -2
- package/lib/server/LogsRequestHandler.js +55 -0
- package/lib/server/RouteRegister.js +40 -7
- package/lib/server/Router.js +44 -20
- package/lib/server/SettingsRequestHandler.js +34 -0
- package/lib/server/StatsRequestHandler.js +2 -2
- package/lib/server/WebServer.js +28 -8
- package/lib/services/Application.js +135 -81
- package/lib/services/ApplicationInstance.js +272 -0
- package/lib/services/Client.js +27 -22
- package/lib/services/ConfigLoader.js +2 -1
- package/lib/services/ConfigParser.js +13 -2
- package/lib/services/Engine.js +51 -6
- package/lib/services/EngineEvents.js +42 -0
- package/lib/services/FailureChecker.js +78 -0
- package/lib/services/WorkersAllocator.js +2 -2
- package/lib/utils/EnvResolver.js +22 -0
- package/lib/utils/HtmlElementParser.js +5 -4
- package/lib/utils/HtmlParser.js +4 -4
- package/lib/utils/PromiseAggregator.js +11 -5
- package/lib/utils/logging/BufferedLogger.js +8 -0
- package/lib/utils/logging/LogBuffer.js +25 -5
- package/lib/utils/logging/LogBufferCollection.js +56 -0
- package/lib/utils/logging/LogContext.js +51 -0
- package/lib/utils/logging/LogFilter.js +34 -0
- package/package.json +1 -1
- package/static/assets/index-BfggoMvd.js +11 -0
- package/static/assets/{index-DBkz-poX.css → index-CuWjW0sT.css} +1 -1
- package/static/index.html +2 -2
- package/static/assets/index-BF8_ow13.js +0 -11
- /package/lib/{models → background}/Job.js +0 -0
- /package/lib/{registry → background}/WorkersRegistry.js +0 -0
- /package/lib/{models → enqueuers}/ActionsEnqueuer.js +0 -0
- /package/lib/{factories → factory}/Factory.js +0 -0
package/README.md
CHANGED
|
@@ -62,6 +62,9 @@ workers:
|
|
|
62
62
|
log:
|
|
63
63
|
size: 100 # max number of log entries kept in memory (default: 100)
|
|
64
64
|
|
|
65
|
+
failure:
|
|
66
|
+
threshold: 10.0 # optional: exit with failure if > 10% of jobs are dead
|
|
67
|
+
|
|
65
68
|
web:
|
|
66
69
|
port: 3000 # port for the monitoring web UI (omit to disable)
|
|
67
70
|
|
|
@@ -113,6 +116,7 @@ resources:
|
|
|
113
116
|
| `workers.sleep` | Milliseconds the engine waits between allocation ticks. Defaults to `500`. |
|
|
114
117
|
| `workers.max-retries` | Maximum number of times a job is retried before being moved to the dead queue. Defaults to `3`. |
|
|
115
118
|
| `log.size` | Maximum number of log entries kept in the in-memory log buffer. Defaults to `100`. |
|
|
119
|
+
| `failure.threshold` | Optional. Percentage (0–100) of dead jobs that triggers a non-zero exit code. When absent, Navi always exits successfully. |
|
|
116
120
|
| `web.port` | Port for the local monitoring web UI. Omit the `web` key entirely to run Navi without the web server. The web UI frontend is bundled with the package and served from `source/static/`. After frontend code changes, run `yarn build` in the `navi_frontend` service to update the bundled assets. |
|
|
117
121
|
|
|
118
122
|
When the web server is enabled, the following screens are available:
|
package/bin/navi.js
CHANGED
|
@@ -4,6 +4,6 @@ import { Application } from '../lib/services/Application.js';
|
|
|
4
4
|
import { ArgumentsParser } from '../lib/services/ArgumentsParser.js';
|
|
5
5
|
|
|
6
6
|
const { config } = ArgumentsParser.parse(process.argv.slice(2));
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
7
|
+
Application.build();
|
|
8
|
+
Application.loadConfig(config);
|
|
9
|
+
Application.run();
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { Factory } from '
|
|
2
|
-
import { ResourceRequestJob } from '../
|
|
1
|
+
import { Factory } from '../factory/Factory.js';
|
|
2
|
+
import { ResourceRequestJob } from '../jobs/ResourceRequestJob.js';
|
|
3
3
|
import { IdGenerator } from '../utils/generators/IdGenerator.js';
|
|
4
4
|
|
|
5
5
|
/**
|
|
@@ -32,6 +32,14 @@ class JobRegistry {
|
|
|
32
32
|
JobRegistry.#instance = null;
|
|
33
33
|
}
|
|
34
34
|
|
|
35
|
+
/**
|
|
36
|
+
* Clears all pending queues (enqueued, retryQueue, and failed).
|
|
37
|
+
* @returns {void}
|
|
38
|
+
*/
|
|
39
|
+
static clearQueues() {
|
|
40
|
+
return JobRegistry.#getInstance().clearQueues();
|
|
41
|
+
}
|
|
42
|
+
|
|
35
43
|
/**
|
|
36
44
|
* Enqueues a new job using the factory registered under the given key.
|
|
37
45
|
* @param {string} factoryKey - The factory key to use.
|
|
@@ -77,6 +85,15 @@ class JobRegistry {
|
|
|
77
85
|
return JobRegistry.#getInstance().requeue(job);
|
|
78
86
|
}
|
|
79
87
|
|
|
88
|
+
/**
|
|
89
|
+
* Moves a job directly to the retry queue, removing it from the failed or dead collection.
|
|
90
|
+
* @param {string} id - The ID of the job to retry.
|
|
91
|
+
* @returns {Job|null} The job if found and moved, or null if not in a retryable state.
|
|
92
|
+
*/
|
|
93
|
+
static retryJob(id) {
|
|
94
|
+
return JobRegistry.#getInstance().retryJob(id);
|
|
95
|
+
}
|
|
96
|
+
|
|
80
97
|
/**
|
|
81
98
|
* Promotes cooling-down failed jobs that are ready to retry.
|
|
82
99
|
* @returns {void}
|
|
@@ -121,7 +138,7 @@ class JobRegistry {
|
|
|
121
138
|
|
|
122
139
|
/**
|
|
123
140
|
* Returns counts of jobs in each state.
|
|
124
|
-
* @returns {{ enqueued: number, processing: number, failed: number, retryQueue: number, finished: number, dead: number }} Job counts per state.
|
|
141
|
+
* @returns {{ enqueued: number, processing: number, failed: number, retryQueue: number, finished: number, dead: number, total: number }} Job counts per state, plus total (dead + finished).
|
|
125
142
|
*/
|
|
126
143
|
static stats() {
|
|
127
144
|
return JobRegistry.#getInstance().stats();
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { JobFactory } from '
|
|
1
|
+
import { JobFactory } from './JobFactory.js';
|
|
2
2
|
import { IdentifyableCollection } from '../utils/collections/IdentifyableCollection.js';
|
|
3
3
|
import { Queue } from '../utils/collections/Queue.js';
|
|
4
4
|
import { SortedCollection } from '../utils/collections/SortedCollection.js';
|
|
@@ -104,6 +104,34 @@ class JobRegistryInstance {
|
|
|
104
104
|
this.#enqueued.push(job);
|
|
105
105
|
}
|
|
106
106
|
|
|
107
|
+
/**
|
|
108
|
+
* Moves a job directly to the retry queue, removing it from the failed or dead collection.
|
|
109
|
+
* Returns the job if found and moved, or null if the job is not in a retryable state.
|
|
110
|
+
* @param {string} id - The ID of the job to retry.
|
|
111
|
+
* @returns {Job|null} The job instance, or null if not found in failed or dead.
|
|
112
|
+
*/
|
|
113
|
+
retryJob(id) {
|
|
114
|
+
const allFailed = this.#failed.list();
|
|
115
|
+
const failedJob = allFailed.find(j => j.id === id);
|
|
116
|
+
if (failedJob) {
|
|
117
|
+
this.#failed = new SortedCollection(
|
|
118
|
+
allFailed.filter(j => j.id !== id),
|
|
119
|
+
{ sortBy: FAILED_SORT_BY }
|
|
120
|
+
);
|
|
121
|
+
this.#retryQueue.push(failedJob);
|
|
122
|
+
return failedJob;
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
const deadJob = this.#dead.get(id);
|
|
126
|
+
if (deadJob) {
|
|
127
|
+
this.#dead.remove(id);
|
|
128
|
+
this.#retryQueue.push(deadJob);
|
|
129
|
+
return deadJob;
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
return null;
|
|
133
|
+
}
|
|
134
|
+
|
|
107
135
|
/**
|
|
108
136
|
* Promotes jobs from the failed queue to the retryQueue once their cooldown has elapsed.
|
|
109
137
|
* @returns {void}
|
|
@@ -117,6 +145,19 @@ class JobRegistryInstance {
|
|
|
117
145
|
ready.forEach(job => this.#retryQueue.push(job));
|
|
118
146
|
}
|
|
119
147
|
|
|
148
|
+
/**
|
|
149
|
+
* Clears all queues (enqueued, retryQueue, failed, finished, and dead), leaving
|
|
150
|
+
* only the processing collection intact.
|
|
151
|
+
* @returns {void}
|
|
152
|
+
*/
|
|
153
|
+
clearQueues() {
|
|
154
|
+
this.#enqueued = new Queue();
|
|
155
|
+
this.#retryQueue = new Queue();
|
|
156
|
+
this.#failed = new SortedCollection([], { sortBy: FAILED_SORT_BY });
|
|
157
|
+
this.#finished = new IdentifyableCollection();
|
|
158
|
+
this.#dead = new IdentifyableCollection();
|
|
159
|
+
}
|
|
160
|
+
|
|
120
161
|
/**
|
|
121
162
|
* Returns whether the registry has any jobs pending (including those in cooldown).
|
|
122
163
|
* @returns {boolean} True if any of enqueued, failed, or retryQueue is non-empty.
|
|
@@ -161,16 +202,20 @@ class JobRegistryInstance {
|
|
|
161
202
|
|
|
162
203
|
/**
|
|
163
204
|
* Returns counts of jobs in each state.
|
|
164
|
-
* @returns {{ enqueued: number, processing: number, failed: number, retryQueue: number, finished: number, dead: number }} Counts of jobs in each state.
|
|
205
|
+
* @returns {{ enqueued: number, processing: number, failed: number, retryQueue: number, finished: number, dead: number, total: number }} Counts of jobs in each state, plus total (dead + finished).
|
|
165
206
|
*/
|
|
166
207
|
stats() {
|
|
208
|
+
const dead = this.#dead.size();
|
|
209
|
+
const finished = this.#finished.size();
|
|
210
|
+
|
|
167
211
|
return {
|
|
168
|
-
enqueued:
|
|
212
|
+
enqueued: this.#enqueued.size(),
|
|
169
213
|
processing: this.#processing.size(),
|
|
170
|
-
failed:
|
|
214
|
+
failed: this.#failed.size(),
|
|
171
215
|
retryQueue: this.#retryQueue.size(),
|
|
172
|
-
finished
|
|
173
|
-
dead
|
|
216
|
+
finished,
|
|
217
|
+
dead,
|
|
218
|
+
total: dead + finished,
|
|
174
219
|
};
|
|
175
220
|
}
|
|
176
221
|
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { LogContext } from '../utils/logging/LogContext.js';
|
|
2
2
|
|
|
3
3
|
/**
|
|
4
4
|
* Worker processes jobs pulled from a JobRegistry.
|
|
@@ -41,11 +41,13 @@ class Worker {
|
|
|
41
41
|
throw new Error('No job assigned to worker');
|
|
42
42
|
}
|
|
43
43
|
|
|
44
|
+
const logContext = new LogContext({ workerId: this.id, jobId: this.job.id });
|
|
45
|
+
|
|
44
46
|
try {
|
|
45
|
-
await this.job.perform();
|
|
47
|
+
await this.job.perform(logContext);
|
|
46
48
|
this.#jobRegistry.finish(this.job);
|
|
47
49
|
} catch (error) {
|
|
48
|
-
|
|
50
|
+
logContext.error(`Error occurred while performing job: #${this.job.id} - ${error}`);
|
|
49
51
|
this.#jobRegistry.fail(this.job);
|
|
50
52
|
} finally {
|
|
51
53
|
this.job = undefined;
|
|
@@ -1,4 +1,5 @@
|
|
|
1
|
-
import { JobRegistry as DefaultJobRegistry } from '../
|
|
1
|
+
import { JobRegistry as DefaultJobRegistry } from '../background/JobRegistry.js';
|
|
2
|
+
import { Application } from '../services/Application.js';
|
|
2
3
|
|
|
3
4
|
/**
|
|
4
5
|
* ActionEnqueuer enqueues one ActionProcessingJob per item for a single Action.
|
|
@@ -22,9 +23,11 @@ class ActionEnqueuer {
|
|
|
22
23
|
|
|
23
24
|
/**
|
|
24
25
|
* Enqueues one ActionProcessingJob per item for the configured action.
|
|
26
|
+
* Does nothing if the application is in 'stopped' status.
|
|
25
27
|
* @returns {void}
|
|
26
28
|
*/
|
|
27
29
|
enqueue() {
|
|
30
|
+
if (Application.isStopped()) return;
|
|
28
31
|
for (const item of this.#items) {
|
|
29
32
|
this.#jobRegistry.enqueue('Action', { action: this.#action, item });
|
|
30
33
|
}
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { Application } from '../services/Application.js';
|
|
1
2
|
import { HtmlParser } from '../utils/HtmlParser.js';
|
|
2
3
|
|
|
3
4
|
/**
|
|
@@ -28,10 +29,13 @@ class AssetRequestEnqueuer {
|
|
|
28
29
|
/**
|
|
29
30
|
* Parses the HTML body for URLs matching the asset request rule, resolves each URL
|
|
30
31
|
* to absolute form, and enqueues one AssetDownloadJob per URL.
|
|
32
|
+
* Does nothing if the application is in 'stopped' status.
|
|
33
|
+
* @param {LogContext} logContext Context carrying workerId/jobId for log entries.
|
|
31
34
|
* @returns {void}
|
|
32
35
|
*/
|
|
33
|
-
enqueue() {
|
|
34
|
-
|
|
36
|
+
enqueue(logContext) {
|
|
37
|
+
if (Application.isStopped()) return;
|
|
38
|
+
const urls = HtmlParser.parse(this.#rawHtml, this.#assetRequest.selector, this.#assetRequest.attribute, logContext);
|
|
35
39
|
for (const url of urls) {
|
|
36
40
|
this.#jobRegistry.enqueue('AssetDownload', {
|
|
37
41
|
url: this.#resolveUrl(url),
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
import { AppError } from './AppError.js';
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* ConflictError is thrown when a requested state transition is not valid
|
|
5
|
+
* given the current application state (e.g. pausing an already-paused engine).
|
|
6
|
+
* @author darthjee
|
|
7
|
+
*/
|
|
8
|
+
class ConflictError extends AppError {
|
|
9
|
+
constructor() {
|
|
10
|
+
super('Conflict');
|
|
11
|
+
}
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
export { ConflictError };
|
|
@@ -1,5 +1,4 @@
|
|
|
1
|
-
import { Job } from '
|
|
2
|
-
import { Logger } from '../utils/logging/Logger.js';
|
|
1
|
+
import { Job } from '../background/Job.js';
|
|
3
2
|
|
|
4
3
|
/**
|
|
5
4
|
* ActionProcessingJob is a Job that processes a single Action for a given item.
|
|
@@ -44,10 +43,11 @@ class ActionProcessingJob extends Job {
|
|
|
44
43
|
|
|
45
44
|
/**
|
|
46
45
|
* Performs the action for the given item.
|
|
46
|
+
* @param {LogContext} logContext - Context carrying workerId/jobId for log entries.
|
|
47
47
|
* @returns {Promise<void>}
|
|
48
48
|
*/
|
|
49
|
-
async perform() {
|
|
50
|
-
|
|
49
|
+
async perform(logContext) {
|
|
50
|
+
logContext.debug(`ActionProcessingJob #${this.id} performing`);
|
|
51
51
|
try {
|
|
52
52
|
this.lastError = undefined;
|
|
53
53
|
await this.#action.execute(this.#item);
|
|
@@ -1,5 +1,4 @@
|
|
|
1
|
-
import { Job } from '
|
|
2
|
-
import { Logger } from '../utils/logging/Logger.js';
|
|
1
|
+
import { Job } from '../background/Job.js';
|
|
3
2
|
|
|
4
3
|
/**
|
|
5
4
|
* AssetDownloadJob is a Job that fetches a single fully-resolved asset URL and validates
|
|
@@ -40,16 +39,17 @@ class AssetDownloadJob extends Job {
|
|
|
40
39
|
|
|
41
40
|
/**
|
|
42
41
|
* Fetches the asset URL using the named client and validates the HTTP status.
|
|
42
|
+
* @param {LogContext} logContext - Context carrying workerId/jobId for log entries.
|
|
43
43
|
* @returns {Promise<object>} The HTTP response.
|
|
44
44
|
*/
|
|
45
|
-
async perform() {
|
|
46
|
-
|
|
45
|
+
async perform(logContext) {
|
|
46
|
+
logContext.debug(`AssetDownloadJob #${this.id} performing`);
|
|
47
47
|
try {
|
|
48
48
|
this.lastError = undefined;
|
|
49
49
|
const client = this.#clientRegistry.getClient(this.#clientName);
|
|
50
|
-
return await client.performUrl(this.#url, this.#status);
|
|
50
|
+
return await client.performUrl(this.#url, this.#status, logContext);
|
|
51
51
|
} catch (error) {
|
|
52
|
-
|
|
52
|
+
logContext.error(`AssetDownloadJob #${this.id} failed: ${error}`);
|
|
53
53
|
this._fail(error);
|
|
54
54
|
}
|
|
55
55
|
}
|
|
@@ -1,6 +1,5 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
3
|
-
import { Logger } from '../utils/logging/Logger.js';
|
|
1
|
+
import { Job } from '../background/Job.js';
|
|
2
|
+
import { AssetRequestEnqueuer } from '../enqueuers/AssetRequestEnqueuer.js';
|
|
4
3
|
|
|
5
4
|
/**
|
|
6
5
|
* HtmlParseJob is a Job that parses an HTML response body, extracts asset URLs using
|
|
@@ -53,14 +52,15 @@ class HtmlParseJob extends Job {
|
|
|
53
52
|
|
|
54
53
|
/**
|
|
55
54
|
* Parses the HTML body, resolves asset URLs, and enqueues one AssetDownloadJob per URL.
|
|
55
|
+
* @param {LogContext} logContext - Context carrying workerId/jobId for log entries.
|
|
56
56
|
* @returns {Promise<void>}
|
|
57
57
|
*/
|
|
58
|
-
async perform() {
|
|
59
|
-
|
|
58
|
+
async perform(logContext) {
|
|
59
|
+
logContext.debug(`HtmlParseJob #${this.id} performing`);
|
|
60
60
|
try {
|
|
61
61
|
this.lastError = undefined;
|
|
62
62
|
for (const assetRequest of this.#assetRequests) {
|
|
63
|
-
new AssetRequestEnqueuer(this.#rawHtml, assetRequest, this.#jobRegistry, this.#clientRegistry).enqueue();
|
|
63
|
+
new AssetRequestEnqueuer(this.#rawHtml, assetRequest, this.#jobRegistry, this.#clientRegistry).enqueue(logContext);
|
|
64
64
|
}
|
|
65
65
|
} catch (error) {
|
|
66
66
|
this._fail(error);
|
|
@@ -1,7 +1,6 @@
|
|
|
1
|
-
import { Job } from '
|
|
2
|
-
import {
|
|
3
|
-
import {
|
|
4
|
-
import { Logger } from '../utils/logging/Logger.js';
|
|
1
|
+
import { Job } from '../background/Job.js';
|
|
2
|
+
import { JobRegistry } from '../background/JobRegistry.js';
|
|
3
|
+
import { ResponseWrapper } from '../models/ResponseWrapper.js';
|
|
5
4
|
|
|
6
5
|
/**
|
|
7
6
|
* ResourceRequestJob is a Job that performs an HTTP request for a ResourceRequest.
|
|
@@ -30,24 +29,26 @@ class ResourceRequestJob extends Job {
|
|
|
30
29
|
|
|
31
30
|
/**
|
|
32
31
|
* Returns the job-specific arguments for serialization.
|
|
33
|
-
*
|
|
32
|
+
* The URL is fully resolved with all parameter placeholders substituted.
|
|
33
|
+
* @returns {{ url: string }} The job arguments.
|
|
34
34
|
*/
|
|
35
35
|
get arguments() {
|
|
36
|
-
return { url: this.#resourceRequest.
|
|
36
|
+
return { url: this.#resourceRequest.resolveUrl(this.#parameters) };
|
|
37
37
|
}
|
|
38
38
|
|
|
39
39
|
/**
|
|
40
40
|
* Performs the HTTP request for the resource request.
|
|
41
|
+
* @param {LogContext} logContext - Context carrying workerId/jobId for log entries.
|
|
41
42
|
* @returns {Promise} A promise that resolves with the HTTP response.
|
|
42
43
|
*/
|
|
43
|
-
async perform() {
|
|
44
|
-
|
|
44
|
+
async perform(logContext) {
|
|
45
|
+
logContext.debug(`ResourceRequestJob #${this.id} performing`);
|
|
45
46
|
try {
|
|
46
47
|
this.lastError = undefined;
|
|
47
|
-
const response = await this.#getClient().perform(this.#resourceRequest, this.#parameters);
|
|
48
|
+
const response = await this.#getClient().perform(this.#resourceRequest, this.#parameters, logContext);
|
|
48
49
|
return this.#handleResponse(response);
|
|
49
50
|
} catch (error) {
|
|
50
|
-
|
|
51
|
+
logContext.error(`Job #${this.id} failed: ${error}`);
|
|
51
52
|
this._fail(error);
|
|
52
53
|
}
|
|
53
54
|
}
|
|
@@ -90,4 +91,3 @@ class ResourceRequestJob extends Job {
|
|
|
90
91
|
}
|
|
91
92
|
|
|
92
93
|
export { ResourceRequestJob };
|
|
93
|
-
|
package/lib/models/Config.js
CHANGED
|
@@ -16,13 +16,15 @@ class Config {
|
|
|
16
16
|
* @param {WorkersConfig} params.workersConfig - The configuration for worker instances.
|
|
17
17
|
* @param {WebConfig|null} [params.webConfig] - Optional web server configuration.
|
|
18
18
|
* @param {LogConfig} [params.logConfig] - Optional log configuration.
|
|
19
|
+
* @param {FailureConfig|null} [params.failureConfig] - Optional failure threshold configuration.
|
|
19
20
|
*/
|
|
20
|
-
constructor({ resources, clients, workersConfig, webConfig, logConfig }) {
|
|
21
|
+
constructor({ resources, clients, workersConfig, webConfig, logConfig, failureConfig }) {
|
|
21
22
|
this.resourceRegistry = ResourceRegistry.build(resources);
|
|
22
|
-
this.clientRegistry =
|
|
23
|
+
this.clientRegistry = ClientRegistry.build(clients);
|
|
23
24
|
this.workersConfig = workersConfig;
|
|
24
25
|
this.webConfig = webConfig ?? null;
|
|
25
26
|
this.logConfig = logConfig ?? new LogConfig();
|
|
27
|
+
this.failureConfig = failureConfig ?? null;
|
|
26
28
|
}
|
|
27
29
|
|
|
28
30
|
/**
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Represents the failure threshold configuration for the application.
|
|
3
|
+
* @author darthjee
|
|
4
|
+
*/
|
|
5
|
+
class FailureConfig {
|
|
6
|
+
#threshold;
|
|
7
|
+
|
|
8
|
+
/**
|
|
9
|
+
* Creates an instance of FailureConfig.
|
|
10
|
+
* @param {object} config - The configuration object.
|
|
11
|
+
* @param {number} config.threshold - The maximum acceptable percentage of dead jobs (0–100).
|
|
12
|
+
*/
|
|
13
|
+
constructor({ threshold }) {
|
|
14
|
+
this.#threshold = threshold;
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
/**
|
|
18
|
+
* Gets the failure threshold percentage.
|
|
19
|
+
* @returns {number} The threshold value.
|
|
20
|
+
*/
|
|
21
|
+
get threshold() {
|
|
22
|
+
return this.#threshold;
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
/**
|
|
26
|
+
* Creates a FailureConfig instance from a raw YAML object, or returns null if absent.
|
|
27
|
+
* @param {object|null|undefined} obj - The raw failure config object from YAML.
|
|
28
|
+
* @returns {FailureConfig|null} A new FailureConfig instance, or null if not configured.
|
|
29
|
+
*/
|
|
30
|
+
static fromObject(obj) {
|
|
31
|
+
if (!obj) return null;
|
|
32
|
+
return new FailureConfig(obj);
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
export { FailureConfig };
|
|
@@ -1,7 +1,8 @@
|
|
|
1
|
-
import { ActionsEnqueuer } from './ActionsEnqueuer.js';
|
|
2
1
|
import { AssetRequest } from './AssetRequest.js';
|
|
3
2
|
import { ResourceRequestAction } from './ResourceRequestAction.js';
|
|
4
|
-
import { JobRegistry as DefaultJobRegistry } from '../
|
|
3
|
+
import { JobRegistry as DefaultJobRegistry } from '../background/JobRegistry.js';
|
|
4
|
+
import { ActionsEnqueuer } from '../enqueuers/ActionsEnqueuer.js';
|
|
5
|
+
import { Application } from '../services/Application.js';
|
|
5
6
|
|
|
6
7
|
/**
|
|
7
8
|
* ResourceRequest represents a request to a specific URL with an expected status code.
|
|
@@ -50,12 +51,14 @@ class ResourceRequest {
|
|
|
50
51
|
|
|
51
52
|
/**
|
|
52
53
|
* Enqueues one HtmlParseJob for this resource request's asset extraction rules.
|
|
54
|
+
* Does nothing if the application is in 'stopped' status.
|
|
53
55
|
* @param {string} rawHtml The raw HTML response body string.
|
|
54
56
|
* @param {object} [jobRegistry=JobRegistry] The job registry used to enqueue the HtmlParseJob.
|
|
55
57
|
* @param {object} clientRegistry The client registry for URL resolution inside HtmlParseJob.
|
|
56
58
|
* @returns {void}
|
|
57
59
|
*/
|
|
58
60
|
enqueueAssets(rawHtml, jobRegistry = DefaultJobRegistry, clientRegistry) {
|
|
61
|
+
if (Application.isStopped()) return;
|
|
59
62
|
jobRegistry.enqueue('HtmlParse', { rawHtml, assetRequests: this.assets, clientRegistry });
|
|
60
63
|
}
|
|
61
64
|
|
|
@@ -1,8 +1,9 @@
|
|
|
1
1
|
import { ParametersMapper } from './ParametersMapper.js';
|
|
2
|
+
import { JobRegistry as DefaultJobRegistry } from '../background/JobRegistry.js';
|
|
2
3
|
import { MissingActionResource } from '../exceptions/MissingActionResource.js';
|
|
3
|
-
import {
|
|
4
|
+
import { LogRegistry } from '../registry/LogRegistry.js';
|
|
4
5
|
import { ResourceRegistry as DefaultResourceRegistry } from '../registry/ResourceRegistry.js';
|
|
5
|
-
import {
|
|
6
|
+
import { Application } from '../services/Application.js';
|
|
6
7
|
|
|
7
8
|
/**
|
|
8
9
|
* Represents a single action to execute after a successful resource request response.
|
|
@@ -39,11 +40,13 @@ class ResourceRequestAction {
|
|
|
39
40
|
/**
|
|
40
41
|
* Maps the response wrapper to variables, looks up the target resource, and
|
|
41
42
|
* enqueues one ResourceRequestJob per ResourceRequest in that resource.
|
|
43
|
+
* Does nothing if the application is in 'stopped' status.
|
|
42
44
|
* @param {ResponseWrapper} responseWrapper A ResponseWrapper instance exposing
|
|
43
45
|
* parsedBody and headers.
|
|
44
46
|
* @returns {void}
|
|
45
47
|
*/
|
|
46
48
|
execute(responseWrapper) {
|
|
49
|
+
if (Application.isStopped()) return;
|
|
47
50
|
const vars = this.#mapper.map(responseWrapper);
|
|
48
51
|
const resource = this.#resourceRegistry.getItem(this.resource);
|
|
49
52
|
|
|
@@ -63,7 +66,7 @@ class ResourceRequestAction {
|
|
|
63
66
|
try {
|
|
64
67
|
return [new ResourceRequestAction(attrs)];
|
|
65
68
|
} catch (error) {
|
|
66
|
-
|
|
69
|
+
LogRegistry.error(`Skipping action: ${error}`);
|
|
67
70
|
return [];
|
|
68
71
|
}
|
|
69
72
|
});
|
package/lib/models/WebConfig.js
CHANGED
|
@@ -6,9 +6,13 @@ class WebConfig {
|
|
|
6
6
|
/**
|
|
7
7
|
* @param {object} config - Configuration object.
|
|
8
8
|
* @param {number} config.port - The port number to listen on.
|
|
9
|
+
* @param {number} [config.logs_page_size=20] - Maximum number of log entries per page.
|
|
10
|
+
* @param {boolean} [config.enable_shutdown=true] - Whether the shutdown button is enabled.
|
|
9
11
|
*/
|
|
10
|
-
constructor({ port }) {
|
|
12
|
+
constructor({ port, logs_page_size: logsPageSize = 20, enable_shutdown: enableShutdown = true }) {
|
|
11
13
|
this.port = port;
|
|
14
|
+
this.logsPageSize = logsPageSize;
|
|
15
|
+
this.enableShutdown = enableShutdown;
|
|
12
16
|
}
|
|
13
17
|
}
|
|
14
18
|
|
|
@@ -2,8 +2,10 @@ import { NamedRegistry } from './NamedRegistry.js';
|
|
|
2
2
|
import { ClientNotFound } from '../exceptions/ClientNotFound.js';
|
|
3
3
|
|
|
4
4
|
/**
|
|
5
|
-
* ClientRegistry
|
|
6
|
-
*
|
|
5
|
+
* ClientRegistry manages client retrieval logic and exposes a static singleton facade.
|
|
6
|
+
*
|
|
7
|
+
* Call `ClientRegistry.build(clients)` once during application bootstrap.
|
|
8
|
+
* Use `ClientRegistry.reset()` in tests to restore a clean state between examples.
|
|
7
9
|
*
|
|
8
10
|
* @example
|
|
9
11
|
* const clientRegistry = new ClientRegistry({
|
|
@@ -38,6 +40,62 @@ class ClientRegistry extends NamedRegistry {
|
|
|
38
40
|
*/
|
|
39
41
|
static notFoundException = ClientNotFound;
|
|
40
42
|
|
|
43
|
+
static #instance = null;
|
|
44
|
+
|
|
45
|
+
/**
|
|
46
|
+
* Creates and stores the singleton instance.
|
|
47
|
+
* @param {object} clients - An object mapping client names to Client instances.
|
|
48
|
+
* @returns {ClientRegistry} The created instance.
|
|
49
|
+
* @throws {Error} If `build()` has already been called without a preceding `reset()`.
|
|
50
|
+
*/
|
|
51
|
+
static build(clients) {
|
|
52
|
+
if (ClientRegistry.#instance) {
|
|
53
|
+
throw new Error('ClientRegistry.build() has already been called. Call reset() first.');
|
|
54
|
+
}
|
|
55
|
+
ClientRegistry.#instance = new ClientRegistry(clients);
|
|
56
|
+
return ClientRegistry.#instance;
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
/**
|
|
60
|
+
* Destroys the singleton instance. Intended for test teardown.
|
|
61
|
+
* @returns {void}
|
|
62
|
+
*/
|
|
63
|
+
static reset() {
|
|
64
|
+
ClientRegistry.#instance = null;
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
/**
|
|
68
|
+
* Retrieves a client by name from the singleton instance.
|
|
69
|
+
* @param {string} [name] - The name of the client to retrieve.
|
|
70
|
+
* @returns {Client} The requested client instance.
|
|
71
|
+
* @throws {ClientNotFound} If the client with the specified name does not exist.
|
|
72
|
+
* @throws {Error} If `build()` has not been called.
|
|
73
|
+
*/
|
|
74
|
+
static getClient(name) {
|
|
75
|
+
return ClientRegistry.#getInstance().getClient(name);
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
/**
|
|
79
|
+
* Returns all registered client instances from the singleton.
|
|
80
|
+
* @returns {Client[]} Array of all registered Client instances.
|
|
81
|
+
* @throws {Error} If `build()` has not been called.
|
|
82
|
+
*/
|
|
83
|
+
static all() {
|
|
84
|
+
return Object.values(ClientRegistry.#getInstance().items);
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
/**
|
|
88
|
+
* Returns the singleton instance, throwing if not yet built.
|
|
89
|
+
* @returns {ClientRegistry} The singleton instance.
|
|
90
|
+
* @throws {Error} If `build()` has not been called.
|
|
91
|
+
*/
|
|
92
|
+
static #getInstance() {
|
|
93
|
+
if (!ClientRegistry.#instance) {
|
|
94
|
+
throw new Error('ClientRegistry has not been built. Call ClientRegistry.build() first.');
|
|
95
|
+
}
|
|
96
|
+
return ClientRegistry.#instance;
|
|
97
|
+
}
|
|
98
|
+
|
|
41
99
|
/**
|
|
42
100
|
* Retrieves a client by name or the default client if no name is provided.
|
|
43
101
|
*
|