navi-hey 0.5.2 → 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -0
- package/bin/navi.js +3 -3
- package/lib/{factories → background}/JobFactory.js +2 -2
- package/lib/{registry → background}/JobRegistry.js +18 -1
- package/lib/{registry → background}/JobRegistryInstance.js +51 -6
- package/lib/{models → background}/Worker.js +5 -3
- package/lib/{factories → background}/WorkerFactory.js +2 -2
- package/lib/{registry → background}/WorkersRegistryInstance.js +1 -1
- package/lib/{models → enqueuers}/ActionEnqueuer.js +4 -1
- package/lib/{models → enqueuers}/AssetRequestEnqueuer.js +6 -2
- package/lib/exceptions/ConflictError.js +14 -0
- package/lib/{models → jobs}/ActionProcessingJob.js +4 -4
- package/lib/{models → jobs}/AssetDownloadJob.js +6 -6
- package/lib/{models → jobs}/HtmlParseJob.js +6 -6
- package/lib/{models → jobs}/ResourceRequestJob.js +11 -11
- package/lib/models/Config.js +4 -2
- package/lib/models/FailureConfig.js +36 -0
- package/lib/models/ResourceRequest.js +5 -2
- package/lib/models/ResourceRequestAction.js +6 -3
- package/lib/models/WebConfig.js +5 -1
- package/lib/registry/ClientRegistry.js +60 -2
- package/lib/registry/LogRegistry.js +146 -0
- package/lib/registry/LogRegistryInstance.js +156 -0
- package/lib/serializers/BaseUrlsSerializer.js +18 -0
- package/lib/serializers/JobIndexSerializer.js +10 -3
- package/lib/serializers/LogSerializer.js +26 -0
- package/lib/server/BaseUrlsRequestHandler.js +33 -0
- package/lib/server/EngineContinueRequestHandler.js +34 -0
- package/lib/server/EnginePauseRequestHandler.js +34 -0
- package/lib/server/EngineRestartRequestHandler.js +34 -0
- package/lib/server/EngineShutdownRequestHandler.js +29 -0
- package/lib/server/EngineStartRequestHandler.js +34 -0
- package/lib/server/EngineStatusRequestHandler.js +27 -0
- package/lib/server/EngineStopRequestHandler.js +34 -0
- package/lib/server/JobLogsRequestHandler.js +39 -0
- package/lib/server/JobRequestHandler.js +1 -1
- package/lib/server/JobRetryRequestHandler.js +40 -0
- package/lib/server/JobsFilter.js +55 -0
- package/lib/server/JobsRequestHandler.js +5 -2
- package/lib/server/LogsRequestHandler.js +55 -0
- package/lib/server/RouteRegister.js +40 -7
- package/lib/server/Router.js +44 -20
- package/lib/server/SettingsRequestHandler.js +34 -0
- package/lib/server/StatsRequestHandler.js +2 -2
- package/lib/server/WebServer.js +28 -8
- package/lib/services/Application.js +135 -81
- package/lib/services/ApplicationInstance.js +272 -0
- package/lib/services/Client.js +27 -22
- package/lib/services/ConfigLoader.js +2 -1
- package/lib/services/ConfigParser.js +13 -2
- package/lib/services/Engine.js +51 -6
- package/lib/services/EngineEvents.js +42 -0
- package/lib/services/FailureChecker.js +78 -0
- package/lib/services/WorkersAllocator.js +2 -2
- package/lib/utils/EnvResolver.js +22 -0
- package/lib/utils/HtmlElementParser.js +5 -4
- package/lib/utils/HtmlParser.js +4 -4
- package/lib/utils/PromiseAggregator.js +11 -5
- package/lib/utils/logging/BufferedLogger.js +8 -0
- package/lib/utils/logging/LogBuffer.js +25 -5
- package/lib/utils/logging/LogBufferCollection.js +56 -0
- package/lib/utils/logging/LogContext.js +51 -0
- package/lib/utils/logging/LogFilter.js +34 -0
- package/package.json +1 -1
- package/static/assets/index-BfggoMvd.js +11 -0
- package/static/assets/{index-DBkz-poX.css → index-CuWjW0sT.css} +1 -1
- package/static/index.html +2 -2
- package/static/assets/index-BF8_ow13.js +0 -11
- /package/lib/{models → background}/Job.js +0 -0
- /package/lib/{registry → background}/WorkersRegistry.js +0 -0
- /package/lib/{models → enqueuers}/ActionsEnqueuer.js +0 -0
- /package/lib/{factories → factory}/Factory.js +0 -0
|
@@ -1,130 +1,184 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import { ConfigurationFileNotProvided } from '../exceptions/ConfigurationFileNotProvided.js';
|
|
3
|
-
import { JobFactory } from '../factories/JobFactory.js';
|
|
4
|
-
import { ActionProcessingJob } from '../models/ActionProcessingJob.js';
|
|
5
|
-
import { AssetDownloadJob } from '../models/AssetDownloadJob.js';
|
|
6
|
-
import { Config } from '../models/Config.js';
|
|
7
|
-
import { HtmlParseJob } from '../models/HtmlParseJob.js';
|
|
8
|
-
import { JobRegistry } from '../registry/JobRegistry.js';
|
|
9
|
-
import { WorkersRegistry } from '../registry/WorkersRegistry.js';
|
|
10
|
-
import { WebServer } from '../server/WebServer.js';
|
|
11
|
-
import { BufferedLogger } from '../utils/logging/BufferedLogger.js';
|
|
12
|
-
import { Logger } from '../utils/logging/Logger.js';
|
|
13
|
-
import { PromiseAggregator } from '../utils/PromiseAggregator.js';
|
|
14
|
-
import { ResourceRequestCollector } from '../utils/ResourceRequestCollector.js';
|
|
1
|
+
import { ApplicationInstance } from './ApplicationInstance.js';
|
|
15
2
|
|
|
16
3
|
/**
|
|
17
|
-
* Application orchestrates the startup of Navi
|
|
18
|
-
* building registries, and running the Engine loop
|
|
4
|
+
* Application is a static singleton facade that orchestrates the startup of Navi
|
|
5
|
+
* by loading configuration, building registries, and running the Engine loop
|
|
6
|
+
* together with the WebServer.
|
|
7
|
+
*
|
|
8
|
+
* Call `Application.build()` once during bootstrap and `Application.reset()` in tests.
|
|
19
9
|
* @author darthjee
|
|
20
10
|
*/
|
|
21
11
|
class Application {
|
|
22
|
-
#
|
|
23
|
-
#bufferedLogger;
|
|
12
|
+
static #instance = null;
|
|
24
13
|
|
|
25
14
|
/**
|
|
26
|
-
* Creates
|
|
27
|
-
* @param {object} [params={}] -
|
|
28
|
-
* @
|
|
15
|
+
* Creates and stores the singleton ApplicationInstance.
|
|
16
|
+
* @param {object} [params={}] - Forwarded to `ApplicationInstance` constructor.
|
|
17
|
+
* @returns {ApplicationInstance} The created instance.
|
|
29
18
|
*/
|
|
30
|
-
|
|
31
|
-
|
|
19
|
+
static build(params = {}) {
|
|
20
|
+
Application.#instance = new ApplicationInstance(params);
|
|
21
|
+
return Application.#instance;
|
|
32
22
|
}
|
|
33
23
|
|
|
34
24
|
/**
|
|
35
|
-
*
|
|
36
|
-
* @param {string} configPath - The path to the configuration file.
|
|
37
|
-
* @throws {ConfigurationFileNotProvided} If the configuration file path is not provided.
|
|
38
|
-
* @throws {ConfigurationFileNotFound} If the configuration file is not found at the specified path.
|
|
25
|
+
* Destroys the singleton instance. Intended for test teardown.
|
|
39
26
|
* @returns {void}
|
|
40
27
|
*/
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
}
|
|
28
|
+
static reset() {
|
|
29
|
+
Application.#instance = null;
|
|
30
|
+
}
|
|
45
31
|
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
32
|
+
/**
|
|
33
|
+
* Returns the singleton instance (for test inspection).
|
|
34
|
+
* @returns {ApplicationInstance|null} The current instance.
|
|
35
|
+
*/
|
|
36
|
+
static get instance() {
|
|
37
|
+
return Application.#instance;
|
|
51
38
|
}
|
|
52
39
|
|
|
53
40
|
/**
|
|
54
|
-
*
|
|
55
|
-
*
|
|
41
|
+
* Returns the current engine status.
|
|
42
|
+
* Returns 'running' if no instance has been built yet.
|
|
43
|
+
* @returns {string} The current status.
|
|
44
|
+
*/
|
|
45
|
+
static status() {
|
|
46
|
+
if (!Application.#instance) return 'running';
|
|
47
|
+
return Application.#instance.status();
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
/**
|
|
51
|
+
* Returns true if the engine is currently running.
|
|
52
|
+
* @returns {boolean} True if the current status is 'running'.
|
|
53
|
+
*/
|
|
54
|
+
static isRunning() {
|
|
55
|
+
return Application.status() === 'running';
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
/**
|
|
59
|
+
* Returns true if the engine is currently paused.
|
|
60
|
+
* @returns {boolean} True if the current status is 'paused'.
|
|
56
61
|
*/
|
|
57
|
-
|
|
58
|
-
|
|
62
|
+
static isPaused() {
|
|
63
|
+
return Application.status() === 'paused';
|
|
64
|
+
}
|
|
59
65
|
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
66
|
+
/**
|
|
67
|
+
* Returns true if the engine is currently stopped.
|
|
68
|
+
* @returns {boolean} True if the current status is 'stopped'.
|
|
69
|
+
*/
|
|
70
|
+
static isStopped() {
|
|
71
|
+
return Application.status() === 'stopped';
|
|
72
|
+
}
|
|
63
73
|
|
|
64
|
-
|
|
65
|
-
|
|
74
|
+
/**
|
|
75
|
+
* Loads the configuration from the specified file path.
|
|
76
|
+
* @param {string} configPath - The path to the configuration file.
|
|
77
|
+
* @returns {void}
|
|
78
|
+
*/
|
|
79
|
+
static loadConfig(configPath) {
|
|
80
|
+
return Application.#getInstance().loadConfig(configPath);
|
|
81
|
+
}
|
|
66
82
|
|
|
67
|
-
|
|
83
|
+
/**
|
|
84
|
+
* Starts the application.
|
|
85
|
+
* @returns {Promise<void>}
|
|
86
|
+
*/
|
|
87
|
+
static async run() {
|
|
88
|
+
return Application.#getInstance().run();
|
|
68
89
|
}
|
|
69
90
|
|
|
70
91
|
/**
|
|
71
|
-
* Builds and returns a new Engine instance
|
|
92
|
+
* Builds and returns a new Engine instance.
|
|
72
93
|
* @returns {Engine} The created Engine instance.
|
|
73
94
|
*/
|
|
74
|
-
buildEngine() {
|
|
75
|
-
return
|
|
95
|
+
static buildEngine() {
|
|
96
|
+
return Application.#getInstance().buildEngine();
|
|
76
97
|
}
|
|
77
98
|
|
|
78
99
|
/**
|
|
79
|
-
* Builds and returns a WebServer
|
|
100
|
+
* Builds and returns a WebServer or null.
|
|
80
101
|
* @returns {WebServer|null} The created WebServer instance or null.
|
|
81
102
|
*/
|
|
82
|
-
buildWebServer() {
|
|
83
|
-
return
|
|
84
|
-
|
|
85
|
-
|
|
103
|
+
static buildWebServer() {
|
|
104
|
+
return Application.#getInstance().buildWebServer();
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
/**
|
|
108
|
+
* Enqueues all parameter-free ResourceRequests.
|
|
109
|
+
* @returns {void}
|
|
110
|
+
*/
|
|
111
|
+
static enqueueFirstJobs() {
|
|
112
|
+
return Application.#getInstance().enqueueFirstJobs();
|
|
86
113
|
}
|
|
87
114
|
|
|
88
115
|
/**
|
|
89
|
-
* Gets the buffered logger
|
|
116
|
+
* Gets the buffered logger from the singleton instance.
|
|
90
117
|
* @returns {BufferedLogger} The buffered logger instance.
|
|
91
118
|
*/
|
|
92
|
-
get bufferedLogger() {
|
|
93
|
-
return
|
|
119
|
+
static get bufferedLogger() {
|
|
120
|
+
return Application.#getInstance().bufferedLogger;
|
|
94
121
|
}
|
|
95
122
|
|
|
96
123
|
/**
|
|
97
|
-
*
|
|
98
|
-
*
|
|
99
|
-
* processed immediately without any external parameters.
|
|
100
|
-
* @returns {void}
|
|
124
|
+
* Pauses the engine and waits for workers to become idle.
|
|
125
|
+
* @returns {Promise<void>}
|
|
101
126
|
*/
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
JobRegistry.enqueue('ResourceRequestJob', { resourceRequest, parameters: {} });
|
|
105
|
-
});
|
|
127
|
+
static async pause() {
|
|
128
|
+
return Application.#getInstance().pause();
|
|
106
129
|
}
|
|
107
130
|
|
|
108
131
|
/**
|
|
109
|
-
*
|
|
110
|
-
* @returns {void}
|
|
132
|
+
* Stops the engine, waits for workers to idle, then clears job queues.
|
|
133
|
+
* @returns {Promise<void>}
|
|
134
|
+
*/
|
|
135
|
+
static async stop() {
|
|
136
|
+
return Application.#getInstance().stop();
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
/**
|
|
140
|
+
* Resumes processing after a pause.
|
|
141
|
+
* @returns {Promise<void>}
|
|
142
|
+
*/
|
|
143
|
+
static async continue() {
|
|
144
|
+
return Application.#getInstance().continue();
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
/**
|
|
148
|
+
* Starts processing from a stopped state.
|
|
149
|
+
* @returns {Promise<void>}
|
|
111
150
|
*/
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
JobFactory.build('HtmlParse', { klass: HtmlParseJob, attributes: { jobRegistry: JobRegistry, clientRegistry: this.config.clientRegistry } });
|
|
116
|
-
JobFactory.build('AssetDownload', { klass: AssetDownloadJob, attributes: { clientRegistry: this.config.clientRegistry } });
|
|
151
|
+
static async start() {
|
|
152
|
+
return Application.#getInstance().start();
|
|
153
|
+
}
|
|
117
154
|
|
|
118
|
-
|
|
155
|
+
/**
|
|
156
|
+
* Restarts processing.
|
|
157
|
+
* @returns {Promise<void>}
|
|
158
|
+
*/
|
|
159
|
+
static async restart() {
|
|
160
|
+
return Application.#getInstance().restart();
|
|
161
|
+
}
|
|
119
162
|
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
163
|
+
/**
|
|
164
|
+
* Shuts down the web server and stops the engine.
|
|
165
|
+
* @returns {Promise<void>}
|
|
166
|
+
*/
|
|
167
|
+
static async shutdown() {
|
|
168
|
+
return Application.#getInstance().shutdown();
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
/**
|
|
172
|
+
* Returns the singleton instance, throwing if not yet built.
|
|
173
|
+
* @returns {ApplicationInstance} The singleton instance.
|
|
174
|
+
* @throws {Error} If `build()` has not been called.
|
|
175
|
+
*/
|
|
176
|
+
static #getInstance() {
|
|
177
|
+
if (!Application.#instance) {
|
|
178
|
+
throw new Error('Application has not been initialized. Call Application.build() before calling static methods that require an instance.');
|
|
179
|
+
}
|
|
180
|
+
return Application.#instance;
|
|
127
181
|
}
|
|
128
182
|
}
|
|
129
183
|
|
|
130
|
-
export { Application };
|
|
184
|
+
export { Application };
|
|
@@ -0,0 +1,272 @@
|
|
|
1
|
+
import { Engine } from './Engine.js';
|
|
2
|
+
import { EngineEvents } from './EngineEvents.js';
|
|
3
|
+
import { FailureChecker } from './FailureChecker.js';
|
|
4
|
+
import { JobFactory } from '../background/JobFactory.js';
|
|
5
|
+
import { JobRegistry } from '../background/JobRegistry.js';
|
|
6
|
+
import { WorkersRegistry } from '../background/WorkersRegistry.js';
|
|
7
|
+
import { ConfigurationFileNotProvided } from '../exceptions/ConfigurationFileNotProvided.js';
|
|
8
|
+
import { ActionProcessingJob } from '../jobs/ActionProcessingJob.js';
|
|
9
|
+
import { AssetDownloadJob } from '../jobs/AssetDownloadJob.js';
|
|
10
|
+
import { HtmlParseJob } from '../jobs/HtmlParseJob.js';
|
|
11
|
+
import { Config } from '../models/Config.js';
|
|
12
|
+
import { LogRegistry } from '../registry/LogRegistry.js';
|
|
13
|
+
import { WebServer } from '../server/WebServer.js';
|
|
14
|
+
import { PromiseAggregator } from '../utils/PromiseAggregator.js';
|
|
15
|
+
import { ResourceRequestCollector } from '../utils/ResourceRequestCollector.js';
|
|
16
|
+
|
|
17
|
+
const DEFAULT_POLL_SLEEP_MS = 10;
|
|
18
|
+
|
|
19
|
+
/**
|
|
20
|
+
* ApplicationInstance holds all instance-level state and logic for a single
|
|
21
|
+
* application lifecycle. Use the static `Application` facade to access it.
|
|
22
|
+
* @author darthjee
|
|
23
|
+
*/
|
|
24
|
+
class ApplicationInstance {
|
|
25
|
+
#workers;
|
|
26
|
+
#bufferedLogger;
|
|
27
|
+
#engineStatus;
|
|
28
|
+
#aggregator;
|
|
29
|
+
#enginePromise;
|
|
30
|
+
#sleepMs;
|
|
31
|
+
|
|
32
|
+
/**
|
|
33
|
+
* @param {object} [params={}] - Optional parameters for dependency injection.
|
|
34
|
+
* @param {IdentifyableCollection} [params.workers] - Workers collection (injected for testing).
|
|
35
|
+
*/
|
|
36
|
+
constructor({ workers } = {}) {
|
|
37
|
+
this.#workers = workers;
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
/**
|
|
41
|
+
* Loads the configuration from the specified file path.
|
|
42
|
+
* @param {string} configPath - The path to the configuration file.
|
|
43
|
+
* @throws {ConfigurationFileNotProvided} If the configuration file path is not provided.
|
|
44
|
+
* @throws {ConfigurationFileNotFound} If the configuration file is not found at the specified path.
|
|
45
|
+
* @returns {void}
|
|
46
|
+
*/
|
|
47
|
+
loadConfig(configPath) {
|
|
48
|
+
if (!configPath) {
|
|
49
|
+
throw new ConfigurationFileNotProvided();
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
this.config = Config.fromFile(configPath);
|
|
53
|
+
const logRegistry = LogRegistry.build({ retention: this.config.logConfig.size });
|
|
54
|
+
this.#bufferedLogger = logRegistry.bufferedLogger;
|
|
55
|
+
this.#initRegistries();
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
/**
|
|
59
|
+
* Starts the application by building the engine, web server, enqueueing initial jobs, and starting both.
|
|
60
|
+
* After the engine finishes, checks the dead-job ratio against the configured failure threshold.
|
|
61
|
+
* @returns {Promise<void>}
|
|
62
|
+
*/
|
|
63
|
+
async run() {
|
|
64
|
+
this.#aggregator = new PromiseAggregator();
|
|
65
|
+
this.#sleepMs = this.config.workersConfig.sleep;
|
|
66
|
+
|
|
67
|
+
this.engine = this.buildEngine();
|
|
68
|
+
this.webServer = this.buildWebServer();
|
|
69
|
+
this.enqueueFirstJobs();
|
|
70
|
+
|
|
71
|
+
this.#engineStatus = 'running';
|
|
72
|
+
this.#aggregator.add(this.webServer?.start());
|
|
73
|
+
this.#enginePromise = this.engine.start();
|
|
74
|
+
this.#aggregator.add(this.#enginePromise);
|
|
75
|
+
|
|
76
|
+
await this.#aggregator.wait();
|
|
77
|
+
this.#engineStatus = 'stopped';
|
|
78
|
+
EngineEvents.emit('stop');
|
|
79
|
+
|
|
80
|
+
new FailureChecker({ failureConfig: this.config.failureConfig }).check();
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
/**
|
|
84
|
+
* Builds and returns a new Engine instance wired to the current registries.
|
|
85
|
+
* @returns {Engine} The created Engine instance.
|
|
86
|
+
*/
|
|
87
|
+
buildEngine() {
|
|
88
|
+
return new Engine({
|
|
89
|
+
sleepMs: this.#sleepMs ?? this.config.workersConfig.sleep,
|
|
90
|
+
keepAlive: !!this.config.webConfig,
|
|
91
|
+
});
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
/**
|
|
95
|
+
* Builds and returns a WebServer if web configuration is present, otherwise null.
|
|
96
|
+
* @returns {WebServer|null} The created WebServer instance or null.
|
|
97
|
+
*/
|
|
98
|
+
buildWebServer() {
|
|
99
|
+
return WebServer.build({
|
|
100
|
+
webConfig: this.config.webConfig,
|
|
101
|
+
});
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
/**
|
|
105
|
+
* Gets the buffered logger instance created during config loading.
|
|
106
|
+
* @returns {BufferedLogger} The buffered logger instance.
|
|
107
|
+
*/
|
|
108
|
+
get bufferedLogger() {
|
|
109
|
+
return this.#bufferedLogger;
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
/**
|
|
113
|
+
* Enqueues all parameter-free ResourceRequests into the job registry.
|
|
114
|
+
* These are requests whose URLs contain no {:placeholder} tokens and can be
|
|
115
|
+
* processed immediately without any external parameters.
|
|
116
|
+
* @returns {void}
|
|
117
|
+
*/
|
|
118
|
+
enqueueFirstJobs() {
|
|
119
|
+
new ResourceRequestCollector(this.config.resourceRegistry).requestsNeedingNoParams().forEach((resourceRequest) => {
|
|
120
|
+
JobRegistry.enqueue('ResourceRequestJob', { resourceRequest, parameters: {} });
|
|
121
|
+
});
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
/**
|
|
125
|
+
* Returns the current engine status string.
|
|
126
|
+
* @returns {string|undefined} The current status.
|
|
127
|
+
*/
|
|
128
|
+
status() {
|
|
129
|
+
return this.#engineStatus;
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
/**
|
|
133
|
+
* Returns true if the engine is currently running.
|
|
134
|
+
* @returns {boolean} True if the current status is 'running'.
|
|
135
|
+
*/
|
|
136
|
+
isRunning() {
|
|
137
|
+
return this.#engineStatus === 'running';
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
/**
|
|
141
|
+
* Returns true if the engine is currently paused.
|
|
142
|
+
* @returns {boolean} True if the current status is 'paused'.
|
|
143
|
+
*/
|
|
144
|
+
isPaused() {
|
|
145
|
+
return this.#engineStatus === 'paused';
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
/**
|
|
149
|
+
* Returns true if the engine is currently stopped.
|
|
150
|
+
* @returns {boolean} True if the current status is 'stopped'.
|
|
151
|
+
*/
|
|
152
|
+
isStopped() {
|
|
153
|
+
return this.#engineStatus === 'stopped';
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
/**
|
|
157
|
+
* Sets the engine status string.
|
|
158
|
+
* @param {string} value - The new status value.
|
|
159
|
+
* @returns {void}
|
|
160
|
+
*/
|
|
161
|
+
setStatus(value) {
|
|
162
|
+
this.#engineStatus = value;
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
/**
|
|
166
|
+
* Pauses processing: pauses the engine (without stopping it) and waits for workers to become idle.
|
|
167
|
+
* @returns {Promise<void>}
|
|
168
|
+
*/
|
|
169
|
+
async pause() {
|
|
170
|
+
this.#engineStatus = 'pausing';
|
|
171
|
+
this.engine.pause();
|
|
172
|
+
await this.#waitForWorkersIdle();
|
|
173
|
+
this.#engineStatus = 'paused';
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
/**
|
|
177
|
+
* Stops processing: pauses the engine, waits for workers to idle, then clears job queues.
|
|
178
|
+
* The engine instance is preserved and its loop continues running in the background.
|
|
179
|
+
* @returns {Promise<void>}
|
|
180
|
+
*/
|
|
181
|
+
async stop() {
|
|
182
|
+
this.#engineStatus = 'stopping';
|
|
183
|
+
this.engine.pause();
|
|
184
|
+
await this.#waitForWorkersIdle();
|
|
185
|
+
JobRegistry.clearQueues();
|
|
186
|
+
this.#engineStatus = 'stopped';
|
|
187
|
+
EngineEvents.emit('stop');
|
|
188
|
+
}
|
|
189
|
+
|
|
190
|
+
/**
|
|
191
|
+
* Resumes processing after a pause by calling engine.resume().
|
|
192
|
+
* No new engine is created; the existing loop continues.
|
|
193
|
+
* Only valid when status is 'paused'.
|
|
194
|
+
* @returns {Promise<void>}
|
|
195
|
+
*/
|
|
196
|
+
async continue() {
|
|
197
|
+
if (this.#engineStatus !== 'paused') return;
|
|
198
|
+
this.engine.resume();
|
|
199
|
+
this.#engineStatus = 'running';
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
/**
|
|
203
|
+
* Starts processing from a stopped state by calling engine.resume() and re-enqueueing initial jobs.
|
|
204
|
+
* No new engine is created; the existing loop continues.
|
|
205
|
+
* Only valid when status is 'stopped'.
|
|
206
|
+
* @returns {Promise<void>}
|
|
207
|
+
*/
|
|
208
|
+
async start() {
|
|
209
|
+
if (this.#engineStatus !== 'stopped') return;
|
|
210
|
+
this.engine.resume();
|
|
211
|
+
this.enqueueFirstJobs();
|
|
212
|
+
this.#engineStatus = 'running';
|
|
213
|
+
EngineEvents.emit('start');
|
|
214
|
+
}
|
|
215
|
+
|
|
216
|
+
/**
|
|
217
|
+
* Restarts processing: stops then starts the engine.
|
|
218
|
+
* Only valid when status is 'running'.
|
|
219
|
+
* @returns {Promise<void>}
|
|
220
|
+
*/
|
|
221
|
+
async restart() {
|
|
222
|
+
if (this.#engineStatus !== 'running') return;
|
|
223
|
+
await this.stop();
|
|
224
|
+
await this.start();
|
|
225
|
+
}
|
|
226
|
+
|
|
227
|
+
/**
|
|
228
|
+
* Shuts down the web server and stops the engine loop.
|
|
229
|
+
* If running, pauses first and waits for workers to idle.
|
|
230
|
+
* @returns {Promise<void>}
|
|
231
|
+
*/
|
|
232
|
+
async shutdown() {
|
|
233
|
+
this.webServer?.shutdown();
|
|
234
|
+
if (this.#engineStatus === 'running') {
|
|
235
|
+
await this.stop();
|
|
236
|
+
}
|
|
237
|
+
this.engine.stop();
|
|
238
|
+
}
|
|
239
|
+
|
|
240
|
+
/**
|
|
241
|
+
* Polls until all workers are idle.
|
|
242
|
+
* @returns {Promise<void>}
|
|
243
|
+
*/
|
|
244
|
+
async #waitForWorkersIdle() {
|
|
245
|
+
while (WorkersRegistry.hasBusyWorker()) {
|
|
246
|
+
await new Promise(resolve => setTimeout(resolve, this.#sleepMs ?? DEFAULT_POLL_SLEEP_MS));
|
|
247
|
+
}
|
|
248
|
+
}
|
|
249
|
+
|
|
250
|
+
/**
|
|
251
|
+
* Initializes the job factory, job registry, and workers registry from the loaded configuration.
|
|
252
|
+
* @returns {void}
|
|
253
|
+
*/
|
|
254
|
+
#initRegistries() {
|
|
255
|
+
JobFactory.build('ResourceRequestJob', { attributes: { clients: this.config.clientRegistry } });
|
|
256
|
+
JobFactory.build('Action', { klass: ActionProcessingJob });
|
|
257
|
+
JobFactory.build('HtmlParse', { klass: HtmlParseJob, attributes: { jobRegistry: JobRegistry, clientRegistry: this.config.clientRegistry } });
|
|
258
|
+
JobFactory.build('AssetDownload', { klass: AssetDownloadJob, attributes: { clientRegistry: this.config.clientRegistry } });
|
|
259
|
+
|
|
260
|
+
JobRegistry.build({ cooldown: this.config.workersConfig.retryCooldown, maxRetries: this.config.workersConfig.maxRetries });
|
|
261
|
+
|
|
262
|
+
WorkersRegistry.build({
|
|
263
|
+
workers: this.#workers,
|
|
264
|
+
jobRegistry: JobRegistry,
|
|
265
|
+
workersRegistry: WorkersRegistry,
|
|
266
|
+
...this.config.workersConfig,
|
|
267
|
+
});
|
|
268
|
+
WorkersRegistry.initWorkers();
|
|
269
|
+
}
|
|
270
|
+
}
|
|
271
|
+
|
|
272
|
+
export { ApplicationInstance };
|
package/lib/services/Client.js
CHANGED
|
@@ -1,7 +1,5 @@
|
|
|
1
1
|
import axios from 'axios';
|
|
2
2
|
import { RequestFailed } from '../exceptions/RequestFailed.js';
|
|
3
|
-
import { EnvResolver } from '../utils/EnvResolver.js';
|
|
4
|
-
import { Logger } from '../utils/logging/Logger.js';
|
|
5
3
|
|
|
6
4
|
/**
|
|
7
5
|
* Client performs HTTP requests for resource paths using a configured base URL.
|
|
@@ -30,16 +28,14 @@ class Client {
|
|
|
30
28
|
*
|
|
31
29
|
* @param {string} name The name identifying the client.
|
|
32
30
|
* @param {object} config Client configuration object.
|
|
33
|
-
* @param {string} config.base_url Base URL for the client.
|
|
34
|
-
* `$VAR` or `${VAR}` are resolved from `process.env` at parse time.
|
|
31
|
+
* @param {string} config.base_url Base URL for the client.
|
|
35
32
|
* @param {number} [config.timeout] Optional request timeout in milliseconds.
|
|
36
|
-
* @param {object} [config.headers] Optional HTTP headers.
|
|
37
|
-
* `$VAR` or `${VAR}` are resolved from `process.env` at parse time.
|
|
33
|
+
* @param {object} [config.headers] Optional HTTP headers.
|
|
38
34
|
* @returns {Client} A new Client instance.
|
|
39
35
|
*/
|
|
40
36
|
static fromObject(name, config) {
|
|
41
|
-
const headers =
|
|
42
|
-
const baseUrl =
|
|
37
|
+
const headers = config.headers || {};
|
|
38
|
+
const baseUrl = config.base_url;
|
|
43
39
|
return new Client({ name, baseUrl, timeout: config.timeout, headers });
|
|
44
40
|
}
|
|
45
41
|
|
|
@@ -58,16 +54,17 @@ class Client {
|
|
|
58
54
|
* @param {ResourceRequest} resourceRequest Information about the URL path to request
|
|
59
55
|
* and the expected status code.
|
|
60
56
|
* @param {object} [parameters={}] Key-value map used to resolve {:placeholder} tokens in the URL.
|
|
57
|
+
* @param {LogContext} logContext Context carrying workerId/jobId for log entries.
|
|
61
58
|
* @returns {Promise<boolean>} Returns true if the response status matches the expected status.
|
|
62
59
|
* @throws {RequestFailed} Throws an error if the request fails or the status does not match.
|
|
63
60
|
*/
|
|
64
|
-
async perform(resourceRequest, parameters = {}) {
|
|
61
|
+
async perform(resourceRequest, parameters = {}, logContext) {
|
|
65
62
|
const requestUrl = this.#buildUrl(resourceRequest.resolveUrl(parameters));
|
|
66
|
-
|
|
63
|
+
logContext.info(`[Client:${this.name}] Requesting ${requestUrl}`);
|
|
67
64
|
try {
|
|
68
|
-
return await this.#request(resourceRequest, requestUrl);
|
|
65
|
+
return await this.#request(resourceRequest, requestUrl, logContext);
|
|
69
66
|
} catch (error) {
|
|
70
|
-
this.#handleError(error, requestUrl);
|
|
67
|
+
this.#handleError(error, requestUrl, logContext);
|
|
71
68
|
}
|
|
72
69
|
}
|
|
73
70
|
|
|
@@ -76,15 +73,16 @@ class Client {
|
|
|
76
73
|
* Unlike `perform`, this method accepts an absolute URL directly and does not prepend `baseUrl`.
|
|
77
74
|
* @param {string} absoluteUrl The fully-resolved URL to request.
|
|
78
75
|
* @param {number} expectedStatus The expected HTTP response status code.
|
|
76
|
+
* @param {LogContext} logContext Context carrying workerId/jobId for log entries.
|
|
79
77
|
* @returns {Promise<object>} The Axios response object.
|
|
80
78
|
* @throws {RequestFailed} If the response status does not match `expectedStatus`.
|
|
81
79
|
*/
|
|
82
|
-
async performUrl(absoluteUrl, expectedStatus) {
|
|
83
|
-
|
|
80
|
+
async performUrl(absoluteUrl, expectedStatus, logContext) {
|
|
81
|
+
logContext.info(`[Client:${this.name}] Requesting ${absoluteUrl}`);
|
|
84
82
|
try {
|
|
85
|
-
return await this.#requestUrl(absoluteUrl, expectedStatus);
|
|
83
|
+
return await this.#requestUrl(absoluteUrl, expectedStatus, logContext);
|
|
86
84
|
} catch (error) {
|
|
87
|
-
this.#handleError(error, absoluteUrl);
|
|
85
|
+
this.#handleError(error, absoluteUrl, logContext);
|
|
88
86
|
}
|
|
89
87
|
}
|
|
90
88
|
|
|
@@ -93,21 +91,23 @@ class Client {
|
|
|
93
91
|
* @param {ResourceRequest} resourceRequest Information about the URL path to request
|
|
94
92
|
* and the expected status code.
|
|
95
93
|
* @param {string} requestUrl The full URL for the request.
|
|
94
|
+
* @param {LogContext} logContext Context carrying workerId/jobId for log entries.
|
|
96
95
|
* @returns {Promise<boolean>} Returns true if the response status matches the expected status.
|
|
97
96
|
* @throws {RequestFailed} Throws an error if the response status does not match.
|
|
98
97
|
*/
|
|
99
|
-
async #request(resourceRequest, requestUrl) {
|
|
100
|
-
return this.#requestUrl(requestUrl, resourceRequest.status);
|
|
98
|
+
async #request(resourceRequest, requestUrl, logContext) {
|
|
99
|
+
return this.#requestUrl(requestUrl, resourceRequest.status, logContext);
|
|
101
100
|
}
|
|
102
101
|
|
|
103
102
|
/**
|
|
104
103
|
* Performs the HTTP GET request to the given URL and validates the response status.
|
|
105
104
|
* @param {string} requestUrl The full URL to request.
|
|
106
105
|
* @param {number} expectedStatus The expected HTTP response status code.
|
|
106
|
+
* @param {LogContext} logContext Context carrying workerId/jobId for log entries.
|
|
107
107
|
* @returns {Promise<object>} The Axios response object.
|
|
108
108
|
* @throws {RequestFailed} Throws an error if the response status does not match.
|
|
109
109
|
*/
|
|
110
|
-
async #requestUrl(requestUrl, expectedStatus) {
|
|
110
|
+
async #requestUrl(requestUrl, expectedStatus, logContext) {
|
|
111
111
|
const response = await axios.get(requestUrl, {
|
|
112
112
|
timeout: this.timeout,
|
|
113
113
|
responseType: 'text',
|
|
@@ -116,10 +116,14 @@ class Client {
|
|
|
116
116
|
validateStatus: () => true,
|
|
117
117
|
});
|
|
118
118
|
|
|
119
|
+
logContext.info(`[Client:${this.name}] Response ${requestUrl} → ${response.status}`);
|
|
120
|
+
|
|
119
121
|
if (response.status !== expectedStatus) {
|
|
122
|
+
logContext.info(`[Client:${this.name}] ${requestUrl} did not match (got ${response.status}, expected ${expectedStatus})`);
|
|
120
123
|
throw new RequestFailed(response.status, requestUrl);
|
|
121
124
|
}
|
|
122
125
|
|
|
126
|
+
logContext.info(`[Client:${this.name}] ${requestUrl} matched (expected ${expectedStatus})`);
|
|
123
127
|
return response;
|
|
124
128
|
}
|
|
125
129
|
|
|
@@ -127,11 +131,12 @@ class Client {
|
|
|
127
131
|
* Handles a caught request error, logging it and re-throwing as RequestFailed when applicable.
|
|
128
132
|
* @param {Error} error The caught error.
|
|
129
133
|
* @param {string} requestUrl The URL that was requested.
|
|
134
|
+
* @param {LogContext} logContext Context carrying workerId/jobId for log entries.
|
|
130
135
|
* @throws {RequestFailed} If the error has a response object.
|
|
131
136
|
* @throws {Error} Re-throws the original error otherwise.
|
|
132
137
|
*/
|
|
133
|
-
#handleError(error, requestUrl) {
|
|
134
|
-
|
|
138
|
+
#handleError(error, requestUrl, logContext) {
|
|
139
|
+
logContext.error(`Request failed: ${error}`);
|
|
135
140
|
if (error.response) {
|
|
136
141
|
throw new RequestFailed(error.response.status, requestUrl);
|
|
137
142
|
}
|
|
@@ -148,4 +153,4 @@ class Client {
|
|
|
148
153
|
}
|
|
149
154
|
}
|
|
150
155
|
|
|
151
|
-
export { Client };
|
|
156
|
+
export { Client };
|
|
@@ -2,6 +2,7 @@ import { readFileSync } from 'node:fs';
|
|
|
2
2
|
import YAML from 'yaml';
|
|
3
3
|
import { ConfigParser } from './ConfigParser.js';
|
|
4
4
|
import { ConfigurationFileNotFound } from '../exceptions/ConfigurationFileNotFound.js';
|
|
5
|
+
import { EnvResolver } from '../utils/EnvResolver.js';
|
|
5
6
|
import { Logger } from '../utils/logging/Logger.js';
|
|
6
7
|
|
|
7
8
|
/**
|
|
@@ -54,7 +55,7 @@ class ConfigLoader {
|
|
|
54
55
|
* @throws {ConfigurationFileNotFound} If the configuration file is not found at the specified path.
|
|
55
56
|
*/
|
|
56
57
|
#parseYaml() {
|
|
57
|
-
return YAML.parse(this.#yamlContent());
|
|
58
|
+
return YAML.parse(EnvResolver.resolveString(this.#yamlContent()));
|
|
58
59
|
}
|
|
59
60
|
|
|
60
61
|
/**
|