navi-hey 1.0.0 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +38 -4
- package/bin/navi.js +3 -3
- package/lib/{factories → background}/JobFactory.js +2 -2
- package/lib/{registry → background}/JobRegistry.js +17 -0
- package/lib/{registry → background}/JobRegistryInstance.js +42 -1
- package/lib/{models → background}/Worker.js +5 -3
- package/lib/{factories → background}/WorkerFactory.js +2 -2
- package/lib/{registry → background}/WorkersRegistryInstance.js +1 -1
- package/lib/enqueuers/ActionEnqueuer.js +53 -0
- package/lib/{models → enqueuers}/ActionsEnqueuer.js +6 -3
- package/lib/{models → enqueuers}/AssetRequestEnqueuer.js +6 -2
- package/lib/enqueuers/PaginatedActionEnqueuer.js +53 -0
- package/lib/enqueuers/PaginatedActionsEnqueuer.js +47 -0
- package/lib/exceptions/{ConfigurationFileNotFound.js → config/ConfigurationFileNotFound.js} +1 -1
- package/lib/exceptions/{ConfigurationFileNotProvided.js → config/ConfigurationFileNotProvided.js} +1 -1
- package/lib/exceptions/{MissingTopLevelConfgKey.js → config/MissingTopLevelConfgKey.js} +1 -1
- package/lib/exceptions/http/ConflictError.js +14 -0
- package/lib/exceptions/{ForbiddenError.js → http/ForbiddenError.js} +1 -1
- package/lib/exceptions/{NotFoundError.js → http/NotFoundError.js} +1 -1
- package/lib/exceptions/{ItemNotFound.js → registry/ItemNotFound.js} +1 -1
- package/lib/exceptions/{MissingActionResource.js → registry/MissingActionResource.js} +1 -1
- package/lib/exceptions/{MissingMappingVariable.js → registry/MissingMappingVariable.js} +1 -1
- package/lib/exceptions/{InvalidHtmlResponseBody.js → request/InvalidHtmlResponseBody.js} +1 -1
- package/lib/exceptions/{InvalidResponseBody.js → request/InvalidResponseBody.js} +1 -1
- package/lib/exceptions/{NullResponse.js → request/NullResponse.js} +1 -1
- package/lib/exceptions/{RequestFailed.js → request/RequestFailed.js} +1 -1
- package/lib/{models → jobs}/ActionProcessingJob.js +21 -7
- package/lib/{models → jobs}/AssetDownloadJob.js +6 -6
- package/lib/{models → jobs}/HtmlParseJob.js +23 -10
- package/lib/jobs/PaginatedActionProcessingJob.js +77 -0
- package/lib/{models → jobs}/ResourceRequestJob.js +18 -15
- package/lib/models/{Config.js → configs/Config.js} +3 -3
- package/lib/models/configs/PaginationConfig.js +63 -0
- package/lib/models/configs/WebConfig.js +19 -0
- package/lib/models/{ResourceRequest.js → request/ResourceRequest.js} +32 -7
- package/lib/models/{ResourceRequestAction.js → request/ResourceRequestAction.js} +9 -6
- package/lib/models/request/ResourceRequestPaginatedAction.js +81 -0
- package/lib/models/{PathSegmentTraverser.js → response/PathSegmentTraverser.js} +1 -1
- package/lib/models/{ResponseParser.js → response/ResponseParser.js} +1 -1
- package/lib/registry/ClientRegistry.js +1 -1
- package/lib/registry/LogRegistry.js +146 -0
- package/lib/registry/LogRegistryInstance.js +156 -0
- package/lib/registry/NamedRegistry.js +1 -1
- package/lib/registry/ResourceRegistry.js +1 -1
- package/lib/serializers/JobIndexSerializer.js +13 -3
- package/lib/serializers/LogSerializer.js +26 -0
- package/lib/server/PathValidator.js +1 -1
- package/lib/server/RouteRegister.js +42 -9
- package/lib/server/Router.js +49 -31
- package/lib/server/WebServer.js +28 -8
- package/lib/server/{AssetsRequestHandler.js → handlers/AssetsRequestHandler.js} +3 -3
- package/lib/server/{BaseUrlsRequestHandler.js → handlers/BaseUrlsRequestHandler.js} +3 -3
- package/lib/server/{IndexRequestHandler.js → handlers/IndexRequestHandler.js} +2 -2
- package/lib/server/handlers/JobsFilter.js +55 -0
- package/lib/server/handlers/LogsRequestHandler.js +55 -0
- package/lib/server/handlers/SettingsRequestHandler.js +35 -0
- package/lib/server/{StatsRequestHandler.js → handlers/StatsRequestHandler.js} +3 -3
- package/lib/server/handlers/engine/EngineContinueRequestHandler.js +34 -0
- package/lib/server/handlers/engine/EnginePauseRequestHandler.js +34 -0
- package/lib/server/handlers/engine/EngineRestartRequestHandler.js +34 -0
- package/lib/server/handlers/engine/EngineShutdownRequestHandler.js +29 -0
- package/lib/server/handlers/engine/EngineStartRequestHandler.js +34 -0
- package/lib/server/handlers/engine/EngineStatusRequestHandler.js +27 -0
- package/lib/server/handlers/engine/EngineStopRequestHandler.js +34 -0
- package/lib/server/handlers/jobs/JobLogsRequestHandler.js +39 -0
- package/lib/server/{JobRequestHandler.js → handlers/jobs/JobRequestHandler.js} +4 -4
- package/lib/server/handlers/jobs/JobRetryRequestHandler.js +40 -0
- package/lib/server/{JobsRequestHandler.js → handlers/jobs/JobsRequestHandler.js} +7 -4
- package/lib/services/Application.js +138 -84
- package/lib/services/ApplicationInstance.js +274 -0
- package/lib/services/Client.js +23 -19
- package/lib/services/ConfigLoader.js +1 -1
- package/lib/services/ConfigParser.js +7 -7
- package/lib/services/Engine.js +51 -6
- package/lib/services/EngineEvents.js +42 -0
- package/lib/services/FailureChecker.js +4 -4
- package/lib/services/WorkersAllocator.js +2 -2
- package/lib/utils/HtmlElementParser.js +5 -4
- package/lib/utils/HtmlParser.js +5 -5
- package/lib/utils/PromiseAggregator.js +11 -5
- package/lib/utils/logging/BufferedLogger.js +8 -0
- package/lib/utils/logging/LogBuffer.js +25 -5
- package/lib/utils/logging/LogBufferCollection.js +56 -0
- package/lib/utils/logging/LogContext.js +51 -0
- package/lib/utils/logging/LogFilter.js +34 -0
- package/package.json +1 -1
- package/static/assets/{index-DBkz-poX.css → index-CuWjW0sT.css} +1 -1
- package/static/assets/index-D5tOOLpl.js +11 -0
- package/static/index.html +2 -2
- package/lib/exceptions/LockedByOtherWorker.js +0 -14
- package/lib/models/ActionEnqueuer.js +0 -34
- package/lib/models/WebConfig.js +0 -15
- /package/lib/{models → background}/Job.js +0 -0
- /package/lib/{registry → background}/WorkersRegistry.js +0 -0
- /package/lib/exceptions/{MissingClientsConfig.js → config/MissingClientsConfig.js} +0 -0
- /package/lib/exceptions/{MissingResourceConfig.js → config/MissingResourceConfig.js} +0 -0
- /package/lib/exceptions/{ClientNotFound.js → registry/ClientNotFound.js} +0 -0
- /package/lib/exceptions/{ResourceNotFound.js → registry/ResourceNotFound.js} +0 -0
- /package/lib/{factories → factory}/Factory.js +0 -0
- /package/lib/models/{FailureConfig.js → configs/FailureConfig.js} +0 -0
- /package/lib/models/{LogConfig.js → configs/LogConfig.js} +0 -0
- /package/lib/models/{WorkersConfig.js → configs/WorkersConfig.js} +0 -0
- /package/lib/models/{AssetRequest.js → request/AssetRequest.js} +0 -0
- /package/lib/models/{Resource.js → request/Resource.js} +0 -0
- /package/lib/models/{ParametersMapper.js → response/ParametersMapper.js} +0 -0
- /package/lib/models/{PathResolver.js → response/PathResolver.js} +0 -0
- /package/lib/models/{ResponseWrapper.js → response/ResponseWrapper.js} +0 -0
|
@@ -1,130 +1,184 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import { FailureChecker } from './FailureChecker.js';
|
|
3
|
-
import { ConfigurationFileNotProvided } from '../exceptions/ConfigurationFileNotProvided.js';
|
|
4
|
-
import { JobFactory } from '../factories/JobFactory.js';
|
|
5
|
-
import { ActionProcessingJob } from '../models/ActionProcessingJob.js';
|
|
6
|
-
import { AssetDownloadJob } from '../models/AssetDownloadJob.js';
|
|
7
|
-
import { Config } from '../models/Config.js';
|
|
8
|
-
import { HtmlParseJob } from '../models/HtmlParseJob.js';
|
|
9
|
-
import { JobRegistry } from '../registry/JobRegistry.js';
|
|
10
|
-
import { WorkersRegistry } from '../registry/WorkersRegistry.js';
|
|
11
|
-
import { WebServer } from '../server/WebServer.js';
|
|
12
|
-
import { BufferedLogger } from '../utils/logging/BufferedLogger.js';
|
|
13
|
-
import { Logger } from '../utils/logging/Logger.js';
|
|
14
|
-
import { PromiseAggregator } from '../utils/PromiseAggregator.js';
|
|
15
|
-
import { ResourceRequestCollector } from '../utils/ResourceRequestCollector.js';
|
|
1
|
+
import { ApplicationInstance } from './ApplicationInstance.js';
|
|
16
2
|
|
|
17
3
|
/**
|
|
18
|
-
* Application orchestrates the startup of Navi
|
|
19
|
-
* building registries, and running the Engine loop
|
|
4
|
+
* Application is a static singleton facade that orchestrates the startup of Navi
|
|
5
|
+
* by loading configuration, building registries, and running the Engine loop
|
|
6
|
+
* together with the WebServer.
|
|
7
|
+
*
|
|
8
|
+
* Call `Application.build()` once during bootstrap and `Application.reset()` in tests.
|
|
20
9
|
* @author darthjee
|
|
21
10
|
*/
|
|
22
11
|
class Application {
|
|
23
|
-
#
|
|
24
|
-
#bufferedLogger;
|
|
12
|
+
static #instance = null;
|
|
25
13
|
|
|
26
14
|
/**
|
|
27
|
-
* Creates
|
|
28
|
-
* @param {object} [params={}] -
|
|
29
|
-
* @
|
|
15
|
+
* Creates and stores the singleton ApplicationInstance.
|
|
16
|
+
* @param {object} [params={}] - Forwarded to `ApplicationInstance` constructor.
|
|
17
|
+
* @returns {ApplicationInstance} The created instance.
|
|
30
18
|
*/
|
|
31
|
-
|
|
32
|
-
|
|
19
|
+
static build(params = {}) {
|
|
20
|
+
Application.#instance = new ApplicationInstance(params);
|
|
21
|
+
return Application.#instance;
|
|
33
22
|
}
|
|
34
23
|
|
|
35
24
|
/**
|
|
36
|
-
*
|
|
37
|
-
* @param {string} configPath - The path to the configuration file.
|
|
38
|
-
* @throws {ConfigurationFileNotProvided} If the configuration file path is not provided.
|
|
39
|
-
* @throws {ConfigurationFileNotFound} If the configuration file is not found at the specified path.
|
|
25
|
+
* Destroys the singleton instance. Intended for test teardown.
|
|
40
26
|
* @returns {void}
|
|
41
27
|
*/
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
}
|
|
28
|
+
static reset() {
|
|
29
|
+
Application.#instance = null;
|
|
30
|
+
}
|
|
46
31
|
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
32
|
+
/**
|
|
33
|
+
* Returns the singleton instance (for test inspection).
|
|
34
|
+
* @returns {ApplicationInstance|null} The current instance.
|
|
35
|
+
*/
|
|
36
|
+
static get instance() {
|
|
37
|
+
return Application.#instance;
|
|
52
38
|
}
|
|
53
39
|
|
|
54
40
|
/**
|
|
55
|
-
*
|
|
56
|
-
*
|
|
57
|
-
* @returns {
|
|
41
|
+
* Returns the current engine status.
|
|
42
|
+
* Returns 'running' if no instance has been built yet.
|
|
43
|
+
* @returns {string} The current status.
|
|
44
|
+
*/
|
|
45
|
+
static status() {
|
|
46
|
+
if (!Application.#instance) return 'running';
|
|
47
|
+
return Application.#instance.status();
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
/**
|
|
51
|
+
* Returns true if the engine is currently running.
|
|
52
|
+
* @returns {boolean} True if the current status is 'running'.
|
|
58
53
|
*/
|
|
59
|
-
|
|
60
|
-
|
|
54
|
+
static isRunning() {
|
|
55
|
+
return Application.status() === 'running';
|
|
56
|
+
}
|
|
61
57
|
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
58
|
+
/**
|
|
59
|
+
* Returns true if the engine is currently paused.
|
|
60
|
+
* @returns {boolean} True if the current status is 'paused'.
|
|
61
|
+
*/
|
|
62
|
+
static isPaused() {
|
|
63
|
+
return Application.status() === 'paused';
|
|
64
|
+
}
|
|
65
65
|
|
|
66
|
-
|
|
67
|
-
|
|
66
|
+
/**
|
|
67
|
+
* Returns true if the engine is currently stopped.
|
|
68
|
+
* @returns {boolean} True if the current status is 'stopped'.
|
|
69
|
+
*/
|
|
70
|
+
static isStopped() {
|
|
71
|
+
return Application.status() === 'stopped';
|
|
72
|
+
}
|
|
68
73
|
|
|
69
|
-
|
|
74
|
+
/**
|
|
75
|
+
* Loads the configuration from the specified file path.
|
|
76
|
+
* @param {string} configPath - The path to the configuration file.
|
|
77
|
+
* @returns {void}
|
|
78
|
+
*/
|
|
79
|
+
static loadConfig(configPath) {
|
|
80
|
+
return Application.#getInstance().loadConfig(configPath);
|
|
81
|
+
}
|
|
70
82
|
|
|
71
|
-
|
|
83
|
+
/**
|
|
84
|
+
* Starts the application.
|
|
85
|
+
* @returns {Promise<void>}
|
|
86
|
+
*/
|
|
87
|
+
static async run() {
|
|
88
|
+
return Application.#getInstance().run();
|
|
72
89
|
}
|
|
73
90
|
|
|
74
91
|
/**
|
|
75
|
-
* Builds and returns a new Engine instance
|
|
92
|
+
* Builds and returns a new Engine instance.
|
|
76
93
|
* @returns {Engine} The created Engine instance.
|
|
77
94
|
*/
|
|
78
|
-
buildEngine() {
|
|
79
|
-
return
|
|
95
|
+
static buildEngine() {
|
|
96
|
+
return Application.#getInstance().buildEngine();
|
|
80
97
|
}
|
|
98
|
+
|
|
81
99
|
/**
|
|
82
|
-
* Builds and returns a WebServer
|
|
100
|
+
* Builds and returns a WebServer or null.
|
|
83
101
|
* @returns {WebServer|null} The created WebServer instance or null.
|
|
84
102
|
*/
|
|
85
|
-
buildWebServer() {
|
|
86
|
-
return
|
|
87
|
-
webConfig: this.config.webConfig,
|
|
88
|
-
});
|
|
103
|
+
static buildWebServer() {
|
|
104
|
+
return Application.#getInstance().buildWebServer();
|
|
89
105
|
}
|
|
106
|
+
|
|
90
107
|
/**
|
|
91
|
-
*
|
|
108
|
+
* Enqueues all parameter-free ResourceRequests.
|
|
109
|
+
* @returns {void}
|
|
110
|
+
*/
|
|
111
|
+
static enqueueFirstJobs() {
|
|
112
|
+
return Application.#getInstance().enqueueFirstJobs();
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
/**
|
|
116
|
+
* Gets the buffered logger from the singleton instance.
|
|
92
117
|
* @returns {BufferedLogger} The buffered logger instance.
|
|
93
118
|
*/
|
|
94
|
-
get bufferedLogger() {
|
|
95
|
-
return
|
|
119
|
+
static get bufferedLogger() {
|
|
120
|
+
return Application.#getInstance().bufferedLogger;
|
|
96
121
|
}
|
|
122
|
+
|
|
97
123
|
/**
|
|
98
|
-
*
|
|
99
|
-
*
|
|
100
|
-
* processed immediately without any external parameters.
|
|
101
|
-
* @returns {void}
|
|
124
|
+
* Pauses the engine and waits for workers to become idle.
|
|
125
|
+
* @returns {Promise<void>}
|
|
102
126
|
*/
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
JobRegistry.enqueue('ResourceRequestJob', { resourceRequest, parameters: {} });
|
|
106
|
-
});
|
|
127
|
+
static async pause() {
|
|
128
|
+
return Application.#getInstance().pause();
|
|
107
129
|
}
|
|
130
|
+
|
|
108
131
|
/**
|
|
109
|
-
*
|
|
110
|
-
* @returns {void}
|
|
132
|
+
* Stops the engine, waits for workers to idle, then clears job queues.
|
|
133
|
+
* @returns {Promise<void>}
|
|
111
134
|
*/
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
JobFactory.build('HtmlParse', { klass: HtmlParseJob, attributes: { jobRegistry: JobRegistry, clientRegistry: this.config.clientRegistry } });
|
|
116
|
-
JobFactory.build('AssetDownload', { klass: AssetDownloadJob, attributes: { clientRegistry: this.config.clientRegistry } });
|
|
135
|
+
static async stop() {
|
|
136
|
+
return Application.#getInstance().stop();
|
|
137
|
+
}
|
|
117
138
|
|
|
118
|
-
|
|
139
|
+
/**
|
|
140
|
+
* Resumes processing after a pause.
|
|
141
|
+
* @returns {Promise<void>}
|
|
142
|
+
*/
|
|
143
|
+
static async continue() {
|
|
144
|
+
return Application.#getInstance().continue();
|
|
145
|
+
}
|
|
119
146
|
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
147
|
+
/**
|
|
148
|
+
* Starts processing from a stopped state.
|
|
149
|
+
* @returns {Promise<void>}
|
|
150
|
+
*/
|
|
151
|
+
static async start() {
|
|
152
|
+
return Application.#getInstance().start();
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
/**
|
|
156
|
+
* Restarts processing.
|
|
157
|
+
* @returns {Promise<void>}
|
|
158
|
+
*/
|
|
159
|
+
static async restart() {
|
|
160
|
+
return Application.#getInstance().restart();
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
/**
|
|
164
|
+
* Shuts down the web server and stops the engine.
|
|
165
|
+
* @returns {Promise<void>}
|
|
166
|
+
*/
|
|
167
|
+
static async shutdown() {
|
|
168
|
+
return Application.#getInstance().shutdown();
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
/**
|
|
172
|
+
* Returns the singleton instance, throwing if not yet built.
|
|
173
|
+
* @returns {ApplicationInstance} The singleton instance.
|
|
174
|
+
* @throws {Error} If `build()` has not been called.
|
|
175
|
+
*/
|
|
176
|
+
static #getInstance() {
|
|
177
|
+
if (!Application.#instance) {
|
|
178
|
+
throw new Error('Application has not been initialized. Call Application.build() before calling static methods that require an instance.');
|
|
179
|
+
}
|
|
180
|
+
return Application.#instance;
|
|
127
181
|
}
|
|
128
182
|
}
|
|
129
183
|
|
|
130
|
-
export { Application };
|
|
184
|
+
export { Application };
|
|
@@ -0,0 +1,274 @@
|
|
|
1
|
+
import { Engine } from './Engine.js';
|
|
2
|
+
import { EngineEvents } from './EngineEvents.js';
|
|
3
|
+
import { FailureChecker } from './FailureChecker.js';
|
|
4
|
+
import { JobFactory } from '../background/JobFactory.js';
|
|
5
|
+
import { JobRegistry } from '../background/JobRegistry.js';
|
|
6
|
+
import { WorkersRegistry } from '../background/WorkersRegistry.js';
|
|
7
|
+
import { ConfigurationFileNotProvided } from '../exceptions/config/ConfigurationFileNotProvided.js';
|
|
8
|
+
import { ActionProcessingJob } from '../jobs/ActionProcessingJob.js';
|
|
9
|
+
import { AssetDownloadJob } from '../jobs/AssetDownloadJob.js';
|
|
10
|
+
import { HtmlParseJob } from '../jobs/HtmlParseJob.js';
|
|
11
|
+
import { PaginatedActionProcessingJob } from '../jobs/PaginatedActionProcessingJob.js';
|
|
12
|
+
import { Config } from '../models/configs/Config.js';
|
|
13
|
+
import { LogRegistry } from '../registry/LogRegistry.js';
|
|
14
|
+
import { WebServer } from '../server/WebServer.js';
|
|
15
|
+
import { PromiseAggregator } from '../utils/PromiseAggregator.js';
|
|
16
|
+
import { ResourceRequestCollector } from '../utils/ResourceRequestCollector.js';
|
|
17
|
+
|
|
18
|
+
const DEFAULT_POLL_SLEEP_MS = 10;
|
|
19
|
+
|
|
20
|
+
/**
|
|
21
|
+
* ApplicationInstance holds all instance-level state and logic for a single
|
|
22
|
+
* application lifecycle. Use the static `Application` facade to access it.
|
|
23
|
+
* @author darthjee
|
|
24
|
+
*/
|
|
25
|
+
class ApplicationInstance {
|
|
26
|
+
#workers;
|
|
27
|
+
#bufferedLogger;
|
|
28
|
+
#engineStatus;
|
|
29
|
+
#aggregator;
|
|
30
|
+
#enginePromise;
|
|
31
|
+
#sleepMs;
|
|
32
|
+
|
|
33
|
+
/**
|
|
34
|
+
* @param {object} [params={}] - Optional parameters for dependency injection.
|
|
35
|
+
* @param {IdentifyableCollection} [params.workers] - Workers collection (injected for testing).
|
|
36
|
+
*/
|
|
37
|
+
constructor({ workers } = {}) {
|
|
38
|
+
this.#workers = workers;
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
/**
|
|
42
|
+
* Loads the configuration from the specified file path.
|
|
43
|
+
* @param {string} configPath - The path to the configuration file.
|
|
44
|
+
* @throws {ConfigurationFileNotProvided} If the configuration file path is not provided.
|
|
45
|
+
* @throws {ConfigurationFileNotFound} If the configuration file is not found at the specified path.
|
|
46
|
+
* @returns {void}
|
|
47
|
+
*/
|
|
48
|
+
loadConfig(configPath) {
|
|
49
|
+
if (!configPath) {
|
|
50
|
+
throw new ConfigurationFileNotProvided();
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
this.config = Config.fromFile(configPath);
|
|
54
|
+
const logRegistry = LogRegistry.build({ retention: this.config.logConfig.size });
|
|
55
|
+
this.#bufferedLogger = logRegistry.bufferedLogger;
|
|
56
|
+
this.#initRegistries();
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
/**
|
|
60
|
+
* Starts the application by building the engine, web server, enqueueing initial jobs, and starting both.
|
|
61
|
+
* After the engine finishes, checks the dead-job ratio against the configured failure threshold.
|
|
62
|
+
* @returns {Promise<void>}
|
|
63
|
+
*/
|
|
64
|
+
async run() {
|
|
65
|
+
this.#aggregator = new PromiseAggregator();
|
|
66
|
+
this.#sleepMs = this.config.workersConfig.sleep;
|
|
67
|
+
|
|
68
|
+
this.engine = this.buildEngine();
|
|
69
|
+
this.webServer = this.buildWebServer();
|
|
70
|
+
this.enqueueFirstJobs();
|
|
71
|
+
|
|
72
|
+
this.#engineStatus = 'running';
|
|
73
|
+
this.#aggregator.add(this.webServer?.start());
|
|
74
|
+
this.#enginePromise = this.engine.start();
|
|
75
|
+
this.#aggregator.add(this.#enginePromise);
|
|
76
|
+
|
|
77
|
+
await this.#aggregator.wait();
|
|
78
|
+
this.#engineStatus = 'stopped';
|
|
79
|
+
EngineEvents.emit('stop');
|
|
80
|
+
|
|
81
|
+
new FailureChecker({ failureConfig: this.config.failureConfig }).check();
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
/**
|
|
85
|
+
* Builds and returns a new Engine instance wired to the current registries.
|
|
86
|
+
* @returns {Engine} The created Engine instance.
|
|
87
|
+
*/
|
|
88
|
+
buildEngine() {
|
|
89
|
+
return new Engine({
|
|
90
|
+
sleepMs: this.#sleepMs ?? this.config.workersConfig.sleep,
|
|
91
|
+
keepAlive: !!this.config.webConfig,
|
|
92
|
+
});
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
/**
|
|
96
|
+
* Builds and returns a WebServer if web configuration is present, otherwise null.
|
|
97
|
+
* @returns {WebServer|null} The created WebServer instance or null.
|
|
98
|
+
*/
|
|
99
|
+
buildWebServer() {
|
|
100
|
+
return WebServer.build({
|
|
101
|
+
webConfig: this.config.webConfig,
|
|
102
|
+
});
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
/**
|
|
106
|
+
* Gets the buffered logger instance created during config loading.
|
|
107
|
+
* @returns {BufferedLogger} The buffered logger instance.
|
|
108
|
+
*/
|
|
109
|
+
get bufferedLogger() {
|
|
110
|
+
return this.#bufferedLogger;
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
/**
|
|
114
|
+
* Enqueues all parameter-free ResourceRequests into the job registry.
|
|
115
|
+
* These are requests whose URLs contain no {:placeholder} tokens and can be
|
|
116
|
+
* processed immediately without any external parameters.
|
|
117
|
+
* @returns {void}
|
|
118
|
+
*/
|
|
119
|
+
enqueueFirstJobs() {
|
|
120
|
+
new ResourceRequestCollector(this.config.resourceRegistry).requestsNeedingNoParams().forEach((resourceRequest) => {
|
|
121
|
+
JobRegistry.enqueue('ResourceRequestJob', { resourceRequest, parameters: {} });
|
|
122
|
+
});
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
/**
|
|
126
|
+
* Returns the current engine status string.
|
|
127
|
+
* @returns {string|undefined} The current status.
|
|
128
|
+
*/
|
|
129
|
+
status() {
|
|
130
|
+
return this.#engineStatus;
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
/**
|
|
134
|
+
* Returns true if the engine is currently running.
|
|
135
|
+
* @returns {boolean} True if the current status is 'running'.
|
|
136
|
+
*/
|
|
137
|
+
isRunning() {
|
|
138
|
+
return this.#engineStatus === 'running';
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
/**
|
|
142
|
+
* Returns true if the engine is currently paused.
|
|
143
|
+
* @returns {boolean} True if the current status is 'paused'.
|
|
144
|
+
*/
|
|
145
|
+
isPaused() {
|
|
146
|
+
return this.#engineStatus === 'paused';
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
/**
|
|
150
|
+
* Returns true if the engine is currently stopped.
|
|
151
|
+
* @returns {boolean} True if the current status is 'stopped'.
|
|
152
|
+
*/
|
|
153
|
+
isStopped() {
|
|
154
|
+
return this.#engineStatus === 'stopped';
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
/**
|
|
158
|
+
* Sets the engine status string.
|
|
159
|
+
* @param {string} value - The new status value.
|
|
160
|
+
* @returns {void}
|
|
161
|
+
*/
|
|
162
|
+
setStatus(value) {
|
|
163
|
+
this.#engineStatus = value;
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
/**
|
|
167
|
+
* Pauses processing: pauses the engine (without stopping it) and waits for workers to become idle.
|
|
168
|
+
* @returns {Promise<void>}
|
|
169
|
+
*/
|
|
170
|
+
async pause() {
|
|
171
|
+
this.#engineStatus = 'pausing';
|
|
172
|
+
this.engine.pause();
|
|
173
|
+
await this.#waitForWorkersIdle();
|
|
174
|
+
this.#engineStatus = 'paused';
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
/**
|
|
178
|
+
* Stops processing: pauses the engine, waits for workers to idle, then clears job queues.
|
|
179
|
+
* The engine instance is preserved and its loop continues running in the background.
|
|
180
|
+
* @returns {Promise<void>}
|
|
181
|
+
*/
|
|
182
|
+
async stop() {
|
|
183
|
+
this.#engineStatus = 'stopping';
|
|
184
|
+
this.engine.pause();
|
|
185
|
+
await this.#waitForWorkersIdle();
|
|
186
|
+
JobRegistry.clearQueues();
|
|
187
|
+
this.#engineStatus = 'stopped';
|
|
188
|
+
EngineEvents.emit('stop');
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
/**
|
|
192
|
+
* Resumes processing after a pause by calling engine.resume().
|
|
193
|
+
* No new engine is created; the existing loop continues.
|
|
194
|
+
* Only valid when status is 'paused'.
|
|
195
|
+
* @returns {Promise<void>}
|
|
196
|
+
*/
|
|
197
|
+
async continue() {
|
|
198
|
+
if (this.#engineStatus !== 'paused') return;
|
|
199
|
+
this.engine.resume();
|
|
200
|
+
this.#engineStatus = 'running';
|
|
201
|
+
}
|
|
202
|
+
|
|
203
|
+
/**
|
|
204
|
+
* Starts processing from a stopped state by calling engine.resume() and re-enqueueing initial jobs.
|
|
205
|
+
* No new engine is created; the existing loop continues.
|
|
206
|
+
* Only valid when status is 'stopped'.
|
|
207
|
+
* @returns {Promise<void>}
|
|
208
|
+
*/
|
|
209
|
+
async start() {
|
|
210
|
+
if (this.#engineStatus !== 'stopped') return;
|
|
211
|
+
this.engine.resume();
|
|
212
|
+
this.enqueueFirstJobs();
|
|
213
|
+
this.#engineStatus = 'running';
|
|
214
|
+
EngineEvents.emit('start');
|
|
215
|
+
}
|
|
216
|
+
|
|
217
|
+
/**
|
|
218
|
+
* Restarts processing: stops then starts the engine.
|
|
219
|
+
* Only valid when status is 'running'.
|
|
220
|
+
* @returns {Promise<void>}
|
|
221
|
+
*/
|
|
222
|
+
async restart() {
|
|
223
|
+
if (this.#engineStatus !== 'running') return;
|
|
224
|
+
await this.stop();
|
|
225
|
+
await this.start();
|
|
226
|
+
}
|
|
227
|
+
|
|
228
|
+
/**
|
|
229
|
+
* Shuts down the web server and stops the engine loop.
|
|
230
|
+
* If running, pauses first and waits for workers to idle.
|
|
231
|
+
* @returns {Promise<void>}
|
|
232
|
+
*/
|
|
233
|
+
async shutdown() {
|
|
234
|
+
this.webServer?.shutdown();
|
|
235
|
+
if (this.#engineStatus === 'running') {
|
|
236
|
+
await this.stop();
|
|
237
|
+
}
|
|
238
|
+
this.engine.stop();
|
|
239
|
+
}
|
|
240
|
+
|
|
241
|
+
/**
|
|
242
|
+
* Polls until all workers are idle.
|
|
243
|
+
* @returns {Promise<void>}
|
|
244
|
+
*/
|
|
245
|
+
async #waitForWorkersIdle() {
|
|
246
|
+
while (WorkersRegistry.hasBusyWorker()) {
|
|
247
|
+
await new Promise(resolve => setTimeout(resolve, this.#sleepMs ?? DEFAULT_POLL_SLEEP_MS));
|
|
248
|
+
}
|
|
249
|
+
}
|
|
250
|
+
|
|
251
|
+
/**
|
|
252
|
+
* Initializes the job factory, job registry, and workers registry from the loaded configuration.
|
|
253
|
+
* @returns {void}
|
|
254
|
+
*/
|
|
255
|
+
#initRegistries() {
|
|
256
|
+
JobFactory.build('ResourceRequestJob', { attributes: { clients: this.config.clientRegistry } });
|
|
257
|
+
JobFactory.build('Action', { klass: ActionProcessingJob });
|
|
258
|
+
JobFactory.build('PaginatedAction', { klass: PaginatedActionProcessingJob });
|
|
259
|
+
JobFactory.build('HtmlParse', { klass: HtmlParseJob, attributes: { jobRegistry: JobRegistry, clientRegistry: this.config.clientRegistry } });
|
|
260
|
+
JobFactory.build('AssetDownload', { klass: AssetDownloadJob, attributes: { clientRegistry: this.config.clientRegistry } });
|
|
261
|
+
|
|
262
|
+
JobRegistry.build({ cooldown: this.config.workersConfig.retryCooldown, maxRetries: this.config.workersConfig.maxRetries });
|
|
263
|
+
|
|
264
|
+
WorkersRegistry.build({
|
|
265
|
+
workers: this.#workers,
|
|
266
|
+
jobRegistry: JobRegistry,
|
|
267
|
+
workersRegistry: WorkersRegistry,
|
|
268
|
+
...this.config.workersConfig,
|
|
269
|
+
});
|
|
270
|
+
WorkersRegistry.initWorkers();
|
|
271
|
+
}
|
|
272
|
+
}
|
|
273
|
+
|
|
274
|
+
export { ApplicationInstance };
|
package/lib/services/Client.js
CHANGED
|
@@ -1,6 +1,5 @@
|
|
|
1
1
|
import axios from 'axios';
|
|
2
|
-
import { RequestFailed } from '../exceptions/RequestFailed.js';
|
|
3
|
-
import { Logger } from '../utils/logging/Logger.js';
|
|
2
|
+
import { RequestFailed } from '../exceptions/request/RequestFailed.js';
|
|
4
3
|
|
|
5
4
|
/**
|
|
6
5
|
* Client performs HTTP requests for resource paths using a configured base URL.
|
|
@@ -55,16 +54,17 @@ class Client {
|
|
|
55
54
|
* @param {ResourceRequest} resourceRequest Information about the URL path to request
|
|
56
55
|
* and the expected status code.
|
|
57
56
|
* @param {object} [parameters={}] Key-value map used to resolve {:placeholder} tokens in the URL.
|
|
57
|
+
* @param {LogContext} logContext Context carrying workerId/jobId for log entries.
|
|
58
58
|
* @returns {Promise<boolean>} Returns true if the response status matches the expected status.
|
|
59
59
|
* @throws {RequestFailed} Throws an error if the request fails or the status does not match.
|
|
60
60
|
*/
|
|
61
|
-
async perform(resourceRequest, parameters = {}) {
|
|
61
|
+
async perform(resourceRequest, parameters = {}, logContext) {
|
|
62
62
|
const requestUrl = this.#buildUrl(resourceRequest.resolveUrl(parameters));
|
|
63
|
-
|
|
63
|
+
logContext.info(`[Client:${this.name}] Requesting ${requestUrl}`);
|
|
64
64
|
try {
|
|
65
|
-
return await this.#request(resourceRequest, requestUrl);
|
|
65
|
+
return await this.#request(resourceRequest, requestUrl, logContext);
|
|
66
66
|
} catch (error) {
|
|
67
|
-
this.#handleError(error, requestUrl);
|
|
67
|
+
this.#handleError(error, requestUrl, logContext);
|
|
68
68
|
}
|
|
69
69
|
}
|
|
70
70
|
|
|
@@ -73,15 +73,16 @@ class Client {
|
|
|
73
73
|
* Unlike `perform`, this method accepts an absolute URL directly and does not prepend `baseUrl`.
|
|
74
74
|
* @param {string} absoluteUrl The fully-resolved URL to request.
|
|
75
75
|
* @param {number} expectedStatus The expected HTTP response status code.
|
|
76
|
+
* @param {LogContext} logContext Context carrying workerId/jobId for log entries.
|
|
76
77
|
* @returns {Promise<object>} The Axios response object.
|
|
77
78
|
* @throws {RequestFailed} If the response status does not match `expectedStatus`.
|
|
78
79
|
*/
|
|
79
|
-
async performUrl(absoluteUrl, expectedStatus) {
|
|
80
|
-
|
|
80
|
+
async performUrl(absoluteUrl, expectedStatus, logContext) {
|
|
81
|
+
logContext.info(`[Client:${this.name}] Requesting ${absoluteUrl}`);
|
|
81
82
|
try {
|
|
82
|
-
return await this.#requestUrl(absoluteUrl, expectedStatus);
|
|
83
|
+
return await this.#requestUrl(absoluteUrl, expectedStatus, logContext);
|
|
83
84
|
} catch (error) {
|
|
84
|
-
this.#handleError(error, absoluteUrl);
|
|
85
|
+
this.#handleError(error, absoluteUrl, logContext);
|
|
85
86
|
}
|
|
86
87
|
}
|
|
87
88
|
|
|
@@ -90,21 +91,23 @@ class Client {
|
|
|
90
91
|
* @param {ResourceRequest} resourceRequest Information about the URL path to request
|
|
91
92
|
* and the expected status code.
|
|
92
93
|
* @param {string} requestUrl The full URL for the request.
|
|
94
|
+
* @param {LogContext} logContext Context carrying workerId/jobId for log entries.
|
|
93
95
|
* @returns {Promise<boolean>} Returns true if the response status matches the expected status.
|
|
94
96
|
* @throws {RequestFailed} Throws an error if the response status does not match.
|
|
95
97
|
*/
|
|
96
|
-
async #request(resourceRequest, requestUrl) {
|
|
97
|
-
return this.#requestUrl(requestUrl, resourceRequest.status);
|
|
98
|
+
async #request(resourceRequest, requestUrl, logContext) {
|
|
99
|
+
return this.#requestUrl(requestUrl, resourceRequest.status, logContext);
|
|
98
100
|
}
|
|
99
101
|
|
|
100
102
|
/**
|
|
101
103
|
* Performs the HTTP GET request to the given URL and validates the response status.
|
|
102
104
|
* @param {string} requestUrl The full URL to request.
|
|
103
105
|
* @param {number} expectedStatus The expected HTTP response status code.
|
|
106
|
+
* @param {LogContext} logContext Context carrying workerId/jobId for log entries.
|
|
104
107
|
* @returns {Promise<object>} The Axios response object.
|
|
105
108
|
* @throws {RequestFailed} Throws an error if the response status does not match.
|
|
106
109
|
*/
|
|
107
|
-
async #requestUrl(requestUrl, expectedStatus) {
|
|
110
|
+
async #requestUrl(requestUrl, expectedStatus, logContext) {
|
|
108
111
|
const response = await axios.get(requestUrl, {
|
|
109
112
|
timeout: this.timeout,
|
|
110
113
|
responseType: 'text',
|
|
@@ -113,14 +116,14 @@ class Client {
|
|
|
113
116
|
validateStatus: () => true,
|
|
114
117
|
});
|
|
115
118
|
|
|
116
|
-
|
|
119
|
+
logContext.info(`[Client:${this.name}] Response ${requestUrl} → ${response.status}`);
|
|
117
120
|
|
|
118
121
|
if (response.status !== expectedStatus) {
|
|
119
|
-
|
|
122
|
+
logContext.info(`[Client:${this.name}] ${requestUrl} did not match (got ${response.status}, expected ${expectedStatus})`);
|
|
120
123
|
throw new RequestFailed(response.status, requestUrl);
|
|
121
124
|
}
|
|
122
125
|
|
|
123
|
-
|
|
126
|
+
logContext.info(`[Client:${this.name}] ${requestUrl} matched (expected ${expectedStatus})`);
|
|
124
127
|
return response;
|
|
125
128
|
}
|
|
126
129
|
|
|
@@ -128,11 +131,12 @@ class Client {
|
|
|
128
131
|
* Handles a caught request error, logging it and re-throwing as RequestFailed when applicable.
|
|
129
132
|
* @param {Error} error The caught error.
|
|
130
133
|
* @param {string} requestUrl The URL that was requested.
|
|
134
|
+
* @param {LogContext} logContext Context carrying workerId/jobId for log entries.
|
|
131
135
|
* @throws {RequestFailed} If the error has a response object.
|
|
132
136
|
* @throws {Error} Re-throws the original error otherwise.
|
|
133
137
|
*/
|
|
134
|
-
#handleError(error, requestUrl) {
|
|
135
|
-
|
|
138
|
+
#handleError(error, requestUrl, logContext) {
|
|
139
|
+
logContext.error(`Request failed: ${error}`);
|
|
136
140
|
if (error.response) {
|
|
137
141
|
throw new RequestFailed(error.response.status, requestUrl);
|
|
138
142
|
}
|
|
@@ -149,4 +153,4 @@ class Client {
|
|
|
149
153
|
}
|
|
150
154
|
}
|
|
151
155
|
|
|
152
|
-
export { Client };
|
|
156
|
+
export { Client };
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { readFileSync } from 'node:fs';
|
|
2
2
|
import YAML from 'yaml';
|
|
3
3
|
import { ConfigParser } from './ConfigParser.js';
|
|
4
|
-
import { ConfigurationFileNotFound } from '../exceptions/ConfigurationFileNotFound.js';
|
|
4
|
+
import { ConfigurationFileNotFound } from '../exceptions/config/ConfigurationFileNotFound.js';
|
|
5
5
|
import { EnvResolver } from '../utils/EnvResolver.js';
|
|
6
6
|
import { Logger } from '../utils/logging/Logger.js';
|
|
7
7
|
|