navi-hey 1.0.0 → 1.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (107) hide show
  1. package/README.md +38 -4
  2. package/bin/navi.js +3 -3
  3. package/lib/{factories → background}/JobFactory.js +2 -2
  4. package/lib/{registry → background}/JobRegistry.js +17 -0
  5. package/lib/{registry → background}/JobRegistryInstance.js +42 -1
  6. package/lib/{models → background}/Worker.js +5 -3
  7. package/lib/{factories → background}/WorkerFactory.js +2 -2
  8. package/lib/{registry → background}/WorkersRegistryInstance.js +1 -1
  9. package/lib/enqueuers/ActionEnqueuer.js +53 -0
  10. package/lib/{models → enqueuers}/ActionsEnqueuer.js +6 -3
  11. package/lib/{models → enqueuers}/AssetRequestEnqueuer.js +6 -2
  12. package/lib/enqueuers/PaginatedActionEnqueuer.js +53 -0
  13. package/lib/enqueuers/PaginatedActionsEnqueuer.js +47 -0
  14. package/lib/exceptions/{ConfigurationFileNotFound.js → config/ConfigurationFileNotFound.js} +1 -1
  15. package/lib/exceptions/{ConfigurationFileNotProvided.js → config/ConfigurationFileNotProvided.js} +1 -1
  16. package/lib/exceptions/{MissingTopLevelConfgKey.js → config/MissingTopLevelConfgKey.js} +1 -1
  17. package/lib/exceptions/http/ConflictError.js +14 -0
  18. package/lib/exceptions/{ForbiddenError.js → http/ForbiddenError.js} +1 -1
  19. package/lib/exceptions/{NotFoundError.js → http/NotFoundError.js} +1 -1
  20. package/lib/exceptions/{ItemNotFound.js → registry/ItemNotFound.js} +1 -1
  21. package/lib/exceptions/{MissingActionResource.js → registry/MissingActionResource.js} +1 -1
  22. package/lib/exceptions/{MissingMappingVariable.js → registry/MissingMappingVariable.js} +1 -1
  23. package/lib/exceptions/{InvalidHtmlResponseBody.js → request/InvalidHtmlResponseBody.js} +1 -1
  24. package/lib/exceptions/{InvalidResponseBody.js → request/InvalidResponseBody.js} +1 -1
  25. package/lib/exceptions/{NullResponse.js → request/NullResponse.js} +1 -1
  26. package/lib/exceptions/{RequestFailed.js → request/RequestFailed.js} +1 -1
  27. package/lib/{models → jobs}/ActionProcessingJob.js +21 -7
  28. package/lib/{models → jobs}/AssetDownloadJob.js +6 -6
  29. package/lib/{models → jobs}/HtmlParseJob.js +23 -10
  30. package/lib/jobs/PaginatedActionProcessingJob.js +77 -0
  31. package/lib/{models → jobs}/ResourceRequestJob.js +18 -15
  32. package/lib/models/{Config.js → configs/Config.js} +3 -3
  33. package/lib/models/configs/PaginationConfig.js +63 -0
  34. package/lib/models/configs/WebConfig.js +19 -0
  35. package/lib/models/{ResourceRequest.js → request/ResourceRequest.js} +32 -7
  36. package/lib/models/{ResourceRequestAction.js → request/ResourceRequestAction.js} +9 -6
  37. package/lib/models/request/ResourceRequestPaginatedAction.js +81 -0
  38. package/lib/models/{PathSegmentTraverser.js → response/PathSegmentTraverser.js} +1 -1
  39. package/lib/models/{ResponseParser.js → response/ResponseParser.js} +1 -1
  40. package/lib/registry/ClientRegistry.js +1 -1
  41. package/lib/registry/LogRegistry.js +146 -0
  42. package/lib/registry/LogRegistryInstance.js +156 -0
  43. package/lib/registry/NamedRegistry.js +1 -1
  44. package/lib/registry/ResourceRegistry.js +1 -1
  45. package/lib/serializers/JobIndexSerializer.js +13 -3
  46. package/lib/serializers/LogSerializer.js +26 -0
  47. package/lib/server/PathValidator.js +1 -1
  48. package/lib/server/RouteRegister.js +42 -9
  49. package/lib/server/Router.js +49 -31
  50. package/lib/server/WebServer.js +28 -8
  51. package/lib/server/{AssetsRequestHandler.js → handlers/AssetsRequestHandler.js} +3 -3
  52. package/lib/server/{BaseUrlsRequestHandler.js → handlers/BaseUrlsRequestHandler.js} +3 -3
  53. package/lib/server/{IndexRequestHandler.js → handlers/IndexRequestHandler.js} +2 -2
  54. package/lib/server/handlers/JobsFilter.js +55 -0
  55. package/lib/server/handlers/LogsRequestHandler.js +55 -0
  56. package/lib/server/handlers/SettingsRequestHandler.js +35 -0
  57. package/lib/server/{StatsRequestHandler.js → handlers/StatsRequestHandler.js} +3 -3
  58. package/lib/server/handlers/engine/EngineContinueRequestHandler.js +34 -0
  59. package/lib/server/handlers/engine/EnginePauseRequestHandler.js +34 -0
  60. package/lib/server/handlers/engine/EngineRestartRequestHandler.js +34 -0
  61. package/lib/server/handlers/engine/EngineShutdownRequestHandler.js +29 -0
  62. package/lib/server/handlers/engine/EngineStartRequestHandler.js +34 -0
  63. package/lib/server/handlers/engine/EngineStatusRequestHandler.js +27 -0
  64. package/lib/server/handlers/engine/EngineStopRequestHandler.js +34 -0
  65. package/lib/server/handlers/jobs/JobLogsRequestHandler.js +39 -0
  66. package/lib/server/{JobRequestHandler.js → handlers/jobs/JobRequestHandler.js} +4 -4
  67. package/lib/server/handlers/jobs/JobRetryRequestHandler.js +40 -0
  68. package/lib/server/{JobsRequestHandler.js → handlers/jobs/JobsRequestHandler.js} +7 -4
  69. package/lib/services/Application.js +138 -84
  70. package/lib/services/ApplicationInstance.js +274 -0
  71. package/lib/services/Client.js +23 -19
  72. package/lib/services/ConfigLoader.js +1 -1
  73. package/lib/services/ConfigParser.js +7 -7
  74. package/lib/services/Engine.js +51 -6
  75. package/lib/services/EngineEvents.js +42 -0
  76. package/lib/services/FailureChecker.js +4 -4
  77. package/lib/services/WorkersAllocator.js +2 -2
  78. package/lib/utils/HtmlElementParser.js +5 -4
  79. package/lib/utils/HtmlParser.js +5 -5
  80. package/lib/utils/PromiseAggregator.js +11 -5
  81. package/lib/utils/logging/BufferedLogger.js +8 -0
  82. package/lib/utils/logging/LogBuffer.js +25 -5
  83. package/lib/utils/logging/LogBufferCollection.js +56 -0
  84. package/lib/utils/logging/LogContext.js +51 -0
  85. package/lib/utils/logging/LogFilter.js +34 -0
  86. package/package.json +1 -1
  87. package/static/assets/{index-DBkz-poX.css → index-CuWjW0sT.css} +1 -1
  88. package/static/assets/index-D5tOOLpl.js +11 -0
  89. package/static/index.html +2 -2
  90. package/lib/exceptions/LockedByOtherWorker.js +0 -14
  91. package/lib/models/ActionEnqueuer.js +0 -34
  92. package/lib/models/WebConfig.js +0 -15
  93. /package/lib/{models → background}/Job.js +0 -0
  94. /package/lib/{registry → background}/WorkersRegistry.js +0 -0
  95. /package/lib/exceptions/{MissingClientsConfig.js → config/MissingClientsConfig.js} +0 -0
  96. /package/lib/exceptions/{MissingResourceConfig.js → config/MissingResourceConfig.js} +0 -0
  97. /package/lib/exceptions/{ClientNotFound.js → registry/ClientNotFound.js} +0 -0
  98. /package/lib/exceptions/{ResourceNotFound.js → registry/ResourceNotFound.js} +0 -0
  99. /package/lib/{factories → factory}/Factory.js +0 -0
  100. /package/lib/models/{FailureConfig.js → configs/FailureConfig.js} +0 -0
  101. /package/lib/models/{LogConfig.js → configs/LogConfig.js} +0 -0
  102. /package/lib/models/{WorkersConfig.js → configs/WorkersConfig.js} +0 -0
  103. /package/lib/models/{AssetRequest.js → request/AssetRequest.js} +0 -0
  104. /package/lib/models/{Resource.js → request/Resource.js} +0 -0
  105. /package/lib/models/{ParametersMapper.js → response/ParametersMapper.js} +0 -0
  106. /package/lib/models/{PathResolver.js → response/PathResolver.js} +0 -0
  107. /package/lib/models/{ResponseWrapper.js → response/ResponseWrapper.js} +0 -0
@@ -1,130 +1,184 @@
1
- import { Engine } from './Engine.js';
2
- import { FailureChecker } from './FailureChecker.js';
3
- import { ConfigurationFileNotProvided } from '../exceptions/ConfigurationFileNotProvided.js';
4
- import { JobFactory } from '../factories/JobFactory.js';
5
- import { ActionProcessingJob } from '../models/ActionProcessingJob.js';
6
- import { AssetDownloadJob } from '../models/AssetDownloadJob.js';
7
- import { Config } from '../models/Config.js';
8
- import { HtmlParseJob } from '../models/HtmlParseJob.js';
9
- import { JobRegistry } from '../registry/JobRegistry.js';
10
- import { WorkersRegistry } from '../registry/WorkersRegistry.js';
11
- import { WebServer } from '../server/WebServer.js';
12
- import { BufferedLogger } from '../utils/logging/BufferedLogger.js';
13
- import { Logger } from '../utils/logging/Logger.js';
14
- import { PromiseAggregator } from '../utils/PromiseAggregator.js';
15
- import { ResourceRequestCollector } from '../utils/ResourceRequestCollector.js';
1
+ import { ApplicationInstance } from './ApplicationInstance.js';
16
2
 
17
3
  /**
18
- * Application orchestrates the startup of Navi by loading configuration,
19
- * building registries, and running the Engine loop together with the WebServer.
4
+ * Application is a static singleton facade that orchestrates the startup of Navi
5
+ * by loading configuration, building registries, and running the Engine loop
6
+ * together with the WebServer.
7
+ *
8
+ * Call `Application.build()` once during bootstrap and `Application.reset()` in tests.
20
9
  * @author darthjee
21
10
  */
22
11
  class Application {
23
- #workers;
24
- #bufferedLogger;
12
+ static #instance = null;
25
13
 
26
14
  /**
27
- * Creates a new Application instance.
28
- * @param {object} [params={}] - Optional parameters for dependency injection.
29
- * @param {IdentifyableCollection} [params.workers] - Workers collection (injected for testing).
15
+ * Creates and stores the singleton ApplicationInstance.
16
+ * @param {object} [params={}] - Forwarded to `ApplicationInstance` constructor.
17
+ * @returns {ApplicationInstance} The created instance.
30
18
  */
31
- constructor({ workers } = {}) {
32
- this.#workers = workers;
19
+ static build(params = {}) {
20
+ Application.#instance = new ApplicationInstance(params);
21
+ return Application.#instance;
33
22
  }
34
23
 
35
24
  /**
36
- * Loads the configuration from the specified file path.
37
- * @param {string} configPath - The path to the configuration file.
38
- * @throws {ConfigurationFileNotProvided} If the configuration file path is not provided.
39
- * @throws {ConfigurationFileNotFound} If the configuration file is not found at the specified path.
25
+ * Destroys the singleton instance. Intended for test teardown.
40
26
  * @returns {void}
41
27
  */
42
- loadConfig(configPath) {
43
- if (!configPath) {
44
- throw new ConfigurationFileNotProvided();
45
- }
28
+ static reset() {
29
+ Application.#instance = null;
30
+ }
46
31
 
47
- // Load the configuration from the specified path.
48
- this.config = Config.fromFile(configPath);
49
- this.#bufferedLogger = new BufferedLogger(undefined, this.config.logConfig.size);
50
- Logger.addLogger(this.#bufferedLogger);
51
- this.#initRegistries();
32
+ /**
33
+ * Returns the singleton instance (for test inspection).
34
+ * @returns {ApplicationInstance|null} The current instance.
35
+ */
36
+ static get instance() {
37
+ return Application.#instance;
52
38
  }
53
39
 
54
40
  /**
55
- * Starts the application by building the engine, web server, enqueueing initial jobs, and starting both.
56
- * After the engine finishes, checks the dead-job ratio against the configured failure threshold.
57
- * @returns {Promise<void>}
41
+ * Returns the current engine status.
42
+ * Returns 'running' if no instance has been built yet.
43
+ * @returns {string} The current status.
44
+ */
45
+ static status() {
46
+ if (!Application.#instance) return 'running';
47
+ return Application.#instance.status();
48
+ }
49
+
50
+ /**
51
+ * Returns true if the engine is currently running.
52
+ * @returns {boolean} True if the current status is 'running'.
58
53
  */
59
- async run() {
60
- const aggregator = new PromiseAggregator();
54
+ static isRunning() {
55
+ return Application.status() === 'running';
56
+ }
61
57
 
62
- this.engine = this.buildEngine();
63
- this.webServer = this.buildWebServer();
64
- this.enqueueFirstJobs();
58
+ /**
59
+ * Returns true if the engine is currently paused.
60
+ * @returns {boolean} True if the current status is 'paused'.
61
+ */
62
+ static isPaused() {
63
+ return Application.status() === 'paused';
64
+ }
65
65
 
66
- aggregator.add(this.webServer?.start());
67
- aggregator.add(this.engine.start());
66
+ /**
67
+ * Returns true if the engine is currently stopped.
68
+ * @returns {boolean} True if the current status is 'stopped'.
69
+ */
70
+ static isStopped() {
71
+ return Application.status() === 'stopped';
72
+ }
68
73
 
69
- await aggregator.wait();
74
+ /**
75
+ * Loads the configuration from the specified file path.
76
+ * @param {string} configPath - The path to the configuration file.
77
+ * @returns {void}
78
+ */
79
+ static loadConfig(configPath) {
80
+ return Application.#getInstance().loadConfig(configPath);
81
+ }
70
82
 
71
- new FailureChecker({ failureConfig: this.config.failureConfig }).check();
83
+ /**
84
+ * Starts the application.
85
+ * @returns {Promise<void>}
86
+ */
87
+ static async run() {
88
+ return Application.#getInstance().run();
72
89
  }
73
90
 
74
91
  /**
75
- * Builds and returns a new Engine instance wired to the current registries.
92
+ * Builds and returns a new Engine instance.
76
93
  * @returns {Engine} The created Engine instance.
77
94
  */
78
- buildEngine() {
79
- return new Engine({ sleepMs: this.config.workersConfig.sleep });
95
+ static buildEngine() {
96
+ return Application.#getInstance().buildEngine();
80
97
  }
98
+
81
99
  /**
82
- * Builds and returns a WebServer if web configuration is present, otherwise null.
100
+ * Builds and returns a WebServer or null.
83
101
  * @returns {WebServer|null} The created WebServer instance or null.
84
102
  */
85
- buildWebServer() {
86
- return WebServer.build({
87
- webConfig: this.config.webConfig,
88
- });
103
+ static buildWebServer() {
104
+ return Application.#getInstance().buildWebServer();
89
105
  }
106
+
90
107
  /**
91
- * Gets the buffered logger instance created during config loading.
108
+ * Enqueues all parameter-free ResourceRequests.
109
+ * @returns {void}
110
+ */
111
+ static enqueueFirstJobs() {
112
+ return Application.#getInstance().enqueueFirstJobs();
113
+ }
114
+
115
+ /**
116
+ * Gets the buffered logger from the singleton instance.
92
117
  * @returns {BufferedLogger} The buffered logger instance.
93
118
  */
94
- get bufferedLogger() {
95
- return this.#bufferedLogger;
119
+ static get bufferedLogger() {
120
+ return Application.#getInstance().bufferedLogger;
96
121
  }
122
+
97
123
  /**
98
- * Enqueues all parameter-free ResourceRequests into the job registry.
99
- * These are requests whose URLs contain no {:placeholder} tokens and can be
100
- * processed immediately without any external parameters.
101
- * @returns {void}
124
+ * Pauses the engine and waits for workers to become idle.
125
+ * @returns {Promise<void>}
102
126
  */
103
- enqueueFirstJobs() {
104
- new ResourceRequestCollector(this.config.resourceRegistry).requestsNeedingNoParams().forEach((resourceRequest) => {
105
- JobRegistry.enqueue('ResourceRequestJob', { resourceRequest, parameters: {} });
106
- });
127
+ static async pause() {
128
+ return Application.#getInstance().pause();
107
129
  }
130
+
108
131
  /**
109
- * Initializes the job factory, job registry, and workers registry from the loaded configuration.
110
- * @returns {void}
132
+ * Stops the engine, waits for workers to idle, then clears job queues.
133
+ * @returns {Promise<void>}
111
134
  */
112
- #initRegistries() {
113
- JobFactory.build('ResourceRequestJob', { attributes: { clients: this.config.clientRegistry } });
114
- JobFactory.build('Action', { klass: ActionProcessingJob });
115
- JobFactory.build('HtmlParse', { klass: HtmlParseJob, attributes: { jobRegistry: JobRegistry, clientRegistry: this.config.clientRegistry } });
116
- JobFactory.build('AssetDownload', { klass: AssetDownloadJob, attributes: { clientRegistry: this.config.clientRegistry } });
135
+ static async stop() {
136
+ return Application.#getInstance().stop();
137
+ }
117
138
 
118
- JobRegistry.build({ cooldown: this.config.workersConfig.retryCooldown, maxRetries: this.config.workersConfig.maxRetries });
139
+ /**
140
+ * Resumes processing after a pause.
141
+ * @returns {Promise<void>}
142
+ */
143
+ static async continue() {
144
+ return Application.#getInstance().continue();
145
+ }
119
146
 
120
- WorkersRegistry.build({
121
- workers: this.#workers,
122
- jobRegistry: JobRegistry,
123
- workersRegistry: WorkersRegistry,
124
- ...this.config.workersConfig,
125
- });
126
- WorkersRegistry.initWorkers();
147
+ /**
148
+ * Starts processing from a stopped state.
149
+ * @returns {Promise<void>}
150
+ */
151
+ static async start() {
152
+ return Application.#getInstance().start();
153
+ }
154
+
155
+ /**
156
+ * Restarts processing.
157
+ * @returns {Promise<void>}
158
+ */
159
+ static async restart() {
160
+ return Application.#getInstance().restart();
161
+ }
162
+
163
+ /**
164
+ * Shuts down the web server and stops the engine.
165
+ * @returns {Promise<void>}
166
+ */
167
+ static async shutdown() {
168
+ return Application.#getInstance().shutdown();
169
+ }
170
+
171
+ /**
172
+ * Returns the singleton instance, throwing if not yet built.
173
+ * @returns {ApplicationInstance} The singleton instance.
174
+ * @throws {Error} If `build()` has not been called.
175
+ */
176
+ static #getInstance() {
177
+ if (!Application.#instance) {
178
+ throw new Error('Application has not been initialized. Call Application.build() before calling static methods that require an instance.');
179
+ }
180
+ return Application.#instance;
127
181
  }
128
182
  }
129
183
 
130
- export { Application };
184
+ export { Application };
@@ -0,0 +1,274 @@
1
+ import { Engine } from './Engine.js';
2
+ import { EngineEvents } from './EngineEvents.js';
3
+ import { FailureChecker } from './FailureChecker.js';
4
+ import { JobFactory } from '../background/JobFactory.js';
5
+ import { JobRegistry } from '../background/JobRegistry.js';
6
+ import { WorkersRegistry } from '../background/WorkersRegistry.js';
7
+ import { ConfigurationFileNotProvided } from '../exceptions/config/ConfigurationFileNotProvided.js';
8
+ import { ActionProcessingJob } from '../jobs/ActionProcessingJob.js';
9
+ import { AssetDownloadJob } from '../jobs/AssetDownloadJob.js';
10
+ import { HtmlParseJob } from '../jobs/HtmlParseJob.js';
11
+ import { PaginatedActionProcessingJob } from '../jobs/PaginatedActionProcessingJob.js';
12
+ import { Config } from '../models/configs/Config.js';
13
+ import { LogRegistry } from '../registry/LogRegistry.js';
14
+ import { WebServer } from '../server/WebServer.js';
15
+ import { PromiseAggregator } from '../utils/PromiseAggregator.js';
16
+ import { ResourceRequestCollector } from '../utils/ResourceRequestCollector.js';
17
+
18
+ const DEFAULT_POLL_SLEEP_MS = 10;
19
+
20
+ /**
21
+ * ApplicationInstance holds all instance-level state and logic for a single
22
+ * application lifecycle. Use the static `Application` facade to access it.
23
+ * @author darthjee
24
+ */
25
+ class ApplicationInstance {
26
+ #workers;
27
+ #bufferedLogger;
28
+ #engineStatus;
29
+ #aggregator;
30
+ #enginePromise;
31
+ #sleepMs;
32
+
33
+ /**
34
+ * @param {object} [params={}] - Optional parameters for dependency injection.
35
+ * @param {IdentifyableCollection} [params.workers] - Workers collection (injected for testing).
36
+ */
37
+ constructor({ workers } = {}) {
38
+ this.#workers = workers;
39
+ }
40
+
41
+ /**
42
+ * Loads the configuration from the specified file path.
43
+ * @param {string} configPath - The path to the configuration file.
44
+ * @throws {ConfigurationFileNotProvided} If the configuration file path is not provided.
45
+ * @throws {ConfigurationFileNotFound} If the configuration file is not found at the specified path.
46
+ * @returns {void}
47
+ */
48
+ loadConfig(configPath) {
49
+ if (!configPath) {
50
+ throw new ConfigurationFileNotProvided();
51
+ }
52
+
53
+ this.config = Config.fromFile(configPath);
54
+ const logRegistry = LogRegistry.build({ retention: this.config.logConfig.size });
55
+ this.#bufferedLogger = logRegistry.bufferedLogger;
56
+ this.#initRegistries();
57
+ }
58
+
59
+ /**
60
+ * Starts the application by building the engine, web server, enqueueing initial jobs, and starting both.
61
+ * After the engine finishes, checks the dead-job ratio against the configured failure threshold.
62
+ * @returns {Promise<void>}
63
+ */
64
+ async run() {
65
+ this.#aggregator = new PromiseAggregator();
66
+ this.#sleepMs = this.config.workersConfig.sleep;
67
+
68
+ this.engine = this.buildEngine();
69
+ this.webServer = this.buildWebServer();
70
+ this.enqueueFirstJobs();
71
+
72
+ this.#engineStatus = 'running';
73
+ this.#aggregator.add(this.webServer?.start());
74
+ this.#enginePromise = this.engine.start();
75
+ this.#aggregator.add(this.#enginePromise);
76
+
77
+ await this.#aggregator.wait();
78
+ this.#engineStatus = 'stopped';
79
+ EngineEvents.emit('stop');
80
+
81
+ new FailureChecker({ failureConfig: this.config.failureConfig }).check();
82
+ }
83
+
84
+ /**
85
+ * Builds and returns a new Engine instance wired to the current registries.
86
+ * @returns {Engine} The created Engine instance.
87
+ */
88
+ buildEngine() {
89
+ return new Engine({
90
+ sleepMs: this.#sleepMs ?? this.config.workersConfig.sleep,
91
+ keepAlive: !!this.config.webConfig,
92
+ });
93
+ }
94
+
95
+ /**
96
+ * Builds and returns a WebServer if web configuration is present, otherwise null.
97
+ * @returns {WebServer|null} The created WebServer instance or null.
98
+ */
99
+ buildWebServer() {
100
+ return WebServer.build({
101
+ webConfig: this.config.webConfig,
102
+ });
103
+ }
104
+
105
+ /**
106
+ * Gets the buffered logger instance created during config loading.
107
+ * @returns {BufferedLogger} The buffered logger instance.
108
+ */
109
+ get bufferedLogger() {
110
+ return this.#bufferedLogger;
111
+ }
112
+
113
+ /**
114
+ * Enqueues all parameter-free ResourceRequests into the job registry.
115
+ * These are requests whose URLs contain no {:placeholder} tokens and can be
116
+ * processed immediately without any external parameters.
117
+ * @returns {void}
118
+ */
119
+ enqueueFirstJobs() {
120
+ new ResourceRequestCollector(this.config.resourceRegistry).requestsNeedingNoParams().forEach((resourceRequest) => {
121
+ JobRegistry.enqueue('ResourceRequestJob', { resourceRequest, parameters: {} });
122
+ });
123
+ }
124
+
125
+ /**
126
+ * Returns the current engine status string.
127
+ * @returns {string|undefined} The current status.
128
+ */
129
+ status() {
130
+ return this.#engineStatus;
131
+ }
132
+
133
+ /**
134
+ * Returns true if the engine is currently running.
135
+ * @returns {boolean} True if the current status is 'running'.
136
+ */
137
+ isRunning() {
138
+ return this.#engineStatus === 'running';
139
+ }
140
+
141
+ /**
142
+ * Returns true if the engine is currently paused.
143
+ * @returns {boolean} True if the current status is 'paused'.
144
+ */
145
+ isPaused() {
146
+ return this.#engineStatus === 'paused';
147
+ }
148
+
149
+ /**
150
+ * Returns true if the engine is currently stopped.
151
+ * @returns {boolean} True if the current status is 'stopped'.
152
+ */
153
+ isStopped() {
154
+ return this.#engineStatus === 'stopped';
155
+ }
156
+
157
+ /**
158
+ * Sets the engine status string.
159
+ * @param {string} value - The new status value.
160
+ * @returns {void}
161
+ */
162
+ setStatus(value) {
163
+ this.#engineStatus = value;
164
+ }
165
+
166
+ /**
167
+ * Pauses processing: pauses the engine (without stopping it) and waits for workers to become idle.
168
+ * @returns {Promise<void>}
169
+ */
170
+ async pause() {
171
+ this.#engineStatus = 'pausing';
172
+ this.engine.pause();
173
+ await this.#waitForWorkersIdle();
174
+ this.#engineStatus = 'paused';
175
+ }
176
+
177
+ /**
178
+ * Stops processing: pauses the engine, waits for workers to idle, then clears job queues.
179
+ * The engine instance is preserved and its loop continues running in the background.
180
+ * @returns {Promise<void>}
181
+ */
182
+ async stop() {
183
+ this.#engineStatus = 'stopping';
184
+ this.engine.pause();
185
+ await this.#waitForWorkersIdle();
186
+ JobRegistry.clearQueues();
187
+ this.#engineStatus = 'stopped';
188
+ EngineEvents.emit('stop');
189
+ }
190
+
191
+ /**
192
+ * Resumes processing after a pause by calling engine.resume().
193
+ * No new engine is created; the existing loop continues.
194
+ * Only valid when status is 'paused'.
195
+ * @returns {Promise<void>}
196
+ */
197
+ async continue() {
198
+ if (this.#engineStatus !== 'paused') return;
199
+ this.engine.resume();
200
+ this.#engineStatus = 'running';
201
+ }
202
+
203
+ /**
204
+ * Starts processing from a stopped state by calling engine.resume() and re-enqueueing initial jobs.
205
+ * No new engine is created; the existing loop continues.
206
+ * Only valid when status is 'stopped'.
207
+ * @returns {Promise<void>}
208
+ */
209
+ async start() {
210
+ if (this.#engineStatus !== 'stopped') return;
211
+ this.engine.resume();
212
+ this.enqueueFirstJobs();
213
+ this.#engineStatus = 'running';
214
+ EngineEvents.emit('start');
215
+ }
216
+
217
+ /**
218
+ * Restarts processing: stops then starts the engine.
219
+ * Only valid when status is 'running'.
220
+ * @returns {Promise<void>}
221
+ */
222
+ async restart() {
223
+ if (this.#engineStatus !== 'running') return;
224
+ await this.stop();
225
+ await this.start();
226
+ }
227
+
228
+ /**
229
+ * Shuts down the web server and stops the engine loop.
230
+ * If running, pauses first and waits for workers to idle.
231
+ * @returns {Promise<void>}
232
+ */
233
+ async shutdown() {
234
+ this.webServer?.shutdown();
235
+ if (this.#engineStatus === 'running') {
236
+ await this.stop();
237
+ }
238
+ this.engine.stop();
239
+ }
240
+
241
+ /**
242
+ * Polls until all workers are idle.
243
+ * @returns {Promise<void>}
244
+ */
245
+ async #waitForWorkersIdle() {
246
+ while (WorkersRegistry.hasBusyWorker()) {
247
+ await new Promise(resolve => setTimeout(resolve, this.#sleepMs ?? DEFAULT_POLL_SLEEP_MS));
248
+ }
249
+ }
250
+
251
+ /**
252
+ * Initializes the job factory, job registry, and workers registry from the loaded configuration.
253
+ * @returns {void}
254
+ */
255
+ #initRegistries() {
256
+ JobFactory.build('ResourceRequestJob', { attributes: { clients: this.config.clientRegistry } });
257
+ JobFactory.build('Action', { klass: ActionProcessingJob });
258
+ JobFactory.build('PaginatedAction', { klass: PaginatedActionProcessingJob });
259
+ JobFactory.build('HtmlParse', { klass: HtmlParseJob, attributes: { jobRegistry: JobRegistry, clientRegistry: this.config.clientRegistry } });
260
+ JobFactory.build('AssetDownload', { klass: AssetDownloadJob, attributes: { clientRegistry: this.config.clientRegistry } });
261
+
262
+ JobRegistry.build({ cooldown: this.config.workersConfig.retryCooldown, maxRetries: this.config.workersConfig.maxRetries });
263
+
264
+ WorkersRegistry.build({
265
+ workers: this.#workers,
266
+ jobRegistry: JobRegistry,
267
+ workersRegistry: WorkersRegistry,
268
+ ...this.config.workersConfig,
269
+ });
270
+ WorkersRegistry.initWorkers();
271
+ }
272
+ }
273
+
274
+ export { ApplicationInstance };
@@ -1,6 +1,5 @@
1
1
  import axios from 'axios';
2
- import { RequestFailed } from '../exceptions/RequestFailed.js';
3
- import { Logger } from '../utils/logging/Logger.js';
2
+ import { RequestFailed } from '../exceptions/request/RequestFailed.js';
4
3
 
5
4
  /**
6
5
  * Client performs HTTP requests for resource paths using a configured base URL.
@@ -55,16 +54,17 @@ class Client {
55
54
  * @param {ResourceRequest} resourceRequest Information about the URL path to request
56
55
  * and the expected status code.
57
56
  * @param {object} [parameters={}] Key-value map used to resolve {:placeholder} tokens in the URL.
57
+ * @param {LogContext} logContext Context carrying workerId/jobId for log entries.
58
58
  * @returns {Promise<boolean>} Returns true if the response status matches the expected status.
59
59
  * @throws {RequestFailed} Throws an error if the request fails or the status does not match.
60
60
  */
61
- async perform(resourceRequest, parameters = {}) {
61
+ async perform(resourceRequest, parameters = {}, logContext) {
62
62
  const requestUrl = this.#buildUrl(resourceRequest.resolveUrl(parameters));
63
- Logger.info(`[Client:${this.name}] Requesting ${requestUrl}`);
63
+ logContext.info(`[Client:${this.name}] Requesting ${requestUrl}`);
64
64
  try {
65
- return await this.#request(resourceRequest, requestUrl);
65
+ return await this.#request(resourceRequest, requestUrl, logContext);
66
66
  } catch (error) {
67
- this.#handleError(error, requestUrl);
67
+ this.#handleError(error, requestUrl, logContext);
68
68
  }
69
69
  }
70
70
 
@@ -73,15 +73,16 @@ class Client {
73
73
  * Unlike `perform`, this method accepts an absolute URL directly and does not prepend `baseUrl`.
74
74
  * @param {string} absoluteUrl The fully-resolved URL to request.
75
75
  * @param {number} expectedStatus The expected HTTP response status code.
76
+ * @param {LogContext} logContext Context carrying workerId/jobId for log entries.
76
77
  * @returns {Promise<object>} The Axios response object.
77
78
  * @throws {RequestFailed} If the response status does not match `expectedStatus`.
78
79
  */
79
- async performUrl(absoluteUrl, expectedStatus) {
80
- Logger.info(`[Client:${this.name}] Requesting ${absoluteUrl}`);
80
+ async performUrl(absoluteUrl, expectedStatus, logContext) {
81
+ logContext.info(`[Client:${this.name}] Requesting ${absoluteUrl}`);
81
82
  try {
82
- return await this.#requestUrl(absoluteUrl, expectedStatus);
83
+ return await this.#requestUrl(absoluteUrl, expectedStatus, logContext);
83
84
  } catch (error) {
84
- this.#handleError(error, absoluteUrl);
85
+ this.#handleError(error, absoluteUrl, logContext);
85
86
  }
86
87
  }
87
88
 
@@ -90,21 +91,23 @@ class Client {
90
91
  * @param {ResourceRequest} resourceRequest Information about the URL path to request
91
92
  * and the expected status code.
92
93
  * @param {string} requestUrl The full URL for the request.
94
+ * @param {LogContext} logContext Context carrying workerId/jobId for log entries.
93
95
  * @returns {Promise<boolean>} Returns true if the response status matches the expected status.
94
96
  * @throws {RequestFailed} Throws an error if the response status does not match.
95
97
  */
96
- async #request(resourceRequest, requestUrl) {
97
- return this.#requestUrl(requestUrl, resourceRequest.status);
98
+ async #request(resourceRequest, requestUrl, logContext) {
99
+ return this.#requestUrl(requestUrl, resourceRequest.status, logContext);
98
100
  }
99
101
 
100
102
  /**
101
103
  * Performs the HTTP GET request to the given URL and validates the response status.
102
104
  * @param {string} requestUrl The full URL to request.
103
105
  * @param {number} expectedStatus The expected HTTP response status code.
106
+ * @param {LogContext} logContext Context carrying workerId/jobId for log entries.
104
107
  * @returns {Promise<object>} The Axios response object.
105
108
  * @throws {RequestFailed} Throws an error if the response status does not match.
106
109
  */
107
- async #requestUrl(requestUrl, expectedStatus) {
110
+ async #requestUrl(requestUrl, expectedStatus, logContext) {
108
111
  const response = await axios.get(requestUrl, {
109
112
  timeout: this.timeout,
110
113
  responseType: 'text',
@@ -113,14 +116,14 @@ class Client {
113
116
  validateStatus: () => true,
114
117
  });
115
118
 
116
- Logger.info(`[Client:${this.name}] Response ${requestUrl} → ${response.status}`);
119
+ logContext.info(`[Client:${this.name}] Response ${requestUrl} → ${response.status}`);
117
120
 
118
121
  if (response.status !== expectedStatus) {
119
- Logger.info(`[Client:${this.name}] ${requestUrl} did not match (got ${response.status}, expected ${expectedStatus})`);
122
+ logContext.info(`[Client:${this.name}] ${requestUrl} did not match (got ${response.status}, expected ${expectedStatus})`);
120
123
  throw new RequestFailed(response.status, requestUrl);
121
124
  }
122
125
 
123
- Logger.info(`[Client:${this.name}] ${requestUrl} matched (expected ${expectedStatus})`);
126
+ logContext.info(`[Client:${this.name}] ${requestUrl} matched (expected ${expectedStatus})`);
124
127
  return response;
125
128
  }
126
129
 
@@ -128,11 +131,12 @@ class Client {
128
131
  * Handles a caught request error, logging it and re-throwing as RequestFailed when applicable.
129
132
  * @param {Error} error The caught error.
130
133
  * @param {string} requestUrl The URL that was requested.
134
+ * @param {LogContext} logContext Context carrying workerId/jobId for log entries.
131
135
  * @throws {RequestFailed} If the error has a response object.
132
136
  * @throws {Error} Re-throws the original error otherwise.
133
137
  */
134
- #handleError(error, requestUrl) {
135
- Logger.error(`Request failed: ${error}`);
138
+ #handleError(error, requestUrl, logContext) {
139
+ logContext.error(`Request failed: ${error}`);
136
140
  if (error.response) {
137
141
  throw new RequestFailed(error.response.status, requestUrl);
138
142
  }
@@ -149,4 +153,4 @@ class Client {
149
153
  }
150
154
  }
151
155
 
152
- export { Client };
156
+ export { Client };
@@ -1,7 +1,7 @@
1
1
  import { readFileSync } from 'node:fs';
2
2
  import YAML from 'yaml';
3
3
  import { ConfigParser } from './ConfigParser.js';
4
- import { ConfigurationFileNotFound } from '../exceptions/ConfigurationFileNotFound.js';
4
+ import { ConfigurationFileNotFound } from '../exceptions/config/ConfigurationFileNotFound.js';
5
5
  import { EnvResolver } from '../utils/EnvResolver.js';
6
6
  import { Logger } from '../utils/logging/Logger.js';
7
7