navi-hey 1.1.0 → 1.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +38 -4
- package/lib/common/utils/EnvResolver.js +26 -0
- package/lib/common/utils/env_resolver/EnvStringResolver.js +52 -0
- package/lib/common/utils/logging/BaseLogger.js +98 -0
- package/lib/common/utils/logging/BufferedLogger.js +100 -0
- package/lib/common/utils/logging/ConsoleLogger.js +20 -0
- package/lib/common/utils/logging/Log.js +82 -0
- package/lib/common/utils/logging/LogBuffer.js +122 -0
- package/lib/common/utils/logging/LogBufferCollection.js +56 -0
- package/lib/common/utils/logging/LogContext.js +51 -0
- package/lib/common/utils/logging/LogFactory.js +33 -0
- package/lib/common/utils/logging/LogFilter.js +34 -0
- package/lib/common/utils/logging/Logger.js +138 -0
- package/lib/common/utils/logging/LoggerGroup.js +104 -0
- package/lib/enqueuers/ActionEnqueuer.js +18 -2
- package/lib/enqueuers/ActionsEnqueuer.js +6 -3
- package/lib/enqueuers/PaginatedActionEnqueuer.js +53 -0
- package/lib/enqueuers/PaginatedActionsEnqueuer.js +47 -0
- package/lib/exceptions/{ConfigurationFileNotFound.js → config/ConfigurationFileNotFound.js} +1 -1
- package/lib/exceptions/{ConfigurationFileNotProvided.js → config/ConfigurationFileNotProvided.js} +1 -1
- package/lib/exceptions/{MissingTopLevelConfgKey.js → config/MissingTopLevelConfgKey.js} +1 -1
- package/lib/exceptions/{ConflictError.js → http/ConflictError.js} +1 -1
- package/lib/exceptions/{ForbiddenError.js → http/ForbiddenError.js} +1 -1
- package/lib/exceptions/{NotFoundError.js → http/NotFoundError.js} +1 -1
- package/lib/exceptions/{ItemNotFound.js → registry/ItemNotFound.js} +1 -1
- package/lib/exceptions/{MissingActionResource.js → registry/MissingActionResource.js} +1 -1
- package/lib/exceptions/{MissingMappingVariable.js → registry/MissingMappingVariable.js} +1 -1
- package/lib/exceptions/{InvalidHtmlResponseBody.js → request/InvalidHtmlResponseBody.js} +1 -1
- package/lib/exceptions/{InvalidResponseBody.js → request/InvalidResponseBody.js} +1 -1
- package/lib/exceptions/{NullResponse.js → request/NullResponse.js} +1 -1
- package/lib/exceptions/{RequestFailed.js → request/RequestFailed.js} +1 -1
- package/lib/jobs/ActionProcessingJob.js +17 -3
- package/lib/jobs/HtmlParseJob.js +17 -4
- package/lib/jobs/PaginatedActionProcessingJob.js +77 -0
- package/lib/jobs/ResourceRequestJob.js +8 -5
- package/lib/models/{Config.js → configs/Config.js} +3 -3
- package/lib/models/configs/Link.js +36 -0
- package/lib/models/configs/PaginationConfig.js +63 -0
- package/lib/models/{WebConfig.js → configs/WebConfig.js} +5 -1
- package/lib/models/{ResourceRequest.js → request/ResourceRequest.js} +30 -8
- package/lib/models/{ResourceRequestAction.js → request/ResourceRequestAction.js} +6 -6
- package/lib/models/request/ResourceRequestPaginatedAction.js +81 -0
- package/lib/models/{PathSegmentTraverser.js → response/PathSegmentTraverser.js} +1 -1
- package/lib/models/{ResponseParser.js → response/ResponseParser.js} +1 -1
- package/lib/registry/ClientRegistry.js +1 -1
- package/lib/registry/LogRegistry.js +13 -13
- package/lib/registry/LogRegistryInstance.js +16 -16
- package/lib/registry/NamedRegistry.js +1 -1
- package/lib/registry/ResourceRegistry.js +1 -1
- package/lib/serializers/JobIndexSerializer.js +3 -0
- package/lib/serializers/LogSerializer.js +1 -1
- package/lib/server/PathValidator.js +1 -1
- package/lib/server/RouteRegister.js +3 -3
- package/lib/server/Router.js +20 -18
- package/lib/server/{AssetsRequestHandler.js → handlers/AssetsRequestHandler.js} +3 -3
- package/lib/server/{BaseUrlsRequestHandler.js → handlers/BaseUrlsRequestHandler.js} +3 -3
- package/lib/server/{IndexRequestHandler.js → handlers/IndexRequestHandler.js} +2 -2
- package/lib/server/{JobsFilter.js → handlers/JobsFilter.js} +1 -1
- package/lib/server/handlers/LinksRequestHandler.js +32 -0
- package/lib/server/{LogsRequestHandler.js → handlers/LogsRequestHandler.js} +4 -4
- package/lib/server/{SettingsRequestHandler.js → handlers/SettingsRequestHandler.js} +4 -3
- package/lib/server/{StatsRequestHandler.js → handlers/StatsRequestHandler.js} +3 -3
- package/lib/server/{EngineContinueRequestHandler.js → handlers/engine/EngineContinueRequestHandler.js} +3 -3
- package/lib/server/{EnginePauseRequestHandler.js → handlers/engine/EnginePauseRequestHandler.js} +3 -3
- package/lib/server/{EngineRestartRequestHandler.js → handlers/engine/EngineRestartRequestHandler.js} +3 -3
- package/lib/server/{EngineShutdownRequestHandler.js → handlers/engine/EngineShutdownRequestHandler.js} +2 -2
- package/lib/server/{EngineStartRequestHandler.js → handlers/engine/EngineStartRequestHandler.js} +3 -3
- package/lib/server/{EngineStatusRequestHandler.js → handlers/engine/EngineStatusRequestHandler.js} +2 -2
- package/lib/server/{EngineStopRequestHandler.js → handlers/engine/EngineStopRequestHandler.js} +3 -3
- package/lib/server/{JobLogsRequestHandler.js → handlers/jobs/JobLogsRequestHandler.js} +4 -4
- package/lib/server/{JobRequestHandler.js → handlers/jobs/JobRequestHandler.js} +4 -4
- package/lib/server/{JobRetryRequestHandler.js → handlers/jobs/JobRetryRequestHandler.js} +4 -4
- package/lib/server/{JobsRequestHandler.js → handlers/jobs/JobsRequestHandler.js} +4 -4
- package/lib/services/ApplicationInstance.js +4 -2
- package/lib/services/Client.js +1 -1
- package/lib/services/ConfigLoader.js +4 -4
- package/lib/services/ConfigParser.js +7 -7
- package/lib/services/Engine.js +1 -1
- package/lib/utils/HtmlParser.js +1 -1
- package/lib/utils/logging/BaseLogger.js +1 -98
- package/lib/utils/logging/BufferedLogger.js +1 -100
- package/lib/utils/logging/ConsoleLogger.js +1 -20
- package/lib/utils/logging/Log.js +1 -82
- package/lib/utils/logging/LogBuffer.js +1 -122
- package/lib/utils/logging/LogBufferCollection.js +1 -56
- package/lib/utils/logging/LogContext.js +1 -51
- package/lib/utils/logging/LogFactory.js +1 -33
- package/lib/utils/logging/LogFilter.js +1 -34
- package/lib/utils/logging/Logger.js +1 -138
- package/lib/utils/logging/LoggerGroup.js +1 -104
- package/package.json +1 -1
- package/static/assets/index-CX_uXu4Q.js +11 -0
- package/static/favicon.png +0 -0
- package/static/index.html +2 -1
- package/lib/exceptions/LockedByOtherWorker.js +0 -14
- package/lib/utils/EnvResolver.js +0 -75
- package/static/assets/index-BfggoMvd.js +0 -11
- /package/lib/exceptions/{MissingClientsConfig.js → config/MissingClientsConfig.js} +0 -0
- /package/lib/exceptions/{MissingResourceConfig.js → config/MissingResourceConfig.js} +0 -0
- /package/lib/exceptions/{ClientNotFound.js → registry/ClientNotFound.js} +0 -0
- /package/lib/exceptions/{ResourceNotFound.js → registry/ResourceNotFound.js} +0 -0
- /package/lib/models/{FailureConfig.js → configs/FailureConfig.js} +0 -0
- /package/lib/models/{LogConfig.js → configs/LogConfig.js} +0 -0
- /package/lib/models/{WorkersConfig.js → configs/WorkersConfig.js} +0 -0
- /package/lib/models/{AssetRequest.js → request/AssetRequest.js} +0 -0
- /package/lib/models/{Resource.js → request/Resource.js} +0 -0
- /package/lib/models/{ParametersMapper.js → response/ParametersMapper.js} +0 -0
- /package/lib/models/{PathResolver.js → response/PathResolver.js} +0 -0
- /package/lib/models/{ResponseWrapper.js → response/ResponseWrapper.js} +0 -0
|
@@ -31,7 +31,7 @@ class LogRegistryInstance {
|
|
|
31
31
|
|
|
32
32
|
/**
|
|
33
33
|
* Returns the underlying BufferedLogger instance.
|
|
34
|
-
* @returns {BufferedLogger}
|
|
34
|
+
* @returns {BufferedLogger} The underlying BufferedLogger instance.
|
|
35
35
|
*/
|
|
36
36
|
get bufferedLogger() {
|
|
37
37
|
return this.#bufferedLogger;
|
|
@@ -59,8 +59,8 @@ class LogRegistryInstance {
|
|
|
59
59
|
|
|
60
60
|
/**
|
|
61
61
|
* Gets a specific log by ID.
|
|
62
|
-
* @param {number} id
|
|
63
|
-
* @returns {import('../utils/logging/Log.js').Log|undefined}
|
|
62
|
+
* @param {number} id - The log entry ID to look up.
|
|
63
|
+
* @returns {import('../utils/logging/Log.js').Log|undefined} The matching log entry, or undefined if not found.
|
|
64
64
|
*/
|
|
65
65
|
getLogById(id) {
|
|
66
66
|
return this.#bufferedLogger.getLogById(id);
|
|
@@ -68,10 +68,10 @@ class LogRegistryInstance {
|
|
|
68
68
|
|
|
69
69
|
/**
|
|
70
70
|
* Gets logs in chronological order (oldest first), optionally filtered to entries newer than lastId.
|
|
71
|
-
* @param {object} [options={}]
|
|
71
|
+
* @param {object} [options={}] - Query options.
|
|
72
72
|
* @param {number|string} [options.lastId] - When provided, returns only logs newer than this ID.
|
|
73
73
|
* Returns an empty array if the ID is not found.
|
|
74
|
-
* @returns {Array<import('../utils/logging/Log.js').Log>}
|
|
74
|
+
* @returns {Array<import('../utils/logging/Log.js').Log>} Array of log entries.
|
|
75
75
|
*/
|
|
76
76
|
getLogs({ lastId } = {}) {
|
|
77
77
|
return new LogFilter(this.bufferedLogger.getLogs()).filter({ lastId });
|
|
@@ -80,10 +80,10 @@ class LogRegistryInstance {
|
|
|
80
80
|
/**
|
|
81
81
|
* Gets logs stored in the per-job buffer for the given job ID,
|
|
82
82
|
* optionally filtered to entries newer than lastId.
|
|
83
|
-
* @param {string|number} jobId
|
|
84
|
-
* @param {object} [options={}]
|
|
83
|
+
* @param {string|number} jobId - The job ID to look up logs for.
|
|
84
|
+
* @param {object} [options={}] - Query options.
|
|
85
85
|
* @param {number|string} [options.lastId] - When provided, returns only logs newer than this ID.
|
|
86
|
-
* @returns {Array<import('../utils/logging/Log.js').Log>}
|
|
86
|
+
* @returns {Array<import('../utils/logging/Log.js').Log>} Array of log entries.
|
|
87
87
|
*/
|
|
88
88
|
getLogsByJobId(jobId, { lastId } = {}) {
|
|
89
89
|
return new LogFilter(this.#jobLogs.getLogs(jobId)).filter({ lastId });
|
|
@@ -91,8 +91,8 @@ class LogRegistryInstance {
|
|
|
91
91
|
|
|
92
92
|
/**
|
|
93
93
|
* Gets logs stored in the per-worker buffer for the given worker ID.
|
|
94
|
-
* @param {string|number} workerId
|
|
95
|
-
* @returns {Array<import('../utils/logging/Log.js').Log>}
|
|
94
|
+
* @param {string|number} workerId - The worker ID to look up logs for.
|
|
95
|
+
* @returns {Array<import('../utils/logging/Log.js').Log>} Array of log entries.
|
|
96
96
|
*/
|
|
97
97
|
getLogsByWorkerId(workerId) {
|
|
98
98
|
return this.#workerLogs.getLogs(workerId);
|
|
@@ -100,8 +100,8 @@ class LogRegistryInstance {
|
|
|
100
100
|
|
|
101
101
|
/**
|
|
102
102
|
* Gets logs filtered by level.
|
|
103
|
-
* @param {string} level
|
|
104
|
-
* @returns {Array<import('../utils/logging/Log.js').Log>}
|
|
103
|
+
* @param {string} level - The log level to filter by.
|
|
104
|
+
* @returns {Array<import('../utils/logging/Log.js').Log>} Array of log entries matching the level.
|
|
105
105
|
*/
|
|
106
106
|
getLogsByLevel(level) {
|
|
107
107
|
return this.#bufferedLogger.getLogsByLevel(level);
|
|
@@ -109,7 +109,7 @@ class LogRegistryInstance {
|
|
|
109
109
|
|
|
110
110
|
/**
|
|
111
111
|
* Returns all logs as plain JSON objects.
|
|
112
|
-
* @returns {Array<object>}
|
|
112
|
+
* @returns {Array<object>} Array of plain log objects.
|
|
113
113
|
*/
|
|
114
114
|
getLogsJSON() {
|
|
115
115
|
return this.#bufferedLogger.getLogsJSON();
|
|
@@ -138,9 +138,9 @@ class LogRegistryInstance {
|
|
|
138
138
|
/**
|
|
139
139
|
* Fans out the log call to the logger group and, if a new log was buffered,
|
|
140
140
|
* routes it to the per-job and per-worker collections based on attributes.
|
|
141
|
-
* @param {string} level
|
|
142
|
-
* @param {string} message
|
|
143
|
-
* @param {object} attributes
|
|
141
|
+
* @param {string} level - The log level.
|
|
142
|
+
* @param {string} message - The log message.
|
|
143
|
+
* @param {object} attributes - Optional structured metadata.
|
|
144
144
|
* @returns {void}
|
|
145
145
|
*/
|
|
146
146
|
#dispatch(level, message, attributes) {
|
|
@@ -8,6 +8,7 @@ class JobIndexSerializer extends Serializer {
|
|
|
8
8
|
/**
|
|
9
9
|
* Serializes a single job instance into a `{ id, status, attempts, jobClass[, url] }` object.
|
|
10
10
|
* For `ResourceRequestJob` instances, the resolved URL is included as `url`.
|
|
11
|
+
* For other job instances that carry an `originUrl` argument, that value is included as `url`.
|
|
11
12
|
*
|
|
12
13
|
* @param {object} job - A job instance.
|
|
13
14
|
* @param {object} options - Serialization options.
|
|
@@ -24,6 +25,8 @@ class JobIndexSerializer extends Serializer {
|
|
|
24
25
|
|
|
25
26
|
if (job.constructor.name === 'ResourceRequestJob') {
|
|
26
27
|
result.url = job.arguments?.url;
|
|
28
|
+
} else if (job.arguments?.originUrl) {
|
|
29
|
+
result.url = job.arguments.originUrl;
|
|
27
30
|
}
|
|
28
31
|
|
|
29
32
|
return result;
|
|
@@ -10,7 +10,7 @@ class LogSerializer extends Serializer {
|
|
|
10
10
|
* Serializes a single Log instance.
|
|
11
11
|
*
|
|
12
12
|
* @param {import('../utils/logging/Log.js').Log} log - The log entry to serialize.
|
|
13
|
-
* @returns {{ id: number, level: string, message: string, attributes: object, timestamp: string }}
|
|
13
|
+
* @returns {{ id: number, level: string, message: string, attributes: object, timestamp: string }} Plain object representation of the log entry.
|
|
14
14
|
*/
|
|
15
15
|
static _serializeObject(log) {
|
|
16
16
|
return {
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import { ConflictError } from '../exceptions/ConflictError.js';
|
|
2
|
-
import { ForbiddenError } from '../exceptions/ForbiddenError.js';
|
|
3
|
-
import { NotFoundError } from '../exceptions/NotFoundError.js';
|
|
1
|
+
import { ConflictError } from '../exceptions/http/ConflictError.js';
|
|
2
|
+
import { ForbiddenError } from '../exceptions/http/ForbiddenError.js';
|
|
3
|
+
import { NotFoundError } from '../exceptions/http/NotFoundError.js';
|
|
4
4
|
import { Logger } from '../utils/logging/Logger.js';
|
|
5
5
|
|
|
6
6
|
/**
|
package/lib/server/Router.js
CHANGED
|
@@ -1,24 +1,25 @@
|
|
|
1
1
|
import path from 'path';
|
|
2
2
|
import { fileURLToPath } from 'url';
|
|
3
3
|
import express from 'express';
|
|
4
|
-
import { AssetsRequestHandler } from './AssetsRequestHandler.js';
|
|
5
|
-
import { BaseUrlsRequestHandler } from './BaseUrlsRequestHandler.js';
|
|
6
|
-
import { EngineContinueRequestHandler } from './EngineContinueRequestHandler.js';
|
|
7
|
-
import { EnginePauseRequestHandler } from './EnginePauseRequestHandler.js';
|
|
8
|
-
import { EngineRestartRequestHandler } from './EngineRestartRequestHandler.js';
|
|
9
|
-
import { EngineShutdownRequestHandler } from './EngineShutdownRequestHandler.js';
|
|
10
|
-
import { EngineStartRequestHandler } from './EngineStartRequestHandler.js';
|
|
11
|
-
import { EngineStatusRequestHandler } from './EngineStatusRequestHandler.js';
|
|
12
|
-
import { EngineStopRequestHandler } from './EngineStopRequestHandler.js';
|
|
13
|
-
import { IndexRequestHandler } from './IndexRequestHandler.js';
|
|
14
|
-
import { JobLogsRequestHandler } from './JobLogsRequestHandler.js';
|
|
15
|
-
import { JobRequestHandler } from './JobRequestHandler.js';
|
|
16
|
-
import { JobRetryRequestHandler } from './JobRetryRequestHandler.js';
|
|
17
|
-
import { JobsRequestHandler } from './JobsRequestHandler.js';
|
|
18
|
-
import {
|
|
4
|
+
import { AssetsRequestHandler } from './handlers/AssetsRequestHandler.js';
|
|
5
|
+
import { BaseUrlsRequestHandler } from './handlers/BaseUrlsRequestHandler.js';
|
|
6
|
+
import { EngineContinueRequestHandler } from './handlers/engine/EngineContinueRequestHandler.js';
|
|
7
|
+
import { EnginePauseRequestHandler } from './handlers/engine/EnginePauseRequestHandler.js';
|
|
8
|
+
import { EngineRestartRequestHandler } from './handlers/engine/EngineRestartRequestHandler.js';
|
|
9
|
+
import { EngineShutdownRequestHandler } from './handlers/engine/EngineShutdownRequestHandler.js';
|
|
10
|
+
import { EngineStartRequestHandler } from './handlers/engine/EngineStartRequestHandler.js';
|
|
11
|
+
import { EngineStatusRequestHandler } from './handlers/engine/EngineStatusRequestHandler.js';
|
|
12
|
+
import { EngineStopRequestHandler } from './handlers/engine/EngineStopRequestHandler.js';
|
|
13
|
+
import { IndexRequestHandler } from './handlers/IndexRequestHandler.js';
|
|
14
|
+
import { JobLogsRequestHandler } from './handlers/jobs/JobLogsRequestHandler.js';
|
|
15
|
+
import { JobRequestHandler } from './handlers/jobs/JobRequestHandler.js';
|
|
16
|
+
import { JobRetryRequestHandler } from './handlers/jobs/JobRetryRequestHandler.js';
|
|
17
|
+
import { JobsRequestHandler } from './handlers/jobs/JobsRequestHandler.js';
|
|
18
|
+
import { LinksRequestHandler } from './handlers/LinksRequestHandler.js';
|
|
19
|
+
import { LogsRequestHandler } from './handlers/LogsRequestHandler.js';
|
|
20
|
+
import { SettingsRequestHandler } from './handlers/SettingsRequestHandler.js';
|
|
21
|
+
import { StatsRequestHandler } from './handlers/StatsRequestHandler.js';
|
|
19
22
|
import { RouteRegister } from './RouteRegister.js';
|
|
20
|
-
import { SettingsRequestHandler } from './SettingsRequestHandler.js';
|
|
21
|
-
import { StatsRequestHandler } from './StatsRequestHandler.js';
|
|
22
23
|
|
|
23
24
|
const { Router: ExpressRouter } = express;
|
|
24
25
|
const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
|
@@ -33,7 +34,7 @@ class Router {
|
|
|
33
34
|
|
|
34
35
|
/**
|
|
35
36
|
* Creates a new Router instance.
|
|
36
|
-
* @param {object} [options={}]
|
|
37
|
+
* @param {object} [options={}] - Constructor options.
|
|
37
38
|
* @param {object} [options.webConfig={}] - Web configuration, used by handlers that need it.
|
|
38
39
|
*/
|
|
39
40
|
constructor({ webConfig = {} } = {}) {
|
|
@@ -57,6 +58,7 @@ class Router {
|
|
|
57
58
|
'/job/:id.json': new JobRequestHandler(),
|
|
58
59
|
'/engine/status': new EngineStatusRequestHandler(),
|
|
59
60
|
'/logs.json': new LogsRequestHandler({ pageSize: this.#webConfig.logsPageSize }),
|
|
61
|
+
'/links.json': new LinksRequestHandler({ links: this.#webConfig.links }),
|
|
60
62
|
'/': new IndexRequestHandler(),
|
|
61
63
|
'/assets/*path': new AssetsRequestHandler(),
|
|
62
64
|
};
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import path from 'path';
|
|
2
2
|
import { fileURLToPath } from 'url';
|
|
3
|
-
import { PathValidator } from '
|
|
4
|
-
import { RequestHandler } from '
|
|
3
|
+
import { PathValidator } from '../PathValidator.js';
|
|
4
|
+
import { RequestHandler } from '../RequestHandler.js';
|
|
5
5
|
|
|
6
6
|
const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
|
7
7
|
|
|
@@ -16,7 +16,7 @@ class AssetsRequestHandler extends RequestHandler {
|
|
|
16
16
|
*/
|
|
17
17
|
constructor() {
|
|
18
18
|
super();
|
|
19
|
-
this.assetsDir = path.resolve(__dirname, '
|
|
19
|
+
this.assetsDir = path.resolve(__dirname, '../../../static/assets');
|
|
20
20
|
this.validator = new PathValidator(this.assetsDir);
|
|
21
21
|
}
|
|
22
22
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
3
|
-
import {
|
|
1
|
+
import { ClientRegistry } from '../../registry/ClientRegistry.js';
|
|
2
|
+
import { BaseUrlsSerializer } from '../../serializers/BaseUrlsSerializer.js';
|
|
3
|
+
import { RequestHandler } from '../RequestHandler.js';
|
|
4
4
|
|
|
5
5
|
/**
|
|
6
6
|
* Handles GET /clients/base_urls.json requests.
|
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
import path from 'path';
|
|
2
2
|
import { fileURLToPath } from 'url';
|
|
3
|
-
import { RequestHandler } from '
|
|
3
|
+
import { RequestHandler } from '../RequestHandler.js';
|
|
4
4
|
|
|
5
5
|
const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
|
6
|
-
const staticDir = path.resolve(__dirname, '
|
|
6
|
+
const staticDir = path.resolve(__dirname, '../../../static');
|
|
7
7
|
|
|
8
8
|
/**
|
|
9
9
|
* Handles GET / and SPA catch-all requests by serving index.html.
|
|
@@ -32,7 +32,7 @@ class JobsFilter {
|
|
|
32
32
|
* Returns true when the job's class is listed in the requested classes.
|
|
33
33
|
* @param {object} job - A job instance.
|
|
34
34
|
* @param {string[]} classes - The list of accepted class names.
|
|
35
|
-
* @returns {boolean}
|
|
35
|
+
* @returns {boolean} True if the job's class is in the accepted list.
|
|
36
36
|
*/
|
|
37
37
|
#matchesClass(job, classes) {
|
|
38
38
|
return classes.includes(job.constructor.name);
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
import { RequestHandler } from '../RequestHandler.js';
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Handles GET /links.json requests.
|
|
5
|
+
* Responds with the configured web links.
|
|
6
|
+
* @author darthjee
|
|
7
|
+
*/
|
|
8
|
+
class LinksRequestHandler extends RequestHandler {
|
|
9
|
+
#links;
|
|
10
|
+
|
|
11
|
+
/**
|
|
12
|
+
* Creates a new LinksRequestHandler instance.
|
|
13
|
+
* @param {object} [params={}] - Constructor params.
|
|
14
|
+
* @param {Array<object>} [params.links=[]] - Links from web configuration.
|
|
15
|
+
*/
|
|
16
|
+
constructor({ links = [] } = {}) {
|
|
17
|
+
super();
|
|
18
|
+
this.#links = links;
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
/**
|
|
22
|
+
* Responds with configured links.
|
|
23
|
+
* @param {object} _req - The Express request object.
|
|
24
|
+
* @param {object} res - The Express response object.
|
|
25
|
+
* @returns {void}
|
|
26
|
+
*/
|
|
27
|
+
handle(_req, res) {
|
|
28
|
+
res.json({ links: this.#links.map((link) => link.toJSON()) });
|
|
29
|
+
}
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
export { LinksRequestHandler };
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
3
|
-
import {
|
|
1
|
+
import { LogRegistry } from '../../registry/LogRegistry.js';
|
|
2
|
+
import { LogSerializer } from '../../serializers/LogSerializer.js';
|
|
3
|
+
import { RequestHandler } from '../RequestHandler.js';
|
|
4
4
|
|
|
5
5
|
/**
|
|
6
6
|
* Handles GET /logs.json requests.
|
|
@@ -13,7 +13,7 @@ class LogsRequestHandler extends RequestHandler {
|
|
|
13
13
|
|
|
14
14
|
/**
|
|
15
15
|
* Creates a new LogsRequestHandler.
|
|
16
|
-
* @param {object} [options={}]
|
|
16
|
+
* @param {object} [options={}] - Constructor options.
|
|
17
17
|
* @param {number} [options.pageSize=20] - Maximum number of log entries to return.
|
|
18
18
|
*/
|
|
19
19
|
constructor({ pageSize = 20 } = {}) {
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
1
|
+
import { ForbiddenError } from '../../exceptions/http/ForbiddenError.js';
|
|
2
|
+
import { RequestHandler } from '../RequestHandler.js';
|
|
3
3
|
|
|
4
4
|
/**
|
|
5
5
|
* Handles GET /settings.json requests.
|
|
@@ -11,7 +11,8 @@ class SettingsRequestHandler extends RequestHandler {
|
|
|
11
11
|
#enableShutdown;
|
|
12
12
|
|
|
13
13
|
/**
|
|
14
|
-
*
|
|
14
|
+
* Creates a new SettingsRequestHandler.
|
|
15
|
+
* @param {object} [options={}] - Constructor options.
|
|
15
16
|
* @param {boolean} [options.enableShutdown=true] - Whether shutdown is permitted.
|
|
16
17
|
*/
|
|
17
18
|
constructor({ enableShutdown = true } = {}) {
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
3
|
-
import {
|
|
1
|
+
import { JobRegistry } from '../../background/JobRegistry.js';
|
|
2
|
+
import { WorkersRegistry } from '../../background/WorkersRegistry.js';
|
|
3
|
+
import { RequestHandler } from '../RequestHandler.js';
|
|
4
4
|
|
|
5
5
|
/**
|
|
6
6
|
* Handles GET /stats.json requests.
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
3
|
-
import {
|
|
1
|
+
import { ConflictError } from '../../../exceptions/http/ConflictError.js';
|
|
2
|
+
import { Application } from '../../../services/Application.js';
|
|
3
|
+
import { RequestHandler } from '../../RequestHandler.js';
|
|
4
4
|
|
|
5
5
|
/**
|
|
6
6
|
* Handles PATCH /engine/continue requests.
|
package/lib/server/{EnginePauseRequestHandler.js → handlers/engine/EnginePauseRequestHandler.js}
RENAMED
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
3
|
-
import {
|
|
1
|
+
import { ConflictError } from '../../../exceptions/http/ConflictError.js';
|
|
2
|
+
import { Application } from '../../../services/Application.js';
|
|
3
|
+
import { RequestHandler } from '../../RequestHandler.js';
|
|
4
4
|
|
|
5
5
|
/**
|
|
6
6
|
* Handles PATCH /engine/pause requests.
|
package/lib/server/{EngineRestartRequestHandler.js → handlers/engine/EngineRestartRequestHandler.js}
RENAMED
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
3
|
-
import {
|
|
1
|
+
import { ConflictError } from '../../../exceptions/http/ConflictError.js';
|
|
2
|
+
import { Application } from '../../../services/Application.js';
|
|
3
|
+
import { RequestHandler } from '../../RequestHandler.js';
|
|
4
4
|
|
|
5
5
|
/**
|
|
6
6
|
* Handles PATCH /engine/restart requests.
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
1
|
+
import { Application } from '../../../services/Application.js';
|
|
2
|
+
import { RequestHandler } from '../../RequestHandler.js';
|
|
3
3
|
|
|
4
4
|
/**
|
|
5
5
|
* Handles PATCH /engine/shutdown requests.
|
package/lib/server/{EngineStartRequestHandler.js → handlers/engine/EngineStartRequestHandler.js}
RENAMED
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
3
|
-
import {
|
|
1
|
+
import { ConflictError } from '../../../exceptions/http/ConflictError.js';
|
|
2
|
+
import { Application } from '../../../services/Application.js';
|
|
3
|
+
import { RequestHandler } from '../../RequestHandler.js';
|
|
4
4
|
|
|
5
5
|
/**
|
|
6
6
|
* Handles PATCH /engine/start requests.
|
package/lib/server/{EngineStatusRequestHandler.js → handlers/engine/EngineStatusRequestHandler.js}
RENAMED
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
1
|
+
import { Application } from '../../../services/Application.js';
|
|
2
|
+
import { RequestHandler } from '../../RequestHandler.js';
|
|
3
3
|
|
|
4
4
|
/**
|
|
5
5
|
* Handles GET /engine/status requests.
|
package/lib/server/{EngineStopRequestHandler.js → handlers/engine/EngineStopRequestHandler.js}
RENAMED
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
3
|
-
import {
|
|
1
|
+
import { ConflictError } from '../../../exceptions/http/ConflictError.js';
|
|
2
|
+
import { Application } from '../../../services/Application.js';
|
|
3
|
+
import { RequestHandler } from '../../RequestHandler.js';
|
|
4
4
|
|
|
5
5
|
/**
|
|
6
6
|
* Handles PATCH /engine/stop requests.
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
3
|
-
import {
|
|
1
|
+
import { LogRegistry } from '../../../registry/LogRegistry.js';
|
|
2
|
+
import { LogSerializer } from '../../../serializers/LogSerializer.js';
|
|
3
|
+
import { RequestHandler } from '../../RequestHandler.js';
|
|
4
4
|
|
|
5
5
|
/**
|
|
6
6
|
* Handles GET /jobs/:job_id/logs.json requests.
|
|
@@ -14,7 +14,7 @@ class JobLogsRequestHandler extends RequestHandler {
|
|
|
14
14
|
|
|
15
15
|
/**
|
|
16
16
|
* Creates a new JobLogsRequestHandler.
|
|
17
|
-
* @param {object} [options={}]
|
|
17
|
+
* @param {object} [options={}] - Constructor options.
|
|
18
18
|
* @param {number} [options.pageSize=20] - Maximum number of log entries to return.
|
|
19
19
|
*/
|
|
20
20
|
constructor({ pageSize = 20 } = {}) {
|
|
@@ -1,7 +1,7 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
3
|
-
import {
|
|
4
|
-
import {
|
|
1
|
+
import { JobRegistry } from '../../../background/JobRegistry.js';
|
|
2
|
+
import { NotFoundError } from '../../../exceptions/http/NotFoundError.js';
|
|
3
|
+
import { JobSerializer } from '../../../serializers/JobSerializer.js';
|
|
4
|
+
import { RequestHandler } from '../../RequestHandler.js';
|
|
5
5
|
|
|
6
6
|
/**
|
|
7
7
|
* Handles GET /job/:id.json requests.
|
|
@@ -1,7 +1,7 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
3
|
-
import {
|
|
4
|
-
import {
|
|
1
|
+
import { JobRegistry } from '../../../background/JobRegistry.js';
|
|
2
|
+
import { ConflictError } from '../../../exceptions/http/ConflictError.js';
|
|
3
|
+
import { NotFoundError } from '../../../exceptions/http/NotFoundError.js';
|
|
4
|
+
import { RequestHandler } from '../../RequestHandler.js';
|
|
5
5
|
|
|
6
6
|
const RETRYABLE_STATUSES = new Set(['failed', 'dead']);
|
|
7
7
|
|
|
@@ -1,7 +1,7 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
3
|
-
import {
|
|
4
|
-
import {
|
|
1
|
+
import { JobRegistry } from '../../../background/JobRegistry.js';
|
|
2
|
+
import { JobSerializer } from '../../../serializers/JobSerializer.js';
|
|
3
|
+
import { RequestHandler } from '../../RequestHandler.js';
|
|
4
|
+
import { JobsFilter } from '../JobsFilter.js';
|
|
5
5
|
|
|
6
6
|
/**
|
|
7
7
|
* Handles GET /jobs/:status.json requests.
|
|
@@ -4,11 +4,12 @@ import { FailureChecker } from './FailureChecker.js';
|
|
|
4
4
|
import { JobFactory } from '../background/JobFactory.js';
|
|
5
5
|
import { JobRegistry } from '../background/JobRegistry.js';
|
|
6
6
|
import { WorkersRegistry } from '../background/WorkersRegistry.js';
|
|
7
|
-
import { ConfigurationFileNotProvided } from '../exceptions/ConfigurationFileNotProvided.js';
|
|
7
|
+
import { ConfigurationFileNotProvided } from '../exceptions/config/ConfigurationFileNotProvided.js';
|
|
8
8
|
import { ActionProcessingJob } from '../jobs/ActionProcessingJob.js';
|
|
9
9
|
import { AssetDownloadJob } from '../jobs/AssetDownloadJob.js';
|
|
10
10
|
import { HtmlParseJob } from '../jobs/HtmlParseJob.js';
|
|
11
|
-
import {
|
|
11
|
+
import { PaginatedActionProcessingJob } from '../jobs/PaginatedActionProcessingJob.js';
|
|
12
|
+
import { Config } from '../models/configs/Config.js';
|
|
12
13
|
import { LogRegistry } from '../registry/LogRegistry.js';
|
|
13
14
|
import { WebServer } from '../server/WebServer.js';
|
|
14
15
|
import { PromiseAggregator } from '../utils/PromiseAggregator.js';
|
|
@@ -254,6 +255,7 @@ class ApplicationInstance {
|
|
|
254
255
|
#initRegistries() {
|
|
255
256
|
JobFactory.build('ResourceRequestJob', { attributes: { clients: this.config.clientRegistry } });
|
|
256
257
|
JobFactory.build('Action', { klass: ActionProcessingJob });
|
|
258
|
+
JobFactory.build('PaginatedAction', { klass: PaginatedActionProcessingJob });
|
|
257
259
|
JobFactory.build('HtmlParse', { klass: HtmlParseJob, attributes: { jobRegistry: JobRegistry, clientRegistry: this.config.clientRegistry } });
|
|
258
260
|
JobFactory.build('AssetDownload', { klass: AssetDownloadJob, attributes: { clientRegistry: this.config.clientRegistry } });
|
|
259
261
|
|
package/lib/services/Client.js
CHANGED
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
import { readFileSync } from 'node:fs';
|
|
2
2
|
import YAML from 'yaml';
|
|
3
3
|
import { ConfigParser } from './ConfigParser.js';
|
|
4
|
-
import {
|
|
5
|
-
import {
|
|
4
|
+
import { EnvStringResolver } from '../common/utils/env_resolver/EnvStringResolver.js';
|
|
5
|
+
import { ConfigurationFileNotFound } from '../exceptions/config/ConfigurationFileNotFound.js';
|
|
6
6
|
import { Logger } from '../utils/logging/Logger.js';
|
|
7
7
|
|
|
8
8
|
/**
|
|
@@ -55,7 +55,7 @@ class ConfigLoader {
|
|
|
55
55
|
* @throws {ConfigurationFileNotFound} If the configuration file is not found at the specified path.
|
|
56
56
|
*/
|
|
57
57
|
#parseYaml() {
|
|
58
|
-
return YAML.parse(
|
|
58
|
+
return YAML.parse(EnvStringResolver.resolve(this.#yamlContent()));
|
|
59
59
|
}
|
|
60
60
|
|
|
61
61
|
/**
|
|
@@ -73,4 +73,4 @@ class ConfigLoader {
|
|
|
73
73
|
}
|
|
74
74
|
}
|
|
75
75
|
|
|
76
|
-
export { ConfigLoader };
|
|
76
|
+
export { ConfigLoader };
|
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
import { Client } from './Client.js';
|
|
2
|
-
import { MissingClientsConfig } from '../exceptions/MissingClientsConfig.js';
|
|
3
|
-
import { MissingResourceConfig } from '../exceptions/MissingResourceConfig.js';
|
|
4
|
-
import { FailureConfig } from '../models/FailureConfig.js';
|
|
5
|
-
import { LogConfig } from '../models/LogConfig.js';
|
|
6
|
-
import {
|
|
7
|
-
import {
|
|
8
|
-
import {
|
|
2
|
+
import { MissingClientsConfig } from '../exceptions/config/MissingClientsConfig.js';
|
|
3
|
+
import { MissingResourceConfig } from '../exceptions/config/MissingResourceConfig.js';
|
|
4
|
+
import { FailureConfig } from '../models/configs/FailureConfig.js';
|
|
5
|
+
import { LogConfig } from '../models/configs/LogConfig.js';
|
|
6
|
+
import { WebConfig } from '../models/configs/WebConfig.js';
|
|
7
|
+
import { WorkersConfig } from '../models/configs/WorkersConfig.js';
|
|
8
|
+
import { Resource } from '../models/request/Resource.js';
|
|
9
9
|
|
|
10
10
|
/**
|
|
11
11
|
* ConfigParser receives a parsed YAML configuration object and maps it to
|
package/lib/services/Engine.js
CHANGED
|
@@ -84,7 +84,7 @@ class Engine {
|
|
|
84
84
|
* Determines whether the engine loop should continue.
|
|
85
85
|
* In keepAlive mode, always returns true. Otherwise, returns true only
|
|
86
86
|
* while there are jobs or busy workers.
|
|
87
|
-
* @returns {boolean}
|
|
87
|
+
* @returns {boolean} True if the loop should continue.
|
|
88
88
|
*/
|
|
89
89
|
#shouldContinue() {
|
|
90
90
|
return this.#keepAlive || this.#continueAllocating();
|
package/lib/utils/HtmlParser.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { parse } from 'node-html-parser';
|
|
2
2
|
import { HtmlElementParser } from './HtmlElementParser.js';
|
|
3
|
-
import { InvalidHtmlResponseBody } from '../exceptions/InvalidHtmlResponseBody.js';
|
|
3
|
+
import { InvalidHtmlResponseBody } from '../exceptions/request/InvalidHtmlResponseBody.js';
|
|
4
4
|
|
|
5
5
|
/**
|
|
6
6
|
* HtmlParser parses a raw HTML string and extracts attribute values using CSS selectors.
|