navi-hey 1.1.0 → 1.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +38 -4
- package/lib/common/utils/EnvResolver.js +26 -0
- package/lib/common/utils/env_resolver/EnvStringResolver.js +52 -0
- package/lib/common/utils/logging/BaseLogger.js +98 -0
- package/lib/common/utils/logging/BufferedLogger.js +100 -0
- package/lib/common/utils/logging/ConsoleLogger.js +20 -0
- package/lib/common/utils/logging/Log.js +82 -0
- package/lib/common/utils/logging/LogBuffer.js +122 -0
- package/lib/common/utils/logging/LogBufferCollection.js +56 -0
- package/lib/common/utils/logging/LogContext.js +51 -0
- package/lib/common/utils/logging/LogFactory.js +33 -0
- package/lib/common/utils/logging/LogFilter.js +34 -0
- package/lib/common/utils/logging/Logger.js +138 -0
- package/lib/common/utils/logging/LoggerGroup.js +104 -0
- package/lib/enqueuers/ActionEnqueuer.js +18 -2
- package/lib/enqueuers/ActionsEnqueuer.js +6 -3
- package/lib/enqueuers/PaginatedActionEnqueuer.js +53 -0
- package/lib/enqueuers/PaginatedActionsEnqueuer.js +47 -0
- package/lib/exceptions/{ConfigurationFileNotFound.js → config/ConfigurationFileNotFound.js} +1 -1
- package/lib/exceptions/{ConfigurationFileNotProvided.js → config/ConfigurationFileNotProvided.js} +1 -1
- package/lib/exceptions/{MissingTopLevelConfgKey.js → config/MissingTopLevelConfgKey.js} +1 -1
- package/lib/exceptions/{ConflictError.js → http/ConflictError.js} +1 -1
- package/lib/exceptions/{ForbiddenError.js → http/ForbiddenError.js} +1 -1
- package/lib/exceptions/{NotFoundError.js → http/NotFoundError.js} +1 -1
- package/lib/exceptions/{ItemNotFound.js → registry/ItemNotFound.js} +1 -1
- package/lib/exceptions/{MissingActionResource.js → registry/MissingActionResource.js} +1 -1
- package/lib/exceptions/{MissingMappingVariable.js → registry/MissingMappingVariable.js} +1 -1
- package/lib/exceptions/{InvalidHtmlResponseBody.js → request/InvalidHtmlResponseBody.js} +1 -1
- package/lib/exceptions/{InvalidResponseBody.js → request/InvalidResponseBody.js} +1 -1
- package/lib/exceptions/{NullResponse.js → request/NullResponse.js} +1 -1
- package/lib/exceptions/{RequestFailed.js → request/RequestFailed.js} +1 -1
- package/lib/jobs/ActionProcessingJob.js +17 -3
- package/lib/jobs/HtmlParseJob.js +17 -4
- package/lib/jobs/PaginatedActionProcessingJob.js +77 -0
- package/lib/jobs/ResourceRequestJob.js +8 -5
- package/lib/models/{Config.js → configs/Config.js} +3 -3
- package/lib/models/configs/Link.js +36 -0
- package/lib/models/configs/PaginationConfig.js +63 -0
- package/lib/models/{WebConfig.js → configs/WebConfig.js} +5 -1
- package/lib/models/{ResourceRequest.js → request/ResourceRequest.js} +30 -8
- package/lib/models/{ResourceRequestAction.js → request/ResourceRequestAction.js} +6 -6
- package/lib/models/request/ResourceRequestPaginatedAction.js +81 -0
- package/lib/models/{PathSegmentTraverser.js → response/PathSegmentTraverser.js} +1 -1
- package/lib/models/{ResponseParser.js → response/ResponseParser.js} +1 -1
- package/lib/registry/ClientRegistry.js +1 -1
- package/lib/registry/LogRegistry.js +13 -13
- package/lib/registry/LogRegistryInstance.js +16 -16
- package/lib/registry/NamedRegistry.js +1 -1
- package/lib/registry/ResourceRegistry.js +1 -1
- package/lib/serializers/JobIndexSerializer.js +3 -0
- package/lib/serializers/LogSerializer.js +1 -1
- package/lib/server/PathValidator.js +1 -1
- package/lib/server/RouteRegister.js +3 -3
- package/lib/server/Router.js +20 -18
- package/lib/server/{AssetsRequestHandler.js → handlers/AssetsRequestHandler.js} +3 -3
- package/lib/server/{BaseUrlsRequestHandler.js → handlers/BaseUrlsRequestHandler.js} +3 -3
- package/lib/server/{IndexRequestHandler.js → handlers/IndexRequestHandler.js} +2 -2
- package/lib/server/{JobsFilter.js → handlers/JobsFilter.js} +1 -1
- package/lib/server/handlers/LinksRequestHandler.js +32 -0
- package/lib/server/{LogsRequestHandler.js → handlers/LogsRequestHandler.js} +4 -4
- package/lib/server/{SettingsRequestHandler.js → handlers/SettingsRequestHandler.js} +4 -3
- package/lib/server/{StatsRequestHandler.js → handlers/StatsRequestHandler.js} +3 -3
- package/lib/server/{EngineContinueRequestHandler.js → handlers/engine/EngineContinueRequestHandler.js} +3 -3
- package/lib/server/{EnginePauseRequestHandler.js → handlers/engine/EnginePauseRequestHandler.js} +3 -3
- package/lib/server/{EngineRestartRequestHandler.js → handlers/engine/EngineRestartRequestHandler.js} +3 -3
- package/lib/server/{EngineShutdownRequestHandler.js → handlers/engine/EngineShutdownRequestHandler.js} +2 -2
- package/lib/server/{EngineStartRequestHandler.js → handlers/engine/EngineStartRequestHandler.js} +3 -3
- package/lib/server/{EngineStatusRequestHandler.js → handlers/engine/EngineStatusRequestHandler.js} +2 -2
- package/lib/server/{EngineStopRequestHandler.js → handlers/engine/EngineStopRequestHandler.js} +3 -3
- package/lib/server/{JobLogsRequestHandler.js → handlers/jobs/JobLogsRequestHandler.js} +4 -4
- package/lib/server/{JobRequestHandler.js → handlers/jobs/JobRequestHandler.js} +4 -4
- package/lib/server/{JobRetryRequestHandler.js → handlers/jobs/JobRetryRequestHandler.js} +4 -4
- package/lib/server/{JobsRequestHandler.js → handlers/jobs/JobsRequestHandler.js} +4 -4
- package/lib/services/ApplicationInstance.js +4 -2
- package/lib/services/Client.js +1 -1
- package/lib/services/ConfigLoader.js +4 -4
- package/lib/services/ConfigParser.js +7 -7
- package/lib/services/Engine.js +1 -1
- package/lib/utils/HtmlParser.js +1 -1
- package/lib/utils/logging/BaseLogger.js +1 -98
- package/lib/utils/logging/BufferedLogger.js +1 -100
- package/lib/utils/logging/ConsoleLogger.js +1 -20
- package/lib/utils/logging/Log.js +1 -82
- package/lib/utils/logging/LogBuffer.js +1 -122
- package/lib/utils/logging/LogBufferCollection.js +1 -56
- package/lib/utils/logging/LogContext.js +1 -51
- package/lib/utils/logging/LogFactory.js +1 -33
- package/lib/utils/logging/LogFilter.js +1 -34
- package/lib/utils/logging/Logger.js +1 -138
- package/lib/utils/logging/LoggerGroup.js +1 -104
- package/package.json +1 -1
- package/static/assets/index-CX_uXu4Q.js +11 -0
- package/static/favicon.png +0 -0
- package/static/index.html +2 -1
- package/lib/exceptions/LockedByOtherWorker.js +0 -14
- package/lib/utils/EnvResolver.js +0 -75
- package/static/assets/index-BfggoMvd.js +0 -11
- /package/lib/exceptions/{MissingClientsConfig.js → config/MissingClientsConfig.js} +0 -0
- /package/lib/exceptions/{MissingResourceConfig.js → config/MissingResourceConfig.js} +0 -0
- /package/lib/exceptions/{ClientNotFound.js → registry/ClientNotFound.js} +0 -0
- /package/lib/exceptions/{ResourceNotFound.js → registry/ResourceNotFound.js} +0 -0
- /package/lib/models/{FailureConfig.js → configs/FailureConfig.js} +0 -0
- /package/lib/models/{LogConfig.js → configs/LogConfig.js} +0 -0
- /package/lib/models/{WorkersConfig.js → configs/WorkersConfig.js} +0 -0
- /package/lib/models/{AssetRequest.js → request/AssetRequest.js} +0 -0
- /package/lib/models/{Resource.js → request/Resource.js} +0 -0
- /package/lib/models/{ParametersMapper.js → response/ParametersMapper.js} +0 -0
- /package/lib/models/{PathResolver.js → response/PathResolver.js} +0 -0
- /package/lib/models/{ResponseWrapper.js → response/ResponseWrapper.js} +0 -0
package/lib/jobs/HtmlParseJob.js
CHANGED
|
@@ -14,6 +14,7 @@ class HtmlParseJob extends Job {
|
|
|
14
14
|
#assetRequests;
|
|
15
15
|
#jobRegistry;
|
|
16
16
|
#clientRegistry;
|
|
17
|
+
#originUrl;
|
|
17
18
|
|
|
18
19
|
/**
|
|
19
20
|
* Creates a new HtmlParseJob instance.
|
|
@@ -23,21 +24,23 @@ class HtmlParseJob extends Job {
|
|
|
23
24
|
* @param {Array} params.assetRequests - List of AssetRequest instances.
|
|
24
25
|
* @param {object} params.jobRegistry - The job registry used to enqueue AssetDownloadJobs.
|
|
25
26
|
* @param {object} params.clientRegistry - The client registry used for URL resolution.
|
|
27
|
+
* @param {string|null} [params.originUrl=null] - The URL of the ResourceRequestJob that triggered this job.
|
|
26
28
|
*/
|
|
27
|
-
constructor({ id, rawHtml, assetRequests, jobRegistry, clientRegistry }) {
|
|
29
|
+
constructor({ id, rawHtml, assetRequests, jobRegistry, clientRegistry, originUrl = null }) {
|
|
28
30
|
super({ id });
|
|
29
31
|
this.#rawHtml = rawHtml;
|
|
30
32
|
this.#assetRequests = assetRequests;
|
|
31
33
|
this.#jobRegistry = jobRegistry;
|
|
32
34
|
this.#clientRegistry = clientRegistry;
|
|
35
|
+
this.#originUrl = originUrl;
|
|
33
36
|
}
|
|
34
37
|
|
|
35
38
|
/**
|
|
36
39
|
* Returns the job-specific arguments for serialization.
|
|
37
|
-
* @returns {{ assetCount: number }} The job arguments.
|
|
40
|
+
* @returns {{ assetCount: number, originUrl?: string }} The job arguments.
|
|
38
41
|
*/
|
|
39
42
|
get arguments() {
|
|
40
|
-
return { assetCount: this.#assetRequests.length };
|
|
43
|
+
return { assetCount: this.#assetRequests.length, ...this.#originUrlField() };
|
|
41
44
|
}
|
|
42
45
|
|
|
43
46
|
/**
|
|
@@ -49,7 +52,6 @@ class HtmlParseJob extends Job {
|
|
|
49
52
|
get maxRetries() {
|
|
50
53
|
return 1;
|
|
51
54
|
}
|
|
52
|
-
|
|
53
55
|
/**
|
|
54
56
|
* Parses the HTML body, resolves asset URLs, and enqueues one AssetDownloadJob per URL.
|
|
55
57
|
* @param {LogContext} logContext - Context carrying workerId/jobId for log entries.
|
|
@@ -66,6 +68,17 @@ class HtmlParseJob extends Job {
|
|
|
66
68
|
this._fail(error);
|
|
67
69
|
}
|
|
68
70
|
}
|
|
71
|
+
|
|
72
|
+
/**
|
|
73
|
+
* Returns an object containing the originUrl field when an origin URL is set,
|
|
74
|
+
* or an empty object otherwise.
|
|
75
|
+
* @returns {{ originUrl: string }|{}} The origin URL field or empty object.
|
|
76
|
+
* @private
|
|
77
|
+
*/
|
|
78
|
+
#originUrlField() {
|
|
79
|
+
return this.#originUrl !== null ? { originUrl: this.#originUrl } : {};
|
|
80
|
+
}
|
|
81
|
+
|
|
69
82
|
}
|
|
70
83
|
|
|
71
84
|
export { HtmlParseJob };
|
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
import { Job } from '../background/Job.js';
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Processes a single paginated action for a given response wrapper.
|
|
5
|
+
* Exhausted after the first failure — no retry rights.
|
|
6
|
+
* @author darthjee
|
|
7
|
+
* @augments Job
|
|
8
|
+
*/
|
|
9
|
+
class PaginatedActionProcessingJob extends Job {
|
|
10
|
+
#paginatedAction;
|
|
11
|
+
#responseWrapper;
|
|
12
|
+
#parameters;
|
|
13
|
+
#originUrl;
|
|
14
|
+
|
|
15
|
+
/**
|
|
16
|
+
* Creates a new PaginatedActionProcessingJob instance.
|
|
17
|
+
* @param {object} params The parameters for creating a PaginatedActionProcessingJob instance.
|
|
18
|
+
* @param {string} params.id Unique job identifier.
|
|
19
|
+
* @param {ResourceRequestPaginatedAction} params.paginatedAction The paginated action to execute.
|
|
20
|
+
* @param {ResponseWrapper} params.responseWrapper The response wrapper carrying response data.
|
|
21
|
+
* @param {object} [params.parameters={}] The original request parameters (key-value map for URL template resolution).
|
|
22
|
+
* @param {string|null} [params.originUrl=null] The URL of the ResourceRequestJob that triggered this job.
|
|
23
|
+
*/
|
|
24
|
+
constructor({ id, paginatedAction, responseWrapper, parameters = {}, originUrl = null }) {
|
|
25
|
+
super({ id });
|
|
26
|
+
this.#paginatedAction = paginatedAction;
|
|
27
|
+
this.#responseWrapper = responseWrapper;
|
|
28
|
+
this.#parameters = parameters;
|
|
29
|
+
this.#originUrl = originUrl;
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
/**
|
|
33
|
+
* Returns the job-specific arguments for serialization.
|
|
34
|
+
* @returns {{ responseWrapper: ResponseWrapper, parameters: object, originUrl?: string }} The job arguments.
|
|
35
|
+
*/
|
|
36
|
+
get arguments() {
|
|
37
|
+
return { responseWrapper: this.#responseWrapper, parameters: this.#parameters, ...this.#originUrlField() };
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
/**
|
|
41
|
+
* Returns the maximum number of retries for this job type.
|
|
42
|
+
* PaginatedActionProcessingJob is exhausted after the first failure.
|
|
43
|
+
* @returns {number} Always 1.
|
|
44
|
+
* @override
|
|
45
|
+
*/
|
|
46
|
+
get maxRetries() {
|
|
47
|
+
return 1;
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
/**
|
|
51
|
+
* Performs the paginated action for the given response wrapper and parameters.
|
|
52
|
+
* @param {LogContext} logContext - Context carrying workerId/jobId for log entries.
|
|
53
|
+
* @returns {Promise<void>}
|
|
54
|
+
*/
|
|
55
|
+
async perform(logContext) {
|
|
56
|
+
logContext.debug(`PaginatedActionProcessingJob #${this.id} performing`);
|
|
57
|
+
try {
|
|
58
|
+
this.lastError = undefined;
|
|
59
|
+
await this.#paginatedAction.execute(this.#responseWrapper, this.#parameters);
|
|
60
|
+
} catch (error) {
|
|
61
|
+
this._fail(error);
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
/**
|
|
66
|
+
* Returns an object containing the originUrl field when an origin URL is set,
|
|
67
|
+
* or an empty object otherwise.
|
|
68
|
+
* @returns {{ originUrl: string }|{}} The origin URL field or empty object.
|
|
69
|
+
* @private
|
|
70
|
+
*/
|
|
71
|
+
#originUrlField() {
|
|
72
|
+
return this.#originUrl !== null ? { originUrl: this.#originUrl } : {};
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
export { PaginatedActionProcessingJob };
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { Job } from '../background/Job.js';
|
|
2
2
|
import { JobRegistry } from '../background/JobRegistry.js';
|
|
3
|
-
import { ResponseWrapper } from '../models/ResponseWrapper.js';
|
|
3
|
+
import { ResponseWrapper } from '../models/response/ResponseWrapper.js';
|
|
4
4
|
|
|
5
5
|
/**
|
|
6
6
|
* ResourceRequestJob is a Job that performs an HTTP request for a ResourceRequest.
|
|
@@ -60,20 +60,23 @@ class ResourceRequestJob extends Job {
|
|
|
60
60
|
* @private
|
|
61
61
|
*/
|
|
62
62
|
#handleResponse(response) {
|
|
63
|
-
this.#
|
|
63
|
+
const originUrl = this.#resourceRequest.resolveUrl(this.#parameters);
|
|
64
|
+
this.#enqueueAssets(response, originUrl);
|
|
64
65
|
const wrapper = new ResponseWrapper(response, this.#parameters);
|
|
65
|
-
this.#resourceRequest.enqueueActions(wrapper);
|
|
66
|
+
this.#resourceRequest.enqueueActions(wrapper, originUrl);
|
|
67
|
+
this.#resourceRequest.enqueuePaginatedActions(wrapper, this.#parameters, originUrl);
|
|
66
68
|
return response;
|
|
67
69
|
}
|
|
68
70
|
|
|
69
71
|
/**
|
|
70
72
|
* Enqueues asset download jobs when the resource request declares asset rules.
|
|
71
73
|
* @param {object} response The HTTP response object.
|
|
74
|
+
* @param {string} originUrl The resolved URL of this job, forwarded as origin to child jobs.
|
|
72
75
|
* @private
|
|
73
76
|
*/
|
|
74
|
-
#enqueueAssets(response) {
|
|
77
|
+
#enqueueAssets(response, originUrl) {
|
|
75
78
|
if (this.#resourceRequest.hasAssets()) {
|
|
76
|
-
this.#resourceRequest.enqueueAssets(response.data, JobRegistry, this.#clients);
|
|
79
|
+
this.#resourceRequest.enqueueAssets(response.data, JobRegistry, this.#clients, originUrl);
|
|
77
80
|
}
|
|
78
81
|
}
|
|
79
82
|
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { LogConfig } from './LogConfig.js';
|
|
2
|
-
import { ClientRegistry } from '
|
|
3
|
-
import { ResourceRegistry } from '
|
|
4
|
-
import { ConfigLoader } from '
|
|
2
|
+
import { ClientRegistry } from '../../registry/ClientRegistry.js';
|
|
3
|
+
import { ResourceRegistry } from '../../registry/ResourceRegistry.js';
|
|
4
|
+
import { ConfigLoader } from '../../services/ConfigLoader.js';
|
|
5
5
|
|
|
6
6
|
/**
|
|
7
7
|
* Config is a class that represents the configuration for the application.
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Represents a web link item configured for the UI.
|
|
3
|
+
* @author darthjee
|
|
4
|
+
*/
|
|
5
|
+
class Link {
|
|
6
|
+
/**
|
|
7
|
+
* Creates a Link instance.
|
|
8
|
+
* @param {object} params - Constructor params.
|
|
9
|
+
* @param {string} params.url - Link URL.
|
|
10
|
+
* @param {string} [params.text] - Link display text.
|
|
11
|
+
*/
|
|
12
|
+
constructor({ url, text }) {
|
|
13
|
+
this.url = url;
|
|
14
|
+
this.text = text ?? url;
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
/**
|
|
18
|
+
* Creates a Link instance from a YAML entry.
|
|
19
|
+
* @param {string|object} entry - A URL string or an object with text/url fields.
|
|
20
|
+
* @returns {Link} The parsed Link instance.
|
|
21
|
+
*/
|
|
22
|
+
static fromObject(entry) {
|
|
23
|
+
if (typeof entry === 'string') return new Link({ url: entry, text: entry });
|
|
24
|
+
return new Link({ url: entry.url, text: entry.text });
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
/**
|
|
28
|
+
* Serializes this link for JSON responses.
|
|
29
|
+
* @returns {{url: string, text: string}} The serialized link.
|
|
30
|
+
*/
|
|
31
|
+
toJSON() {
|
|
32
|
+
return { url: this.url, text: this.text };
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
export { Link };
|
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
import { PathResolver } from '../response/PathResolver.js';
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Parses the pagination block from YAML and resolves the total page count
|
|
5
|
+
* against a response wrapper.
|
|
6
|
+
* @author darthjee
|
|
7
|
+
*/
|
|
8
|
+
class PaginationConfig {
|
|
9
|
+
#pagesResolver;
|
|
10
|
+
#pageKey;
|
|
11
|
+
#zeroIndexed;
|
|
12
|
+
|
|
13
|
+
/**
|
|
14
|
+
* @param {object} params Pagination configuration attributes.
|
|
15
|
+
* @param {string} params.pages Path expression to resolve the total page count.
|
|
16
|
+
* @param {string} params.page_key The parameter name to inject the page number.
|
|
17
|
+
* @param {boolean} [params.zero_indexed=false] Whether pages are zero-indexed.
|
|
18
|
+
*/
|
|
19
|
+
constructor({ pages, page_key, zero_indexed = false }) {
|
|
20
|
+
this.#pagesResolver = PathResolver.fromExpression(pages);
|
|
21
|
+
this.#pageKey = page_key;
|
|
22
|
+
this.#zeroIndexed = zero_indexed;
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
/**
|
|
26
|
+
* @returns {string} The parameter name used to inject the page number.
|
|
27
|
+
*/
|
|
28
|
+
get pageKey() {
|
|
29
|
+
return this.#pageKey;
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
/**
|
|
33
|
+
* Evaluates the `pages` expression against the response wrapper.
|
|
34
|
+
* @param {ResponseWrapper} responseWrapper The response wrapper to resolve against.
|
|
35
|
+
* @returns {number} Total number of pages.
|
|
36
|
+
*/
|
|
37
|
+
resolvePages(responseWrapper) {
|
|
38
|
+
return this.#pagesResolver.resolve(responseWrapper);
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
/**
|
|
42
|
+
* Returns an array of page numbers to iterate over.
|
|
43
|
+
* @param {number} count Total number of pages.
|
|
44
|
+
* @returns {Array<number>} Array of page numbers.
|
|
45
|
+
*/
|
|
46
|
+
pageNumbers(count) {
|
|
47
|
+
const start = this.#zeroIndexed ? 0 : 1;
|
|
48
|
+
const end = this.#zeroIndexed ? count - 1 : count;
|
|
49
|
+
return Array.from({ length: end - start + 1 }, (_, i) => start + i);
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
/**
|
|
53
|
+
* Creates a PaginationConfig from a raw YAML list entry.
|
|
54
|
+
* @param {Array<object>} list List of single-key maps from YAML.
|
|
55
|
+
* @returns {PaginationConfig} A new PaginationConfig instance.
|
|
56
|
+
*/
|
|
57
|
+
static fromList(list) {
|
|
58
|
+
const attrs = Object.assign({}, ...list);
|
|
59
|
+
return new PaginationConfig(attrs);
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
export { PaginationConfig };
|
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
import { Link } from './Link.js';
|
|
2
|
+
|
|
1
3
|
/**
|
|
2
4
|
* Represents the web server configuration.
|
|
3
5
|
* @author darthjee
|
|
@@ -8,11 +10,13 @@ class WebConfig {
|
|
|
8
10
|
* @param {number} config.port - The port number to listen on.
|
|
9
11
|
* @param {number} [config.logs_page_size=20] - Maximum number of log entries per page.
|
|
10
12
|
* @param {boolean} [config.enable_shutdown=true] - Whether the shutdown button is enabled.
|
|
13
|
+
* @param {Array<string|object>} [config.links=[]] - Links shown in the web UI.
|
|
11
14
|
*/
|
|
12
|
-
constructor({ port, logs_page_size: logsPageSize = 20, enable_shutdown: enableShutdown = true }) {
|
|
15
|
+
constructor({ port, logs_page_size: logsPageSize = 20, enable_shutdown: enableShutdown = true, links = [] }) {
|
|
13
16
|
this.port = port;
|
|
14
17
|
this.logsPageSize = logsPageSize;
|
|
15
18
|
this.enableShutdown = enableShutdown;
|
|
19
|
+
this.links = links.map((link) => Link.fromObject(link));
|
|
16
20
|
}
|
|
17
21
|
}
|
|
18
22
|
|
|
@@ -1,8 +1,10 @@
|
|
|
1
1
|
import { AssetRequest } from './AssetRequest.js';
|
|
2
2
|
import { ResourceRequestAction } from './ResourceRequestAction.js';
|
|
3
|
-
import {
|
|
4
|
-
import {
|
|
5
|
-
import {
|
|
3
|
+
import { ResourceRequestPaginatedAction } from './ResourceRequestPaginatedAction.js';
|
|
4
|
+
import { JobRegistry as DefaultJobRegistry } from '../../background/JobRegistry.js';
|
|
5
|
+
import { ActionsEnqueuer } from '../../enqueuers/ActionsEnqueuer.js';
|
|
6
|
+
import { PaginatedActionsEnqueuer } from '../../enqueuers/PaginatedActionsEnqueuer.js';
|
|
7
|
+
import { Application } from '../../services/Application.js';
|
|
6
8
|
|
|
7
9
|
/**
|
|
8
10
|
* ResourceRequest represents a request to a specific URL with an expected status code.
|
|
@@ -18,13 +20,15 @@ class ResourceRequest {
|
|
|
18
20
|
* @param {string} [attributes.clientName] The name of the client to use for this request.
|
|
19
21
|
* @param {Array} [attributes.actions=[]] List of raw action config objects.
|
|
20
22
|
* @param {Array} [attributes.assets=[]] List of raw asset extraction rule objects.
|
|
23
|
+
* @param {Array} [attributes.paginated_actions=[]] List of raw paginated action config objects.
|
|
21
24
|
*/
|
|
22
|
-
constructor({ url, status, clientName, actions = [], assets = [] }) {
|
|
25
|
+
constructor({ url, status, clientName, actions = [], assets = [], paginated_actions = [] }) {
|
|
23
26
|
this.url = url;
|
|
24
27
|
this.status = status;
|
|
25
28
|
this.#clientName = clientName;
|
|
26
29
|
this.actions = ResourceRequestAction.fromList(actions);
|
|
27
30
|
this.assets = AssetRequest.fromListObject(assets);
|
|
31
|
+
this.paginatedActions = ResourceRequestPaginatedAction.fromList(paginated_actions);
|
|
28
32
|
}
|
|
29
33
|
|
|
30
34
|
/**
|
|
@@ -40,13 +44,28 @@ class ResourceRequest {
|
|
|
40
44
|
* Enqueues one ActionProcessingJob per action associated with the resource request.
|
|
41
45
|
* Returns immediately if there are no actions.
|
|
42
46
|
* @param {ResponseWrapper} responseWrapper The ResponseWrapper for the HTTP response.
|
|
47
|
+
* @param {string|null} [originUrl=null] The URL of the ResourceRequestJob that triggered this enqueue.
|
|
43
48
|
* @returns {void}
|
|
44
49
|
*/
|
|
45
|
-
enqueueActions(responseWrapper) {
|
|
50
|
+
enqueueActions(responseWrapper, originUrl = null) {
|
|
46
51
|
if (this.actions.length === 0) return;
|
|
47
52
|
|
|
48
53
|
const itemWrappers = responseWrapper.toItemWrappers();
|
|
49
|
-
new ActionsEnqueuer(this.actions, itemWrappers).enqueue();
|
|
54
|
+
new ActionsEnqueuer(this.actions, itemWrappers, undefined, originUrl).enqueue();
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
/**
|
|
58
|
+
* Enqueues one PaginatedActionProcessingJob per paginated action.
|
|
59
|
+
* Returns immediately if there are no paginated actions.
|
|
60
|
+
* @param {ResponseWrapper} responseWrapper The ResponseWrapper for the HTTP response.
|
|
61
|
+
* @param {object} [parameters={}] The original request parameters (key-value map for URL template resolution).
|
|
62
|
+
* @param {string|null} [originUrl=null] The URL of the ResourceRequestJob that triggered this enqueue.
|
|
63
|
+
* @returns {void}
|
|
64
|
+
*/
|
|
65
|
+
enqueuePaginatedActions(responseWrapper, parameters = {}, originUrl = null) {
|
|
66
|
+
if (this.paginatedActions.length === 0) return;
|
|
67
|
+
|
|
68
|
+
new PaginatedActionsEnqueuer(this.paginatedActions, responseWrapper, parameters, undefined, originUrl).enqueue();
|
|
50
69
|
}
|
|
51
70
|
|
|
52
71
|
/**
|
|
@@ -55,11 +74,14 @@ class ResourceRequest {
|
|
|
55
74
|
* @param {string} rawHtml The raw HTML response body string.
|
|
56
75
|
* @param {object} [jobRegistry=JobRegistry] The job registry used to enqueue the HtmlParseJob.
|
|
57
76
|
* @param {object} clientRegistry The client registry for URL resolution inside HtmlParseJob.
|
|
77
|
+
* @param {string|null} [originUrl=null] The URL of the ResourceRequestJob that triggered this enqueue.
|
|
58
78
|
* @returns {void}
|
|
59
79
|
*/
|
|
60
|
-
enqueueAssets(rawHtml, jobRegistry = DefaultJobRegistry, clientRegistry) {
|
|
80
|
+
enqueueAssets(rawHtml, jobRegistry = DefaultJobRegistry, clientRegistry, originUrl = null) {
|
|
61
81
|
if (Application.isStopped()) return;
|
|
62
|
-
|
|
82
|
+
const params = { rawHtml, assetRequests: this.assets, clientRegistry };
|
|
83
|
+
if (originUrl !== null) params.originUrl = originUrl;
|
|
84
|
+
jobRegistry.enqueue('HtmlParse', params);
|
|
63
85
|
}
|
|
64
86
|
|
|
65
87
|
/**
|
|
@@ -1,9 +1,9 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
3
|
-
import {
|
|
4
|
-
import {
|
|
5
|
-
import {
|
|
6
|
-
import {
|
|
1
|
+
import { JobRegistry as DefaultJobRegistry } from '../../background/JobRegistry.js';
|
|
2
|
+
import { MissingActionResource } from '../../exceptions/registry/MissingActionResource.js';
|
|
3
|
+
import { LogRegistry } from '../../registry/LogRegistry.js';
|
|
4
|
+
import { ResourceRegistry as DefaultResourceRegistry } from '../../registry/ResourceRegistry.js';
|
|
5
|
+
import { Application } from '../../services/Application.js';
|
|
6
|
+
import { ParametersMapper } from '../response/ParametersMapper.js';
|
|
7
7
|
|
|
8
8
|
/**
|
|
9
9
|
* Represents a single action to execute after a successful resource request response.
|
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
import { JobRegistry as DefaultJobRegistry } from '../../background/JobRegistry.js';
|
|
2
|
+
import { MissingActionResource } from '../../exceptions/registry/MissingActionResource.js';
|
|
3
|
+
import { LogRegistry } from '../../registry/LogRegistry.js';
|
|
4
|
+
import { ResourceRegistry as DefaultResourceRegistry } from '../../registry/ResourceRegistry.js';
|
|
5
|
+
import { Application } from '../../services/Application.js';
|
|
6
|
+
import { PaginationConfig } from '../configs/PaginationConfig.js';
|
|
7
|
+
|
|
8
|
+
/**
|
|
9
|
+
* Represents a single paginated action to execute after a successful resource request response.
|
|
10
|
+
*
|
|
11
|
+
* Iterates over pages and enqueues one ResourceRequestJob per page, merging the page number
|
|
12
|
+
* into the item's existing parameters.
|
|
13
|
+
* @author darthjee
|
|
14
|
+
*/
|
|
15
|
+
class ResourceRequestPaginatedAction {
|
|
16
|
+
#resource;
|
|
17
|
+
#pagination;
|
|
18
|
+
#jobRegistry;
|
|
19
|
+
#resourceRegistry;
|
|
20
|
+
|
|
21
|
+
/**
|
|
22
|
+
* @param {object} attributes Action attributes.
|
|
23
|
+
* @param {string} attributes.resource Name of the resource to act upon.
|
|
24
|
+
* @param {Array<object>} attributes.pagination Pagination config list.
|
|
25
|
+
* @param {object} [attributes.jobRegistry=JobRegistry] The job registry to enqueue jobs to.
|
|
26
|
+
* @param {object} [attributes.resourceRegistry=ResourceRegistry] The resource registry to look up resources.
|
|
27
|
+
* @throws {MissingActionResource} If resource is absent or falsy.
|
|
28
|
+
*/
|
|
29
|
+
constructor({
|
|
30
|
+
resource,
|
|
31
|
+
pagination,
|
|
32
|
+
jobRegistry = DefaultJobRegistry,
|
|
33
|
+
resourceRegistry = DefaultResourceRegistry,
|
|
34
|
+
}) {
|
|
35
|
+
if (!resource) throw new MissingActionResource();
|
|
36
|
+
this.#resource = resource;
|
|
37
|
+
this.#pagination = PaginationConfig.fromList(pagination);
|
|
38
|
+
this.#jobRegistry = jobRegistry;
|
|
39
|
+
this.#resourceRegistry = resourceRegistry;
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
/**
|
|
43
|
+
* Evaluates the page count from the response, then enqueues one
|
|
44
|
+
* ResourceRequestJob per page for the target resource.
|
|
45
|
+
* @param {ResponseWrapper} responseWrapper A ResponseWrapper instance exposing parsedBody and headers.
|
|
46
|
+
* @param {object} [parameters={}] The original request parameters to merge with the page number.
|
|
47
|
+
* @returns {void}
|
|
48
|
+
*/
|
|
49
|
+
execute(responseWrapper, parameters = {}) {
|
|
50
|
+
if (Application.isStopped()) return;
|
|
51
|
+
|
|
52
|
+
const count = this.#pagination.resolvePages(responseWrapper);
|
|
53
|
+
const pages = this.#pagination.pageNumbers(count);
|
|
54
|
+
const resource = this.#resourceRegistry.getItem(this.#resource);
|
|
55
|
+
|
|
56
|
+
for (const page of pages) {
|
|
57
|
+
const pageParameters = { ...parameters, [this.#pagination.pageKey]: page };
|
|
58
|
+
for (const resourceRequest of resource.resourceRequests) {
|
|
59
|
+
this.#jobRegistry.enqueue('ResourceRequestJob', { resourceRequest, parameters: pageParameters });
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
/**
|
|
65
|
+
* Creates instances from a raw YAML list, skipping invalid entries.
|
|
66
|
+
* @param {Array<object>} [array=[]] Raw paginated action config entries.
|
|
67
|
+
* @returns {Array<ResourceRequestPaginatedAction>} Valid paginated action instances.
|
|
68
|
+
*/
|
|
69
|
+
static fromList(array = []) {
|
|
70
|
+
return array.flatMap((attrs) => {
|
|
71
|
+
try {
|
|
72
|
+
return [new ResourceRequestPaginatedAction(attrs)];
|
|
73
|
+
} catch (error) {
|
|
74
|
+
LogRegistry.error(`Skipping paginated_action: ${error}`);
|
|
75
|
+
return [];
|
|
76
|
+
}
|
|
77
|
+
});
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
export { ResourceRequestPaginatedAction };
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { NamedRegistry } from './NamedRegistry.js';
|
|
2
|
-
import { ClientNotFound } from '../exceptions/ClientNotFound.js';
|
|
2
|
+
import { ClientNotFound } from '../exceptions/registry/ClientNotFound.js';
|
|
3
3
|
|
|
4
4
|
/**
|
|
5
5
|
* ClientRegistry manages client retrieval logic and exposes a static singleton facade.
|
|
@@ -54,8 +54,8 @@ class LogRegistry {
|
|
|
54
54
|
|
|
55
55
|
/**
|
|
56
56
|
* Gets a specific log by ID.
|
|
57
|
-
* @param {number} id
|
|
58
|
-
* @returns {import('../utils/logging/Log.js').Log|undefined}
|
|
57
|
+
* @param {number} id - The log entry ID to look up.
|
|
58
|
+
* @returns {import('../utils/logging/Log.js').Log|undefined} The matching log entry, or undefined if not found.
|
|
59
59
|
*/
|
|
60
60
|
static getLogById(id) {
|
|
61
61
|
return LogRegistry.#getInstance().getLogById(id);
|
|
@@ -63,10 +63,10 @@ class LogRegistry {
|
|
|
63
63
|
|
|
64
64
|
/**
|
|
65
65
|
* Gets logs in chronological order (oldest first), optionally filtered to entries newer than lastId.
|
|
66
|
-
* @param {object} [options={}]
|
|
66
|
+
* @param {object} [options={}] - Query options.
|
|
67
67
|
* @param {number|string} [options.lastId] - When provided, returns only logs newer than this ID.
|
|
68
68
|
* Returns an empty array if the ID is not found.
|
|
69
|
-
* @returns {Array<import('../utils/logging/Log.js').Log>}
|
|
69
|
+
* @returns {Array<import('../utils/logging/Log.js').Log>} Array of log entries.
|
|
70
70
|
*/
|
|
71
71
|
static getLogs({ lastId } = {}) {
|
|
72
72
|
return LogRegistry.#getInstance().getLogs({ lastId });
|
|
@@ -75,10 +75,10 @@ class LogRegistry {
|
|
|
75
75
|
/**
|
|
76
76
|
* Gets logs stored in the per-job buffer for the given job ID,
|
|
77
77
|
* optionally filtered to entries newer than lastId.
|
|
78
|
-
* @param {string|number} jobId
|
|
79
|
-
* @param {object} [options={}]
|
|
78
|
+
* @param {string|number} jobId - The job ID to look up logs for.
|
|
79
|
+
* @param {object} [options={}] - Query options.
|
|
80
80
|
* @param {number|string} [options.lastId] - When provided, returns only logs newer than this ID.
|
|
81
|
-
* @returns {Array<import('../utils/logging/Log.js').Log>}
|
|
81
|
+
* @returns {Array<import('../utils/logging/Log.js').Log>} Array of log entries.
|
|
82
82
|
*/
|
|
83
83
|
static getLogsByJobId(jobId, { lastId } = {}) {
|
|
84
84
|
return LogRegistry.#getInstance().getLogsByJobId(jobId, { lastId });
|
|
@@ -86,8 +86,8 @@ class LogRegistry {
|
|
|
86
86
|
|
|
87
87
|
/**
|
|
88
88
|
* Gets logs stored in the per-worker buffer for the given worker ID.
|
|
89
|
-
* @param {string|number} workerId
|
|
90
|
-
* @returns {Array<import('../utils/logging/Log.js').Log>}
|
|
89
|
+
* @param {string|number} workerId - The worker ID to look up logs for.
|
|
90
|
+
* @returns {Array<import('../utils/logging/Log.js').Log>} Array of log entries.
|
|
91
91
|
*/
|
|
92
92
|
static getLogsByWorkerId(workerId) {
|
|
93
93
|
return LogRegistry.#getInstance().getLogsByWorkerId(workerId);
|
|
@@ -95,8 +95,8 @@ class LogRegistry {
|
|
|
95
95
|
|
|
96
96
|
/**
|
|
97
97
|
* Gets logs filtered by level.
|
|
98
|
-
* @param {string} level
|
|
99
|
-
* @returns {Array<import('../utils/logging/Log.js').Log>}
|
|
98
|
+
* @param {string} level - The log level to filter by.
|
|
99
|
+
* @returns {Array<import('../utils/logging/Log.js').Log>} Array of log entries matching the level.
|
|
100
100
|
*/
|
|
101
101
|
static getLogsByLevel(level) {
|
|
102
102
|
return LogRegistry.#getInstance().getLogsByLevel(level);
|
|
@@ -104,7 +104,7 @@ class LogRegistry {
|
|
|
104
104
|
|
|
105
105
|
/**
|
|
106
106
|
* Returns all logs as plain JSON objects.
|
|
107
|
-
* @returns {Array<object>}
|
|
107
|
+
* @returns {Array<object>} Array of plain log objects.
|
|
108
108
|
*/
|
|
109
109
|
static getLogsJSON() {
|
|
110
110
|
return LogRegistry.#getInstance().getLogsJSON();
|
|
@@ -132,7 +132,7 @@ class LogRegistry {
|
|
|
132
132
|
|
|
133
133
|
/**
|
|
134
134
|
* Returns the singleton instance, throwing if not yet built.
|
|
135
|
-
* @returns {LogRegistryInstance}
|
|
135
|
+
* @returns {LogRegistryInstance} The singleton instance.
|
|
136
136
|
* @throws {Error} If build() has not been called.
|
|
137
137
|
*/
|
|
138
138
|
static #getInstance() {
|