navi-hey 1.8.0 → 1.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/navi.js +2 -2
- package/lib/{services → client}/Client.js +14 -6
- package/lib/common/utils/env_resolver/EnvStringResolver.js +38 -2
- package/lib/common/utils/logging/LogBufferCollection.js +2 -3
- package/lib/common/utils/parser/HtmlRootParser.js +25 -0
- package/lib/enqueuers/ActionEnqueuer.js +1 -1
- package/lib/enqueuers/AssetRequestEnqueuer.js +1 -1
- package/lib/enqueuers/EmitEnqueuer.js +46 -0
- package/lib/enqueuers/PaginatedActionEnqueuer.js +1 -1
- package/lib/exceptions/config/ConfigurationFileNotFound.js +1 -1
- package/lib/exceptions/config/ConfigurationFileNotProvided.js +1 -1
- package/lib/exceptions/config/InvalidEmitBodyTemplate.js +16 -0
- package/lib/exceptions/config/InvalidEmitCooldown.js +15 -0
- package/lib/exceptions/config/InvalidEmitHeaders.js +16 -0
- package/lib/exceptions/config/InvalidEmitMethod.js +1 -1
- package/lib/exceptions/config/InvalidEmitRetries.js +16 -0
- package/lib/exceptions/config/InvalidMemoryThresholds.js +1 -1
- package/lib/exceptions/config/InvalidParserMatch.js +1 -1
- package/lib/exceptions/config/InvalidParserType.js +3 -3
- package/lib/exceptions/config/MissingEmitUrl.js +1 -1
- package/lib/exceptions/config/MissingParserField.js +1 -1
- package/lib/exceptions/config/MissingParserFields.js +1 -1
- package/lib/exceptions/config/MissingParserMatch.js +1 -1
- package/lib/exceptions/config/MissingTopLevelConfgKey.js +1 -1
- package/lib/exceptions/http/ConflictError.js +1 -1
- package/lib/exceptions/http/ForbiddenError.js +1 -1
- package/lib/exceptions/http/NotFoundError.js +1 -1
- package/lib/exceptions/registry/ItemNotFound.js +1 -1
- package/lib/exceptions/registry/MissingActionResource.js +1 -1
- package/lib/exceptions/registry/MissingMappingVariable.js +1 -1
- package/lib/exceptions/request/InvalidResponseBody.js +1 -1
- package/lib/exceptions/request/NullResponse.js +1 -1
- package/lib/exceptions/request/RequestFailed.js +11 -3
- package/lib/jobs/AssetDownloadJob.js +6 -2
- package/lib/jobs/EmitJob.js +123 -5
- package/lib/jobs/ExtractionJob.js +32 -3
- package/lib/jobs/ResourceRequestJob.js +7 -3
- package/lib/models/configs/Config.js +10 -2
- package/lib/models/configs/EmitConfig.js +39 -0
- package/lib/models/configs/ExtractionConfig.js +38 -0
- package/lib/models/configs/MemoryConfig.js +11 -1
- package/lib/models/configs/WebConfig.js +1 -1
- package/lib/models/request/BodyTemplateRenderer.js +35 -0
- package/lib/models/request/ResourceRequest.js +16 -4
- package/lib/models/request/ResourceRequestAction.js +1 -1
- package/lib/models/request/ResourceRequestEmit.js +123 -2
- package/lib/models/request/ResourceRequestPaginatedAction.js +1 -1
- package/lib/models/request/ResourceRequestParser.js +3 -3
- package/lib/models/request/TemplateStringRenderer.js +35 -0
- package/lib/models/request/TokenResolver.js +20 -0
- package/lib/models/request/UrlTokenResolver.js +19 -0
- package/lib/parsers/CssSelectorParser.js +68 -0
- package/lib/parsers/JsonPathParser.js +19 -39
- package/lib/parsers/RegexParser.js +8 -11
- package/lib/parsers/css_selector_parser/ArrayValueResolver.js +43 -0
- package/lib/parsers/css_selector_parser/ConditionMatcher.js +54 -0
- package/lib/parsers/css_selector_parser/FieldsMapper.js +35 -0
- package/lib/parsers/css_selector_parser/FilterMatcher.js +47 -0
- package/lib/parsers/css_selector_parser/ValueExtractor.js +34 -0
- package/lib/parsers/css_selector_parser/ValueResolver.js +36 -0
- package/lib/parsers/json_path/ConditionMatcher.js +33 -0
- package/lib/parsers/json_path/FieldMapper.js +30 -0
- package/lib/parsers/json_path/FilterMatcher.js +30 -0
- package/lib/parsers/json_path/MatchResolver.js +38 -0
- package/lib/parsers/regex_parser/AttributesValidator.js +31 -0
- package/lib/parsers/regex_parser/MatchValueExtractor.js +17 -0
- package/lib/parsers/regex_parser/PatternMatcher.js +27 -0
- package/lib/registry/EmissionRegistry.js +113 -0
- package/lib/registry/EmissionRegistryInstance.js +85 -0
- package/lib/registry/ExtractionRegistry.js +103 -0
- package/lib/registry/ExtractionRegistryInstance.js +76 -0
- package/lib/registry/LogRegistry.js +8 -0
- package/lib/registry/LogRegistryInstance.js +9 -0
- package/lib/registry/NamespaceMap.js +1 -1
- package/lib/serializers/EmissionSerializer.js +30 -0
- package/lib/serializers/ExtractionSerializer.js +26 -0
- package/lib/server/Router.js +4 -0
- package/lib/server/SecuredRequestHandler.js +8 -0
- package/lib/server/handlers/StatsHandler.js +5 -3
- package/lib/server/handlers/api/ApiConfigHandler.js +2 -2
- package/lib/server/handlers/api/ApiEngineStartHandler.js +1 -1
- package/lib/server/handlers/api/ApiEngineStopHandler.js +1 -1
- package/lib/server/handlers/emissions/EmissionsHandler.js +43 -0
- package/lib/server/handlers/engine/EngineContinueHandler.js +1 -1
- package/lib/server/handlers/engine/EnginePauseHandler.js +1 -1
- package/lib/server/handlers/engine/EngineReloadHandler.js +1 -1
- package/lib/server/handlers/engine/EngineRestartHandler.js +1 -1
- package/lib/server/handlers/engine/EngineShutdownHandler.js +1 -1
- package/lib/server/handlers/engine/EngineStartHandler.js +1 -1
- package/lib/server/handlers/engine/EngineStatusHandler.js +1 -1
- package/lib/server/handlers/engine/EngineStopHandler.js +1 -1
- package/lib/server/handlers/extractions/ExtractionsHandler.js +43 -0
- package/lib/services/{Application.js → application/Application.js} +0 -16
- package/lib/services/application/ApplicationConfigurator.js +39 -0
- package/lib/services/application/ApplicationInstance.js +245 -0
- package/lib/services/application/ConfigStore.js +54 -0
- package/lib/services/application/ResourceQueueFacade.js +39 -0
- package/lib/services/application/StartupCoordinator.js +43 -0
- package/lib/services/{NamespaceMapBuilder.js → builders/NamespaceMapBuilder.js} +3 -3
- package/lib/services/builders/RegistriesBuilder.js +56 -0
- package/lib/services/{ConfigIncluder.js → config/ConfigIncluder.js} +47 -5
- package/lib/services/{ConfigLoader.js → config/ConfigLoader.js} +8 -2
- package/lib/services/{ConfigParser.js → config/ConfigParser.js} +32 -9
- package/lib/services/engine/EngineController.js +239 -0
- package/lib/services/engine/EngineState.js +51 -0
- package/lib/services/{EngineStopService.js → engine/EngineStopService.js} +7 -5
- package/lib/services/engine/ServerController.js +60 -0
- package/lib/services/{FailureChecker.js → execution/FailureChecker.js} +1 -1
- package/lib/services/execution/RunReporter.js +31 -0
- package/lib/utils/HtmlParser.js +2 -10
- package/lib/utils/emissions/EmissionRecord.js +135 -0
- package/lib/utils/emissions/EmissionRecordFactory.js +40 -0
- package/lib/utils/emissions/EmissionStore.js +133 -0
- package/lib/utils/extractions/ExtractionRecord.js +84 -0
- package/lib/utils/extractions/ExtractionRecordFactory.js +35 -0
- package/lib/utils/extractions/ExtractionStore.js +111 -0
- package/lib/utils/memory/MemoryData.js +70 -0
- package/lib/utils/memory/MemoryDataFactory.js +32 -0
- package/lib/utils/memory/MemoryDataStore.js +92 -0
- package/package.json +2 -2
- package/static/assets/index-C81vj0N0.js +11 -0
- package/static/index.html +1 -1
- package/lib/services/ApplicationInstance.js +0 -376
- package/lib/services/EngineEvents.js +0 -42
- package/static/assets/index-DGgF9Hti.js +0 -11
- /package/lib/{exceptions → common/exceptions}/AppError.js +0 -0
- /package/lib/{exceptions → common/exceptions}/request/InvalidHtmlResponseBody.js +0 -0
- /package/lib/services/{ArgumentsParser.js → application/ArgumentsParser.js} +0 -0
- /package/lib/services/{RunSummary.js → execution/RunSummary.js} +0 -0
package/lib/jobs/EmitJob.js
CHANGED
|
@@ -1,33 +1,108 @@
|
|
|
1
1
|
import { Job } from 'deku-swarm';
|
|
2
|
+
import { RequestFailed } from '../exceptions/request/RequestFailed.js';
|
|
3
|
+
import { EmissionRegistry } from '../registry/EmissionRegistry.js';
|
|
4
|
+
|
|
5
|
+
/**
|
|
6
|
+
* The set of HTTP status codes considered retryable by EmitJob's default policy:
|
|
7
|
+
* any 5xx server error, 429 (Too Many Requests), and 408 (Request Timeout).
|
|
8
|
+
* @param {number} statusCode The HTTP status code to check.
|
|
9
|
+
* @returns {boolean} True when the status code is retryable.
|
|
10
|
+
*/
|
|
11
|
+
const isRetryableStatus = (statusCode) => (statusCode >= 500 && statusCode < 600) || statusCode === 429 || statusCode === 408;
|
|
2
12
|
|
|
3
13
|
/**
|
|
4
14
|
* EmitJob is a Job that sends a single extracted item to an external endpoint, as
|
|
5
15
|
* declared by a resource request's `emit` configuration.
|
|
16
|
+
*
|
|
17
|
+
* Unlike other job classes, EmitJob defines its own retry policy — distinct from the
|
|
18
|
+
* global `workersConfig` policy — overridable per resource via `emit.retries`/`emit.cooldown`,
|
|
19
|
+
* that skips retrying non-retryable 4xx failures and honors a capped `Retry-After` header
|
|
20
|
+
* on HTTP 429 responses.
|
|
6
21
|
* @author darthjee
|
|
7
22
|
*/
|
|
8
23
|
class EmitJob extends Job {
|
|
24
|
+
/**
|
|
25
|
+
* The default maximum number of retries for an EmitJob, when the resource's
|
|
26
|
+
* `emit.retries` is not set.
|
|
27
|
+
* @type {number}
|
|
28
|
+
*/
|
|
29
|
+
static DEFAULT_MAX_RETRIES = 5;
|
|
30
|
+
|
|
31
|
+
/**
|
|
32
|
+
* The default cooldown, in milliseconds, applied between EmitJob retries, when the
|
|
33
|
+
* resource's `emit.cooldown` is not set.
|
|
34
|
+
* @type {number}
|
|
35
|
+
*/
|
|
36
|
+
static DEFAULT_COOLDOWN = 5000;
|
|
37
|
+
|
|
38
|
+
/**
|
|
39
|
+
* The maximum cooldown, in milliseconds, honored from a `Retry-After` response header.
|
|
40
|
+
* @type {number}
|
|
41
|
+
*/
|
|
42
|
+
static RETRY_AFTER_CAP_MS = 60000;
|
|
43
|
+
|
|
9
44
|
#item;
|
|
10
45
|
#emit;
|
|
11
46
|
#parameters;
|
|
12
47
|
#clients;
|
|
48
|
+
#maxRetries;
|
|
49
|
+
#cooldown;
|
|
50
|
+
#extractionId;
|
|
13
51
|
|
|
14
52
|
/**
|
|
15
53
|
* Creates a new EmitJob instance.
|
|
16
54
|
* @param {object} params - The parameters for creating an EmitJob instance.
|
|
17
55
|
* @param {string} params.id - The unique identifier for this job.
|
|
18
|
-
* @param {
|
|
56
|
+
* @param {ExtractedItem} params.item - The extracted item to send, a single element of a
|
|
19
57
|
* parser's `extract()` output array.
|
|
20
58
|
* @param {ResourceRequestEmit} params.emit - The resource request's declared emit configuration.
|
|
21
59
|
* @param {object} [params.parameters] - Key-value map used to resolve {:placeholder} tokens
|
|
22
60
|
* in the emit URL.
|
|
23
61
|
* @param {NamespaceMap} params.clients - The namespace map used to resolve the client for this emit.
|
|
62
|
+
* @param {number|null} [params.extractionId=null] - The id of the extraction record whose
|
|
63
|
+
* items produced this emission, or null when it cannot be traced.
|
|
24
64
|
*/
|
|
25
|
-
constructor({ id, item, emit, parameters, clients }) {
|
|
65
|
+
constructor({ id, item, emit, parameters, clients, extractionId = null }) {
|
|
26
66
|
super({ id });
|
|
27
67
|
this.#item = item;
|
|
28
68
|
this.#emit = emit;
|
|
29
69
|
this.#parameters = parameters;
|
|
30
70
|
this.#clients = clients;
|
|
71
|
+
this.#maxRetries = emit.retries ?? EmitJob.DEFAULT_MAX_RETRIES;
|
|
72
|
+
this.#cooldown = emit.cooldown ?? EmitJob.DEFAULT_COOLDOWN;
|
|
73
|
+
this.#extractionId = extractionId;
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
/**
|
|
77
|
+
* Returns the effective maximum number of retries for this job.
|
|
78
|
+
* When the last error is a non-retryable HTTP failure (any 4xx other than 429/408),
|
|
79
|
+
* forces immediate exhaustion by returning the current attempt count. A network-level
|
|
80
|
+
* error (not a RequestFailed) is always retryable.
|
|
81
|
+
* @returns {number} The maximum number of retries.
|
|
82
|
+
*/
|
|
83
|
+
get maxRetries() {
|
|
84
|
+
if (this.lastError instanceof RequestFailed && !isRetryableStatus(this.lastError.statusCode)) {
|
|
85
|
+
return this._attempts;
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
return this.#maxRetries;
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
/**
|
|
92
|
+
* Returns the effective cooldown, in milliseconds, to apply before the next retry.
|
|
93
|
+
* When the last error is a 429 RequestFailed carrying a parseable numeric `Retry-After`
|
|
94
|
+
* header (delta-seconds form), returns that value in milliseconds, capped at
|
|
95
|
+
* `RETRY_AFTER_CAP_MS`. Otherwise returns the configured/default cooldown.
|
|
96
|
+
* @returns {number} The cooldown in milliseconds.
|
|
97
|
+
*/
|
|
98
|
+
get cooldown() {
|
|
99
|
+
const retryAfterMs = this.#retryAfterMs();
|
|
100
|
+
|
|
101
|
+
if (retryAfterMs !== undefined) {
|
|
102
|
+
return Math.min(retryAfterMs, EmitJob.RETRY_AFTER_CAP_MS);
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
return this.#cooldown;
|
|
31
106
|
}
|
|
32
107
|
|
|
33
108
|
/**
|
|
@@ -40,23 +115,48 @@ class EmitJob extends Job {
|
|
|
40
115
|
}
|
|
41
116
|
|
|
42
117
|
/**
|
|
43
|
-
* Emits the extracted item to the configured external endpoint
|
|
118
|
+
* Emits the extracted item to the configured external endpoint, rendered through the
|
|
119
|
+
* emit's `bodyTemplate` when one is configured (the bare item is sent otherwise).
|
|
44
120
|
* @param {LogContext} logContext - Context carrying workerId/jobId for log entries.
|
|
45
121
|
* @returns {Promise<object>} A promise that resolves with the HTTP response.
|
|
46
122
|
*/
|
|
47
123
|
async perform(logContext) {
|
|
48
124
|
logContext.debug(`EmitJob #${this.id} performing`);
|
|
125
|
+
const url = this.#emit.resolveUrl(this.#parameters);
|
|
126
|
+
const method = this.#emit.method;
|
|
49
127
|
try {
|
|
50
128
|
this.lastError = undefined;
|
|
51
|
-
const
|
|
52
|
-
|
|
129
|
+
const response = await this.#getClient().emit(
|
|
130
|
+
method, url, this.#emit.resolveBody(this.#item), this.#emit.status, logContext, this.#emit.headers,
|
|
131
|
+
);
|
|
132
|
+
EmissionRegistry.recordEmission({
|
|
133
|
+
status: 'success', url, method, httpStatus: response?.status ?? null,
|
|
134
|
+
itemRef: this.#itemRef(), extractionId: this.#extractionId,
|
|
135
|
+
});
|
|
53
136
|
return response;
|
|
54
137
|
} catch (error) {
|
|
55
138
|
logContext.error(`EmitJob #${this.id} failed: ${error}`);
|
|
139
|
+
this.lastError = error;
|
|
140
|
+
const dead = this._attempts + 1 >= this.maxRetries;
|
|
141
|
+
const httpStatus = error instanceof RequestFailed ? error.statusCode : null;
|
|
142
|
+
EmissionRegistry.recordEmission({
|
|
143
|
+
status: dead ? 'dead' : 'failed', url, method, httpStatus, error: String(error),
|
|
144
|
+
itemRef: this.#itemRef(), extractionId: this.#extractionId,
|
|
145
|
+
});
|
|
56
146
|
this._fail(error);
|
|
57
147
|
}
|
|
58
148
|
}
|
|
59
149
|
|
|
150
|
+
/**
|
|
151
|
+
* Derives a compact reference to the emitted item for the emission record. Never returns
|
|
152
|
+
* the full payload; falls back to null when the item has no `id` field.
|
|
153
|
+
* @returns {string|number|null} The item's `id`, or null.
|
|
154
|
+
* @private
|
|
155
|
+
*/
|
|
156
|
+
#itemRef() {
|
|
157
|
+
return this.#item?.id ?? null;
|
|
158
|
+
}
|
|
159
|
+
|
|
60
160
|
/**
|
|
61
161
|
* Gets the client associated with this job's emit configuration.
|
|
62
162
|
* @returns {Client} The client associated with this job's emit configuration.
|
|
@@ -65,6 +165,24 @@ class EmitJob extends Job {
|
|
|
65
165
|
#getClient() {
|
|
66
166
|
return this.#clients.getClient('default', this.#emit.clientName, this.#emit.clientNamespace);
|
|
67
167
|
}
|
|
168
|
+
|
|
169
|
+
/**
|
|
170
|
+
* Reads and parses a `Retry-After` header, in delta-seconds form, from the last error,
|
|
171
|
+
* when it is a 429 RequestFailed.
|
|
172
|
+
* @returns {number|undefined} The parsed Retry-After delay in milliseconds, or undefined
|
|
173
|
+
* when there is none, or it isn't a parseable delta-seconds value.
|
|
174
|
+
* @private
|
|
175
|
+
*/
|
|
176
|
+
#retryAfterMs() {
|
|
177
|
+
if (!(this.lastError instanceof RequestFailed) || this.lastError.statusCode !== 429) return undefined;
|
|
178
|
+
|
|
179
|
+
const headers = this.lastError.headers || {};
|
|
180
|
+
const retryAfter = headers['retry-after'] ?? headers['Retry-After'];
|
|
181
|
+
|
|
182
|
+
if (!/^\d+$/.test(String(retryAfter))) return undefined;
|
|
183
|
+
|
|
184
|
+
return Number(retryAfter) * 1000;
|
|
185
|
+
}
|
|
68
186
|
}
|
|
69
187
|
|
|
70
188
|
export { EmitJob };
|
|
@@ -1,8 +1,12 @@
|
|
|
1
1
|
import { Job } from 'deku-swarm';
|
|
2
|
+
import { EmitEnqueuer } from '../enqueuers/EmitEnqueuer.js';
|
|
3
|
+
import { EmissionRegistry } from '../registry/EmissionRegistry.js';
|
|
4
|
+
import { ExtractionRegistry } from '../registry/ExtractionRegistry.js';
|
|
2
5
|
|
|
3
6
|
/**
|
|
4
7
|
* ExtractionJob is a Job that extracts structured items from a raw response body
|
|
5
|
-
* using a parser implementation resolved from a ParserRegistry
|
|
8
|
+
* using a parser implementation resolved from a ParserRegistry, and, when the
|
|
9
|
+
* resource declares an `emit`, enqueues one EmitJob per extracted item.
|
|
6
10
|
*
|
|
7
11
|
* Unlike ResourceRequestJob, this job is exhausted after the first failure — it has no
|
|
8
12
|
* retry rights, analogous to ActionProcessingJob/HtmlParseJob.
|
|
@@ -12,6 +16,9 @@ class ExtractionJob extends Job {
|
|
|
12
16
|
#rawBody;
|
|
13
17
|
#parser;
|
|
14
18
|
#parserRegistry;
|
|
19
|
+
#jobRegistry;
|
|
20
|
+
#emit;
|
|
21
|
+
#parameters;
|
|
15
22
|
#originUrl;
|
|
16
23
|
|
|
17
24
|
/**
|
|
@@ -21,13 +28,21 @@ class ExtractionJob extends Job {
|
|
|
21
28
|
* @param {string} params.rawBody - The raw HTTP response body to extract data from.
|
|
22
29
|
* @param {ResourceRequestParser} params.parser - The resource's declared parser rule.
|
|
23
30
|
* @param {object} params.parserRegistry - The registry used to resolve the parser implementation.
|
|
31
|
+
* @param {object} params.jobRegistry - The job registry used to enqueue EmitJobs.
|
|
32
|
+
* @param {ResourceRequestEmit} [params.emit] - The resource request's declared emit
|
|
33
|
+
* configuration. When absent, extracted items are logged but not emitted.
|
|
34
|
+
* @param {object} [params.parameters] - The original resource-request parameters, forwarded
|
|
35
|
+
* to each EmitJob for URL placeholder resolution.
|
|
24
36
|
* @param {string|null} [params.originUrl=null] - The URL of the ResourceRequestJob that triggered this job.
|
|
25
37
|
*/
|
|
26
|
-
constructor({ id, rawBody, parser, parserRegistry, originUrl = null }) {
|
|
38
|
+
constructor({ id, rawBody, parser, parserRegistry, jobRegistry, emit, parameters, originUrl = null }) {
|
|
27
39
|
super({ id });
|
|
28
40
|
this.#rawBody = rawBody;
|
|
29
41
|
this.#parser = parser;
|
|
30
42
|
this.#parserRegistry = parserRegistry;
|
|
43
|
+
this.#jobRegistry = jobRegistry;
|
|
44
|
+
this.#emit = emit;
|
|
45
|
+
this.#parameters = parameters;
|
|
31
46
|
this.#originUrl = originUrl;
|
|
32
47
|
}
|
|
33
48
|
|
|
@@ -51,7 +66,8 @@ class ExtractionJob extends Job {
|
|
|
51
66
|
|
|
52
67
|
/**
|
|
53
68
|
* Resolves the parser implementation and extracts items from the raw body, logging
|
|
54
|
-
* the resulting items via `logContext.debug`.
|
|
69
|
+
* the resulting items via `logContext.debug`. When an `emit` configuration is present,
|
|
70
|
+
* delegates to EmitEnqueuer to enqueue one EmitJob per extracted item.
|
|
55
71
|
* @param {LogContext} logContext - Context carrying workerId/jobId for log entries.
|
|
56
72
|
* @returns {Promise<void>}
|
|
57
73
|
*/
|
|
@@ -62,6 +78,19 @@ class ExtractionJob extends Job {
|
|
|
62
78
|
const parserImpl = this.#parserRegistry.getItem(this.#parser.type);
|
|
63
79
|
const items = parserImpl.extract(this.#rawBody, this.#parser.attributes);
|
|
64
80
|
logContext.debug(`ExtractionJob #${this.id} extracted ${items.length} item(s)`, { items });
|
|
81
|
+
EmissionRegistry.incExtracted(items.length);
|
|
82
|
+
const extraction = ExtractionRegistry.recordExtraction({
|
|
83
|
+
parserType: this.#parser.type,
|
|
84
|
+
originUrl: this.#originUrl,
|
|
85
|
+
itemCount: items.length,
|
|
86
|
+
});
|
|
87
|
+
const extractionId = extraction?.id ?? null;
|
|
88
|
+
|
|
89
|
+
if (this.#emit) {
|
|
90
|
+
new EmitEnqueuer(
|
|
91
|
+
items, this.#emit, this.#parameters, this.#jobRegistry, extractionId
|
|
92
|
+
).enqueue();
|
|
93
|
+
}
|
|
65
94
|
} catch (error) {
|
|
66
95
|
this._fail(error);
|
|
67
96
|
}
|
|
@@ -18,9 +18,13 @@ class ResourceRequestJob extends Job {
|
|
|
18
18
|
* @param {ResourceRequest} params.resourceRequest - The resource request associated with this job.
|
|
19
19
|
* @param {object} params.parameters - Additional parameters for the request.
|
|
20
20
|
* @param {NamespaceMap} params.clients - The namespace map used to resolve the client for this request.
|
|
21
|
+
* @param {number} [params.maxRetries] - The maximum number of retries allowed for this job,
|
|
22
|
+
* normally injected from the global `workers.max-retries` config.
|
|
23
|
+
* @param {number} [params.cooldown] - The cooldown, in milliseconds, applied on failure,
|
|
24
|
+
* normally injected from the global `workers.retry_cooldown` config.
|
|
21
25
|
*/
|
|
22
|
-
constructor({ id, resourceRequest, parameters, clients }) {
|
|
23
|
-
super({ id });
|
|
26
|
+
constructor({ id, resourceRequest, parameters, clients, maxRetries, cooldown }) {
|
|
27
|
+
super({ id, maxRetries, cooldown });
|
|
24
28
|
this.#resourceRequest = resourceRequest;
|
|
25
29
|
this.#parameters = parameters;
|
|
26
30
|
this.#clients = clients;
|
|
@@ -88,7 +92,7 @@ class ResourceRequestJob extends Job {
|
|
|
88
92
|
*/
|
|
89
93
|
#enqueueExtraction(response, originUrl) {
|
|
90
94
|
if (this.#resourceRequest.hasParser()) {
|
|
91
|
-
this.#resourceRequest.enqueueExtraction(response.data, JobRegistry, originUrl);
|
|
95
|
+
this.#resourceRequest.enqueueExtraction(response.data, JobRegistry, this.#parameters, originUrl);
|
|
92
96
|
}
|
|
93
97
|
}
|
|
94
98
|
|
|
@@ -1,8 +1,10 @@
|
|
|
1
|
+
import { EmitConfig } from './EmitConfig.js';
|
|
2
|
+
import { ExtractionConfig } from './ExtractionConfig.js';
|
|
1
3
|
import { LogConfig } from './LogConfig.js';
|
|
2
4
|
import { NamespaceNotFound } from '../../exceptions/registry/NamespaceNotFound.js';
|
|
3
5
|
import { NamespaceMap } from '../../registry/NamespaceMap.js';
|
|
4
6
|
import { ResourceRegistry } from '../../registry/ResourceRegistry.js';
|
|
5
|
-
import { ConfigLoader } from '../../services/ConfigLoader.js';
|
|
7
|
+
import { ConfigLoader } from '../../services/config/ConfigLoader.js';
|
|
6
8
|
|
|
7
9
|
const DEFAULT_NAMESPACE = 'default';
|
|
8
10
|
|
|
@@ -19,14 +21,20 @@ class Config {
|
|
|
19
21
|
* @param {WorkersConfig} params.workersConfig - The configuration for worker instances.
|
|
20
22
|
* @param {WebConfig|null} [params.webConfig] - Optional web server configuration.
|
|
21
23
|
* @param {LogConfig} [params.logConfig] - Optional log configuration.
|
|
24
|
+
* @param {EmitConfig} [params.emitConfig] - Optional emission-tracking configuration.
|
|
25
|
+
* @param {ExtractionConfig} [params.extractionConfig] - Optional extraction-tracking configuration.
|
|
22
26
|
* @param {FailureConfig|null} [params.failureConfig] - Optional failure threshold configuration.
|
|
23
27
|
*/
|
|
24
|
-
constructor({
|
|
28
|
+
constructor({
|
|
29
|
+
namespaceMap, workersConfig, webConfig, logConfig, emitConfig, extractionConfig, failureConfig
|
|
30
|
+
}) {
|
|
25
31
|
this.namespaceMap = NamespaceMap.build(namespaceMap);
|
|
26
32
|
|
|
27
33
|
this.workersConfig = workersConfig;
|
|
28
34
|
this.webConfig = webConfig ?? null;
|
|
29
35
|
this.logConfig = logConfig ?? new LogConfig();
|
|
36
|
+
this.emitConfig = emitConfig ?? new EmitConfig();
|
|
37
|
+
this.extractionConfig = extractionConfig ?? new ExtractionConfig();
|
|
30
38
|
this.failureConfig = failureConfig ?? null;
|
|
31
39
|
}
|
|
32
40
|
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Represents the top-level emission-tracking configuration for the application.
|
|
3
|
+
*
|
|
4
|
+
* This `emit:` key is a sibling of `resources` / `web` / `log` and is distinct from the
|
|
5
|
+
* per-resource `resources.*.emit` block: it only carries the retention limit for the
|
|
6
|
+
* emission-tracking store.
|
|
7
|
+
* @author darthjee
|
|
8
|
+
*/
|
|
9
|
+
class EmitConfig {
|
|
10
|
+
#size;
|
|
11
|
+
|
|
12
|
+
/**
|
|
13
|
+
* Creates an instance of EmitConfig.
|
|
14
|
+
* @param {object} [config={}] - The configuration object.
|
|
15
|
+
* @param {number} [config.size=100] - The maximum number of emission records to retain.
|
|
16
|
+
*/
|
|
17
|
+
constructor({ size = 100 } = {}) {
|
|
18
|
+
this.#size = size;
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
/**
|
|
22
|
+
* Gets the maximum number of emission records to retain.
|
|
23
|
+
* @returns {number} The emission store size.
|
|
24
|
+
*/
|
|
25
|
+
get size() {
|
|
26
|
+
return this.#size;
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
/**
|
|
30
|
+
* Creates an EmitConfig instance from a raw YAML object.
|
|
31
|
+
* @param {object|null|undefined} obj - The raw emit config object from YAML.
|
|
32
|
+
* @returns {EmitConfig} A new EmitConfig instance.
|
|
33
|
+
*/
|
|
34
|
+
static fromObject(obj) {
|
|
35
|
+
return new EmitConfig(obj ?? {});
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
export { EmitConfig };
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Represents the top-level extraction-tracking configuration for the application.
|
|
3
|
+
*
|
|
4
|
+
* This `extraction:` key is a sibling of `resources` / `web` / `log` / `emit` and only
|
|
5
|
+
* carries the retention limit for the extraction-tracking store.
|
|
6
|
+
* @author darthjee
|
|
7
|
+
*/
|
|
8
|
+
class ExtractionConfig {
|
|
9
|
+
#size;
|
|
10
|
+
|
|
11
|
+
/**
|
|
12
|
+
* Creates an instance of ExtractionConfig.
|
|
13
|
+
* @param {object} [config={}] - The configuration object.
|
|
14
|
+
* @param {number} [config.size=100] - The maximum number of extraction records to retain.
|
|
15
|
+
*/
|
|
16
|
+
constructor({ size = 100 } = {}) {
|
|
17
|
+
this.#size = size;
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
/**
|
|
21
|
+
* Gets the maximum number of extraction records to retain.
|
|
22
|
+
* @returns {number} The extraction store size.
|
|
23
|
+
*/
|
|
24
|
+
get size() {
|
|
25
|
+
return this.#size;
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
/**
|
|
29
|
+
* Creates an ExtractionConfig instance from a raw YAML object.
|
|
30
|
+
* @param {object|null|undefined} obj - The raw extraction config object from YAML.
|
|
31
|
+
* @returns {ExtractionConfig} A new ExtractionConfig instance.
|
|
32
|
+
*/
|
|
33
|
+
static fromObject(obj) {
|
|
34
|
+
return new ExtractionConfig(obj ?? {});
|
|
35
|
+
}
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
export { ExtractionConfig };
|
|
@@ -11,21 +11,24 @@ const THRESHOLD_KEYS = ['low', 'medium', 'high', 'over'];
|
|
|
11
11
|
*/
|
|
12
12
|
class MemoryConfig {
|
|
13
13
|
#thresholds;
|
|
14
|
+
#dataStoreSize;
|
|
14
15
|
|
|
15
16
|
/**
|
|
16
17
|
* @param {object} [config={}] - Raw `web.memory` configuration object.
|
|
17
18
|
* @param {number} [config.maximum] - Configured memory maximum, in bytes.
|
|
18
19
|
* @param {object} [config.thresholds={}] - Percentage thresholds (`low`, `medium`, `high`, `over`).
|
|
20
|
+
* @param {object} [config.data_store={}] - Configuration for the memory readings store (`{ size }`).
|
|
19
21
|
* @param {object} [options={}] - Additional options.
|
|
20
22
|
* @param {{resolve: Function}} [options.resolver] - Resolver used to compute the memory maximum,
|
|
21
23
|
* defaulting to {@link MemoryMaximumResolver}.
|
|
22
24
|
* @throws {InvalidMemoryThresholds} When thresholds are not in strictly ascending order.
|
|
23
25
|
*/
|
|
24
|
-
constructor({ maximum, thresholds = {} } = {}, { resolver = MemoryMaximumResolver } = {}) {
|
|
26
|
+
constructor({ maximum, thresholds = {}, data_store: dataStore = {} } = {}, { resolver = MemoryMaximumResolver } = {}) {
|
|
25
27
|
this.#thresholds = { ...DEFAULT_THRESHOLDS, ...thresholds };
|
|
26
28
|
this.#validateThresholds();
|
|
27
29
|
|
|
28
30
|
this.maximum = resolver.resolve(maximum);
|
|
31
|
+
this.#dataStoreSize = { size: 100, ...dataStore }.size;
|
|
29
32
|
}
|
|
30
33
|
|
|
31
34
|
/**
|
|
@@ -35,6 +38,13 @@ class MemoryConfig {
|
|
|
35
38
|
return this.#thresholds;
|
|
36
39
|
}
|
|
37
40
|
|
|
41
|
+
/**
|
|
42
|
+
* @returns {number} The maximum number of memory readings to retain in the data store.
|
|
43
|
+
*/
|
|
44
|
+
get dataStoreSize() {
|
|
45
|
+
return this.#dataStoreSize;
|
|
46
|
+
}
|
|
47
|
+
|
|
38
48
|
/**
|
|
39
49
|
* Derives a status label from a memory usage percentage, using inclusive
|
|
40
50
|
* (`>=`) boundaries checked from the top down.
|
|
@@ -15,7 +15,7 @@ class WebConfig {
|
|
|
15
15
|
* @param {number} [config.idle_timeout=0] - Seconds of inactivity (no busy workers, no queued jobs) before the application auto-shuts-down. `0` disables auto-shutdown.
|
|
16
16
|
* @param {Array<string|object>} [config.links=[]] - Links shown in the web UI.
|
|
17
17
|
* @param {object} [config.api={}] - Configuration for the token-secured `/api` namespace (`{ token }`).
|
|
18
|
-
* @param {object} [config.memory={}] - Configuration for memory monitoring (`{ maximum, thresholds }`).
|
|
18
|
+
* @param {object} [config.memory={}] - Configuration for memory monitoring (`{ maximum, thresholds, data_store }`).
|
|
19
19
|
*/
|
|
20
20
|
constructor({ port, logs_page_size: logsPageSize = 20, enable_shutdown: enableShutdown = true, autostart = true, idle_timeout: idleTimeout = 0, links = [], api = {}, memory = {} }) {
|
|
21
21
|
this.port = port;
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
import { TemplateStringRenderer } from './TemplateStringRenderer.js';
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* BodyTemplateRenderer recursively renders a body template node against an item, used by
|
|
5
|
+
* `ResourceRequestEmit#resolveBody` to wrap/re-shape an extracted item before it is sent
|
|
6
|
+
* as the emit request body.
|
|
7
|
+
* @author darthjee
|
|
8
|
+
*/
|
|
9
|
+
class BodyTemplateRenderer {
|
|
10
|
+
/**
|
|
11
|
+
* Recursively renders a body template node against the given item.
|
|
12
|
+
* @param {*} node The template node to render (array, plain object, string, or scalar).
|
|
13
|
+
* @param {*} item The item to resolve tokens against.
|
|
14
|
+
* @returns {*} The rendered node.
|
|
15
|
+
*/
|
|
16
|
+
static render(node, item) {
|
|
17
|
+
if (Array.isArray(node)) {
|
|
18
|
+
return node.map((element) => BodyTemplateRenderer.render(element, item));
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
if (node !== null && typeof node === 'object') {
|
|
22
|
+
return Object.fromEntries(
|
|
23
|
+
Object.entries(node).map(([key, value]) => [key, BodyTemplateRenderer.render(value, item)])
|
|
24
|
+
);
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
if (typeof node === 'string') {
|
|
28
|
+
return TemplateStringRenderer.render(node, item);
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
return node;
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
export { BodyTemplateRenderer };
|
|
@@ -5,10 +5,11 @@ import { ResourceRequestAction } from './ResourceRequestAction.js';
|
|
|
5
5
|
import { ResourceRequestEmit } from './ResourceRequestEmit.js';
|
|
6
6
|
import { ResourceRequestPaginatedAction } from './ResourceRequestPaginatedAction.js';
|
|
7
7
|
import { ResourceRequestParser } from './ResourceRequestParser.js';
|
|
8
|
+
import { UrlTokenResolver } from './UrlTokenResolver.js';
|
|
8
9
|
import { ActionsEnqueuer } from '../../enqueuers/ActionsEnqueuer.js';
|
|
9
10
|
import { PaginatedActionsEnqueuer } from '../../enqueuers/PaginatedActionsEnqueuer.js';
|
|
10
11
|
import { LogRegistry } from '../../registry/LogRegistry.js';
|
|
11
|
-
import { Application } from '../../services/Application.js';
|
|
12
|
+
import { Application } from '../../services/application/Application.js';
|
|
12
13
|
|
|
13
14
|
/**
|
|
14
15
|
* ResourceRequest represents a request to a specific URL with an expected status code.
|
|
@@ -175,12 +176,15 @@ class ResourceRequest {
|
|
|
175
176
|
* Does nothing if the application is in 'stopped' status.
|
|
176
177
|
* @param {string} rawBody The raw HTTP response body.
|
|
177
178
|
* @param {object} [jobRegistry=JobRegistry] The job registry used to enqueue the ExtractionJob.
|
|
179
|
+
* @param {object} [parameters] The original request parameters, forwarded to the ExtractionJob
|
|
180
|
+
* for downstream {:placeholder} resolution in any emitted EmitJobs.
|
|
178
181
|
* @param {string|null} [originUrl=null] The URL of the ResourceRequestJob that triggered this enqueue.
|
|
179
182
|
* @returns {void}
|
|
180
183
|
*/
|
|
181
|
-
enqueueExtraction(rawBody, jobRegistry = DefaultJobRegistry, originUrl = null) {
|
|
184
|
+
enqueueExtraction(rawBody, jobRegistry = DefaultJobRegistry, parameters, originUrl = null) {
|
|
182
185
|
if (Application.isStopped()) return;
|
|
183
|
-
const params = { rawBody, parser: this.parser };
|
|
186
|
+
const params = { rawBody, parser: this.parser, parameters };
|
|
187
|
+
if (this.hasEmit()) params.emit = this.emit;
|
|
184
188
|
if (originUrl !== null) params.originUrl = originUrl;
|
|
185
189
|
jobRegistry.enqueue('Extraction', params);
|
|
186
190
|
}
|
|
@@ -193,6 +197,14 @@ class ResourceRequest {
|
|
|
193
197
|
return !!this.parser;
|
|
194
198
|
}
|
|
195
199
|
|
|
200
|
+
/**
|
|
201
|
+
* Returns true when the resource request declares a follow-up emit configuration.
|
|
202
|
+
* @returns {boolean} True if emit is configured.
|
|
203
|
+
*/
|
|
204
|
+
hasEmit() {
|
|
205
|
+
return !!this.emit;
|
|
206
|
+
}
|
|
207
|
+
|
|
196
208
|
/**
|
|
197
209
|
* Returns the URL with every {:placeholder} token replaced by the
|
|
198
210
|
* corresponding value from the parameters object.
|
|
@@ -201,7 +213,7 @@ class ResourceRequest {
|
|
|
201
213
|
* @returns {string} The resolved URL.
|
|
202
214
|
*/
|
|
203
215
|
resolveUrl(parameters = {}) {
|
|
204
|
-
return this.url
|
|
216
|
+
return UrlTokenResolver.resolve(this.url, parameters);
|
|
205
217
|
}
|
|
206
218
|
|
|
207
219
|
/**
|
|
@@ -2,7 +2,7 @@ import { JobRegistry as DefaultJobRegistry } from 'deku-swarm';
|
|
|
2
2
|
import { MissingActionResource } from '../../exceptions/registry/MissingActionResource.js';
|
|
3
3
|
import { LogRegistry } from '../../registry/LogRegistry.js';
|
|
4
4
|
import { NamespaceMap as DefaultNamespaceMap } from '../../registry/NamespaceMap.js';
|
|
5
|
-
import { Application } from '../../services/Application.js';
|
|
5
|
+
import { Application } from '../../services/application/Application.js';
|
|
6
6
|
import { ParametersMapper } from '../response/ParametersMapper.js';
|
|
7
7
|
|
|
8
8
|
/**
|