navi-hey 1.8.1 → 1.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/navi.js +2 -2
- package/lib/{services → client}/Client.js +14 -6
- package/lib/common/utils/env_resolver/EnvStringResolver.js +38 -2
- package/lib/common/utils/logging/LogBufferCollection.js +2 -3
- package/lib/common/utils/parser/HtmlRootParser.js +25 -0
- package/lib/enqueuers/ActionEnqueuer.js +1 -1
- package/lib/enqueuers/AssetRequestEnqueuer.js +1 -1
- package/lib/enqueuers/EmitEnqueuer.js +8 -3
- package/lib/enqueuers/PaginatedActionEnqueuer.js +1 -1
- package/lib/exceptions/config/ConfigurationFileNotFound.js +1 -1
- package/lib/exceptions/config/ConfigurationFileNotProvided.js +1 -1
- package/lib/exceptions/config/InvalidEmitBodyTemplate.js +16 -0
- package/lib/exceptions/config/InvalidEmitCooldown.js +15 -0
- package/lib/exceptions/config/InvalidEmitHeaders.js +16 -0
- package/lib/exceptions/config/InvalidEmitMethod.js +1 -1
- package/lib/exceptions/config/InvalidEmitRetries.js +16 -0
- package/lib/exceptions/config/InvalidMemoryThresholds.js +1 -1
- package/lib/exceptions/config/InvalidParserMatch.js +1 -1
- package/lib/exceptions/config/InvalidParserType.js +3 -3
- package/lib/exceptions/config/MissingEmitUrl.js +1 -1
- package/lib/exceptions/config/MissingParserField.js +1 -1
- package/lib/exceptions/config/MissingParserFields.js +1 -1
- package/lib/exceptions/config/MissingParserMatch.js +1 -1
- package/lib/exceptions/config/MissingTopLevelConfgKey.js +1 -1
- package/lib/exceptions/http/ConflictError.js +1 -1
- package/lib/exceptions/http/ForbiddenError.js +1 -1
- package/lib/exceptions/http/NotFoundError.js +1 -1
- package/lib/exceptions/registry/ItemNotFound.js +1 -1
- package/lib/exceptions/registry/MissingActionResource.js +1 -1
- package/lib/exceptions/registry/MissingMappingVariable.js +1 -1
- package/lib/exceptions/request/InvalidResponseBody.js +1 -1
- package/lib/exceptions/request/NullResponse.js +1 -1
- package/lib/exceptions/request/RequestFailed.js +11 -3
- package/lib/jobs/AssetDownloadJob.js +6 -2
- package/lib/jobs/EmitJob.js +122 -4
- package/lib/jobs/ExtractionJob.js +12 -1
- package/lib/jobs/ResourceRequestJob.js +6 -2
- package/lib/models/configs/Config.js +10 -2
- package/lib/models/configs/EmitConfig.js +39 -0
- package/lib/models/configs/ExtractionConfig.js +38 -0
- package/lib/models/configs/MemoryConfig.js +11 -1
- package/lib/models/configs/WebConfig.js +1 -1
- package/lib/models/request/BodyTemplateRenderer.js +35 -0
- package/lib/models/request/ResourceRequest.js +3 -2
- package/lib/models/request/ResourceRequestAction.js +1 -1
- package/lib/models/request/ResourceRequestEmit.js +123 -2
- package/lib/models/request/ResourceRequestPaginatedAction.js +1 -1
- package/lib/models/request/ResourceRequestParser.js +3 -3
- package/lib/models/request/TemplateStringRenderer.js +35 -0
- package/lib/models/request/TokenResolver.js +20 -0
- package/lib/models/request/UrlTokenResolver.js +19 -0
- package/lib/parsers/CssSelectorParser.js +68 -0
- package/lib/parsers/JsonPathParser.js +6 -36
- package/lib/parsers/RegexParser.js +6 -9
- package/lib/parsers/css_selector_parser/ArrayValueResolver.js +43 -0
- package/lib/parsers/css_selector_parser/ConditionMatcher.js +54 -0
- package/lib/parsers/css_selector_parser/FieldsMapper.js +35 -0
- package/lib/parsers/css_selector_parser/FilterMatcher.js +47 -0
- package/lib/parsers/css_selector_parser/ValueExtractor.js +34 -0
- package/lib/parsers/css_selector_parser/ValueResolver.js +36 -0
- package/lib/parsers/json_path/ConditionMatcher.js +33 -0
- package/lib/parsers/json_path/FieldMapper.js +30 -0
- package/lib/parsers/json_path/FilterMatcher.js +30 -0
- package/lib/parsers/json_path/MatchResolver.js +38 -0
- package/lib/parsers/regex_parser/AttributesValidator.js +31 -0
- package/lib/parsers/regex_parser/MatchValueExtractor.js +17 -0
- package/lib/parsers/regex_parser/PatternMatcher.js +27 -0
- package/lib/registry/EmissionRegistry.js +113 -0
- package/lib/registry/EmissionRegistryInstance.js +85 -0
- package/lib/registry/ExtractionRegistry.js +103 -0
- package/lib/registry/ExtractionRegistryInstance.js +76 -0
- package/lib/registry/LogRegistry.js +8 -0
- package/lib/registry/LogRegistryInstance.js +9 -0
- package/lib/registry/NamespaceMap.js +1 -1
- package/lib/serializers/EmissionSerializer.js +30 -0
- package/lib/serializers/ExtractionSerializer.js +26 -0
- package/lib/server/Router.js +4 -0
- package/lib/server/SecuredRequestHandler.js +8 -0
- package/lib/server/handlers/StatsHandler.js +5 -3
- package/lib/server/handlers/api/ApiConfigHandler.js +2 -2
- package/lib/server/handlers/api/ApiEngineStartHandler.js +1 -1
- package/lib/server/handlers/api/ApiEngineStopHandler.js +1 -1
- package/lib/server/handlers/emissions/EmissionsHandler.js +43 -0
- package/lib/server/handlers/engine/EngineContinueHandler.js +1 -1
- package/lib/server/handlers/engine/EnginePauseHandler.js +1 -1
- package/lib/server/handlers/engine/EngineReloadHandler.js +1 -1
- package/lib/server/handlers/engine/EngineRestartHandler.js +1 -1
- package/lib/server/handlers/engine/EngineShutdownHandler.js +1 -1
- package/lib/server/handlers/engine/EngineStartHandler.js +1 -1
- package/lib/server/handlers/engine/EngineStatusHandler.js +1 -1
- package/lib/server/handlers/engine/EngineStopHandler.js +1 -1
- package/lib/server/handlers/extractions/ExtractionsHandler.js +43 -0
- package/lib/services/{Application.js → application/Application.js} +0 -16
- package/lib/services/application/ApplicationConfigurator.js +39 -0
- package/lib/services/application/ApplicationInstance.js +245 -0
- package/lib/services/application/ConfigStore.js +54 -0
- package/lib/services/application/ResourceQueueFacade.js +39 -0
- package/lib/services/application/StartupCoordinator.js +43 -0
- package/lib/services/{NamespaceMapBuilder.js → builders/NamespaceMapBuilder.js} +3 -3
- package/lib/services/builders/RegistriesBuilder.js +56 -0
- package/lib/services/{ConfigIncluder.js → config/ConfigIncluder.js} +47 -5
- package/lib/services/{ConfigLoader.js → config/ConfigLoader.js} +8 -2
- package/lib/services/{ConfigParser.js → config/ConfigParser.js} +32 -9
- package/lib/services/engine/EngineController.js +239 -0
- package/lib/services/engine/EngineState.js +51 -0
- package/lib/services/{EngineStopService.js → engine/EngineStopService.js} +7 -5
- package/lib/services/engine/ServerController.js +60 -0
- package/lib/services/{FailureChecker.js → execution/FailureChecker.js} +1 -1
- package/lib/services/execution/RunReporter.js +31 -0
- package/lib/utils/HtmlParser.js +2 -10
- package/lib/utils/emissions/EmissionRecord.js +135 -0
- package/lib/utils/emissions/EmissionRecordFactory.js +40 -0
- package/lib/utils/emissions/EmissionStore.js +133 -0
- package/lib/utils/extractions/ExtractionRecord.js +84 -0
- package/lib/utils/extractions/ExtractionRecordFactory.js +35 -0
- package/lib/utils/extractions/ExtractionStore.js +111 -0
- package/lib/utils/memory/MemoryData.js +70 -0
- package/lib/utils/memory/MemoryDataFactory.js +32 -0
- package/lib/utils/memory/MemoryDataStore.js +92 -0
- package/package.json +2 -2
- package/static/assets/index-C81vj0N0.js +11 -0
- package/static/index.html +1 -1
- package/lib/services/ApplicationInstance.js +0 -376
- package/lib/services/EngineEvents.js +0 -42
- package/static/assets/index-3lOQ6IAs.js +0 -11
- /package/lib/{exceptions → common/exceptions}/AppError.js +0 -0
- /package/lib/{exceptions → common/exceptions}/request/InvalidHtmlResponseBody.js +0 -0
- /package/lib/services/{ArgumentsParser.js → application/ArgumentsParser.js} +0 -0
- /package/lib/services/{RunSummary.js → execution/RunSummary.js} +0 -0
package/bin/navi.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
|
|
3
|
-
import { Application } from '../lib/services/Application.js';
|
|
4
|
-
import { ArgumentsParser } from '../lib/services/ArgumentsParser.js';
|
|
3
|
+
import { Application } from '../lib/services/application/Application.js';
|
|
4
|
+
import { ArgumentsParser } from '../lib/services/application/ArgumentsParser.js';
|
|
5
5
|
|
|
6
6
|
const { config } = ArgumentsParser.parse(process.argv.slice(2));
|
|
7
7
|
Application.build();
|
|
@@ -104,14 +104,16 @@ class Client {
|
|
|
104
104
|
* @param {number} [expectedStatus] The expected HTTP response status code. When omitted,
|
|
105
105
|
* any 2xx status is treated as success.
|
|
106
106
|
* @param {LogContext} logContext Context carrying workerId/jobId for log entries.
|
|
107
|
+
* @param {object} [headers={}] Extra HTTP headers for this single emit call, merged over
|
|
108
|
+
* the client's configured headers (per-call keys win on collision).
|
|
107
109
|
* @returns {Promise<object>} The Axios response object.
|
|
108
110
|
* @throws {RequestFailed} Throws an error if the request fails or the status does not match.
|
|
109
111
|
*/
|
|
110
|
-
async emit(method, resourceUrl, body, expectedStatus, logContext) {
|
|
112
|
+
async emit(method, resourceUrl, body, expectedStatus, logContext, headers = {}) {
|
|
111
113
|
const requestUrl = this.#buildUrl(resourceUrl);
|
|
112
114
|
logContext.info(`[Client:${this.name}] Emitting ${method} ${requestUrl}`);
|
|
113
115
|
try {
|
|
114
|
-
return await this.#emitRequest(method, requestUrl, body, expectedStatus, logContext);
|
|
116
|
+
return await this.#emitRequest(method, requestUrl, body, expectedStatus, logContext, headers);
|
|
115
117
|
} catch (error) {
|
|
116
118
|
this.#handleError(error, requestUrl, logContext);
|
|
117
119
|
}
|
|
@@ -161,11 +163,17 @@ class Client {
|
|
|
161
163
|
* @param {number} [expectedStatus] The expected HTTP response status code. When omitted,
|
|
162
164
|
* any 2xx status is treated as success.
|
|
163
165
|
* @param {LogContext} logContext Context carrying workerId/jobId for log entries.
|
|
166
|
+
* @param {object} [headers={}] Extra HTTP headers for this single emit call, merged over
|
|
167
|
+
* the client's configured headers (per-call keys win on collision).
|
|
164
168
|
* @returns {Promise<object>} The Axios response object.
|
|
165
169
|
* @throws {RequestFailed} Throws an error if the response status does not match.
|
|
166
170
|
*/
|
|
167
|
-
async #emitRequest(method, requestUrl, body, expectedStatus, logContext) {
|
|
168
|
-
const options = {
|
|
171
|
+
async #emitRequest(method, requestUrl, body, expectedStatus, logContext, headers = {}) {
|
|
172
|
+
const options = {
|
|
173
|
+
timeout: this.timeout,
|
|
174
|
+
headers: { ...this.headers, ...headers },
|
|
175
|
+
validateStatus: () => true,
|
|
176
|
+
};
|
|
169
177
|
|
|
170
178
|
let response;
|
|
171
179
|
switch (method) {
|
|
@@ -203,7 +211,7 @@ class Client {
|
|
|
203
211
|
|
|
204
212
|
if (!matches) {
|
|
205
213
|
logContext.info(`[Client:${this.name}] ${requestUrl} did not match (got ${response.status}, expected ${expectedStatus ?? '2xx'})`);
|
|
206
|
-
throw new RequestFailed(response.status, requestUrl);
|
|
214
|
+
throw new RequestFailed(response.status, requestUrl, 'Request failed', response.headers);
|
|
207
215
|
}
|
|
208
216
|
|
|
209
217
|
logContext.info(`[Client:${this.name}] ${requestUrl} matched (expected ${expectedStatus ?? '2xx'})`);
|
|
@@ -221,7 +229,7 @@ class Client {
|
|
|
221
229
|
#handleError(error, requestUrl, logContext) {
|
|
222
230
|
logContext.error(`Request failed: ${error}`);
|
|
223
231
|
if (error.response) {
|
|
224
|
-
throw new RequestFailed(error.response.status, requestUrl);
|
|
232
|
+
throw new RequestFailed(error.response.status, requestUrl, 'Request failed', error.response.headers);
|
|
225
233
|
}
|
|
226
234
|
throw error;
|
|
227
235
|
}
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { createHash } from 'node:crypto';
|
|
1
2
|
import { Logger } from '../logging/Logger.js';
|
|
2
3
|
|
|
3
4
|
/**
|
|
@@ -7,12 +8,30 @@ import { Logger } from '../logging/Logger.js';
|
|
|
7
8
|
*/
|
|
8
9
|
const ENV_VAR_PATTERN = /\$\{([^}]+)\}|\$([A-Za-z_][A-Za-z0-9_]*)/g;
|
|
9
10
|
|
|
11
|
+
/**
|
|
12
|
+
* Number of leading hex characters of the SHA-256 digest kept in a match's
|
|
13
|
+
* `hash` field — enough to spot value drift between runs without logging the
|
|
14
|
+
* raw value itself.
|
|
15
|
+
* @type {number}
|
|
16
|
+
*/
|
|
17
|
+
const HASH_LENGTH = 12;
|
|
18
|
+
|
|
10
19
|
class EnvStringResolver {
|
|
11
20
|
/**
|
|
12
21
|
* @param {string} string Raw string content.
|
|
13
22
|
*/
|
|
14
23
|
constructor(string) {
|
|
15
24
|
this.string = string;
|
|
25
|
+
|
|
26
|
+
/**
|
|
27
|
+
* Per-occurrence resolution data recorded by the last `resolve()` call,
|
|
28
|
+
* in match order (not deduped). Each entry is
|
|
29
|
+
* `{ varName, defined, length, hash }`, where `length`/`hash` are only
|
|
30
|
+
* present when `defined` is `true`. Consumers (e.g. `ConfigIncluder`)
|
|
31
|
+
* use this for debug logging — the raw resolved value is never recorded.
|
|
32
|
+
* @type {Array<{varName: string, defined: boolean, length?: number, hash?: string}>}
|
|
33
|
+
*/
|
|
34
|
+
this.matches = [];
|
|
16
35
|
}
|
|
17
36
|
|
|
18
37
|
/**
|
|
@@ -30,7 +49,8 @@ class EnvStringResolver {
|
|
|
30
49
|
/**
|
|
31
50
|
* Resolves environment variable references in a raw string (e.g. YAML file content).
|
|
32
51
|
*
|
|
33
|
-
* Replaces all `$VAR` and `${VAR}` occurrences with their environment values
|
|
52
|
+
* Replaces all `$VAR` and `${VAR}` occurrences with their environment values,
|
|
53
|
+
* recording per-occurrence resolution data into `this.matches` as it goes.
|
|
34
54
|
*
|
|
35
55
|
* @returns {string} Resolved string with env var references replaced.
|
|
36
56
|
*/
|
|
@@ -38,8 +58,13 @@ class EnvStringResolver {
|
|
|
38
58
|
return String(this.string).replace(ENV_VAR_PATTERN, (_match, braced, bare) => {
|
|
39
59
|
const varName = braced || bare;
|
|
40
60
|
const resolved = process.env[varName];
|
|
61
|
+
const defined = resolved !== undefined;
|
|
62
|
+
|
|
63
|
+
this.matches.push(defined
|
|
64
|
+
? { varName, defined, length: resolved.length, hash: this.#hash(resolved) }
|
|
65
|
+
: { varName, defined });
|
|
41
66
|
|
|
42
|
-
if (
|
|
67
|
+
if (!defined) {
|
|
43
68
|
Logger.warn(`Environment variable not defined: ${varName}`);
|
|
44
69
|
return '';
|
|
45
70
|
}
|
|
@@ -47,6 +72,17 @@ class EnvStringResolver {
|
|
|
47
72
|
return resolved;
|
|
48
73
|
});
|
|
49
74
|
}
|
|
75
|
+
|
|
76
|
+
/**
|
|
77
|
+
* Hashes a resolved value for safe (non-reversible) debug logging.
|
|
78
|
+
*
|
|
79
|
+
* @param {string} value The resolved value to hash.
|
|
80
|
+
* @returns {string} The first {@link HASH_LENGTH} hex characters of the value's SHA-256 digest.
|
|
81
|
+
* @private
|
|
82
|
+
*/
|
|
83
|
+
#hash(value) {
|
|
84
|
+
return createHash('sha256').update(value).digest('hex').slice(0, HASH_LENGTH);
|
|
85
|
+
}
|
|
50
86
|
}
|
|
51
87
|
|
|
52
88
|
export { EnvStringResolver };
|
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
import { LogBuffer } from './LogBuffer.js';
|
|
2
|
-
import { EngineEvents } from '../../../services/EngineEvents.js';
|
|
3
2
|
|
|
4
3
|
/**
|
|
5
4
|
* LogBufferCollection manages a keyed map of LogBuffer instances.
|
|
6
|
-
* Buffers are created on first use
|
|
5
|
+
* Buffers are created on first use. Callers are responsible for invoking
|
|
6
|
+
* `clear()` when appropriate (e.g. LogRegistryInstance clears buffers on engine stop).
|
|
7
7
|
* @author darthjee
|
|
8
8
|
*/
|
|
9
9
|
class LogBufferCollection {
|
|
@@ -16,7 +16,6 @@ class LogBufferCollection {
|
|
|
16
16
|
constructor(retention = 100) {
|
|
17
17
|
this.#buffers = new Map();
|
|
18
18
|
this.#retention = retention;
|
|
19
|
-
EngineEvents.on('stop', () => this.clear());
|
|
20
19
|
}
|
|
21
20
|
|
|
22
21
|
/**
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
import { parse } from 'node-html-parser';
|
|
2
|
+
import { InvalidHtmlResponseBody } from '../../exceptions/request/InvalidHtmlResponseBody.js';
|
|
3
|
+
|
|
4
|
+
/**
|
|
5
|
+
* HtmlRootParser parses a raw HTML string into a DOM root while normalizing
|
|
6
|
+
* parser failures to {@link InvalidHtmlResponseBody}.
|
|
7
|
+
* @author darthjee
|
|
8
|
+
*/
|
|
9
|
+
class HtmlRootParser {
|
|
10
|
+
/**
|
|
11
|
+
* Parses a raw HTML string into a DOM root.
|
|
12
|
+
* @param {string} rawBody The raw HTML response body to parse.
|
|
13
|
+
* @returns {HTMLElement} The parsed DOM root.
|
|
14
|
+
* @throws {InvalidHtmlResponseBody} If the body cannot be parsed as HTML.
|
|
15
|
+
*/
|
|
16
|
+
parse(rawBody) {
|
|
17
|
+
try {
|
|
18
|
+
return parse(rawBody);
|
|
19
|
+
} catch (cause) {
|
|
20
|
+
throw new InvalidHtmlResponseBody(rawBody, cause);
|
|
21
|
+
}
|
|
22
|
+
}
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
export { HtmlRootParser };
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { JobRegistry as DefaultJobRegistry } from 'deku-swarm';
|
|
2
|
-
import { Application } from '../services/Application.js';
|
|
2
|
+
import { Application } from '../services/application/Application.js';
|
|
3
3
|
|
|
4
4
|
/**
|
|
5
5
|
* ActionEnqueuer enqueues one ActionProcessingJob per item for a single Action.
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { JobRegistry as DefaultJobRegistry } from 'deku-swarm';
|
|
2
|
-
import { Application } from '../services/Application.js';
|
|
2
|
+
import { Application } from '../services/application/Application.js';
|
|
3
3
|
|
|
4
4
|
/**
|
|
5
5
|
* EmitEnqueuer enqueues one EmitJob per extracted item, forwarding the resource
|
|
@@ -11,18 +11,21 @@ class EmitEnqueuer {
|
|
|
11
11
|
#emit;
|
|
12
12
|
#parameters;
|
|
13
13
|
#jobRegistry;
|
|
14
|
+
#extractionId;
|
|
14
15
|
|
|
15
16
|
/**
|
|
16
17
|
* @param {Array<ExtractedItem>} items List of extracted items, as produced by a parser's `extract()`.
|
|
17
18
|
* @param {ResourceRequestEmit} emit The resource request's declared emit configuration.
|
|
18
19
|
* @param {object} [parameters] Key-value map used to resolve {:placeholder} tokens in the emit URL.
|
|
19
20
|
* @param {object} [jobRegistry=JobRegistry] The job registry to enqueue jobs to.
|
|
21
|
+
* @param {number|null} [extractionId=null] The id of the extraction record whose items are being emitted.
|
|
20
22
|
*/
|
|
21
|
-
constructor(items, emit, parameters, jobRegistry = DefaultJobRegistry) {
|
|
23
|
+
constructor(items, emit, parameters, jobRegistry = DefaultJobRegistry, extractionId = null) {
|
|
22
24
|
this.#items = items;
|
|
23
25
|
this.#emit = emit;
|
|
24
26
|
this.#parameters = parameters;
|
|
25
27
|
this.#jobRegistry = jobRegistry;
|
|
28
|
+
this.#extractionId = extractionId;
|
|
26
29
|
}
|
|
27
30
|
|
|
28
31
|
/**
|
|
@@ -33,7 +36,9 @@ class EmitEnqueuer {
|
|
|
33
36
|
enqueue() {
|
|
34
37
|
if (Application.isStopped()) return;
|
|
35
38
|
for (const item of this.#items) {
|
|
36
|
-
this.#jobRegistry.enqueue('Emit', {
|
|
39
|
+
this.#jobRegistry.enqueue('Emit', {
|
|
40
|
+
item, emit: this.#emit, parameters: this.#parameters, extractionId: this.#extractionId,
|
|
41
|
+
});
|
|
37
42
|
}
|
|
38
43
|
}
|
|
39
44
|
}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { JobRegistry as DefaultJobRegistry } from 'deku-swarm';
|
|
2
|
-
import { Application } from '../services/Application.js';
|
|
2
|
+
import { Application } from '../services/application/Application.js';
|
|
3
3
|
|
|
4
4
|
/**
|
|
5
5
|
* Enqueues one PaginatedActionProcessingJob for a single paginated action.
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
import { AppError } from '../../common/exceptions/AppError.js';
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Thrown when an `emit` config entry declares a `body_template` value that is not a
|
|
5
|
+
* plain object or array.
|
|
6
|
+
* @author darthjee
|
|
7
|
+
*/
|
|
8
|
+
class InvalidEmitBodyTemplate extends AppError {
|
|
9
|
+
constructor(bodyTemplate) {
|
|
10
|
+
super(`Invalid emit body_template: ${JSON.stringify(bodyTemplate)}. Expected a plain object or array`);
|
|
11
|
+
|
|
12
|
+
this.bodyTemplate = bodyTemplate;
|
|
13
|
+
}
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
export { InvalidEmitBodyTemplate };
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
import { AppError } from '../../common/exceptions/AppError.js';
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Thrown when an `emit` config entry declares a non-numeric or negative `cooldown` value.
|
|
5
|
+
* @author darthjee
|
|
6
|
+
*/
|
|
7
|
+
class InvalidEmitCooldown extends AppError {
|
|
8
|
+
constructor(cooldown) {
|
|
9
|
+
super(`Invalid emit cooldown: ${JSON.stringify(cooldown)}. Expected a non-negative number`);
|
|
10
|
+
|
|
11
|
+
this.cooldown = cooldown;
|
|
12
|
+
}
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
export { InvalidEmitCooldown };
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
import { AppError } from '../../common/exceptions/AppError.js';
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Thrown when an `emit` config entry declares a `headers` value that is not a plain
|
|
5
|
+
* object mapping keys to string-coercible values.
|
|
6
|
+
* @author darthjee
|
|
7
|
+
*/
|
|
8
|
+
class InvalidEmitHeaders extends AppError {
|
|
9
|
+
constructor(headers) {
|
|
10
|
+
super(`Invalid emit headers: ${JSON.stringify(headers)}. Expected a map of string values`);
|
|
11
|
+
|
|
12
|
+
this.headers = headers;
|
|
13
|
+
}
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
export { InvalidEmitHeaders };
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
import { AppError } from '../../common/exceptions/AppError.js';
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Thrown when an `emit` config entry declares a non-numeric or negative `retries` value.
|
|
5
|
+
* `retries: 0` is valid (one attempt, no retries).
|
|
6
|
+
* @author darthjee
|
|
7
|
+
*/
|
|
8
|
+
class InvalidEmitRetries extends AppError {
|
|
9
|
+
constructor(retries) {
|
|
10
|
+
super(`Invalid emit retries: ${JSON.stringify(retries)}. Expected a non-negative number`);
|
|
11
|
+
|
|
12
|
+
this.retries = retries;
|
|
13
|
+
}
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
export { InvalidEmitRetries };
|
|
@@ -1,13 +1,13 @@
|
|
|
1
|
-
import { AppError } from '
|
|
1
|
+
import { AppError } from '../../common/exceptions/AppError.js';
|
|
2
2
|
|
|
3
3
|
/**
|
|
4
4
|
* Thrown when a `parser` config entry declares a `type` other than the
|
|
5
|
-
* supported set (`regex`, `json_path`), or omits `type` entirely.
|
|
5
|
+
* supported set (`regex`, `json_path`, `css`), or omits `type` entirely.
|
|
6
6
|
* @author darthjee
|
|
7
7
|
*/
|
|
8
8
|
class InvalidParserType extends AppError {
|
|
9
9
|
constructor(type) {
|
|
10
|
-
super(`Invalid parser type: ${JSON.stringify(type)}. Expected one of "regex", "json_path"`);
|
|
10
|
+
super(`Invalid parser type: ${JSON.stringify(type)}. Expected one of "regex", "json_path", "css"`);
|
|
11
11
|
|
|
12
12
|
this.type = type;
|
|
13
13
|
}
|
|
@@ -1,15 +1,23 @@
|
|
|
1
|
-
import { AppError } from '
|
|
1
|
+
import { AppError } from '../../common/exceptions/AppError.js';
|
|
2
2
|
|
|
3
3
|
/**
|
|
4
4
|
* RequestFailed is a custom error class that represents a failed HTTP request.
|
|
5
|
-
* It includes the status code and
|
|
5
|
+
* It includes the status code, URL, and response headers of the failed request for better
|
|
6
|
+
* error handling and debugging (e.g. reading a `Retry-After` header for retry policies).
|
|
6
7
|
* @author darthjee
|
|
7
8
|
*/
|
|
8
9
|
class RequestFailed extends AppError {
|
|
9
|
-
|
|
10
|
+
/**
|
|
11
|
+
* @param {number} statusCode The HTTP status code received (or that triggered the failure).
|
|
12
|
+
* @param {string} url The URL that was requested.
|
|
13
|
+
* @param {string} [message='Request failed'] A custom error message prefix.
|
|
14
|
+
* @param {object} [headers={}] The response headers received, when available.
|
|
15
|
+
*/
|
|
16
|
+
constructor(statusCode, url, message = 'Request failed', headers = {}) {
|
|
10
17
|
super(`${message}: ${statusCode} for ${url}`);
|
|
11
18
|
this.statusCode = statusCode;
|
|
12
19
|
this.url = url;
|
|
20
|
+
this.headers = headers;
|
|
13
21
|
}
|
|
14
22
|
}
|
|
15
23
|
|
|
@@ -21,9 +21,13 @@ class AssetDownloadJob extends Job {
|
|
|
21
21
|
* @param {number} params.status - Expected HTTP status code.
|
|
22
22
|
* @param {NamespaceMap} params.clientRegistry - The namespace map used to look up the client.
|
|
23
23
|
* Resolution always targets the `default` namespace (not namespace-aware).
|
|
24
|
+
* @param {number} [params.maxRetries] - The maximum number of retries allowed for this job,
|
|
25
|
+
* normally injected from the global `workers.max-retries` config.
|
|
26
|
+
* @param {number} [params.cooldown] - The cooldown, in milliseconds, applied on failure,
|
|
27
|
+
* normally injected from the global `workers.retry_cooldown` config.
|
|
24
28
|
*/
|
|
25
|
-
constructor({ id, url, client, status, clientRegistry }) {
|
|
26
|
-
super({ id });
|
|
29
|
+
constructor({ id, url, client, status, clientRegistry, maxRetries, cooldown }) {
|
|
30
|
+
super({ id, maxRetries, cooldown });
|
|
27
31
|
this.#url = url;
|
|
28
32
|
this.#clientName = client;
|
|
29
33
|
this.#status = status;
|