navi-hey 1.7.0 → 1.8.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/lib/enqueuers/EmitEnqueuer.js +41 -0
- package/lib/exceptions/config/InvalidEmitMethod.js +16 -0
- package/lib/exceptions/config/InvalidMemoryThresholds.js +23 -0
- package/lib/exceptions/config/InvalidParserMatch.js +17 -0
- package/lib/exceptions/config/InvalidParserType.js +16 -0
- package/lib/exceptions/config/MissingEmitUrl.js +13 -0
- package/lib/exceptions/config/MissingParserField.js +13 -0
- package/lib/exceptions/config/MissingParserFields.js +13 -0
- package/lib/exceptions/config/MissingParserMatch.js +13 -0
- package/lib/exceptions/registry/ParserNotFound.js +16 -0
- package/lib/jobs/EmitJob.js +70 -0
- package/lib/jobs/ExtractionJob.js +100 -0
- package/lib/jobs/ResourceRequestJob.js +13 -0
- package/lib/models/configs/MemoryConfig.js +70 -0
- package/lib/models/configs/WebConfig.js +4 -1
- package/lib/models/request/ClientReference.js +23 -0
- package/lib/models/request/ResourceRequest.js +44 -15
- package/lib/models/request/ResourceRequestEmit.js +82 -0
- package/lib/models/request/ResourceRequestParser.js +40 -0
- package/lib/parsers/JsonPathParser.js +90 -0
- package/lib/parsers/RegexParser.js +36 -0
- package/lib/registry/ParserRegistry.js +32 -0
- package/lib/server/Router.js +2 -0
- package/lib/server/handlers/memory/MemoryStatusHandler.js +40 -0
- package/lib/services/ApplicationInstance.js +9 -0
- package/lib/services/Client.js +76 -3
- package/lib/services/NamespaceMapBuilder.js +16 -2
- package/lib/utils/memory/CgroupV1MemoryLimitReader.js +40 -0
- package/lib/utils/memory/CgroupV2MemoryLimitReader.js +38 -0
- package/lib/utils/memory/ConfigMemoryLimitReader.js +25 -0
- package/lib/utils/memory/MemoryMaximumResolver.js +55 -0
- package/lib/utils/memory/OsTotalMemoryReader.js +17 -0
- package/lib/utils/memory/ProcessRssReader.js +15 -0
- package/package.json +1 -1
- package/static/assets/index-3lOQ6IAs.js +11 -0
- package/static/assets/{index-CuWjW0sT.css → index-Dcv_oCts.css} +1 -1
- package/static/index.html +2 -2
- package/static/assets/index-DyqqrESe.js +0 -11
package/README.md
CHANGED
|
@@ -207,4 +207,4 @@ docker run --rm \
|
|
|
207
207
|
|
|
208
208
|
GitHub repository: [darthjee/navi](https://github.com/darthjee/navi)
|
|
209
209
|
|
|
210
|
-
Integration guide for developers and AI agents: [How to Use Navi in Your Project](https://github.com/darthjee/navi/blob/main/docs/guides/
|
|
210
|
+
Integration guide for developers and AI agents: [How to Use Navi in Your Project](https://github.com/darthjee/navi/blob/main/docs/guides/how_to_use_navi.md)
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
import { JobRegistry as DefaultJobRegistry } from 'deku-swarm';
|
|
2
|
+
import { Application } from '../services/Application.js';
|
|
3
|
+
|
|
4
|
+
/**
|
|
5
|
+
* EmitEnqueuer enqueues one EmitJob per extracted item, forwarding the resource
|
|
6
|
+
* request's declared `emit` configuration and the original request parameters.
|
|
7
|
+
* @author darthjee
|
|
8
|
+
*/
|
|
9
|
+
class EmitEnqueuer {
|
|
10
|
+
#items;
|
|
11
|
+
#emit;
|
|
12
|
+
#parameters;
|
|
13
|
+
#jobRegistry;
|
|
14
|
+
|
|
15
|
+
/**
|
|
16
|
+
* @param {Array<ExtractedItem>} items List of extracted items, as produced by a parser's `extract()`.
|
|
17
|
+
* @param {ResourceRequestEmit} emit The resource request's declared emit configuration.
|
|
18
|
+
* @param {object} [parameters] Key-value map used to resolve {:placeholder} tokens in the emit URL.
|
|
19
|
+
* @param {object} [jobRegistry=JobRegistry] The job registry to enqueue jobs to.
|
|
20
|
+
*/
|
|
21
|
+
constructor(items, emit, parameters, jobRegistry = DefaultJobRegistry) {
|
|
22
|
+
this.#items = items;
|
|
23
|
+
this.#emit = emit;
|
|
24
|
+
this.#parameters = parameters;
|
|
25
|
+
this.#jobRegistry = jobRegistry;
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
/**
|
|
29
|
+
* Enqueues one EmitJob per extracted item, with the configured emit and parameters.
|
|
30
|
+
* Does nothing if the application is in 'stopped' status.
|
|
31
|
+
* @returns {void}
|
|
32
|
+
*/
|
|
33
|
+
enqueue() {
|
|
34
|
+
if (Application.isStopped()) return;
|
|
35
|
+
for (const item of this.#items) {
|
|
36
|
+
this.#jobRegistry.enqueue('Emit', { item, emit: this.#emit, parameters: this.#parameters });
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
export { EmitEnqueuer };
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
import { AppError } from '../AppError.js';
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Thrown when an `emit` config entry declares a `method` other than the
|
|
5
|
+
* supported set (`POST`, `PUT`, `PATCH`), or omits `method` entirely.
|
|
6
|
+
* @author darthjee
|
|
7
|
+
*/
|
|
8
|
+
class InvalidEmitMethod extends AppError {
|
|
9
|
+
constructor(method) {
|
|
10
|
+
super(`Invalid emit method: ${JSON.stringify(method)}. Expected one of "POST", "PUT", "PATCH"`);
|
|
11
|
+
|
|
12
|
+
this.method = method;
|
|
13
|
+
}
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
export { InvalidEmitMethod };
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
import { AppError } from '../AppError.js';
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Thrown when a `web.memory.thresholds` config block does not declare
|
|
5
|
+
* `low < medium < high < over` in strictly ascending order.
|
|
6
|
+
* @author darthjee
|
|
7
|
+
*/
|
|
8
|
+
class InvalidMemoryThresholds extends AppError {
|
|
9
|
+
/**
|
|
10
|
+
* @param {string} lowerKey - The threshold key expected to be strictly lower.
|
|
11
|
+
* @param {string} higherKey - The threshold key expected to be strictly higher.
|
|
12
|
+
* @param {object} thresholds - The full resolved thresholds object.
|
|
13
|
+
*/
|
|
14
|
+
constructor(lowerKey, higherKey, thresholds) {
|
|
15
|
+
super(`Invalid memory thresholds: "${lowerKey}" (${thresholds[lowerKey]}) must be strictly less than "${higherKey}" (${thresholds[higherKey]})`);
|
|
16
|
+
|
|
17
|
+
this.lowerKey = lowerKey;
|
|
18
|
+
this.higherKey = higherKey;
|
|
19
|
+
this.thresholds = thresholds;
|
|
20
|
+
}
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
export { InvalidMemoryThresholds };
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
import { AppError } from '../AppError.js';
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Thrown when a parser's `match` attribute is present but resolving it against
|
|
5
|
+
* the parsed body does not yield an array (a key missing anywhere along the
|
|
6
|
+
* dot-notation path, or a resolved value that isn't an array).
|
|
7
|
+
* @author darthjee
|
|
8
|
+
*/
|
|
9
|
+
class InvalidParserMatch extends AppError {
|
|
10
|
+
constructor(match) {
|
|
11
|
+
super(`Parser "match" (${JSON.stringify(match)}) did not resolve to an array`);
|
|
12
|
+
|
|
13
|
+
this.match = match;
|
|
14
|
+
}
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
export { InvalidParserMatch };
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
import { AppError } from '../AppError.js';
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Thrown when a `parser` config entry declares a `type` other than the
|
|
5
|
+
* supported set (`regex`, `json_path`), or omits `type` entirely.
|
|
6
|
+
* @author darthjee
|
|
7
|
+
*/
|
|
8
|
+
class InvalidParserType extends AppError {
|
|
9
|
+
constructor(type) {
|
|
10
|
+
super(`Invalid parser type: ${JSON.stringify(type)}. Expected one of "regex", "json_path"`);
|
|
11
|
+
|
|
12
|
+
this.type = type;
|
|
13
|
+
}
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
export { InvalidParserType };
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
import { AppError } from '../AppError.js';
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Thrown when an `emit` config entry is missing the required "url" field.
|
|
5
|
+
* @author darthjee
|
|
6
|
+
*/
|
|
7
|
+
class MissingEmitUrl extends AppError {
|
|
8
|
+
constructor() {
|
|
9
|
+
super('Emit is missing the required "url" field');
|
|
10
|
+
}
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
export { MissingEmitUrl };
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
import { AppError } from '../AppError.js';
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Thrown when a `regex` parser config entry is missing the required "field" field.
|
|
5
|
+
* @author darthjee
|
|
6
|
+
*/
|
|
7
|
+
class MissingParserField extends AppError {
|
|
8
|
+
constructor() {
|
|
9
|
+
super('Regex parser is missing the required "field" field');
|
|
10
|
+
}
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
export { MissingParserField };
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
import { AppError } from '../AppError.js';
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Thrown when a parser config entry is missing the required "fields" mapping.
|
|
5
|
+
* @author darthjee
|
|
6
|
+
*/
|
|
7
|
+
class MissingParserFields extends AppError {
|
|
8
|
+
constructor() {
|
|
9
|
+
super('Parser is missing the required "fields" field');
|
|
10
|
+
}
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
export { MissingParserFields };
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
import { AppError } from '../AppError.js';
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Thrown when a parser config entry is missing the required "match" field.
|
|
5
|
+
* @author darthjee
|
|
6
|
+
*/
|
|
7
|
+
class MissingParserMatch extends AppError {
|
|
8
|
+
constructor() {
|
|
9
|
+
super('Parser is missing the required "match" field');
|
|
10
|
+
}
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
export { MissingParserMatch };
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
import { ItemNotFound } from './ItemNotFound.js';
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* ParserNotFound is a custom error class thrown when a requested parser
|
|
5
|
+
* cannot be found in the registry.
|
|
6
|
+
* @author darthjee
|
|
7
|
+
*/
|
|
8
|
+
class ParserNotFound extends ItemNotFound {
|
|
9
|
+
constructor(type) {
|
|
10
|
+
super(type, { itemType: 'Parser' });
|
|
11
|
+
|
|
12
|
+
this.type = type;
|
|
13
|
+
}
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
export { ParserNotFound };
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
import { Job } from 'deku-swarm';
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* EmitJob is a Job that sends a single extracted item to an external endpoint, as
|
|
5
|
+
* declared by a resource request's `emit` configuration.
|
|
6
|
+
* @author darthjee
|
|
7
|
+
*/
|
|
8
|
+
class EmitJob extends Job {
|
|
9
|
+
#item;
|
|
10
|
+
#emit;
|
|
11
|
+
#parameters;
|
|
12
|
+
#clients;
|
|
13
|
+
|
|
14
|
+
/**
|
|
15
|
+
* Creates a new EmitJob instance.
|
|
16
|
+
* @param {object} params - The parameters for creating an EmitJob instance.
|
|
17
|
+
* @param {string} params.id - The unique identifier for this job.
|
|
18
|
+
* @param {ExtractedItem} params.item - The extracted item to send, a single element of a
|
|
19
|
+
* parser's `extract()` output array.
|
|
20
|
+
* @param {ResourceRequestEmit} params.emit - The resource request's declared emit configuration.
|
|
21
|
+
* @param {object} [params.parameters] - Key-value map used to resolve {:placeholder} tokens
|
|
22
|
+
* in the emit URL.
|
|
23
|
+
* @param {NamespaceMap} params.clients - The namespace map used to resolve the client for this emit.
|
|
24
|
+
*/
|
|
25
|
+
constructor({ id, item, emit, parameters, clients }) {
|
|
26
|
+
super({ id });
|
|
27
|
+
this.#item = item;
|
|
28
|
+
this.#emit = emit;
|
|
29
|
+
this.#parameters = parameters;
|
|
30
|
+
this.#clients = clients;
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
/**
|
|
34
|
+
* Returns the job-specific arguments for serialization.
|
|
35
|
+
* The URL is fully resolved with all parameter placeholders substituted.
|
|
36
|
+
* @returns {{ url: string, method: string }} The job arguments.
|
|
37
|
+
*/
|
|
38
|
+
get arguments() {
|
|
39
|
+
return { url: this.#emit.resolveUrl(this.#parameters), method: this.#emit.method };
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
/**
|
|
43
|
+
* Emits the extracted item to the configured external endpoint.
|
|
44
|
+
* @param {LogContext} logContext - Context carrying workerId/jobId for log entries.
|
|
45
|
+
* @returns {Promise<object>} A promise that resolves with the HTTP response.
|
|
46
|
+
*/
|
|
47
|
+
async perform(logContext) {
|
|
48
|
+
logContext.debug(`EmitJob #${this.id} performing`);
|
|
49
|
+
try {
|
|
50
|
+
this.lastError = undefined;
|
|
51
|
+
const url = this.#emit.resolveUrl(this.#parameters);
|
|
52
|
+
const response = await this.#getClient().emit(this.#emit.method, url, this.#item, this.#emit.status, logContext);
|
|
53
|
+
return response;
|
|
54
|
+
} catch (error) {
|
|
55
|
+
logContext.error(`EmitJob #${this.id} failed: ${error}`);
|
|
56
|
+
this._fail(error);
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
/**
|
|
61
|
+
* Gets the client associated with this job's emit configuration.
|
|
62
|
+
* @returns {Client} The client associated with this job's emit configuration.
|
|
63
|
+
* @private
|
|
64
|
+
*/
|
|
65
|
+
#getClient() {
|
|
66
|
+
return this.#clients.getClient('default', this.#emit.clientName, this.#emit.clientNamespace);
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
export { EmitJob };
|
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
import { Job } from 'deku-swarm';
|
|
2
|
+
import { EmitEnqueuer } from '../enqueuers/EmitEnqueuer.js';
|
|
3
|
+
|
|
4
|
+
/**
|
|
5
|
+
* ExtractionJob is a Job that extracts structured items from a raw response body
|
|
6
|
+
* using a parser implementation resolved from a ParserRegistry, and, when the
|
|
7
|
+
* resource declares an `emit`, enqueues one EmitJob per extracted item.
|
|
8
|
+
*
|
|
9
|
+
* Unlike ResourceRequestJob, this job is exhausted after the first failure — it has no
|
|
10
|
+
* retry rights, analogous to ActionProcessingJob/HtmlParseJob.
|
|
11
|
+
* @author darthjee
|
|
12
|
+
*/
|
|
13
|
+
class ExtractionJob extends Job {
|
|
14
|
+
#rawBody;
|
|
15
|
+
#parser;
|
|
16
|
+
#parserRegistry;
|
|
17
|
+
#jobRegistry;
|
|
18
|
+
#emit;
|
|
19
|
+
#parameters;
|
|
20
|
+
#originUrl;
|
|
21
|
+
|
|
22
|
+
/**
|
|
23
|
+
* Creates a new ExtractionJob instance.
|
|
24
|
+
* @param {object} params - The parameters for creating an ExtractionJob instance.
|
|
25
|
+
* @param {string} params.id - The unique identifier for this job.
|
|
26
|
+
* @param {string} params.rawBody - The raw HTTP response body to extract data from.
|
|
27
|
+
* @param {ResourceRequestParser} params.parser - The resource's declared parser rule.
|
|
28
|
+
* @param {object} params.parserRegistry - The registry used to resolve the parser implementation.
|
|
29
|
+
* @param {object} params.jobRegistry - The job registry used to enqueue EmitJobs.
|
|
30
|
+
* @param {ResourceRequestEmit} [params.emit] - The resource request's declared emit
|
|
31
|
+
* configuration. When absent, extracted items are logged but not emitted.
|
|
32
|
+
* @param {object} [params.parameters] - The original resource-request parameters, forwarded
|
|
33
|
+
* to each EmitJob for URL placeholder resolution.
|
|
34
|
+
* @param {string|null} [params.originUrl=null] - The URL of the ResourceRequestJob that triggered this job.
|
|
35
|
+
*/
|
|
36
|
+
constructor({ id, rawBody, parser, parserRegistry, jobRegistry, emit, parameters, originUrl = null }) {
|
|
37
|
+
super({ id });
|
|
38
|
+
this.#rawBody = rawBody;
|
|
39
|
+
this.#parser = parser;
|
|
40
|
+
this.#parserRegistry = parserRegistry;
|
|
41
|
+
this.#jobRegistry = jobRegistry;
|
|
42
|
+
this.#emit = emit;
|
|
43
|
+
this.#parameters = parameters;
|
|
44
|
+
this.#originUrl = originUrl;
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
/**
|
|
48
|
+
* Returns the job-specific arguments for serialization.
|
|
49
|
+
* @returns {{ parserType: string, originUrl?: string }} The job arguments.
|
|
50
|
+
*/
|
|
51
|
+
get arguments() {
|
|
52
|
+
return { parserType: this.#parser.type, ...this.#originUrlField() };
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
/**
|
|
56
|
+
* Returns the maximum number of retries for this job type.
|
|
57
|
+
* ExtractionJob is exhausted after the first failure.
|
|
58
|
+
* @returns {number} Always 1.
|
|
59
|
+
* @override
|
|
60
|
+
*/
|
|
61
|
+
get maxRetries() {
|
|
62
|
+
return 1;
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
/**
|
|
66
|
+
* Resolves the parser implementation and extracts items from the raw body, logging
|
|
67
|
+
* the resulting items via `logContext.debug`. When an `emit` configuration is present,
|
|
68
|
+
* delegates to EmitEnqueuer to enqueue one EmitJob per extracted item.
|
|
69
|
+
* @param {LogContext} logContext - Context carrying workerId/jobId for log entries.
|
|
70
|
+
* @returns {Promise<void>}
|
|
71
|
+
*/
|
|
72
|
+
async perform(logContext) {
|
|
73
|
+
logContext.debug(`ExtractionJob #${this.id} performing`);
|
|
74
|
+
try {
|
|
75
|
+
this.lastError = undefined;
|
|
76
|
+
const parserImpl = this.#parserRegistry.getItem(this.#parser.type);
|
|
77
|
+
const items = parserImpl.extract(this.#rawBody, this.#parser.attributes);
|
|
78
|
+
logContext.debug(`ExtractionJob #${this.id} extracted ${items.length} item(s)`, { items });
|
|
79
|
+
|
|
80
|
+
if (this.#emit) {
|
|
81
|
+
new EmitEnqueuer(items, this.#emit, this.#parameters, this.#jobRegistry).enqueue();
|
|
82
|
+
}
|
|
83
|
+
} catch (error) {
|
|
84
|
+
this._fail(error);
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
/**
|
|
89
|
+
* Returns an object containing the originUrl field when an origin URL is set,
|
|
90
|
+
* or an empty object otherwise.
|
|
91
|
+
* @returns {{ originUrl: string }|{}} The origin URL field or empty object.
|
|
92
|
+
* @private
|
|
93
|
+
*/
|
|
94
|
+
#originUrlField() {
|
|
95
|
+
return this.#originUrl !== null ? { originUrl: this.#originUrl } : {};
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
export { ExtractionJob };
|
|
@@ -61,6 +61,7 @@ class ResourceRequestJob extends Job {
|
|
|
61
61
|
#handleResponse(response) {
|
|
62
62
|
const originUrl = this.#resourceRequest.resolveUrl(this.#parameters);
|
|
63
63
|
this.#enqueueAssets(response, originUrl);
|
|
64
|
+
this.#enqueueExtraction(response, originUrl);
|
|
64
65
|
const wrapper = new ResponseWrapper(response, this.#parameters);
|
|
65
66
|
this.#resourceRequest.enqueueActions(wrapper, originUrl);
|
|
66
67
|
this.#resourceRequest.enqueuePaginatedActions(wrapper, this.#parameters, originUrl);
|
|
@@ -79,6 +80,18 @@ class ResourceRequestJob extends Job {
|
|
|
79
80
|
}
|
|
80
81
|
}
|
|
81
82
|
|
|
83
|
+
/**
|
|
84
|
+
* Enqueues an extraction job when the resource request declares a response-parsing rule.
|
|
85
|
+
* @param {object} response The HTTP response object.
|
|
86
|
+
* @param {string} originUrl The resolved URL of this job, forwarded as origin to child jobs.
|
|
87
|
+
* @private
|
|
88
|
+
*/
|
|
89
|
+
#enqueueExtraction(response, originUrl) {
|
|
90
|
+
if (this.#resourceRequest.hasParser()) {
|
|
91
|
+
this.#resourceRequest.enqueueExtraction(response.data, JobRegistry, this.#parameters, originUrl);
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
|
|
82
95
|
/**
|
|
83
96
|
* Gets the client associated with this job's resource request.
|
|
84
97
|
* @returns {Client} The client associated with this job's resource request.
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
import { InvalidMemoryThresholds } from '../../exceptions/config/InvalidMemoryThresholds.js';
|
|
2
|
+
import { MemoryMaximumResolver } from '../../utils/memory/MemoryMaximumResolver.js';
|
|
3
|
+
|
|
4
|
+
const DEFAULT_THRESHOLDS = { low: 25.0, medium: 50.0, high: 75.0, over: 100.0 };
|
|
5
|
+
const THRESHOLD_KEYS = ['low', 'medium', 'high', 'over'];
|
|
6
|
+
|
|
7
|
+
/**
|
|
8
|
+
* Represents the `web.memory` configuration: the resolved memory maximum and
|
|
9
|
+
* the percentage thresholds used to derive a status from it.
|
|
10
|
+
* @author darthjee
|
|
11
|
+
*/
|
|
12
|
+
class MemoryConfig {
|
|
13
|
+
#thresholds;
|
|
14
|
+
|
|
15
|
+
/**
|
|
16
|
+
* @param {object} [config={}] - Raw `web.memory` configuration object.
|
|
17
|
+
* @param {number} [config.maximum] - Configured memory maximum, in bytes.
|
|
18
|
+
* @param {object} [config.thresholds={}] - Percentage thresholds (`low`, `medium`, `high`, `over`).
|
|
19
|
+
* @param {object} [options={}] - Additional options.
|
|
20
|
+
* @param {{resolve: Function}} [options.resolver] - Resolver used to compute the memory maximum,
|
|
21
|
+
* defaulting to {@link MemoryMaximumResolver}.
|
|
22
|
+
* @throws {InvalidMemoryThresholds} When thresholds are not in strictly ascending order.
|
|
23
|
+
*/
|
|
24
|
+
constructor({ maximum, thresholds = {} } = {}, { resolver = MemoryMaximumResolver } = {}) {
|
|
25
|
+
this.#thresholds = { ...DEFAULT_THRESHOLDS, ...thresholds };
|
|
26
|
+
this.#validateThresholds();
|
|
27
|
+
|
|
28
|
+
this.maximum = resolver.resolve(maximum);
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
/**
|
|
32
|
+
* @returns {{low: number, medium: number, high: number, over: number}} The percentage thresholds.
|
|
33
|
+
*/
|
|
34
|
+
get thresholds() {
|
|
35
|
+
return this.#thresholds;
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
/**
|
|
39
|
+
* Derives a status label from a memory usage percentage, using inclusive
|
|
40
|
+
* (`>=`) boundaries checked from the top down.
|
|
41
|
+
* @param {number} percentage - The memory usage percentage.
|
|
42
|
+
* @returns {'low'|'medium'|'high'|'over'} The derived status.
|
|
43
|
+
*/
|
|
44
|
+
statusFor(percentage) {
|
|
45
|
+
if (percentage >= this.#thresholds.over) return 'over';
|
|
46
|
+
if (percentage >= this.#thresholds.high) return 'high';
|
|
47
|
+
if (percentage >= this.#thresholds.medium) return 'medium';
|
|
48
|
+
|
|
49
|
+
return 'low';
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
/**
|
|
53
|
+
* Validates that thresholds are in strictly ascending order (`low < medium < high < over`).
|
|
54
|
+
* @returns {void}
|
|
55
|
+
* @throws {InvalidMemoryThresholds} When thresholds are not in strictly ascending order.
|
|
56
|
+
* @private
|
|
57
|
+
*/
|
|
58
|
+
#validateThresholds() {
|
|
59
|
+
for (let i = 0; i < THRESHOLD_KEYS.length - 1; i++) {
|
|
60
|
+
const lowerKey = THRESHOLD_KEYS[i];
|
|
61
|
+
const higherKey = THRESHOLD_KEYS[i + 1];
|
|
62
|
+
|
|
63
|
+
if (!(this.#thresholds[lowerKey] < this.#thresholds[higherKey])) {
|
|
64
|
+
throw new InvalidMemoryThresholds(lowerKey, higherKey, this.#thresholds);
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
export { MemoryConfig };
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { Link } from './Link.js';
|
|
2
|
+
import { MemoryConfig } from './MemoryConfig.js';
|
|
2
3
|
|
|
3
4
|
/**
|
|
4
5
|
* Represents the web server configuration.
|
|
@@ -14,8 +15,9 @@ class WebConfig {
|
|
|
14
15
|
* @param {number} [config.idle_timeout=0] - Seconds of inactivity (no busy workers, no queued jobs) before the application auto-shuts-down. `0` disables auto-shutdown.
|
|
15
16
|
* @param {Array<string|object>} [config.links=[]] - Links shown in the web UI.
|
|
16
17
|
* @param {object} [config.api={}] - Configuration for the token-secured `/api` namespace (`{ token }`).
|
|
18
|
+
* @param {object} [config.memory={}] - Configuration for memory monitoring (`{ maximum, thresholds }`).
|
|
17
19
|
*/
|
|
18
|
-
constructor({ port, logs_page_size: logsPageSize = 20, enable_shutdown: enableShutdown = true, autostart = true, idle_timeout: idleTimeout = 0, links = [], api = {} }) {
|
|
20
|
+
constructor({ port, logs_page_size: logsPageSize = 20, enable_shutdown: enableShutdown = true, autostart = true, idle_timeout: idleTimeout = 0, links = [], api = {}, memory = {} }) {
|
|
19
21
|
this.port = port;
|
|
20
22
|
this.logsPageSize = logsPageSize;
|
|
21
23
|
this.enableShutdown = enableShutdown;
|
|
@@ -23,6 +25,7 @@ class WebConfig {
|
|
|
23
25
|
this.idleTimeout = idleTimeout;
|
|
24
26
|
this.links = links.map((link) => Link.fromObject(link));
|
|
25
27
|
this.apiToken = api?.token ?? null;
|
|
28
|
+
this.memory = new MemoryConfig(memory);
|
|
26
29
|
}
|
|
27
30
|
}
|
|
28
31
|
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* ClientReference parses the raw client reference shape shared by `ResourceRequest`'s
|
|
3
|
+
* top-level `client:` field and `ResourceRequestEmit`'s `emit.client` field, since both
|
|
4
|
+
* accept the exact same two forms: a bare name string, or a `{name, namespace}` object.
|
|
5
|
+
* @author darthjee
|
|
6
|
+
*/
|
|
7
|
+
class ClientReference {
|
|
8
|
+
/**
|
|
9
|
+
* Parses a raw client reference into a name/namespace pair.
|
|
10
|
+
* Accepts either a bare string (shorthand, resolved in the requester's own namespace)
|
|
11
|
+
* or an object with an explicit target `namespace`.
|
|
12
|
+
* @param {string|{name: string, namespace: string}} [client] The raw client reference.
|
|
13
|
+
* @returns {{name: string|undefined, namespace: string|null}} The parsed client name and target namespace.
|
|
14
|
+
*/
|
|
15
|
+
static parse(client) {
|
|
16
|
+
if (client && typeof client === 'object') {
|
|
17
|
+
return { name: client.name, namespace: client.namespace ?? null };
|
|
18
|
+
}
|
|
19
|
+
return { name: client, namespace: null };
|
|
20
|
+
}
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
export { ClientReference };
|
|
@@ -1,7 +1,10 @@
|
|
|
1
1
|
import { JobRegistry as DefaultJobRegistry } from 'deku-swarm';
|
|
2
2
|
import { AssetRequest } from './AssetRequest.js';
|
|
3
|
+
import { ClientReference } from './ClientReference.js';
|
|
3
4
|
import { ResourceRequestAction } from './ResourceRequestAction.js';
|
|
5
|
+
import { ResourceRequestEmit } from './ResourceRequestEmit.js';
|
|
4
6
|
import { ResourceRequestPaginatedAction } from './ResourceRequestPaginatedAction.js';
|
|
7
|
+
import { ResourceRequestParser } from './ResourceRequestParser.js';
|
|
5
8
|
import { ActionsEnqueuer } from '../../enqueuers/ActionsEnqueuer.js';
|
|
6
9
|
import { PaginatedActionsEnqueuer } from '../../enqueuers/PaginatedActionsEnqueuer.js';
|
|
7
10
|
import { LogRegistry } from '../../registry/LogRegistry.js';
|
|
@@ -28,6 +31,8 @@ class ResourceRequest {
|
|
|
28
31
|
* @param {Array} [attributes.actions=[]] List of raw action config objects.
|
|
29
32
|
* @param {Array} [attributes.assets=[]] List of raw asset extraction rule objects.
|
|
30
33
|
* @param {Array} [attributes.paginated_actions=[]] List of raw paginated action config objects.
|
|
34
|
+
* @param {object} [attributes.parser] Raw response-parsing rule config object.
|
|
35
|
+
* @param {object} [attributes.emit] Raw follow-up emit request config object.
|
|
31
36
|
* @param {boolean} [attributes.enabled] Whether this request is enabled. Defaults to enabled
|
|
32
37
|
* when omitted. Ignored when `disabled` is `true`.
|
|
33
38
|
* @param {boolean} [attributes.disabled] Whether this request is disabled. Takes precedence
|
|
@@ -44,6 +49,8 @@ class ResourceRequest {
|
|
|
44
49
|
actions = [],
|
|
45
50
|
assets = [],
|
|
46
51
|
paginated_actions = [],
|
|
52
|
+
parser,
|
|
53
|
+
emit,
|
|
47
54
|
enabled,
|
|
48
55
|
disabled,
|
|
49
56
|
max_page,
|
|
@@ -54,13 +61,15 @@ class ResourceRequest {
|
|
|
54
61
|
this.#disabled = disabled === true || enabled === false;
|
|
55
62
|
this.#maxPage = this.#sanitizeMaxPage(max_page);
|
|
56
63
|
|
|
57
|
-
const parsedClient =
|
|
64
|
+
const parsedClient = ClientReference.parse(clientName);
|
|
58
65
|
this.#clientName = parsedClient.name;
|
|
59
66
|
this.#clientNamespace = parsedClient.namespace;
|
|
60
67
|
|
|
61
68
|
this.actions = ResourceRequestAction.fromList(actions, { originNamespace: namespace });
|
|
62
69
|
this.assets = AssetRequest.fromListObject(assets);
|
|
63
70
|
this.paginatedActions = ResourceRequestPaginatedAction.fromList(paginated_actions, { originNamespace: namespace });
|
|
71
|
+
this.parser = parser ? ResourceRequestParser.fromObject(parser) : parser;
|
|
72
|
+
this.emit = emit ? ResourceRequestEmit.fromObject(emit) : emit;
|
|
64
73
|
}
|
|
65
74
|
|
|
66
75
|
/**
|
|
@@ -161,6 +170,40 @@ class ResourceRequest {
|
|
|
161
170
|
return this.assets.length > 0;
|
|
162
171
|
}
|
|
163
172
|
|
|
173
|
+
/**
|
|
174
|
+
* Enqueues one ExtractionJob for this resource request's declared parser.
|
|
175
|
+
* Does nothing if the application is in 'stopped' status.
|
|
176
|
+
* @param {string} rawBody The raw HTTP response body.
|
|
177
|
+
* @param {object} [jobRegistry=JobRegistry] The job registry used to enqueue the ExtractionJob.
|
|
178
|
+
* @param {object} [parameters] The original request parameters, forwarded to the ExtractionJob
|
|
179
|
+
* for downstream {:placeholder} resolution in any emitted EmitJobs.
|
|
180
|
+
* @param {string|null} [originUrl=null] The URL of the ResourceRequestJob that triggered this enqueue.
|
|
181
|
+
* @returns {void}
|
|
182
|
+
*/
|
|
183
|
+
enqueueExtraction(rawBody, jobRegistry = DefaultJobRegistry, parameters, originUrl = null) {
|
|
184
|
+
if (Application.isStopped()) return;
|
|
185
|
+
const params = { rawBody, parser: this.parser, parameters };
|
|
186
|
+
if (this.hasEmit()) params.emit = this.emit;
|
|
187
|
+
if (originUrl !== null) params.originUrl = originUrl;
|
|
188
|
+
jobRegistry.enqueue('Extraction', params);
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
/**
|
|
192
|
+
* Returns true when the resource request declares a response-parsing rule.
|
|
193
|
+
* @returns {boolean} True if a parser is configured.
|
|
194
|
+
*/
|
|
195
|
+
hasParser() {
|
|
196
|
+
return !!this.parser;
|
|
197
|
+
}
|
|
198
|
+
|
|
199
|
+
/**
|
|
200
|
+
* Returns true when the resource request declares a follow-up emit configuration.
|
|
201
|
+
* @returns {boolean} True if emit is configured.
|
|
202
|
+
*/
|
|
203
|
+
hasEmit() {
|
|
204
|
+
return !!this.emit;
|
|
205
|
+
}
|
|
206
|
+
|
|
164
207
|
/**
|
|
165
208
|
* Returns the URL with every {:placeholder} token replaced by the
|
|
166
209
|
* corresponding value from the parameters object.
|
|
@@ -194,20 +237,6 @@ class ResourceRequest {
|
|
|
194
237
|
return array.map((attrs) => new ResourceRequest({ ...attrs, clientName, namespace }));
|
|
195
238
|
}
|
|
196
239
|
|
|
197
|
-
/**
|
|
198
|
-
* Parses a raw client reference into a name/namespace pair.
|
|
199
|
-
* Accepts either a bare string (shorthand, resolved in the requester's own namespace)
|
|
200
|
-
* or an object with an explicit target `namespace`.
|
|
201
|
-
* @param {string|{name: string, namespace: string}} [client] The raw client reference.
|
|
202
|
-
* @returns {{name: string|undefined, namespace: string|null}} The parsed client name and target namespace.
|
|
203
|
-
*/
|
|
204
|
-
static #parseClient(client) {
|
|
205
|
-
if (client && typeof client === 'object') {
|
|
206
|
-
return { name: client.name, namespace: client.namespace ?? null };
|
|
207
|
-
}
|
|
208
|
-
return { name: client, namespace: null };
|
|
209
|
-
}
|
|
210
|
-
|
|
211
240
|
/**
|
|
212
241
|
* Sanitizes the raw `max_page` value into a positive integer or `null`.
|
|
213
242
|
* Logs a warning when a present value is invalid.
|