navi-hey 1.6.4 → 1.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/lib/exceptions/config/InvalidEmitMethod.js +16 -0
- package/lib/exceptions/config/InvalidMemoryThresholds.js +23 -0
- package/lib/exceptions/config/InvalidParserMatch.js +17 -0
- package/lib/exceptions/config/InvalidParserType.js +16 -0
- package/lib/exceptions/config/MissingEmitUrl.js +13 -0
- package/lib/exceptions/config/MissingParserField.js +13 -0
- package/lib/exceptions/config/MissingParserFields.js +13 -0
- package/lib/exceptions/config/MissingParserMatch.js +13 -0
- package/lib/exceptions/registry/ParserNotFound.js +16 -0
- package/lib/jobs/EmitJob.js +70 -0
- package/lib/jobs/ExtractionJob.js +82 -0
- package/lib/jobs/ResourceRequestJob.js +13 -0
- package/lib/models/configs/MemoryConfig.js +70 -0
- package/lib/models/configs/WebConfig.js +4 -1
- package/lib/models/request/ClientReference.js +23 -0
- package/lib/models/request/ResourceRequest.js +33 -15
- package/lib/models/request/ResourceRequestEmit.js +82 -0
- package/lib/models/request/ResourceRequestParser.js +40 -0
- package/lib/parsers/JsonPathParser.js +80 -0
- package/lib/parsers/RegexParser.js +36 -0
- package/lib/registry/ParserRegistry.js +32 -0
- package/lib/server/Router.js +2 -0
- package/lib/server/handlers/memory/MemoryStatusHandler.js +40 -0
- package/lib/services/ApplicationInstance.js +9 -0
- package/lib/services/Client.js +76 -3
- package/lib/services/NamespaceMapBuilder.js +16 -2
- package/lib/utils/memory/CgroupV1MemoryLimitReader.js +40 -0
- package/lib/utils/memory/CgroupV2MemoryLimitReader.js +38 -0
- package/lib/utils/memory/ConfigMemoryLimitReader.js +25 -0
- package/lib/utils/memory/MemoryMaximumResolver.js +55 -0
- package/lib/utils/memory/OsTotalMemoryReader.js +17 -0
- package/lib/utils/memory/ProcessRssReader.js +15 -0
- package/package.json +2 -2
- package/static/assets/index-DGgF9Hti.js +11 -0
- package/static/assets/{index-CuWjW0sT.css → index-Dcv_oCts.css} +1 -1
- package/static/index.html +2 -2
- package/static/assets/index-DyqqrESe.js +0 -11
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
import { InvalidParserType } from '../../exceptions/config/InvalidParserType.js';
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* The set of supported `parser.type` values.
|
|
5
|
+
* @type {Array<string>}
|
|
6
|
+
*/
|
|
7
|
+
const PARSER_TYPES = ['regex', 'json_path'];
|
|
8
|
+
|
|
9
|
+
/**
|
|
10
|
+
* ResourceRequestParser represents a resource request's declared response-parsing rule.
|
|
11
|
+
* `type` is validated eagerly at construction time; every other key present (e.g. `match`,
|
|
12
|
+
* `filter`, `fields`, `field`) is stored as-is, unvalidated, since interpreting them belongs
|
|
13
|
+
* to the parser-specific implementations (`regex`, `json_path`).
|
|
14
|
+
* @author darthjee
|
|
15
|
+
*/
|
|
16
|
+
class ResourceRequestParser {
|
|
17
|
+
/**
|
|
18
|
+
* @param {object} attributes ResourceRequestParser attributes.
|
|
19
|
+
* @param {string} attributes.type The parser type. Must be one of "regex", "json_path".
|
|
20
|
+
* @param {*} [attributes.rest] Any other parser-type-specific keys (e.g. `match`, `filter`,
|
|
21
|
+
* `fields`, `field`), stored as-is and interpreted by the parser-specific implementations.
|
|
22
|
+
*/
|
|
23
|
+
constructor({ type, ...rest }) {
|
|
24
|
+
if (!PARSER_TYPES.includes(type)) throw new InvalidParserType(type);
|
|
25
|
+
|
|
26
|
+
this.type = type;
|
|
27
|
+
this.attributes = rest;
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
/**
|
|
31
|
+
* Creates a ResourceRequestParser instance from a plain config object.
|
|
32
|
+
* @param {object} obj Raw config object.
|
|
33
|
+
* @returns {ResourceRequestParser} A new ResourceRequestParser instance.
|
|
34
|
+
*/
|
|
35
|
+
static fromObject(obj) {
|
|
36
|
+
return new ResourceRequestParser(obj);
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
export { ResourceRequestParser, PARSER_TYPES };
|
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
import { InvalidParserMatch } from '../exceptions/config/InvalidParserMatch.js';
|
|
2
|
+
import { MissingParserFields } from '../exceptions/config/MissingParserFields.js';
|
|
3
|
+
import { MissingParserMatch } from '../exceptions/config/MissingParserMatch.js';
|
|
4
|
+
|
|
5
|
+
/**
|
|
6
|
+
* JsonPathParser extracts a list of mapped items from a raw JSON response body,
|
|
7
|
+
* navigating to an array via a dot-notation `match` path, optionally filtering
|
|
8
|
+
* items, and remapping each item's keys according to a `fields` declaration in
|
|
9
|
+
* a resource request's `parser` config.
|
|
10
|
+
* @author darthjee
|
|
11
|
+
*/
|
|
12
|
+
class JsonPathParser {
|
|
13
|
+
/**
|
|
14
|
+
* Extracts data from the given raw JSON body using the path declared in
|
|
15
|
+
* `attributes.match`, optionally filtering items via `attributes.filter`,
|
|
16
|
+
* and remapping fields via `attributes.fields`.
|
|
17
|
+
* @param {string} rawBody The raw response body to extract data from (parsed as JSON).
|
|
18
|
+
* @param {object} attributes The parser attributes.
|
|
19
|
+
* @param {string} attributes.match The dot-notation path (e.g. `data.items`) to the array
|
|
20
|
+
* to extract items from, resolved against the parsed body.
|
|
21
|
+
* @param {Array<object>} [attributes.filter] Optional list of AND'ed conditions an item
|
|
22
|
+
* must satisfy to be included. Each condition is either `{ field, equals }` (literal
|
|
23
|
+
* comparison) or `{ field, equals_field }` (field-to-field comparison, both fields read
|
|
24
|
+
* from the same item).
|
|
25
|
+
* @param {object} attributes.fields A `{ sourceKey: outputKey }` mapping used to build
|
|
26
|
+
* each output item from the matched item's keys.
|
|
27
|
+
* @returns {Array<object>} An array of `{ [outputKey]: value, ... }` items, one per matched
|
|
28
|
+
* item that passes `filter` (or all matched items when `filter` is absent). An empty array
|
|
29
|
+
* when no items match or none pass the filter.
|
|
30
|
+
* @throws {MissingParserMatch} If `attributes.match` is absent.
|
|
31
|
+
* @throws {MissingParserFields} If `attributes.fields` is absent.
|
|
32
|
+
* @throws {InvalidParserMatch} If `attributes.match` does not resolve to an array within
|
|
33
|
+
* the parsed body.
|
|
34
|
+
*/
|
|
35
|
+
extract(rawBody, { match, filter, fields } = {}) {
|
|
36
|
+
if (!match) throw new MissingParserMatch();
|
|
37
|
+
if (!fields) throw new MissingParserFields();
|
|
38
|
+
|
|
39
|
+
const parsedBody = JSON.parse(rawBody);
|
|
40
|
+
const items = this.#resolveMatch(parsedBody, match);
|
|
41
|
+
|
|
42
|
+
return items
|
|
43
|
+
.filter((item) => this.#matchesFilter(item, filter))
|
|
44
|
+
.map((item) => this.#mapFields(item, fields));
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
#resolveMatch(parsedBody, match) {
|
|
48
|
+
const resolved = match.split('.').reduce((value, key) => {
|
|
49
|
+
if (value === undefined || value === null) return undefined;
|
|
50
|
+
|
|
51
|
+
return value[key];
|
|
52
|
+
}, parsedBody);
|
|
53
|
+
|
|
54
|
+
if (!Array.isArray(resolved)) throw new InvalidParserMatch(match);
|
|
55
|
+
|
|
56
|
+
return resolved;
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
#matchesFilter(item, filter) {
|
|
60
|
+
if (!filter) return true;
|
|
61
|
+
|
|
62
|
+
return filter.every((condition) => this.#matchesCondition(item, condition));
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
#matchesCondition(item, { field, equals, equals_field: equalsField }) {
|
|
66
|
+
if (equalsField !== undefined) return item[field] === item[equalsField];
|
|
67
|
+
|
|
68
|
+
return item[field] === equals;
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
#mapFields(item, fields) {
|
|
72
|
+
return Object.entries(fields).reduce((mapped, [sourceKey, outputKey]) => {
|
|
73
|
+
mapped[outputKey] = item[sourceKey];
|
|
74
|
+
|
|
75
|
+
return mapped;
|
|
76
|
+
}, {});
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
export { JsonPathParser };
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
import { MissingParserField } from '../exceptions/config/MissingParserField.js';
|
|
2
|
+
import { MissingParserMatch } from '../exceptions/config/MissingParserMatch.js';
|
|
3
|
+
|
|
4
|
+
/**
|
|
5
|
+
* RegexParser extracts a single field from a raw response body using a regular
|
|
6
|
+
* expression declared in a resource request's `parser` config.
|
|
7
|
+
* @author darthjee
|
|
8
|
+
*/
|
|
9
|
+
class RegexParser {
|
|
10
|
+
/**
|
|
11
|
+
* Extracts data from the given raw body using the regex declared in `attributes.match`.
|
|
12
|
+
* @param {string} rawBody The raw response body to extract data from.
|
|
13
|
+
* @param {object} attributes The parser attributes.
|
|
14
|
+
* @param {string} attributes.match The regular expression pattern to match against `rawBody`.
|
|
15
|
+
* @param {string} attributes.field The name of the field to populate with the captured value.
|
|
16
|
+
* @returns {Array<object>} An array with a single `{ [field]: value }` item when the pattern
|
|
17
|
+
* matches, or an empty array when it does not.
|
|
18
|
+
* @throws {MissingParserMatch} If `attributes.match` is absent.
|
|
19
|
+
* @throws {MissingParserField} If `attributes.field` is absent.
|
|
20
|
+
*/
|
|
21
|
+
extract(rawBody, { match, field } = {}) {
|
|
22
|
+
if (!match) throw new MissingParserMatch();
|
|
23
|
+
if (!field) throw new MissingParserField();
|
|
24
|
+
|
|
25
|
+
const regex = new RegExp(match);
|
|
26
|
+
const result = regex.exec(rawBody);
|
|
27
|
+
|
|
28
|
+
if (!result) return [];
|
|
29
|
+
|
|
30
|
+
const value = result.length > 1 ? result[1] : result[0];
|
|
31
|
+
|
|
32
|
+
return [{ [field]: value }];
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
export { RegexParser };
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
import { NamedRegistry } from './NamedRegistry.js';
|
|
2
|
+
import { ParserNotFound } from '../exceptions/registry/ParserNotFound.js';
|
|
3
|
+
|
|
4
|
+
/**
|
|
5
|
+
* ParserRegistry manages parser implementation retrieval logic, mapping a
|
|
6
|
+
* `parser.type` string (e.g. `"regex"`) to a parser implementation instance.
|
|
7
|
+
*
|
|
8
|
+
* A plain `NamedRegistry` subclass, extendable with new parser types (e.g. `json_path`)
|
|
9
|
+
* without changing `ExtractionJob` itself.
|
|
10
|
+
*
|
|
11
|
+
* @example
|
|
12
|
+
* const parserRegistry = new ParserRegistry({
|
|
13
|
+
* regex: new RegexParser(),
|
|
14
|
+
* });
|
|
15
|
+
*
|
|
16
|
+
* parserRegistry.getItem('regex'); // returns the RegexParser instance
|
|
17
|
+
* parserRegistry.getItem('json_path'); // throws ParserNotFound error
|
|
18
|
+
*
|
|
19
|
+
* @author darthjee
|
|
20
|
+
*/
|
|
21
|
+
class ParserRegistry extends NamedRegistry {
|
|
22
|
+
/**
|
|
23
|
+
* The exception class to throw when a parser is not found.
|
|
24
|
+
* @see ParserNotFound
|
|
25
|
+
* @type {class}
|
|
26
|
+
* @see NamedRegistry#notFound
|
|
27
|
+
* @see NamedRegistry#notFoundException
|
|
28
|
+
*/
|
|
29
|
+
static notFoundException = ParserNotFound;
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
export { ParserRegistry };
|
package/lib/server/Router.js
CHANGED
|
@@ -21,6 +21,7 @@ import { JobRetryHandler } from './handlers/jobs/JobRetryHandler.js';
|
|
|
21
21
|
import { JobsHandler } from './handlers/jobs/JobsHandler.js';
|
|
22
22
|
import { LinksHandler } from './handlers/LinksHandler.js';
|
|
23
23
|
import { LogsHandler } from './handlers/LogsHandler.js';
|
|
24
|
+
import { MemoryStatusHandler } from './handlers/memory/MemoryStatusHandler.js';
|
|
24
25
|
import { SettingsHandler } from './handlers/SettingsHandler.js';
|
|
25
26
|
import { StatsHandler } from './handlers/StatsHandler.js';
|
|
26
27
|
import { PathValidator } from './PathValidator.js';
|
|
@@ -65,6 +66,7 @@ class Router {
|
|
|
65
66
|
'/jobs/:job_id/logs.json': new HandlerConfig(JobLogsHandler, this.#webConfig.logsPageSize),
|
|
66
67
|
'/job/:id.json': new HandlerConfig(JobHandler),
|
|
67
68
|
'/engine/status': new HandlerConfig(EngineStatusHandler),
|
|
69
|
+
'/memory/status.json': new HandlerConfig(MemoryStatusHandler, [this.#webConfig.memory]),
|
|
68
70
|
'/logs.json': new HandlerConfig(LogsHandler, this.#webConfig.logsPageSize),
|
|
69
71
|
'/links.json': new HandlerConfig(LinksHandler, [this.#webConfig.links]),
|
|
70
72
|
'/': new HandlerConfig(IndexHandler),
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
import { RequestHandler } from '../../../common/server/RequestHandler.js';
|
|
2
|
+
import { ProcessRssReader } from '../../../utils/memory/ProcessRssReader.js';
|
|
3
|
+
|
|
4
|
+
/**
|
|
5
|
+
* Executes request-handling behaviour for GET /memory/status.json.
|
|
6
|
+
* @author darthjee
|
|
7
|
+
*/
|
|
8
|
+
class MemoryStatusHandler extends RequestHandler {
|
|
9
|
+
#response;
|
|
10
|
+
#memoryConfig;
|
|
11
|
+
#rssReader;
|
|
12
|
+
|
|
13
|
+
/**
|
|
14
|
+
* @param {object} _request - The Express request object.
|
|
15
|
+
* @param {object} response - The Express response object.
|
|
16
|
+
* @param {MemoryConfig} memoryConfig - The resolved `web.memory` configuration.
|
|
17
|
+
* @param {ProcessRssReader} [rssReader] - Reader used to obtain the current process RSS.
|
|
18
|
+
*/
|
|
19
|
+
constructor(_request, response, memoryConfig, rssReader = new ProcessRssReader()) {
|
|
20
|
+
super();
|
|
21
|
+
this.#response = response;
|
|
22
|
+
this.#memoryConfig = memoryConfig;
|
|
23
|
+
this.#rssReader = rssReader;
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
/**
|
|
27
|
+
* Responds with the current process memory usage against the resolved maximum.
|
|
28
|
+
* @returns {void}
|
|
29
|
+
*/
|
|
30
|
+
handle() {
|
|
31
|
+
const current = this.#rssReader.read();
|
|
32
|
+
const maximum = this.#memoryConfig.maximum;
|
|
33
|
+
const percentage = (current / maximum) * 100;
|
|
34
|
+
const status = this.#memoryConfig.statusFor(percentage);
|
|
35
|
+
|
|
36
|
+
this.#response.json({ current, maximum, percentage, status });
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
export { MemoryStatusHandler };
|
|
@@ -6,12 +6,17 @@ import { RunSummary } from './RunSummary.js';
|
|
|
6
6
|
import { ConfigurationFileNotProvided } from '../exceptions/config/ConfigurationFileNotProvided.js';
|
|
7
7
|
import { ActionProcessingJob } from '../jobs/ActionProcessingJob.js';
|
|
8
8
|
import { AssetDownloadJob } from '../jobs/AssetDownloadJob.js';
|
|
9
|
+
import { EmitJob } from '../jobs/EmitJob.js';
|
|
10
|
+
import { ExtractionJob } from '../jobs/ExtractionJob.js';
|
|
9
11
|
import { HtmlParseJob } from '../jobs/HtmlParseJob.js';
|
|
10
12
|
import { PaginatedActionProcessingJob } from '../jobs/PaginatedActionProcessingJob.js';
|
|
11
13
|
import { ResourceRequestJob } from '../jobs/ResourceRequestJob.js';
|
|
12
14
|
import { Config } from '../models/configs/Config.js';
|
|
15
|
+
import { JsonPathParser } from '../parsers/JsonPathParser.js';
|
|
16
|
+
import { RegexParser } from '../parsers/RegexParser.js';
|
|
13
17
|
import { LogRegistry } from '../registry/LogRegistry.js';
|
|
14
18
|
import { NamespaceMap } from '../registry/NamespaceMap.js';
|
|
19
|
+
import { ParserRegistry } from '../registry/ParserRegistry.js';
|
|
15
20
|
import { WebServer } from '../server/WebServer.js';
|
|
16
21
|
import { LogContext } from '../utils/logging/LogContext.js';
|
|
17
22
|
import { PromiseAggregator } from '../utils/PromiseAggregator.js';
|
|
@@ -325,6 +330,10 @@ class ApplicationInstance {
|
|
|
325
330
|
JobFactory.build('HtmlParse', { klass: HtmlParseJob, attributes: { jobRegistry: JobRegistry, clientRegistry: this.config.namespaceMap } });
|
|
326
331
|
JobFactory.build('AssetDownload', { klass: AssetDownloadJob, attributes: { clientRegistry: this.config.namespaceMap } });
|
|
327
332
|
|
|
333
|
+
const parserRegistry = new ParserRegistry({ regex: new RegexParser(), json_path: new JsonPathParser() });
|
|
334
|
+
JobFactory.build('Extraction', { klass: ExtractionJob, attributes: { parserRegistry } });
|
|
335
|
+
JobFactory.build('Emit', { klass: EmitJob, attributes: { clients: this.config.namespaceMap } });
|
|
336
|
+
|
|
328
337
|
JobRegistry.build({ cooldown: this.config.workersConfig.retryCooldown, maxRetries: this.config.workersConfig.maxRetries });
|
|
329
338
|
|
|
330
339
|
const loggerFactory = ({ workerId, jobId }) => new LogContext({ workerId, jobId });
|
package/lib/services/Client.js
CHANGED
|
@@ -96,6 +96,27 @@ class Client {
|
|
|
96
96
|
}
|
|
97
97
|
}
|
|
98
98
|
|
|
99
|
+
/**
|
|
100
|
+
* Performs a POST/PUT/PATCH request with a JSON body and validates the response status.
|
|
101
|
+
* @param {string} method The HTTP method to use. Must be one of "POST", "PUT", "PATCH".
|
|
102
|
+
* @param {string} resourceUrl URL path from the resource request, appended to `baseUrl`.
|
|
103
|
+
* @param {object} body The JSON body to send with the request.
|
|
104
|
+
* @param {number} [expectedStatus] The expected HTTP response status code. When omitted,
|
|
105
|
+
* any 2xx status is treated as success.
|
|
106
|
+
* @param {LogContext} logContext Context carrying workerId/jobId for log entries.
|
|
107
|
+
* @returns {Promise<object>} The Axios response object.
|
|
108
|
+
* @throws {RequestFailed} Throws an error if the request fails or the status does not match.
|
|
109
|
+
*/
|
|
110
|
+
async emit(method, resourceUrl, body, expectedStatus, logContext) {
|
|
111
|
+
const requestUrl = this.#buildUrl(resourceUrl);
|
|
112
|
+
logContext.info(`[Client:${this.name}] Emitting ${method} ${requestUrl}`);
|
|
113
|
+
try {
|
|
114
|
+
return await this.#emitRequest(method, requestUrl, body, expectedStatus, logContext);
|
|
115
|
+
} catch (error) {
|
|
116
|
+
this.#handleError(error, requestUrl, logContext);
|
|
117
|
+
}
|
|
118
|
+
}
|
|
119
|
+
|
|
99
120
|
/**
|
|
100
121
|
* Performs the HTTP request and checks the response status.
|
|
101
122
|
* @param {ResourceRequest} resourceRequest Information about the URL path to request
|
|
@@ -128,12 +149,64 @@ class Client {
|
|
|
128
149
|
|
|
129
150
|
logContext.info(`[Client:${this.name}] Response ${requestUrl} → ${response.status}`);
|
|
130
151
|
|
|
131
|
-
|
|
132
|
-
|
|
152
|
+
return this.#handleResponse(response, expectedStatus, requestUrl, logContext);
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
/**
|
|
156
|
+
* Dispatches a POST/PUT/PATCH request with a JSON body to the given URL and validates
|
|
157
|
+
* the response status.
|
|
158
|
+
* @param {string} method The HTTP method to use. Must be one of "POST", "PUT", "PATCH".
|
|
159
|
+
* @param {string} requestUrl The full URL to request.
|
|
160
|
+
* @param {object} body The JSON body to send with the request.
|
|
161
|
+
* @param {number} [expectedStatus] The expected HTTP response status code. When omitted,
|
|
162
|
+
* any 2xx status is treated as success.
|
|
163
|
+
* @param {LogContext} logContext Context carrying workerId/jobId for log entries.
|
|
164
|
+
* @returns {Promise<object>} The Axios response object.
|
|
165
|
+
* @throws {RequestFailed} Throws an error if the response status does not match.
|
|
166
|
+
*/
|
|
167
|
+
async #emitRequest(method, requestUrl, body, expectedStatus, logContext) {
|
|
168
|
+
const options = { timeout: this.timeout, headers: this.headers, validateStatus: () => true };
|
|
169
|
+
|
|
170
|
+
let response;
|
|
171
|
+
switch (method) {
|
|
172
|
+
case 'POST':
|
|
173
|
+
response = await axios.post(requestUrl, body, options);
|
|
174
|
+
break;
|
|
175
|
+
case 'PUT':
|
|
176
|
+
response = await axios.put(requestUrl, body, options);
|
|
177
|
+
break;
|
|
178
|
+
case 'PATCH':
|
|
179
|
+
response = await axios.patch(requestUrl, body, options);
|
|
180
|
+
break;
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
logContext.info(`[Client:${this.name}] Response ${requestUrl} → ${response.status}`);
|
|
184
|
+
|
|
185
|
+
return this.#handleResponse(response, expectedStatus, requestUrl, logContext);
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
/**
|
|
189
|
+
* Validates a response's status against the expected status, throwing when it does not match.
|
|
190
|
+
* When `expectedStatus` is `undefined`/`null`, any 2xx status is treated as success; otherwise
|
|
191
|
+
* the response status must exactly match `expectedStatus`.
|
|
192
|
+
* @param {object} response The Axios response object.
|
|
193
|
+
* @param {number} [expectedStatus] The expected HTTP response status code.
|
|
194
|
+
* @param {string} requestUrl The full URL that was requested.
|
|
195
|
+
* @param {LogContext} logContext Context carrying workerId/jobId for log entries.
|
|
196
|
+
* @returns {object} The given response, when it matches.
|
|
197
|
+
* @throws {RequestFailed} Throws an error if the response status does not match.
|
|
198
|
+
*/
|
|
199
|
+
#handleResponse(response, expectedStatus, requestUrl, logContext) {
|
|
200
|
+
const matches = expectedStatus === undefined || expectedStatus === null
|
|
201
|
+
? response.status >= 200 && response.status < 300
|
|
202
|
+
: response.status === expectedStatus;
|
|
203
|
+
|
|
204
|
+
if (!matches) {
|
|
205
|
+
logContext.info(`[Client:${this.name}] ${requestUrl} did not match (got ${response.status}, expected ${expectedStatus ?? '2xx'})`);
|
|
133
206
|
throw new RequestFailed(response.status, requestUrl);
|
|
134
207
|
}
|
|
135
208
|
|
|
136
|
-
logContext.info(`[Client:${this.name}] ${requestUrl} matched (expected ${expectedStatus})`);
|
|
209
|
+
logContext.info(`[Client:${this.name}] ${requestUrl} matched (expected ${expectedStatus ?? '2xx'})`);
|
|
137
210
|
return response;
|
|
138
211
|
}
|
|
139
212
|
|
|
@@ -138,8 +138,8 @@ class NamespaceMapBuilder {
|
|
|
138
138
|
|
|
139
139
|
/**
|
|
140
140
|
* Walks every resource in every namespace and validates that its declared client
|
|
141
|
-
* reference, action targets, and paginated_action targets
|
|
142
|
-
* namespace map, raising immediately on the first unresolvable reference.
|
|
141
|
+
* reference, emit.client reference, action targets, and paginated_action targets
|
|
142
|
+
* all resolve via the namespace map, raising immediately on the first unresolvable reference.
|
|
143
143
|
* @param {Record<string, Namespace>} namespaces A map of namespace name to Namespace instance.
|
|
144
144
|
* @returns {void}
|
|
145
145
|
* @private
|
|
@@ -151,6 +151,7 @@ class NamespaceMapBuilder {
|
|
|
151
151
|
namespace.resourceRegistry.filter(() => true).forEach((resource) => {
|
|
152
152
|
resource.resourceRequests.forEach((resourceRequest) => {
|
|
153
153
|
this.#validateClient(resourceRequest, namespaceMap);
|
|
154
|
+
this.#validateEmitClient(resourceRequest, namespaceMap);
|
|
154
155
|
this.#validateActions(resourceRequest, namespaceMap);
|
|
155
156
|
});
|
|
156
157
|
});
|
|
@@ -170,6 +171,19 @@ class NamespaceMapBuilder {
|
|
|
170
171
|
namespaceMap.getClient(resourceRequest.namespace, resourceRequest.clientName, resourceRequest.clientNamespace);
|
|
171
172
|
}
|
|
172
173
|
|
|
174
|
+
/**
|
|
175
|
+
* Validates a ResourceRequest's declared `emit.client` reference, when `emit` is present.
|
|
176
|
+
* @param {ResourceRequest} resourceRequest The resource request to validate.
|
|
177
|
+
* @param {NamespaceMap} namespaceMap The namespace map used to resolve the reference.
|
|
178
|
+
* @returns {void}
|
|
179
|
+
* @private
|
|
180
|
+
*/
|
|
181
|
+
#validateEmitClient(resourceRequest, namespaceMap) {
|
|
182
|
+
if (!resourceRequest.emit) return;
|
|
183
|
+
|
|
184
|
+
namespaceMap.getClient(resourceRequest.namespace, resourceRequest.emit.clientName, resourceRequest.emit.clientNamespace);
|
|
185
|
+
}
|
|
186
|
+
|
|
173
187
|
/**
|
|
174
188
|
* Validates every action/paginated_action target resource declared on a ResourceRequest.
|
|
175
189
|
* @param {ResourceRequest} resourceRequest The resource request to validate.
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
import fs from 'node:fs';
|
|
2
|
+
|
|
3
|
+
const CGROUP_V1_MEMORY_LIMIT_PATH = '/sys/fs/cgroup/memory/memory.limit_in_bytes';
|
|
4
|
+
const UNBOUNDED = 9223372036854771712;
|
|
5
|
+
|
|
6
|
+
/**
|
|
7
|
+
* Reads the cgroup v1 memory limit (`memory.limit_in_bytes`), returning `null`
|
|
8
|
+
* when the file is missing/unreadable (e.g. non-Linux dev machines, bare
|
|
9
|
+
* hosts, or hosts not running under cgroup v1) or reports the kernel's
|
|
10
|
+
* "unbounded" sentinel value.
|
|
11
|
+
* @author darthjee
|
|
12
|
+
*/
|
|
13
|
+
class CgroupV1MemoryLimitReader {
|
|
14
|
+
/**
|
|
15
|
+
* @returns {number|null} The cgroup v1 memory limit in bytes, or `null` when unavailable/unbounded.
|
|
16
|
+
*/
|
|
17
|
+
read() {
|
|
18
|
+
const content = this.#readFile();
|
|
19
|
+
|
|
20
|
+
if (content === null) return null;
|
|
21
|
+
|
|
22
|
+
const value = Number(content);
|
|
23
|
+
|
|
24
|
+
return value === UNBOUNDED ? null : value;
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
/**
|
|
28
|
+
* @returns {string|null} The trimmed file content, or `null` when it cannot be read.
|
|
29
|
+
* @private
|
|
30
|
+
*/
|
|
31
|
+
#readFile() {
|
|
32
|
+
try {
|
|
33
|
+
return fs.readFileSync(CGROUP_V1_MEMORY_LIMIT_PATH, 'utf8').trim();
|
|
34
|
+
} catch {
|
|
35
|
+
return null;
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
export { CgroupV1MemoryLimitReader };
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
import fs from 'node:fs';
|
|
2
|
+
|
|
3
|
+
const CGROUP_V2_MEMORY_MAX_PATH = '/sys/fs/cgroup/memory.max';
|
|
4
|
+
const UNBOUNDED = 'max';
|
|
5
|
+
|
|
6
|
+
/**
|
|
7
|
+
* Reads the cgroup v2 memory limit (`memory.max`), returning `null` when the
|
|
8
|
+
* file is missing/unreadable (e.g. non-Linux dev machines, bare hosts, or
|
|
9
|
+
* hosts not running under cgroup v2) or reports the literal `"max"`, cgroup
|
|
10
|
+
* v2's own way of expressing "unbounded".
|
|
11
|
+
* @author darthjee
|
|
12
|
+
*/
|
|
13
|
+
class CgroupV2MemoryLimitReader {
|
|
14
|
+
/**
|
|
15
|
+
* @returns {number|null} The cgroup v2 memory limit in bytes, or `null` when unavailable/unbounded.
|
|
16
|
+
*/
|
|
17
|
+
read() {
|
|
18
|
+
const content = this.#readFile();
|
|
19
|
+
|
|
20
|
+
if (content === null || content === UNBOUNDED) return null;
|
|
21
|
+
|
|
22
|
+
return Number(content);
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
/**
|
|
26
|
+
* @returns {string|null} The trimmed file content, or `null` when it cannot be read.
|
|
27
|
+
* @private
|
|
28
|
+
*/
|
|
29
|
+
#readFile() {
|
|
30
|
+
try {
|
|
31
|
+
return fs.readFileSync(CGROUP_V2_MEMORY_MAX_PATH, 'utf8').trim();
|
|
32
|
+
} catch {
|
|
33
|
+
return null;
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
export { CgroupV2MemoryLimitReader };
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Wraps the raw `web.memory.maximum` value coming from YAML config, exposing it
|
|
3
|
+
* through the common reader contract (`read()`) shared with the other memory
|
|
4
|
+
* limit readers consumed by {@link MemoryMaximumResolver}.
|
|
5
|
+
* @author darthjee
|
|
6
|
+
*/
|
|
7
|
+
class ConfigMemoryLimitReader {
|
|
8
|
+
#maximum;
|
|
9
|
+
|
|
10
|
+
/**
|
|
11
|
+
* @param {number} [maximum] - The raw `web.memory.maximum` value from config.
|
|
12
|
+
*/
|
|
13
|
+
constructor(maximum) {
|
|
14
|
+
this.#maximum = maximum;
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
/**
|
|
18
|
+
* @returns {number|null} The configured maximum, or `null` when it was not set.
|
|
19
|
+
*/
|
|
20
|
+
read() {
|
|
21
|
+
return this.#maximum ?? null;
|
|
22
|
+
}
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
export { ConfigMemoryLimitReader };
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
import { CgroupV1MemoryLimitReader } from './CgroupV1MemoryLimitReader.js';
|
|
2
|
+
import { CgroupV2MemoryLimitReader } from './CgroupV2MemoryLimitReader.js';
|
|
3
|
+
import { ConfigMemoryLimitReader } from './ConfigMemoryLimitReader.js';
|
|
4
|
+
import { OsTotalMemoryReader } from './OsTotalMemoryReader.js';
|
|
5
|
+
|
|
6
|
+
/**
|
|
7
|
+
* Resolves the effective memory maximum via a fallback chain: configured
|
|
8
|
+
* value → cgroup v2 limit → cgroup v1 limit → OS total memory. Each reader
|
|
9
|
+
* exposes a `read()` method returning a byte count or `null` when that source
|
|
10
|
+
* has no limit to offer; the first non-`null` result wins.
|
|
11
|
+
* @author darthjee
|
|
12
|
+
*/
|
|
13
|
+
class MemoryMaximumResolver {
|
|
14
|
+
#readers;
|
|
15
|
+
|
|
16
|
+
/**
|
|
17
|
+
* @param {number} [maximum] - The configured `web.memory.maximum` value, if any.
|
|
18
|
+
* @param {object} [options={}] - Additional options.
|
|
19
|
+
* @param {Array<{read: Function}>} [options.readers] - Ordered list of readers to consult,
|
|
20
|
+
* defaulting to the config → cgroup v2 → cgroup v1 → OS total memory chain.
|
|
21
|
+
*/
|
|
22
|
+
constructor(maximum, { readers } = {}) {
|
|
23
|
+
this.#readers = readers ?? [
|
|
24
|
+
new ConfigMemoryLimitReader(maximum),
|
|
25
|
+
new CgroupV2MemoryLimitReader(),
|
|
26
|
+
new CgroupV1MemoryLimitReader(),
|
|
27
|
+
new OsTotalMemoryReader(),
|
|
28
|
+
];
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
/**
|
|
32
|
+
* Resolves the memory maximum for the given configured value.
|
|
33
|
+
* @param {number} [maximum] - The configured `web.memory.maximum` value, if any.
|
|
34
|
+
* @returns {number|null} The resolved memory maximum, in bytes.
|
|
35
|
+
*/
|
|
36
|
+
static resolve(maximum) {
|
|
37
|
+
return new MemoryMaximumResolver(maximum).resolve();
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
/**
|
|
41
|
+
* Consults each reader in order, returning the first non-`null` result.
|
|
42
|
+
* @returns {number|null} The resolved memory maximum, in bytes.
|
|
43
|
+
*/
|
|
44
|
+
resolve() {
|
|
45
|
+
for (const reader of this.#readers) {
|
|
46
|
+
const value = reader.read();
|
|
47
|
+
|
|
48
|
+
if (value !== null) return value;
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
return null;
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
export { MemoryMaximumResolver };
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
import os from 'node:os';
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Wraps Node's `os.totalmem()`, the last resort in the memory-maximum
|
|
5
|
+
* resolution chain — it always returns a number, never `null`.
|
|
6
|
+
* @author darthjee
|
|
7
|
+
*/
|
|
8
|
+
class OsTotalMemoryReader {
|
|
9
|
+
/**
|
|
10
|
+
* @returns {number} The total system memory, in bytes.
|
|
11
|
+
*/
|
|
12
|
+
read() {
|
|
13
|
+
return os.totalmem();
|
|
14
|
+
}
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
export { OsTotalMemoryReader };
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Wraps `process.memoryUsage().rss`, so consumers reading the current process
|
|
3
|
+
* resident set size never reach for `process` directly.
|
|
4
|
+
* @author darthjee
|
|
5
|
+
*/
|
|
6
|
+
class ProcessRssReader {
|
|
7
|
+
/**
|
|
8
|
+
* @returns {number} The current process resident set size, in bytes.
|
|
9
|
+
*/
|
|
10
|
+
read() {
|
|
11
|
+
return process.memoryUsage().rss;
|
|
12
|
+
}
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
export { ProcessRssReader };
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "navi-hey",
|
|
3
3
|
"type": "module",
|
|
4
|
-
"version": "1.
|
|
4
|
+
"version": "1.8.0",
|
|
5
5
|
"description": "Navi cache warmer CLI",
|
|
6
6
|
"readme": "README.md",
|
|
7
7
|
"main": "index.js",
|
|
@@ -28,7 +28,7 @@
|
|
|
28
28
|
"license": "MIT",
|
|
29
29
|
"dependencies": {
|
|
30
30
|
"axios": "^1.13.0",
|
|
31
|
-
"deku-swarm": "
|
|
31
|
+
"deku-swarm": "1.7.0",
|
|
32
32
|
"express": "5.2.1",
|
|
33
33
|
"node-html-parser": "7.1.0",
|
|
34
34
|
"yaml": "^2.8.2"
|