navi-hey 1.4.0 → 1.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/enqueuers/AssetRequestEnqueuer.js +4 -2
- package/lib/exceptions/config/ConfigurationIncludeNotFound.js +15 -0
- package/lib/exceptions/registry/DuplicateNamespaceItem.js +23 -0
- package/lib/exceptions/registry/NamespaceNotFound.js +16 -0
- package/lib/jobs/AssetDownloadJob.js +3 -2
- package/lib/jobs/ResourceRequestJob.js +6 -2
- package/lib/models/configs/Config.js +26 -13
- package/lib/models/configs/WebConfig.js +3 -1
- package/lib/models/request/Resource.js +14 -6
- package/lib/models/request/ResourceRequest.js +50 -8
- package/lib/models/request/ResourceRequestAction.js +24 -8
- package/lib/models/request/ResourceRequestPaginatedAction.js +44 -8
- package/lib/registry/Namespace.js +26 -0
- package/lib/registry/NamespaceMap.js +182 -0
- package/lib/services/ApplicationInstance.js +15 -3
- package/lib/services/Client.js +11 -5
- package/lib/services/ConfigIncluder.js +131 -0
- package/lib/services/ConfigLoader.js +38 -43
- package/lib/services/ConfigParser.js +22 -6
- package/lib/services/Engine.js +42 -1
- package/lib/services/NamespaceMapBuilder.js +167 -0
- package/package.json +1 -1
|
@@ -0,0 +1,182 @@
|
|
|
1
|
+
import { NamedRegistry } from './NamedRegistry.js';
|
|
2
|
+
import { ClientNotFound } from '../exceptions/registry/ClientNotFound.js';
|
|
3
|
+
import { NamespaceNotFound } from '../exceptions/registry/NamespaceNotFound.js';
|
|
4
|
+
import { ResourceNotFound } from '../exceptions/registry/ResourceNotFound.js';
|
|
5
|
+
|
|
6
|
+
const DEFAULT_NAMESPACE = 'default';
|
|
7
|
+
|
|
8
|
+
/**
|
|
9
|
+
* NamespaceMap resolves resource/client references across namespaces.
|
|
10
|
+
*
|
|
11
|
+
* Implements a static singleton facade: call `NamespaceMap.build(namespaces)` once
|
|
12
|
+
* during application bootstrap, then use static delegation methods (`getResource`,
|
|
13
|
+
* `getClient`) to access it. Use `NamespaceMap.reset()` in tests to restore a clean
|
|
14
|
+
* state between examples.
|
|
15
|
+
*
|
|
16
|
+
* Resolution rules:
|
|
17
|
+
* - When an explicit target namespace is given and it (or the item within it) cannot
|
|
18
|
+
* be found, the lookup fails immediately — no fallback is attempted.
|
|
19
|
+
* - When no explicit target namespace is given, the lookup is first attempted in the
|
|
20
|
+
* requester's own namespace (`originNamespace`), then falls back to the `default`
|
|
21
|
+
* namespace.
|
|
22
|
+
* @author darthjee
|
|
23
|
+
*/
|
|
24
|
+
class NamespaceMap extends NamedRegistry {
|
|
25
|
+
/**
|
|
26
|
+
* The exception class to throw when a namespace is not found.
|
|
27
|
+
* @see NamespaceNotFound
|
|
28
|
+
* @type {class}
|
|
29
|
+
*/
|
|
30
|
+
static notFoundException = NamespaceNotFound;
|
|
31
|
+
|
|
32
|
+
static #instance = null;
|
|
33
|
+
|
|
34
|
+
/**
|
|
35
|
+
* Creates and stores the singleton instance.
|
|
36
|
+
* @param {Record<string, Namespace>} namespaces - A map of namespace names to Namespace instances.
|
|
37
|
+
* @returns {NamespaceMap} The created instance.
|
|
38
|
+
* @throws {Error} If `build()` has already been called without a preceding `reset()`.
|
|
39
|
+
*/
|
|
40
|
+
static build(namespaces) {
|
|
41
|
+
if (NamespaceMap.#instance) {
|
|
42
|
+
throw new Error('NamespaceMap.build() has already been called. Call reset() first.');
|
|
43
|
+
}
|
|
44
|
+
NamespaceMap.#instance = new NamespaceMap(namespaces);
|
|
45
|
+
return NamespaceMap.#instance;
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
/**
|
|
49
|
+
* Destroys the singleton instance. Intended for test teardown.
|
|
50
|
+
* @returns {void}
|
|
51
|
+
*/
|
|
52
|
+
static reset() {
|
|
53
|
+
NamespaceMap.#instance = null;
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
/**
|
|
57
|
+
* Resolves a resource from the singleton instance.
|
|
58
|
+
* @param {string} originNamespace The namespace of the requester.
|
|
59
|
+
* @param {string} name The resource name.
|
|
60
|
+
* @param {string|null} [namespace=null] Explicit target namespace, or null to resolve
|
|
61
|
+
* against `originNamespace` (falling back to `default`).
|
|
62
|
+
* @returns {Resource} The resolved resource.
|
|
63
|
+
* @throws {Error} If `build()` has not been called.
|
|
64
|
+
*/
|
|
65
|
+
static getResource(originNamespace, name, namespace = null) {
|
|
66
|
+
return NamespaceMap.#getInstance().getResource(originNamespace, name, namespace);
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
/**
|
|
70
|
+
* Resolves a client from the singleton instance.
|
|
71
|
+
* @param {string} originNamespace The namespace of the requester.
|
|
72
|
+
* @param {string} name The client name.
|
|
73
|
+
* @param {string|null} [namespace=null] Explicit target namespace, or null to resolve
|
|
74
|
+
* against `originNamespace` (falling back to `default`).
|
|
75
|
+
* @returns {Client} The resolved client.
|
|
76
|
+
* @throws {Error} If `build()` has not been called.
|
|
77
|
+
*/
|
|
78
|
+
static getClient(originNamespace, name, namespace = null) {
|
|
79
|
+
return NamespaceMap.#getInstance().getClient(originNamespace, name, namespace);
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
/**
|
|
83
|
+
* Returns the singleton instance, throwing if not yet built.
|
|
84
|
+
* @returns {NamespaceMap} The singleton instance.
|
|
85
|
+
* @throws {Error} If `build()` has not been called.
|
|
86
|
+
*/
|
|
87
|
+
static #getInstance() {
|
|
88
|
+
if (!NamespaceMap.#instance) {
|
|
89
|
+
throw new Error('NamespaceMap has not been built. Call NamespaceMap.build() first.');
|
|
90
|
+
}
|
|
91
|
+
return NamespaceMap.#instance;
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
/**
|
|
95
|
+
* Resolves a resource, given the requester's namespace, the resource name, and
|
|
96
|
+
* an optional explicit target namespace.
|
|
97
|
+
* @param {string} originNamespace The namespace of the requester.
|
|
98
|
+
* @param {string} name The resource name.
|
|
99
|
+
* @param {string|null} [namespace=null] Explicit target namespace, or null to resolve
|
|
100
|
+
* against `originNamespace` (falling back to `default`).
|
|
101
|
+
* @returns {Resource} The resolved resource.
|
|
102
|
+
* @throws {NamespaceNotFound} When the target/origin/default namespace cannot be found.
|
|
103
|
+
* @throws {ResourceNotFound} When the resource cannot be found in the resolved namespace(s).
|
|
104
|
+
*/
|
|
105
|
+
getResource(originNamespace, name, namespace = null) {
|
|
106
|
+
return this.#resolve(originNamespace, name, namespace, {
|
|
107
|
+
registryKey: 'resourceRegistry',
|
|
108
|
+
method: 'getItem',
|
|
109
|
+
NotFoundError: ResourceNotFound,
|
|
110
|
+
});
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
/**
|
|
114
|
+
* Resolves a client, given the requester's namespace, the client name, and
|
|
115
|
+
* an optional explicit target namespace.
|
|
116
|
+
* @param {string} originNamespace The namespace of the requester.
|
|
117
|
+
* @param {string} name The client name.
|
|
118
|
+
* @param {string|null} [namespace=null] Explicit target namespace, or null to resolve
|
|
119
|
+
* against `originNamespace` (falling back to `default`).
|
|
120
|
+
* @returns {Client} The resolved client.
|
|
121
|
+
* @throws {NamespaceNotFound} When the target/origin/default namespace cannot be found.
|
|
122
|
+
* @throws {ClientNotFound} When the client cannot be found in the resolved namespace(s).
|
|
123
|
+
*/
|
|
124
|
+
getClient(originNamespace, name, namespace = null) {
|
|
125
|
+
return this.#resolve(originNamespace, name, namespace, {
|
|
126
|
+
registryKey: 'clientRegistry',
|
|
127
|
+
method: 'getClient',
|
|
128
|
+
NotFoundError: ClientNotFound,
|
|
129
|
+
});
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
/**
|
|
133
|
+
* Resolves an item (resource or client) using the namespace resolution rules.
|
|
134
|
+
* @param {string} originNamespace The namespace of the requester.
|
|
135
|
+
* @param {string} name The item name.
|
|
136
|
+
* @param {string|null} namespace Explicit target namespace, or null/undefined.
|
|
137
|
+
* @param {object} strategy Lookup strategy.
|
|
138
|
+
* @param {string} strategy.registryKey Either 'resourceRegistry' or 'clientRegistry'.
|
|
139
|
+
* @param {string} strategy.method Either 'getItem' or 'getClient'.
|
|
140
|
+
* @param {class} strategy.NotFoundError The exception class raised by the underlying registry.
|
|
141
|
+
* @returns {*} The resolved item.
|
|
142
|
+
* @throws {NamespaceNotFound} When the relevant namespace cannot be found.
|
|
143
|
+
* @private
|
|
144
|
+
*/
|
|
145
|
+
#resolve(originNamespace, name, namespace, { registryKey, method, NotFoundError }) {
|
|
146
|
+
if (namespace) {
|
|
147
|
+
const explicitNamespace = this.getItem(namespace);
|
|
148
|
+
return explicitNamespace[registryKey][method](name);
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
const originResult = this.#tryResolve(originNamespace, name, { registryKey, method, NotFoundError });
|
|
152
|
+
if (originResult !== undefined) return originResult;
|
|
153
|
+
|
|
154
|
+
const defaultNamespace = this.getItem(DEFAULT_NAMESPACE);
|
|
155
|
+
return defaultNamespace[registryKey][method](name);
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
/**
|
|
159
|
+
* Attempts to resolve an item within a given namespace without raising when the
|
|
160
|
+
* namespace or item is missing.
|
|
161
|
+
* @param {string} namespaceName The namespace to look into.
|
|
162
|
+
* @param {string} name The item name.
|
|
163
|
+
* @param {object} strategy Lookup strategy (see {@link #resolve}).
|
|
164
|
+
* @param {string} strategy.registryKey Either 'resourceRegistry' or 'clientRegistry'.
|
|
165
|
+
* @param {string} strategy.method Either 'getItem' or 'getClient'.
|
|
166
|
+
* @param {class} strategy.NotFoundError The exception class raised by the underlying registry.
|
|
167
|
+
* @returns {*|undefined} The resolved item, or undefined when it could not be resolved.
|
|
168
|
+
* @private
|
|
169
|
+
*/
|
|
170
|
+
#tryResolve(namespaceName, name, { registryKey, method, NotFoundError }) {
|
|
171
|
+
if (!this.has(namespaceName)) return undefined;
|
|
172
|
+
|
|
173
|
+
try {
|
|
174
|
+
return this.getItem(namespaceName)[registryKey][method](name);
|
|
175
|
+
} catch (error) {
|
|
176
|
+
if (error instanceof NotFoundError) return undefined;
|
|
177
|
+
throw error;
|
|
178
|
+
}
|
|
179
|
+
}
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
export { NamespaceMap };
|
|
@@ -94,6 +94,8 @@ class ApplicationInstance {
|
|
|
94
94
|
return new Engine({
|
|
95
95
|
sleepMs: this.#sleepMs ?? this.config.workersConfig.sleep,
|
|
96
96
|
keepAlive: !!this.config.webConfig,
|
|
97
|
+
idleTimeoutMs: (this.config.webConfig?.idleTimeout ?? 0) * 1000,
|
|
98
|
+
onIdleTimeout: () => this.#handleIdleTimeout(),
|
|
97
99
|
});
|
|
98
100
|
}
|
|
99
101
|
|
|
@@ -270,6 +272,16 @@ class ApplicationInstance {
|
|
|
270
272
|
return this.config.webConfig?.autostart ?? true;
|
|
271
273
|
}
|
|
272
274
|
|
|
275
|
+
/**
|
|
276
|
+
* Invoked by the Engine when `web.idle_timeout` has elapsed with no jobs
|
|
277
|
+
* queued and no busy workers. Shuts the application down exactly like a
|
|
278
|
+
* manual `PATCH /engine/shutdown` call, regardless of `enable_shutdown`.
|
|
279
|
+
* @returns {Promise<void>}
|
|
280
|
+
*/
|
|
281
|
+
async #handleIdleTimeout() {
|
|
282
|
+
await this.shutdown();
|
|
283
|
+
}
|
|
284
|
+
|
|
273
285
|
/**
|
|
274
286
|
* Polls until all workers are idle.
|
|
275
287
|
* @returns {Promise<void>}
|
|
@@ -285,11 +297,11 @@ class ApplicationInstance {
|
|
|
285
297
|
* @returns {void}
|
|
286
298
|
*/
|
|
287
299
|
#initRegistries() {
|
|
288
|
-
JobFactory.build('ResourceRequestJob', { attributes: { clients: this.config.
|
|
300
|
+
JobFactory.build('ResourceRequestJob', { attributes: { clients: this.config.namespaceMap } });
|
|
289
301
|
JobFactory.build('Action', { klass: ActionProcessingJob });
|
|
290
302
|
JobFactory.build('PaginatedAction', { klass: PaginatedActionProcessingJob });
|
|
291
|
-
JobFactory.build('HtmlParse', { klass: HtmlParseJob, attributes: { jobRegistry: JobRegistry, clientRegistry: this.config.
|
|
292
|
-
JobFactory.build('AssetDownload', { klass: AssetDownloadJob, attributes: { clientRegistry: this.config.
|
|
303
|
+
JobFactory.build('HtmlParse', { klass: HtmlParseJob, attributes: { jobRegistry: JobRegistry, clientRegistry: this.config.namespaceMap } });
|
|
304
|
+
JobFactory.build('AssetDownload', { klass: AssetDownloadJob, attributes: { clientRegistry: this.config.namespaceMap } });
|
|
293
305
|
|
|
294
306
|
JobRegistry.build({ cooldown: this.config.workersConfig.retryCooldown, maxRetries: this.config.workersConfig.maxRetries });
|
|
295
307
|
|
package/lib/services/Client.js
CHANGED
|
@@ -16,13 +16,15 @@ class Client {
|
|
|
16
16
|
* @param {string|null} [attributes.linkText] Optional display text for UI links.
|
|
17
17
|
* @param {number} [attributes.timeout] Optional request timeout in milliseconds.
|
|
18
18
|
* @param {object} [attributes.headers] Optional HTTP headers sent with every request.
|
|
19
|
+
* @param {string} [attributes.namespace='default'] The namespace this client was declared in.
|
|
19
20
|
*/
|
|
20
|
-
constructor({ name, baseUrl, linkText = null, timeout = 5000, headers = {} }) {
|
|
21
|
+
constructor({ name, baseUrl, linkText = null, timeout = 5000, headers = {}, namespace = 'default' }) {
|
|
21
22
|
this.name = name;
|
|
22
23
|
this.baseUrl = baseUrl;
|
|
23
24
|
this.linkText = linkText;
|
|
24
25
|
this.timeout = timeout;
|
|
25
26
|
this.headers = headers;
|
|
27
|
+
this.namespace = namespace;
|
|
26
28
|
}
|
|
27
29
|
|
|
28
30
|
/**
|
|
@@ -34,23 +36,27 @@ class Client {
|
|
|
34
36
|
* @param {string} [config.linkText] Optional display text for UI links.
|
|
35
37
|
* @param {number} [config.timeout] Optional request timeout in milliseconds.
|
|
36
38
|
* @param {object} [config.headers] Optional HTTP headers.
|
|
39
|
+
* @param {object} [options={}] Additional options.
|
|
40
|
+
* @param {string} [options.namespace='default'] The namespace this client was declared in.
|
|
37
41
|
* @returns {Client} A new Client instance.
|
|
38
42
|
*/
|
|
39
|
-
static fromObject(name, config) {
|
|
43
|
+
static fromObject(name, config, { namespace = 'default' } = {}) {
|
|
40
44
|
const headers = config.headers || {};
|
|
41
45
|
const baseUrl = config.base_url;
|
|
42
46
|
const linkText = config.linkText ?? null;
|
|
43
|
-
return new Client({ name, baseUrl, linkText, timeout: config.timeout, headers });
|
|
47
|
+
return new Client({ name, baseUrl, linkText, timeout: config.timeout, headers, namespace });
|
|
44
48
|
}
|
|
45
49
|
|
|
46
50
|
/**
|
|
47
51
|
* Creates an array of Client instances from a map of client configurations.
|
|
48
52
|
*
|
|
49
53
|
* @param {object} object Map of client names to client configuration objects.
|
|
54
|
+
* @param {object} [options={}] Additional options.
|
|
55
|
+
* @param {string} [options.namespace='default'] The namespace these clients were declared in.
|
|
50
56
|
* @returns {Array<Client>} List of Client instances.
|
|
51
57
|
*/
|
|
52
|
-
static fromListObject(object) {
|
|
53
|
-
return Object.entries(object).map(([name, config]) => Client.fromObject(name, config));
|
|
58
|
+
static fromListObject(object, { namespace = 'default' } = {}) {
|
|
59
|
+
return Object.entries(object).map(([name, config]) => Client.fromObject(name, config, { namespace }));
|
|
54
60
|
}
|
|
55
61
|
|
|
56
62
|
/**
|
|
@@ -0,0 +1,131 @@
|
|
|
1
|
+
import { readFileSync } from 'node:fs';
|
|
2
|
+
import path from 'node:path';
|
|
3
|
+
import YAML from 'yaml';
|
|
4
|
+
import { EnvStringResolver } from '../common/utils/env_resolver/EnvStringResolver.js';
|
|
5
|
+
import { ConfigurationFileNotFound } from '../exceptions/config/ConfigurationFileNotFound.js';
|
|
6
|
+
import { ConfigurationIncludeNotFound } from '../exceptions/config/ConfigurationIncludeNotFound.js';
|
|
7
|
+
import { Logger } from '../utils/logging/Logger.js';
|
|
8
|
+
|
|
9
|
+
/**
|
|
10
|
+
* ConfigIncluder recursively resolves the `include:` chain starting from an entry
|
|
11
|
+
* configuration file, reading and YAML-parsing every reachable file.
|
|
12
|
+
*
|
|
13
|
+
* Each `include:` path is resolved relative to the directory of the file declaring
|
|
14
|
+
* it, unless it is already an absolute path. Files already visited (by resolved
|
|
15
|
+
* absolute path) are skipped on subsequent encounters, so duplicate/circular
|
|
16
|
+
* `include:` entries are silently de-duplicated rather than raising an error.
|
|
17
|
+
*
|
|
18
|
+
* Only the *entry* file's `workers`/`web`/`log`/`failure` sections are ever
|
|
19
|
+
* consulted (exposed via {@link ConfigIncluder#entryRaw}); included files are only
|
|
20
|
+
* ever expected to contribute `resources`/`clients` to a namespace.
|
|
21
|
+
* @author darthjee
|
|
22
|
+
*/
|
|
23
|
+
class ConfigIncluder {
|
|
24
|
+
#visited;
|
|
25
|
+
#files;
|
|
26
|
+
#entryRaw;
|
|
27
|
+
|
|
28
|
+
/**
|
|
29
|
+
* @param {string} entryFilePath Path to the entry configuration file.
|
|
30
|
+
*/
|
|
31
|
+
constructor(entryFilePath) {
|
|
32
|
+
this.entryFilePath = entryFilePath;
|
|
33
|
+
this.#visited = new Set();
|
|
34
|
+
this.#files = [];
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
/**
|
|
38
|
+
* Resolves the full include chain starting from the given entry file path.
|
|
39
|
+
* @param {string} entryFilePath Path to the entry configuration file.
|
|
40
|
+
* @returns {Array<{namespace: string, resources: object, clients: object, filePath: string}>}
|
|
41
|
+
* The flat list of files contributing to the configuration.
|
|
42
|
+
*/
|
|
43
|
+
static resolve(entryFilePath) {
|
|
44
|
+
return new ConfigIncluder(entryFilePath).resolve();
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
/**
|
|
48
|
+
* Resolves the full include chain starting from the entry file.
|
|
49
|
+
* @returns {Array<{namespace: string, resources: object, clients: object, filePath: string}>}
|
|
50
|
+
* The flat list of files contributing to the configuration.
|
|
51
|
+
*/
|
|
52
|
+
resolve() {
|
|
53
|
+
this.#collect(this.entryFilePath, { isEntry: true });
|
|
54
|
+
return this.#files;
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
/**
|
|
58
|
+
* Returns the entry file's raw parsed YAML object.
|
|
59
|
+
* @returns {object|undefined} The entry file's raw parsed object.
|
|
60
|
+
*/
|
|
61
|
+
get entryRaw() {
|
|
62
|
+
return this.#entryRaw;
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
/**
|
|
66
|
+
* Reads, parses, and registers a single file, then recurses into its `include:` list.
|
|
67
|
+
* Already-visited (resolved) paths are skipped.
|
|
68
|
+
* @param {string} filePath The file path to read (relative or absolute).
|
|
69
|
+
* @param {object} [options={}] Additional options.
|
|
70
|
+
* @param {boolean} [options.isEntry=false] Whether this is the entry file.
|
|
71
|
+
* @returns {void}
|
|
72
|
+
* @private
|
|
73
|
+
*/
|
|
74
|
+
#collect(filePath, { isEntry = false } = {}) {
|
|
75
|
+
const resolvedPath = path.resolve(filePath);
|
|
76
|
+
if (this.#visited.has(resolvedPath)) return;
|
|
77
|
+
this.#visited.add(resolvedPath);
|
|
78
|
+
|
|
79
|
+
const raw = this.#readYaml(resolvedPath, isEntry);
|
|
80
|
+
const namespace = raw?.namespace ?? 'default';
|
|
81
|
+
|
|
82
|
+
this.#files.push({
|
|
83
|
+
namespace,
|
|
84
|
+
resources: raw?.resources,
|
|
85
|
+
clients: raw?.clients,
|
|
86
|
+
filePath: resolvedPath,
|
|
87
|
+
});
|
|
88
|
+
|
|
89
|
+
if (isEntry) this.#entryRaw = raw;
|
|
90
|
+
|
|
91
|
+
this.#resolveIncludes(raw, resolvedPath);
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
/**
|
|
95
|
+
* Resolves and recurses into every path listed under the file's `include:` key.
|
|
96
|
+
* @param {object} raw The file's raw parsed YAML object.
|
|
97
|
+
* @param {string} currentFilePath The absolute path of the file declaring the includes.
|
|
98
|
+
* @returns {void}
|
|
99
|
+
* @private
|
|
100
|
+
*/
|
|
101
|
+
#resolveIncludes(raw, currentFilePath) {
|
|
102
|
+
const includes = raw?.include ?? [];
|
|
103
|
+
const dir = path.dirname(currentFilePath);
|
|
104
|
+
|
|
105
|
+
includes.forEach((includePath) => {
|
|
106
|
+
const resolvedIncludePath = path.isAbsolute(includePath) ? includePath : path.join(dir, includePath);
|
|
107
|
+
this.#collect(resolvedIncludePath);
|
|
108
|
+
});
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
/**
|
|
112
|
+
* Reads and YAML-parses a configuration file, resolving env var references.
|
|
113
|
+
* @param {string} filePath The absolute file path to read.
|
|
114
|
+
* @param {boolean} isEntry Whether this is the entry file (affects the exception raised).
|
|
115
|
+
* @returns {object} The parsed YAML object.
|
|
116
|
+
* @throws {ConfigurationFileNotFound} If the entry file cannot be read.
|
|
117
|
+
* @throws {ConfigurationIncludeNotFound} If an included file cannot be read.
|
|
118
|
+
* @private
|
|
119
|
+
*/
|
|
120
|
+
#readYaml(filePath, isEntry) {
|
|
121
|
+
try {
|
|
122
|
+
const content = readFileSync(filePath, 'utf8');
|
|
123
|
+
return YAML.parse(EnvStringResolver.resolve(content));
|
|
124
|
+
} catch {
|
|
125
|
+
Logger.error(`Configuration file not found: ${filePath}`);
|
|
126
|
+
throw isEntry ? new ConfigurationFileNotFound(filePath) : new ConfigurationIncludeNotFound(filePath);
|
|
127
|
+
}
|
|
128
|
+
}
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
export { ConfigIncluder };
|
|
@@ -1,12 +1,12 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import YAML from 'yaml';
|
|
1
|
+
import { ConfigIncluder } from './ConfigIncluder.js';
|
|
3
2
|
import { ConfigParser } from './ConfigParser.js';
|
|
4
|
-
import {
|
|
5
|
-
import { ConfigurationFileNotFound } from '../exceptions/config/ConfigurationFileNotFound.js';
|
|
6
|
-
import { Logger } from '../utils/logging/Logger.js';
|
|
3
|
+
import { NamespaceMapBuilder } from './NamespaceMapBuilder.js';
|
|
7
4
|
|
|
8
5
|
/**
|
|
9
|
-
* ConfigLoader loads a YAML configuration file and
|
|
6
|
+
* ConfigLoader loads a YAML configuration file (and every file transitively reachable
|
|
7
|
+
* through its `include:` chain), grouping the resources/clients contributed by each
|
|
8
|
+
* file into a NamespaceMap, and sourcing worker/web/log/failure sections from the
|
|
9
|
+
* entry file only.
|
|
10
10
|
* @author darthjee
|
|
11
11
|
*/
|
|
12
12
|
class ConfigLoader {
|
|
@@ -18,58 +18,53 @@ class ConfigLoader {
|
|
|
18
18
|
}
|
|
19
19
|
|
|
20
20
|
/**
|
|
21
|
-
*
|
|
22
|
-
*
|
|
23
|
-
* The YAML must contain a top-level `resources` key.
|
|
21
|
+
* Loads the configuration starting from the given entry file path.
|
|
24
22
|
*
|
|
25
23
|
* @param {string} filePath Path to the YAML configuration file.
|
|
26
24
|
* @returns {{
|
|
27
|
-
*
|
|
28
|
-
*
|
|
29
|
-
*
|
|
30
|
-
*
|
|
31
|
-
*
|
|
25
|
+
* namespaceMap: Record<string, Namespace>,
|
|
26
|
+
* workersConfig: WorkersConfig,
|
|
27
|
+
* webConfig: WebConfig|null,
|
|
28
|
+
* logConfig: LogConfig,
|
|
29
|
+
* failureConfig: FailureConfig|null
|
|
30
|
+
* }} The built namespace map, alongside the entry file's non-resource configuration sections.
|
|
31
|
+
* @throws {MissingTopLevelConfigKey} Throws when a single-file (no `include:`) config is
|
|
32
|
+
* invalid or does not contain required keys.
|
|
32
33
|
* @throws {ConfigurationFileNotFound} If the configuration file is not found at the specified path.
|
|
34
|
+
* @throws {ConfigurationIncludeNotFound} If an included configuration file is not found.
|
|
33
35
|
*/
|
|
34
36
|
static fromFile(filePath) {
|
|
35
37
|
return new ConfigLoader(filePath).load();
|
|
36
38
|
}
|
|
37
39
|
|
|
38
40
|
/**
|
|
39
|
-
*
|
|
41
|
+
* Resolves the include chain, builds the NamespaceMap, and extracts the entry
|
|
42
|
+
* file's worker/web/log/failure configuration sections.
|
|
40
43
|
* @returns {{
|
|
41
|
-
*
|
|
42
|
-
*
|
|
43
|
-
*
|
|
44
|
-
*
|
|
44
|
+
* namespaceMap: Record<string, Namespace>,
|
|
45
|
+
* workersConfig: WorkersConfig,
|
|
46
|
+
* webConfig: WebConfig|null,
|
|
47
|
+
* logConfig: LogConfig,
|
|
48
|
+
* failureConfig: FailureConfig|null
|
|
49
|
+
* }} The built namespace map, alongside the entry file's non-resource configuration sections.
|
|
50
|
+
* @throws {MissingTopLevelConfigKey} Throws when a single-file (no `include:`) config is
|
|
51
|
+
* invalid or does not contain required keys.
|
|
45
52
|
* @throws {ConfigurationFileNotFound} If the configuration file is not found at the specified path.
|
|
46
|
-
* @throws {
|
|
53
|
+
* @throws {ConfigurationIncludeNotFound} If an included configuration file is not found.
|
|
47
54
|
*/
|
|
48
55
|
load() {
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
* Reads and parses the raw YAML file content into a plain object.
|
|
54
|
-
* @returns {object} The raw parsed YAML object.
|
|
55
|
-
* @throws {ConfigurationFileNotFound} If the configuration file is not found at the specified path.
|
|
56
|
-
*/
|
|
57
|
-
#parseYaml() {
|
|
58
|
-
return YAML.parse(EnvStringResolver.resolve(this.#yamlContent()));
|
|
59
|
-
}
|
|
56
|
+
const includer = new ConfigIncluder(this.filePath);
|
|
57
|
+
const files = includer.resolve();
|
|
58
|
+
const namespaceMap = NamespaceMapBuilder.build(files);
|
|
59
|
+
const entryConfig = ConfigParser.fromObject(includer.entryRaw, { strict: false });
|
|
60
60
|
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
return readFileSync(this.filePath, 'utf8');
|
|
69
|
-
} catch (err) {
|
|
70
|
-
Logger.error(`Configuration file not found: ${this.filePath}`);
|
|
71
|
-
throw new ConfigurationFileNotFound(this.filePath);
|
|
72
|
-
}
|
|
61
|
+
return {
|
|
62
|
+
namespaceMap,
|
|
63
|
+
workersConfig: entryConfig.workersConfig,
|
|
64
|
+
webConfig: entryConfig.webConfig,
|
|
65
|
+
logConfig: entryConfig.logConfig,
|
|
66
|
+
failureConfig: entryConfig.failureConfig,
|
|
67
|
+
};
|
|
73
68
|
}
|
|
74
69
|
}
|
|
75
70
|
|
|
@@ -13,17 +13,29 @@ import { Resource } from '../models/request/Resource.js';
|
|
|
13
13
|
* @author darthjee
|
|
14
14
|
*/
|
|
15
15
|
class ConfigParser {
|
|
16
|
+
#namespace;
|
|
17
|
+
#strict;
|
|
18
|
+
|
|
16
19
|
/**
|
|
17
20
|
* @param {object} config Parsed configuration object from a YAML file.
|
|
21
|
+
* @param {object} [options={}] Additional options.
|
|
22
|
+
* @param {string} [options.namespace='default'] The namespace `resources`/`clients` in this
|
|
23
|
+
* file were declared under.
|
|
24
|
+
* @param {boolean} [options.strict=true] When true, missing top-level `resources`/`clients`
|
|
25
|
+
* keys raise. When false (used for files parsed as part of an `include:` chain), missing
|
|
26
|
+
* keys default to an empty map instead of raising.
|
|
18
27
|
*/
|
|
19
|
-
constructor(config) {
|
|
28
|
+
constructor(config, { namespace = 'default', strict = true } = {}) {
|
|
20
29
|
this.config = config;
|
|
30
|
+
this.#namespace = namespace;
|
|
31
|
+
this.#strict = strict;
|
|
21
32
|
}
|
|
22
33
|
|
|
23
34
|
/**
|
|
24
35
|
* Creates a mapped configuration from a plain parsed YAML object.
|
|
25
36
|
*
|
|
26
37
|
* @param {object} config Parsed configuration object from a YAML file.
|
|
38
|
+
* @param {object} [options={}] Additional options, forwarded to the constructor.
|
|
27
39
|
* @returns {{
|
|
28
40
|
* resources: Record<string, Resource>,
|
|
29
41
|
* clients: Record<string, Client>,
|
|
@@ -31,8 +43,8 @@ class ConfigParser {
|
|
|
31
43
|
* }} Mapped resources and clients by name. and workers configuration.
|
|
32
44
|
* @throws {Error} Throws when the config is invalid or missing required keys.
|
|
33
45
|
*/
|
|
34
|
-
static fromObject(config) {
|
|
35
|
-
return new ConfigParser(config).parse();
|
|
46
|
+
static fromObject(config, options = {}) {
|
|
47
|
+
return new ConfigParser(config, options).parse();
|
|
36
48
|
}
|
|
37
49
|
|
|
38
50
|
/**
|
|
@@ -124,7 +136,7 @@ class ConfigParser {
|
|
|
124
136
|
#loadResources() {
|
|
125
137
|
this.#requireKey('resources', MissingResourceConfig);
|
|
126
138
|
|
|
127
|
-
return Resource.fromListObject(this.config
|
|
139
|
+
return Resource.fromListObject(this.config?.resources ?? {}, { namespace: this.#namespace });
|
|
128
140
|
}
|
|
129
141
|
|
|
130
142
|
/**
|
|
@@ -135,17 +147,21 @@ class ConfigParser {
|
|
|
135
147
|
#loadClients() {
|
|
136
148
|
this.#requireKey('clients', MissingClientsConfig);
|
|
137
149
|
|
|
138
|
-
return Client.fromListObject(this.config
|
|
150
|
+
return Client.fromListObject(this.config?.clients ?? {}, { namespace: this.#namespace });
|
|
139
151
|
}
|
|
140
152
|
|
|
141
153
|
/**
|
|
142
154
|
* Validates that the config object contains the specified key.
|
|
155
|
+
* Skipped entirely (no error, missing key defaults to an empty map by the caller)
|
|
156
|
+
* when this parser was built in non-strict mode (see the constructor `strict` option).
|
|
143
157
|
* @param {string} key - The key to check for.
|
|
144
158
|
* @param {Function} ExceptionClass - The exception class to throw if the key is missing.
|
|
145
159
|
* @returns {void}
|
|
146
|
-
* @throws {Error} Throws if the config is invalid or missing the key.
|
|
160
|
+
* @throws {Error} Throws if the config is invalid or missing the key (strict mode only).
|
|
147
161
|
*/
|
|
148
162
|
#requireKey(key, ExceptionClass) {
|
|
163
|
+
if (!this.#strict) return;
|
|
164
|
+
|
|
149
165
|
if (!this.config || typeof this.config !== 'object' || !(key in this.config)) {
|
|
150
166
|
throw new ExceptionClass();
|
|
151
167
|
}
|
package/lib/services/Engine.js
CHANGED
|
@@ -10,12 +10,20 @@ import { Logger } from '../utils/logging/Logger.js';
|
|
|
10
10
|
* jobs to workers until there are no more jobs and no more busy workers.
|
|
11
11
|
* In keepAlive mode (web UI present), the loop continues indefinitely until
|
|
12
12
|
* `stop()` is explicitly called.
|
|
13
|
+
*
|
|
14
|
+
* When `idleTimeoutMs` is set, the loop also tracks how long it has been idle
|
|
15
|
+
* (no jobs queued and no busy workers) and invokes `onIdleTimeout` once that
|
|
16
|
+
* threshold is crossed; the idle window resets as soon as activity resumes.
|
|
13
17
|
*/
|
|
14
18
|
class Engine {
|
|
15
19
|
#sleepMs;
|
|
16
20
|
#stopped = false;
|
|
17
21
|
#paused = false;
|
|
18
22
|
#keepAlive;
|
|
23
|
+
#idleTimeoutMs;
|
|
24
|
+
#onIdleTimeout;
|
|
25
|
+
#idleSince = null;
|
|
26
|
+
#idleTimeoutFired = false;
|
|
19
27
|
|
|
20
28
|
/**
|
|
21
29
|
* Creates an instance of Engine.
|
|
@@ -23,10 +31,14 @@ class Engine {
|
|
|
23
31
|
* @param {WorkersAllocator} param0.allocator - The workers allocator to manage job allocation.
|
|
24
32
|
* @param {number} [param0.sleepMs=500] - Milliseconds to wait when all jobs are in cooldown. Use a negative value to disable sleeping (e.g. in tests).
|
|
25
33
|
* @param {boolean} [param0.keepAlive=false] - When true, the loop runs indefinitely until `stop()` is called (web UI mode).
|
|
34
|
+
* @param {number} [param0.idleTimeoutMs=0] - Milliseconds of inactivity before `onIdleTimeout` fires. `0` disables idle-timeout tracking.
|
|
35
|
+
* @param {Function} [param0.onIdleTimeout] - Callback invoked (without being awaited) once the idle threshold is reached. Fires at most once per idle window.
|
|
26
36
|
*/
|
|
27
|
-
constructor({ allocator, sleepMs = 500, keepAlive = false } = {}) {
|
|
37
|
+
constructor({ allocator, sleepMs = 500, keepAlive = false, idleTimeoutMs = 0, onIdleTimeout = () => {} } = {}) {
|
|
28
38
|
this.#sleepMs = sleepMs;
|
|
29
39
|
this.#keepAlive = keepAlive;
|
|
40
|
+
this.#idleTimeoutMs = idleTimeoutMs;
|
|
41
|
+
this.#onIdleTimeout = onIdleTimeout;
|
|
30
42
|
|
|
31
43
|
this.allocator = allocator || new WorkersAllocator();
|
|
32
44
|
}
|
|
@@ -75,6 +87,8 @@ class Engine {
|
|
|
75
87
|
this.allocator.allocate();
|
|
76
88
|
}
|
|
77
89
|
|
|
90
|
+
this.#checkIdleTimeout();
|
|
91
|
+
|
|
78
92
|
// wait before next iteration so the block runs ~once per second
|
|
79
93
|
await this.#sleep();
|
|
80
94
|
}
|
|
@@ -98,6 +112,33 @@ class Engine {
|
|
|
98
112
|
return JobRegistry.hasJob() || WorkersRegistry.hasBusyWorker();
|
|
99
113
|
}
|
|
100
114
|
|
|
115
|
+
/**
|
|
116
|
+
* Tracks idle time (no queued jobs, no busy workers) across loop ticks and
|
|
117
|
+
* fires `onIdleTimeout` once at most per idle window when `idleTimeoutMs`
|
|
118
|
+
* is exceeded. Any activity resets the window. No-op when idle-timeout
|
|
119
|
+
* tracking is disabled (`idleTimeoutMs <= 0`).
|
|
120
|
+
* @returns {void}
|
|
121
|
+
*/
|
|
122
|
+
#checkIdleTimeout() {
|
|
123
|
+
if (this.#idleTimeoutMs <= 0) return;
|
|
124
|
+
|
|
125
|
+
if (this.#continueAllocating()) {
|
|
126
|
+
this.#idleSince = null;
|
|
127
|
+
this.#idleTimeoutFired = false;
|
|
128
|
+
return;
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
if (this.#idleSince === null) {
|
|
132
|
+
this.#idleSince = Date.now();
|
|
133
|
+
return;
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
if (!this.#idleTimeoutFired && Date.now() - this.#idleSince >= this.#idleTimeoutMs) {
|
|
137
|
+
this.#idleTimeoutFired = true;
|
|
138
|
+
this.#onIdleTimeout();
|
|
139
|
+
}
|
|
140
|
+
}
|
|
141
|
+
|
|
101
142
|
/**
|
|
102
143
|
* Waits for the given number of milliseconds.
|
|
103
144
|
* @returns {Promise<void>}
|