navi-hey 1.6.0 → 1.6.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/lib/enqueuers/ActionEnqueuer.js +1 -1
- package/lib/enqueuers/PaginatedActionEnqueuer.js +1 -1
- package/lib/jobs/ActionProcessingJob.js +1 -1
- package/lib/jobs/AssetDownloadJob.js +1 -1
- package/lib/jobs/HtmlParseJob.js +1 -1
- package/lib/jobs/PaginatedActionProcessingJob.js +1 -1
- package/lib/jobs/ResourceRequestJob.js +1 -2
- package/lib/models/configs/Config.js +16 -3
- package/lib/models/configs/PageRange.js +37 -0
- package/lib/models/configs/PaginationConfig.js +7 -11
- package/lib/models/request/ResourceRequest.js +31 -1
- package/lib/models/request/ResourceRequestAction.js +1 -1
- package/lib/models/request/ResourceRequestPaginatedAction.js +15 -11
- package/lib/server/Router.js +2 -0
- package/lib/server/handlers/StatsHandler.js +1 -2
- package/lib/server/handlers/engine/EngineReloadHandler.js +32 -0
- package/lib/server/handlers/jobs/JobHandler.js +1 -1
- package/lib/server/handlers/jobs/JobRetryHandler.js +1 -1
- package/lib/server/handlers/jobs/JobsHandler.js +1 -1
- package/lib/services/Application.js +9 -0
- package/lib/services/ApplicationInstance.js +30 -11
- package/lib/services/FailureChecker.js +1 -1
- package/lib/utils/ResourceEnqueuer.js +1 -1
- package/package.json +2 -1
- package/static/assets/{index-B02RlFjr.js → index-DyqqrESe.js} +3 -3
- package/static/index.html +1 -1
- package/lib/background/Job.js +0 -97
- package/lib/background/JobFactory.js +0 -82
- package/lib/background/JobRegistry.js +0 -160
- package/lib/background/JobRegistryInstance.js +0 -247
- package/lib/background/Worker.js +0 -59
- package/lib/background/WorkerFactory.js +0 -35
- package/lib/background/WorkersRegistry.js +0 -106
- package/lib/background/WorkersRegistryInstance.js +0 -139
- package/lib/factory/Factory.js +0 -79
- package/lib/services/Engine.js +0 -153
- package/lib/services/WorkersAllocator.js +0 -74
- package/lib/utils/collections/Collection.js +0 -26
- package/lib/utils/collections/IdentifyableCollection.js +0 -95
- package/lib/utils/collections/Queue.js +0 -53
- package/lib/utils/collections/SortedArrayMerger.js +0 -84
- package/lib/utils/collections/SortedArraySearcher.js +0 -101
- package/lib/utils/collections/SortedCollection.js +0 -169
- package/lib/utils/generators/IdGenerator.js +0 -79
- package/lib/utils/generators/UUidGenerator.js +0 -49
package/README.md
CHANGED
|
@@ -207,4 +207,4 @@ docker run --rm \
|
|
|
207
207
|
|
|
208
208
|
GitHub repository: [darthjee/navi](https://github.com/darthjee/navi)
|
|
209
209
|
|
|
210
|
-
Integration guide for developers and AI agents: [How to Use Navi in Your Project](https://github.com/darthjee/navi/blob/main/docs/HOW_TO_USE_NAVI.md)
|
|
210
|
+
Integration guide for developers and AI agents: [How to Use Navi in Your Project](https://github.com/darthjee/navi/blob/main/docs/guides/HOW_TO_USE_NAVI.md)
|
package/lib/jobs/HtmlParseJob.js
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { LogConfig } from './LogConfig.js';
|
|
2
|
+
import { NamespaceNotFound } from '../../exceptions/registry/NamespaceNotFound.js';
|
|
2
3
|
import { NamespaceMap } from '../../registry/NamespaceMap.js';
|
|
3
4
|
import { ResourceRegistry } from '../../registry/ResourceRegistry.js';
|
|
4
5
|
import { ConfigLoader } from '../../services/ConfigLoader.js';
|
|
@@ -23,15 +24,27 @@ class Config {
|
|
|
23
24
|
constructor({ namespaceMap, workersConfig, webConfig, logConfig, failureConfig }) {
|
|
24
25
|
this.namespaceMap = NamespaceMap.build(namespaceMap);
|
|
25
26
|
|
|
26
|
-
const defaultNamespace = namespaceMap[DEFAULT_NAMESPACE];
|
|
27
|
-
this.resourceRegistry = defaultNamespace ? defaultNamespace.resourceRegistry : new ResourceRegistry({});
|
|
28
|
-
|
|
29
27
|
this.workersConfig = workersConfig;
|
|
30
28
|
this.webConfig = webConfig ?? null;
|
|
31
29
|
this.logConfig = logConfig ?? new LogConfig();
|
|
32
30
|
this.failureConfig = failureConfig ?? null;
|
|
33
31
|
}
|
|
34
32
|
|
|
33
|
+
/**
|
|
34
|
+
* Returns the current default namespace's resource registry, resolved live
|
|
35
|
+
* from the `namespaceMap` singleton on every access. Falls back to an empty
|
|
36
|
+
* `ResourceRegistry` when there is no `default` namespace.
|
|
37
|
+
* @returns {ResourceRegistry} The default namespace's resource registry.
|
|
38
|
+
*/
|
|
39
|
+
get resourceRegistry() {
|
|
40
|
+
try {
|
|
41
|
+
return this.namespaceMap.getItem(DEFAULT_NAMESPACE).resourceRegistry;
|
|
42
|
+
} catch (error) {
|
|
43
|
+
if (error instanceof NamespaceNotFound) return new ResourceRegistry({});
|
|
44
|
+
throw error;
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
|
|
35
48
|
/**
|
|
36
49
|
* Returns the resource identified by the given name, resolved as if the caller
|
|
37
50
|
* originated from the `default` namespace.
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Iterates over a range of page numbers, honoring zero-indexing and an optional cap.
|
|
3
|
+
* @author darthjee
|
|
4
|
+
*/
|
|
5
|
+
class PageRange {
|
|
6
|
+
#count;
|
|
7
|
+
#zeroIndexed;
|
|
8
|
+
#maxPage;
|
|
9
|
+
|
|
10
|
+
/**
|
|
11
|
+
* @param {object} params Range parameters.
|
|
12
|
+
* @param {number} params.count Total number of pages available.
|
|
13
|
+
* @param {boolean} [params.zeroIndexed=false] Whether pages start at 0 instead of 1.
|
|
14
|
+
* @param {number|null} [params.maxPage=null] Caps iteration to the first `maxPage` pages. `null` means unlimited.
|
|
15
|
+
*/
|
|
16
|
+
constructor({ count, zeroIndexed = false, maxPage = null }) {
|
|
17
|
+
this.#count = count;
|
|
18
|
+
this.#zeroIndexed = zeroIndexed;
|
|
19
|
+
this.#maxPage = maxPage;
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
/**
|
|
23
|
+
* Invokes callback once per page number, in order, honoring zeroIndexed and maxPage.
|
|
24
|
+
* @param {Function} callback Called with each page number.
|
|
25
|
+
* @returns {void}
|
|
26
|
+
*/
|
|
27
|
+
each(callback) {
|
|
28
|
+
const start = this.#zeroIndexed ? 0 : 1;
|
|
29
|
+
const total = this.#maxPage !== null ? Math.min(this.#count, this.#maxPage) : this.#count;
|
|
30
|
+
|
|
31
|
+
for (let i = 0; i < total; i++) {
|
|
32
|
+
callback(start + i);
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
export { PageRange };
|
|
@@ -29,6 +29,13 @@ class PaginationConfig {
|
|
|
29
29
|
return this.#pageKey;
|
|
30
30
|
}
|
|
31
31
|
|
|
32
|
+
/**
|
|
33
|
+
* @returns {boolean} Whether pages are zero-indexed.
|
|
34
|
+
*/
|
|
35
|
+
get zeroIndexed() {
|
|
36
|
+
return this.#zeroIndexed;
|
|
37
|
+
}
|
|
38
|
+
|
|
32
39
|
/**
|
|
33
40
|
* Evaluates the `pages` expression against the response wrapper.
|
|
34
41
|
* @param {ResponseWrapper} responseWrapper The response wrapper to resolve against.
|
|
@@ -38,17 +45,6 @@ class PaginationConfig {
|
|
|
38
45
|
return this.#pagesResolver.resolve(responseWrapper);
|
|
39
46
|
}
|
|
40
47
|
|
|
41
|
-
/**
|
|
42
|
-
* Returns an array of page numbers to iterate over.
|
|
43
|
-
* @param {number} count Total number of pages.
|
|
44
|
-
* @returns {Array<number>} Array of page numbers.
|
|
45
|
-
*/
|
|
46
|
-
pageNumbers(count) {
|
|
47
|
-
const start = this.#zeroIndexed ? 0 : 1;
|
|
48
|
-
const end = this.#zeroIndexed ? count - 1 : count;
|
|
49
|
-
return Array.from({ length: end - start + 1 }, (_, i) => start + i);
|
|
50
|
-
}
|
|
51
|
-
|
|
52
48
|
/**
|
|
53
49
|
* Creates a PaginationConfig from a raw YAML list entry.
|
|
54
50
|
* @param {Array<object>} list List of single-key maps from YAML.
|
|
@@ -1,9 +1,10 @@
|
|
|
1
|
+
import { JobRegistry as DefaultJobRegistry } from 'deku-swarm';
|
|
1
2
|
import { AssetRequest } from './AssetRequest.js';
|
|
2
3
|
import { ResourceRequestAction } from './ResourceRequestAction.js';
|
|
3
4
|
import { ResourceRequestPaginatedAction } from './ResourceRequestPaginatedAction.js';
|
|
4
|
-
import { JobRegistry as DefaultJobRegistry } from '../../background/JobRegistry.js';
|
|
5
5
|
import { ActionsEnqueuer } from '../../enqueuers/ActionsEnqueuer.js';
|
|
6
6
|
import { PaginatedActionsEnqueuer } from '../../enqueuers/PaginatedActionsEnqueuer.js';
|
|
7
|
+
import { LogRegistry } from '../../registry/LogRegistry.js';
|
|
7
8
|
import { Application } from '../../services/Application.js';
|
|
8
9
|
|
|
9
10
|
/**
|
|
@@ -15,6 +16,7 @@ class ResourceRequest {
|
|
|
15
16
|
#clientNamespace;
|
|
16
17
|
#namespace;
|
|
17
18
|
#disabled;
|
|
19
|
+
#maxPage;
|
|
18
20
|
|
|
19
21
|
/**
|
|
20
22
|
* @param {object} attributes ResourceRequest attributes
|
|
@@ -30,6 +32,9 @@ class ResourceRequest {
|
|
|
30
32
|
* when omitted. Ignored when `disabled` is `true`.
|
|
31
33
|
* @param {boolean} [attributes.disabled] Whether this request is disabled. Takes precedence
|
|
32
34
|
* over `enabled` when `true`.
|
|
35
|
+
* @param {number} [attributes.max_page] Caps how many pages of this resource are ever enqueued
|
|
36
|
+
* when it's the target of another resource's `paginated_actions`. Must be a positive integer;
|
|
37
|
+
* any other value (including omitted, `null`, or `0`) means unlimited.
|
|
33
38
|
*/
|
|
34
39
|
constructor({
|
|
35
40
|
url,
|
|
@@ -41,11 +46,13 @@ class ResourceRequest {
|
|
|
41
46
|
paginated_actions = [],
|
|
42
47
|
enabled,
|
|
43
48
|
disabled,
|
|
49
|
+
max_page,
|
|
44
50
|
}) {
|
|
45
51
|
this.url = url;
|
|
46
52
|
this.status = status;
|
|
47
53
|
this.#namespace = namespace;
|
|
48
54
|
this.#disabled = disabled === true || enabled === false;
|
|
55
|
+
this.#maxPage = this.#sanitizeMaxPage(max_page);
|
|
49
56
|
|
|
50
57
|
const parsedClient = ResourceRequest.#parseClient(clientName);
|
|
51
58
|
this.#clientName = parsedClient.name;
|
|
@@ -93,6 +100,15 @@ class ResourceRequest {
|
|
|
93
100
|
return this.#disabled;
|
|
94
101
|
}
|
|
95
102
|
|
|
103
|
+
/**
|
|
104
|
+
* Returns the cap on how many pages of this resource are ever enqueued when it's the
|
|
105
|
+
* target of another resource's `paginated_actions`. `null` means unlimited.
|
|
106
|
+
* @returns {number|null} The positive integer cap, or null when unlimited.
|
|
107
|
+
*/
|
|
108
|
+
get maxPage() {
|
|
109
|
+
return this.#maxPage;
|
|
110
|
+
}
|
|
111
|
+
|
|
96
112
|
/**
|
|
97
113
|
* Enqueues one ActionProcessingJob per action associated with the resource request.
|
|
98
114
|
* Returns immediately if there are no actions.
|
|
@@ -191,6 +207,20 @@ class ResourceRequest {
|
|
|
191
207
|
}
|
|
192
208
|
return { name: client, namespace: null };
|
|
193
209
|
}
|
|
210
|
+
|
|
211
|
+
/**
|
|
212
|
+
* Sanitizes the raw `max_page` value into a positive integer or `null`.
|
|
213
|
+
* Logs a warning when a present value is invalid.
|
|
214
|
+
* @param {*} value The raw `max_page` value.
|
|
215
|
+
* @returns {number|null} The sanitized positive integer, or null when unlimited.
|
|
216
|
+
*/
|
|
217
|
+
#sanitizeMaxPage(value) {
|
|
218
|
+
if (value === undefined || value === null || value === 0) return null;
|
|
219
|
+
if (Number.isInteger(value) && value > 0) return value;
|
|
220
|
+
|
|
221
|
+
LogRegistry.warn(`Ignoring invalid max_page value: ${JSON.stringify(value)} — treating as unlimited`);
|
|
222
|
+
return null;
|
|
223
|
+
}
|
|
194
224
|
}
|
|
195
225
|
|
|
196
226
|
export { ResourceRequest };
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { JobRegistry as DefaultJobRegistry } from '
|
|
1
|
+
import { JobRegistry as DefaultJobRegistry } from 'deku-swarm';
|
|
2
2
|
import { MissingActionResource } from '../../exceptions/registry/MissingActionResource.js';
|
|
3
3
|
import { LogRegistry } from '../../registry/LogRegistry.js';
|
|
4
4
|
import { NamespaceMap as DefaultNamespaceMap } from '../../registry/NamespaceMap.js';
|
|
@@ -1,8 +1,9 @@
|
|
|
1
|
-
import { JobRegistry as DefaultJobRegistry } from '
|
|
1
|
+
import { JobRegistry as DefaultJobRegistry } from 'deku-swarm';
|
|
2
2
|
import { MissingActionResource } from '../../exceptions/registry/MissingActionResource.js';
|
|
3
3
|
import { LogRegistry } from '../../registry/LogRegistry.js';
|
|
4
4
|
import { NamespaceMap as DefaultNamespaceMap } from '../../registry/NamespaceMap.js';
|
|
5
5
|
import { Application } from '../../services/Application.js';
|
|
6
|
+
import { PageRange } from '../configs/PageRange.js';
|
|
6
7
|
import { PaginationConfig } from '../configs/PaginationConfig.js';
|
|
7
8
|
import { ParametersMapper } from '../response/ParametersMapper.js';
|
|
8
9
|
|
|
@@ -92,21 +93,24 @@ class ResourceRequestPaginatedAction {
|
|
|
92
93
|
if (Application.isStopped()) return;
|
|
93
94
|
|
|
94
95
|
const count = this.#pagination.resolvePages(responseWrapper);
|
|
95
|
-
const pages = this.#pagination.pageNumbers(count);
|
|
96
96
|
const resource = this.#namespaceMap.getResource(this.#originNamespace, this.#resource, this.#namespace);
|
|
97
97
|
const mappedParameters = this.#mapper.map(responseWrapper);
|
|
98
98
|
|
|
99
|
-
for (const
|
|
100
|
-
|
|
101
|
-
...parameters,
|
|
102
|
-
...mappedParameters,
|
|
103
|
-
[this.#pagination.pageKey]: page,
|
|
104
|
-
};
|
|
105
|
-
for (const resourceRequest of resource.resourceRequests) {
|
|
106
|
-
if (resourceRequest.disabled) continue;
|
|
99
|
+
for (const resourceRequest of resource.resourceRequests) {
|
|
100
|
+
if (resourceRequest.disabled) continue;
|
|
107
101
|
|
|
102
|
+
new PageRange({
|
|
103
|
+
count,
|
|
104
|
+
zeroIndexed: this.#pagination.zeroIndexed,
|
|
105
|
+
maxPage: resourceRequest.maxPage,
|
|
106
|
+
}).each((page) => {
|
|
107
|
+
const pageParameters = {
|
|
108
|
+
...parameters,
|
|
109
|
+
...mappedParameters,
|
|
110
|
+
[this.#pagination.pageKey]: page,
|
|
111
|
+
};
|
|
108
112
|
this.#jobRegistry.enqueue('ResourceRequestJob', { resourceRequest, parameters: pageParameters });
|
|
109
|
-
}
|
|
113
|
+
});
|
|
110
114
|
}
|
|
111
115
|
}
|
|
112
116
|
|
package/lib/server/Router.js
CHANGED
|
@@ -8,6 +8,7 @@ import { ApiEngineStartHandler } from './handlers/api/ApiEngineStartHandler.js';
|
|
|
8
8
|
import { ApiEngineStopHandler } from './handlers/api/ApiEngineStopHandler.js';
|
|
9
9
|
import { EngineContinueHandler } from './handlers/engine/EngineContinueHandler.js';
|
|
10
10
|
import { EnginePauseHandler } from './handlers/engine/EnginePauseHandler.js';
|
|
11
|
+
import { EngineReloadHandler } from './handlers/engine/EngineReloadHandler.js';
|
|
11
12
|
import { EngineRestartHandler } from './handlers/engine/EngineRestartHandler.js';
|
|
12
13
|
import { EngineShutdownHandler } from './handlers/engine/EngineShutdownHandler.js';
|
|
13
14
|
import { EngineStartHandler } from './handlers/engine/EngineStartHandler.js';
|
|
@@ -77,6 +78,7 @@ class Router {
|
|
|
77
78
|
'/engine/continue': new HandlerConfig(EngineContinueHandler),
|
|
78
79
|
'/engine/start': new HandlerConfig(EngineStartHandler),
|
|
79
80
|
'/engine/restart': new HandlerConfig(EngineRestartHandler),
|
|
81
|
+
'/engine/reload': new HandlerConfig(EngineReloadHandler),
|
|
80
82
|
'/engine/shutdown': new HandlerConfig(EngineShutdownHandler),
|
|
81
83
|
};
|
|
82
84
|
|
|
@@ -1,5 +1,4 @@
|
|
|
1
|
-
import { JobRegistry } from '
|
|
2
|
-
import { WorkersRegistry } from '../../background/WorkersRegistry.js';
|
|
1
|
+
import { JobRegistry, WorkersRegistry } from 'deku-swarm';
|
|
3
2
|
import { RequestHandler } from '../../common/server/RequestHandler.js';
|
|
4
3
|
|
|
5
4
|
/**
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
import { RequestHandler } from '../../../common/server/RequestHandler.js';
|
|
2
|
+
import { ConflictError } from '../../../exceptions/http/ConflictError.js';
|
|
3
|
+
import { Application } from '../../../services/Application.js';
|
|
4
|
+
|
|
5
|
+
/**
|
|
6
|
+
* Executes request-handling behaviour for PATCH /engine/reload.
|
|
7
|
+
* @author darthjee
|
|
8
|
+
*/
|
|
9
|
+
class EngineReloadHandler extends RequestHandler {
|
|
10
|
+
#response;
|
|
11
|
+
|
|
12
|
+
/**
|
|
13
|
+
* @param {object} _request - The Express request object.
|
|
14
|
+
* @param {object} response - The Express response object.
|
|
15
|
+
*/
|
|
16
|
+
constructor(_request, response) {
|
|
17
|
+
super();
|
|
18
|
+
this.#response = response;
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
/**
|
|
22
|
+
* Initiates a config reload + restart. Responds immediately with the transitional status.
|
|
23
|
+
* @returns {Promise<void>}
|
|
24
|
+
*/
|
|
25
|
+
async handle() {
|
|
26
|
+
if (!Application.isRunning()) throw new ConflictError();
|
|
27
|
+
Application.reload();
|
|
28
|
+
this.#response.json({ status: 'stopping' });
|
|
29
|
+
}
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
export { EngineReloadHandler };
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { JobRegistry } from '
|
|
1
|
+
import { JobRegistry } from 'deku-swarm';
|
|
2
2
|
import { RequestHandler } from '../../../common/server/RequestHandler.js';
|
|
3
3
|
import { NotFoundError } from '../../../exceptions/http/NotFoundError.js';
|
|
4
4
|
import { JobSerializer } from '../../../serializers/JobSerializer.js';
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { JobRegistry } from '
|
|
1
|
+
import { JobRegistry } from 'deku-swarm';
|
|
2
2
|
import { RequestHandler } from '../../../common/server/RequestHandler.js';
|
|
3
3
|
import { ConflictError } from '../../../exceptions/http/ConflictError.js';
|
|
4
4
|
import { NotFoundError } from '../../../exceptions/http/NotFoundError.js';
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { JobRegistry } from '
|
|
1
|
+
import { JobRegistry } from 'deku-swarm';
|
|
2
2
|
import { RequestHandler } from '../../../common/server/RequestHandler.js';
|
|
3
3
|
import { JobSerializer } from '../../../serializers/JobSerializer.js';
|
|
4
4
|
import { JobsFilter } from '../JobsFilter.js';
|
|
@@ -171,6 +171,15 @@ class Application {
|
|
|
171
171
|
return Application.#getInstance().restart();
|
|
172
172
|
}
|
|
173
173
|
|
|
174
|
+
/**
|
|
175
|
+
* Reloads configuration from disk, merging it into the live namespace, then
|
|
176
|
+
* restarts processing.
|
|
177
|
+
* @returns {Promise<void>}
|
|
178
|
+
*/
|
|
179
|
+
static async reload() {
|
|
180
|
+
return Application.#getInstance().reload();
|
|
181
|
+
}
|
|
182
|
+
|
|
174
183
|
/**
|
|
175
184
|
* Shuts down the web server and stops the engine.
|
|
176
185
|
* @returns {Promise<void>}
|
|
@@ -1,21 +1,21 @@
|
|
|
1
|
-
import { Engine } from '
|
|
1
|
+
import { Engine, JobFactory, JobRegistry, WorkerFactory, WorkersRegistry } from 'deku-swarm';
|
|
2
|
+
import { ConfigIncluder } from './ConfigIncluder.js';
|
|
2
3
|
import { EngineEvents } from './EngineEvents.js';
|
|
3
4
|
import { FailureChecker } from './FailureChecker.js';
|
|
4
5
|
import { RunSummary } from './RunSummary.js';
|
|
5
|
-
import { JobFactory } from '../background/JobFactory.js';
|
|
6
|
-
import { JobRegistry } from '../background/JobRegistry.js';
|
|
7
|
-
import { WorkersRegistry } from '../background/WorkersRegistry.js';
|
|
8
6
|
import { ConfigurationFileNotProvided } from '../exceptions/config/ConfigurationFileNotProvided.js';
|
|
9
7
|
import { ActionProcessingJob } from '../jobs/ActionProcessingJob.js';
|
|
10
8
|
import { AssetDownloadJob } from '../jobs/AssetDownloadJob.js';
|
|
11
9
|
import { HtmlParseJob } from '../jobs/HtmlParseJob.js';
|
|
12
10
|
import { PaginatedActionProcessingJob } from '../jobs/PaginatedActionProcessingJob.js';
|
|
11
|
+
import { ResourceRequestJob } from '../jobs/ResourceRequestJob.js';
|
|
13
12
|
import { Config } from '../models/configs/Config.js';
|
|
14
13
|
import { LogRegistry } from '../registry/LogRegistry.js';
|
|
14
|
+
import { NamespaceMap } from '../registry/NamespaceMap.js';
|
|
15
15
|
import { WebServer } from '../server/WebServer.js';
|
|
16
|
+
import { LogContext } from '../utils/logging/LogContext.js';
|
|
16
17
|
import { PromiseAggregator } from '../utils/PromiseAggregator.js';
|
|
17
18
|
import { ResourceEnqueuer } from '../utils/ResourceEnqueuer.js';
|
|
18
|
-
import { ResourceRequestCollector } from '../utils/ResourceRequestCollector.js';
|
|
19
19
|
|
|
20
20
|
const DEFAULT_POLL_SLEEP_MS = 10;
|
|
21
21
|
|
|
@@ -31,6 +31,7 @@ class ApplicationInstance {
|
|
|
31
31
|
#aggregator;
|
|
32
32
|
#enginePromise;
|
|
33
33
|
#sleepMs;
|
|
34
|
+
#configPath;
|
|
34
35
|
|
|
35
36
|
/**
|
|
36
37
|
* @param {object} [params={}] - Optional parameters for dependency injection.
|
|
@@ -52,6 +53,7 @@ class ApplicationInstance {
|
|
|
52
53
|
throw new ConfigurationFileNotProvided();
|
|
53
54
|
}
|
|
54
55
|
|
|
56
|
+
this.#configPath = configPath;
|
|
55
57
|
this.config = Config.fromFile(configPath);
|
|
56
58
|
const logRegistry = LogRegistry.build({ retention: this.config.logConfig.size });
|
|
57
59
|
this.#bufferedLogger = logRegistry.bufferedLogger;
|
|
@@ -92,6 +94,8 @@ class ApplicationInstance {
|
|
|
92
94
|
*/
|
|
93
95
|
buildEngine() {
|
|
94
96
|
return new Engine({
|
|
97
|
+
jobRegistry: JobRegistry,
|
|
98
|
+
workersRegistry: WorkersRegistry,
|
|
95
99
|
sleepMs: this.#sleepMs ?? this.config.workersConfig.sleep,
|
|
96
100
|
keepAlive: !!this.config.webConfig,
|
|
97
101
|
idleTimeoutMs: (this.config.webConfig?.idleTimeout ?? 0) * 1000,
|
|
@@ -121,12 +125,13 @@ class ApplicationInstance {
|
|
|
121
125
|
* Enqueues all parameter-free ResourceRequests into the job registry.
|
|
122
126
|
* These are requests whose URLs contain no {:placeholder} tokens and can be
|
|
123
127
|
* processed immediately without any external parameters.
|
|
128
|
+
* Delegates to `ResourceEnqueuer#enqueueAll` (default namespace), so it always
|
|
129
|
+
* reflects the live state of the `NamespaceMap` singleton, including resources
|
|
130
|
+
* added via `POST /api/config` after boot.
|
|
124
131
|
* @returns {void}
|
|
125
132
|
*/
|
|
126
133
|
enqueueFirstJobs() {
|
|
127
|
-
new
|
|
128
|
-
JobRegistry.enqueue('ResourceRequestJob', { resourceRequest, parameters: {} });
|
|
129
|
-
});
|
|
134
|
+
new ResourceEnqueuer().enqueueAll();
|
|
130
135
|
}
|
|
131
136
|
|
|
132
137
|
/**
|
|
@@ -254,6 +259,19 @@ class ApplicationInstance {
|
|
|
254
259
|
await this.start();
|
|
255
260
|
}
|
|
256
261
|
|
|
262
|
+
/**
|
|
263
|
+
* Reloads processing: stops the engine, re-reads the on-disk config file(s) and
|
|
264
|
+
* merges them into the live `NamespaceMap`, then starts the engine again.
|
|
265
|
+
* Only valid when status is 'running'.
|
|
266
|
+
* @returns {Promise<void>}
|
|
267
|
+
*/
|
|
268
|
+
async reload() {
|
|
269
|
+
if (this.#engineStatus !== 'running') return;
|
|
270
|
+
await this.stop();
|
|
271
|
+
NamespaceMap.include(ConfigIncluder.resolve(this.#configPath));
|
|
272
|
+
await this.start();
|
|
273
|
+
}
|
|
274
|
+
|
|
257
275
|
/**
|
|
258
276
|
* Shuts down the web server and stops the engine loop.
|
|
259
277
|
* If running, pauses first and waits for workers to idle.
|
|
@@ -301,7 +319,7 @@ class ApplicationInstance {
|
|
|
301
319
|
* @returns {void}
|
|
302
320
|
*/
|
|
303
321
|
#initRegistries() {
|
|
304
|
-
JobFactory.build('ResourceRequestJob', { attributes: { clients: this.config.namespaceMap } });
|
|
322
|
+
JobFactory.build('ResourceRequestJob', { klass: ResourceRequestJob, attributes: { clients: this.config.namespaceMap } });
|
|
305
323
|
JobFactory.build('Action', { klass: ActionProcessingJob });
|
|
306
324
|
JobFactory.build('PaginatedAction', { klass: PaginatedActionProcessingJob });
|
|
307
325
|
JobFactory.build('HtmlParse', { klass: HtmlParseJob, attributes: { jobRegistry: JobRegistry, clientRegistry: this.config.namespaceMap } });
|
|
@@ -309,10 +327,11 @@ class ApplicationInstance {
|
|
|
309
327
|
|
|
310
328
|
JobRegistry.build({ cooldown: this.config.workersConfig.retryCooldown, maxRetries: this.config.workersConfig.maxRetries });
|
|
311
329
|
|
|
330
|
+
const loggerFactory = ({ workerId, jobId }) => new LogContext({ workerId, jobId });
|
|
331
|
+
|
|
312
332
|
WorkersRegistry.build({
|
|
313
333
|
workers: this.#workers,
|
|
314
|
-
jobRegistry: JobRegistry,
|
|
315
|
-
workersRegistry: WorkersRegistry,
|
|
334
|
+
factory: new WorkerFactory({ jobRegistry: JobRegistry, workersRegistry: WorkersRegistry, loggerFactory }),
|
|
316
335
|
...this.config.workersConfig,
|
|
317
336
|
});
|
|
318
337
|
WorkersRegistry.initWorkers();
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
+
import { JobRegistry } from 'deku-swarm';
|
|
1
2
|
import { ResourceRequestCollector } from './ResourceRequestCollector.js';
|
|
2
|
-
import { JobRegistry } from '../background/JobRegistry.js';
|
|
3
3
|
import { NamespaceNotFound } from '../exceptions/registry/NamespaceNotFound.js';
|
|
4
4
|
import { ResourceNotFound } from '../exceptions/registry/ResourceNotFound.js';
|
|
5
5
|
import { NamespaceMap } from '../registry/NamespaceMap.js';
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "navi-hey",
|
|
3
3
|
"type": "module",
|
|
4
|
-
"version": "1.6.
|
|
4
|
+
"version": "1.6.3",
|
|
5
5
|
"description": "Navi cache warmer CLI",
|
|
6
6
|
"readme": "README.md",
|
|
7
7
|
"main": "index.js",
|
|
@@ -28,6 +28,7 @@
|
|
|
28
28
|
"license": "MIT",
|
|
29
29
|
"dependencies": {
|
|
30
30
|
"axios": "^1.13.0",
|
|
31
|
+
"deku-swarm": "^worker",
|
|
31
32
|
"express": "5.2.1",
|
|
32
33
|
"node-html-parser": "7.1.0",
|
|
33
34
|
"yaml": "^2.8.2"
|