navi-hey 1.1.0 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +38 -4
- package/lib/enqueuers/ActionEnqueuer.js +18 -2
- package/lib/enqueuers/ActionsEnqueuer.js +6 -3
- package/lib/enqueuers/PaginatedActionEnqueuer.js +53 -0
- package/lib/enqueuers/PaginatedActionsEnqueuer.js +47 -0
- package/lib/exceptions/{ConfigurationFileNotFound.js → config/ConfigurationFileNotFound.js} +1 -1
- package/lib/exceptions/{ConfigurationFileNotProvided.js → config/ConfigurationFileNotProvided.js} +1 -1
- package/lib/exceptions/{MissingTopLevelConfgKey.js → config/MissingTopLevelConfgKey.js} +1 -1
- package/lib/exceptions/{ConflictError.js → http/ConflictError.js} +1 -1
- package/lib/exceptions/{ForbiddenError.js → http/ForbiddenError.js} +1 -1
- package/lib/exceptions/{NotFoundError.js → http/NotFoundError.js} +1 -1
- package/lib/exceptions/{ItemNotFound.js → registry/ItemNotFound.js} +1 -1
- package/lib/exceptions/{MissingActionResource.js → registry/MissingActionResource.js} +1 -1
- package/lib/exceptions/{MissingMappingVariable.js → registry/MissingMappingVariable.js} +1 -1
- package/lib/exceptions/{InvalidHtmlResponseBody.js → request/InvalidHtmlResponseBody.js} +1 -1
- package/lib/exceptions/{InvalidResponseBody.js → request/InvalidResponseBody.js} +1 -1
- package/lib/exceptions/{NullResponse.js → request/NullResponse.js} +1 -1
- package/lib/exceptions/{RequestFailed.js → request/RequestFailed.js} +1 -1
- package/lib/jobs/ActionProcessingJob.js +17 -3
- package/lib/jobs/HtmlParseJob.js +17 -4
- package/lib/jobs/PaginatedActionProcessingJob.js +77 -0
- package/lib/jobs/ResourceRequestJob.js +8 -5
- package/lib/models/{Config.js → configs/Config.js} +3 -3
- package/lib/models/configs/PaginationConfig.js +63 -0
- package/lib/models/{ResourceRequest.js → request/ResourceRequest.js} +30 -8
- package/lib/models/{ResourceRequestAction.js → request/ResourceRequestAction.js} +6 -6
- package/lib/models/request/ResourceRequestPaginatedAction.js +81 -0
- package/lib/models/{PathSegmentTraverser.js → response/PathSegmentTraverser.js} +1 -1
- package/lib/models/{ResponseParser.js → response/ResponseParser.js} +1 -1
- package/lib/registry/ClientRegistry.js +1 -1
- package/lib/registry/LogRegistry.js +13 -13
- package/lib/registry/LogRegistryInstance.js +16 -16
- package/lib/registry/NamedRegistry.js +1 -1
- package/lib/registry/ResourceRegistry.js +1 -1
- package/lib/serializers/JobIndexSerializer.js +3 -0
- package/lib/serializers/LogSerializer.js +1 -1
- package/lib/server/PathValidator.js +1 -1
- package/lib/server/RouteRegister.js +3 -3
- package/lib/server/Router.js +18 -18
- package/lib/server/{AssetsRequestHandler.js → handlers/AssetsRequestHandler.js} +3 -3
- package/lib/server/{BaseUrlsRequestHandler.js → handlers/BaseUrlsRequestHandler.js} +3 -3
- package/lib/server/{IndexRequestHandler.js → handlers/IndexRequestHandler.js} +2 -2
- package/lib/server/{JobsFilter.js → handlers/JobsFilter.js} +1 -1
- package/lib/server/{LogsRequestHandler.js → handlers/LogsRequestHandler.js} +4 -4
- package/lib/server/{SettingsRequestHandler.js → handlers/SettingsRequestHandler.js} +4 -3
- package/lib/server/{StatsRequestHandler.js → handlers/StatsRequestHandler.js} +3 -3
- package/lib/server/{EngineContinueRequestHandler.js → handlers/engine/EngineContinueRequestHandler.js} +3 -3
- package/lib/server/{EnginePauseRequestHandler.js → handlers/engine/EnginePauseRequestHandler.js} +3 -3
- package/lib/server/{EngineRestartRequestHandler.js → handlers/engine/EngineRestartRequestHandler.js} +3 -3
- package/lib/server/{EngineShutdownRequestHandler.js → handlers/engine/EngineShutdownRequestHandler.js} +2 -2
- package/lib/server/{EngineStartRequestHandler.js → handlers/engine/EngineStartRequestHandler.js} +3 -3
- package/lib/server/{EngineStatusRequestHandler.js → handlers/engine/EngineStatusRequestHandler.js} +2 -2
- package/lib/server/{EngineStopRequestHandler.js → handlers/engine/EngineStopRequestHandler.js} +3 -3
- package/lib/server/{JobLogsRequestHandler.js → handlers/jobs/JobLogsRequestHandler.js} +4 -4
- package/lib/server/{JobRequestHandler.js → handlers/jobs/JobRequestHandler.js} +4 -4
- package/lib/server/{JobRetryRequestHandler.js → handlers/jobs/JobRetryRequestHandler.js} +4 -4
- package/lib/server/{JobsRequestHandler.js → handlers/jobs/JobsRequestHandler.js} +4 -4
- package/lib/services/ApplicationInstance.js +4 -2
- package/lib/services/Client.js +1 -1
- package/lib/services/ConfigLoader.js +1 -1
- package/lib/services/ConfigParser.js +7 -7
- package/lib/services/Engine.js +1 -1
- package/lib/utils/HtmlParser.js +1 -1
- package/lib/utils/logging/BufferedLogger.js +1 -1
- package/lib/utils/logging/LogBuffer.js +1 -1
- package/lib/utils/logging/LogBufferCollection.js +1 -1
- package/lib/utils/logging/LogFilter.js +2 -2
- package/package.json +1 -1
- package/static/assets/index-D5tOOLpl.js +11 -0
- package/static/index.html +1 -1
- package/lib/exceptions/LockedByOtherWorker.js +0 -14
- package/static/assets/index-BfggoMvd.js +0 -11
- /package/lib/exceptions/{MissingClientsConfig.js → config/MissingClientsConfig.js} +0 -0
- /package/lib/exceptions/{MissingResourceConfig.js → config/MissingResourceConfig.js} +0 -0
- /package/lib/exceptions/{ClientNotFound.js → registry/ClientNotFound.js} +0 -0
- /package/lib/exceptions/{ResourceNotFound.js → registry/ResourceNotFound.js} +0 -0
- /package/lib/models/{FailureConfig.js → configs/FailureConfig.js} +0 -0
- /package/lib/models/{LogConfig.js → configs/LogConfig.js} +0 -0
- /package/lib/models/{WebConfig.js → configs/WebConfig.js} +0 -0
- /package/lib/models/{WorkersConfig.js → configs/WorkersConfig.js} +0 -0
- /package/lib/models/{AssetRequest.js → request/AssetRequest.js} +0 -0
- /package/lib/models/{Resource.js → request/Resource.js} +0 -0
- /package/lib/models/{ParametersMapper.js → response/ParametersMapper.js} +0 -0
- /package/lib/models/{PathResolver.js → response/PathResolver.js} +0 -0
- /package/lib/models/{ResponseWrapper.js → response/ResponseWrapper.js} +0 -0
package/lib/server/{EngineStatusRequestHandler.js → handlers/engine/EngineStatusRequestHandler.js}
RENAMED
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
1
|
+
import { Application } from '../../../services/Application.js';
|
|
2
|
+
import { RequestHandler } from '../../RequestHandler.js';
|
|
3
3
|
|
|
4
4
|
/**
|
|
5
5
|
* Handles GET /engine/status requests.
|
package/lib/server/{EngineStopRequestHandler.js → handlers/engine/EngineStopRequestHandler.js}
RENAMED
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
3
|
-
import {
|
|
1
|
+
import { ConflictError } from '../../../exceptions/http/ConflictError.js';
|
|
2
|
+
import { Application } from '../../../services/Application.js';
|
|
3
|
+
import { RequestHandler } from '../../RequestHandler.js';
|
|
4
4
|
|
|
5
5
|
/**
|
|
6
6
|
* Handles PATCH /engine/stop requests.
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
3
|
-
import {
|
|
1
|
+
import { LogRegistry } from '../../../registry/LogRegistry.js';
|
|
2
|
+
import { LogSerializer } from '../../../serializers/LogSerializer.js';
|
|
3
|
+
import { RequestHandler } from '../../RequestHandler.js';
|
|
4
4
|
|
|
5
5
|
/**
|
|
6
6
|
* Handles GET /jobs/:job_id/logs.json requests.
|
|
@@ -14,7 +14,7 @@ class JobLogsRequestHandler extends RequestHandler {
|
|
|
14
14
|
|
|
15
15
|
/**
|
|
16
16
|
* Creates a new JobLogsRequestHandler.
|
|
17
|
-
* @param {object} [options={}]
|
|
17
|
+
* @param {object} [options={}] - Constructor options.
|
|
18
18
|
* @param {number} [options.pageSize=20] - Maximum number of log entries to return.
|
|
19
19
|
*/
|
|
20
20
|
constructor({ pageSize = 20 } = {}) {
|
|
@@ -1,7 +1,7 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
3
|
-
import {
|
|
4
|
-
import {
|
|
1
|
+
import { JobRegistry } from '../../../background/JobRegistry.js';
|
|
2
|
+
import { NotFoundError } from '../../../exceptions/http/NotFoundError.js';
|
|
3
|
+
import { JobSerializer } from '../../../serializers/JobSerializer.js';
|
|
4
|
+
import { RequestHandler } from '../../RequestHandler.js';
|
|
5
5
|
|
|
6
6
|
/**
|
|
7
7
|
* Handles GET /job/:id.json requests.
|
|
@@ -1,7 +1,7 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
3
|
-
import {
|
|
4
|
-
import {
|
|
1
|
+
import { JobRegistry } from '../../../background/JobRegistry.js';
|
|
2
|
+
import { ConflictError } from '../../../exceptions/http/ConflictError.js';
|
|
3
|
+
import { NotFoundError } from '../../../exceptions/http/NotFoundError.js';
|
|
4
|
+
import { RequestHandler } from '../../RequestHandler.js';
|
|
5
5
|
|
|
6
6
|
const RETRYABLE_STATUSES = new Set(['failed', 'dead']);
|
|
7
7
|
|
|
@@ -1,7 +1,7 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
3
|
-
import {
|
|
4
|
-
import {
|
|
1
|
+
import { JobRegistry } from '../../../background/JobRegistry.js';
|
|
2
|
+
import { JobSerializer } from '../../../serializers/JobSerializer.js';
|
|
3
|
+
import { RequestHandler } from '../../RequestHandler.js';
|
|
4
|
+
import { JobsFilter } from '../JobsFilter.js';
|
|
5
5
|
|
|
6
6
|
/**
|
|
7
7
|
* Handles GET /jobs/:status.json requests.
|
|
@@ -4,11 +4,12 @@ import { FailureChecker } from './FailureChecker.js';
|
|
|
4
4
|
import { JobFactory } from '../background/JobFactory.js';
|
|
5
5
|
import { JobRegistry } from '../background/JobRegistry.js';
|
|
6
6
|
import { WorkersRegistry } from '../background/WorkersRegistry.js';
|
|
7
|
-
import { ConfigurationFileNotProvided } from '../exceptions/ConfigurationFileNotProvided.js';
|
|
7
|
+
import { ConfigurationFileNotProvided } from '../exceptions/config/ConfigurationFileNotProvided.js';
|
|
8
8
|
import { ActionProcessingJob } from '../jobs/ActionProcessingJob.js';
|
|
9
9
|
import { AssetDownloadJob } from '../jobs/AssetDownloadJob.js';
|
|
10
10
|
import { HtmlParseJob } from '../jobs/HtmlParseJob.js';
|
|
11
|
-
import {
|
|
11
|
+
import { PaginatedActionProcessingJob } from '../jobs/PaginatedActionProcessingJob.js';
|
|
12
|
+
import { Config } from '../models/configs/Config.js';
|
|
12
13
|
import { LogRegistry } from '../registry/LogRegistry.js';
|
|
13
14
|
import { WebServer } from '../server/WebServer.js';
|
|
14
15
|
import { PromiseAggregator } from '../utils/PromiseAggregator.js';
|
|
@@ -254,6 +255,7 @@ class ApplicationInstance {
|
|
|
254
255
|
#initRegistries() {
|
|
255
256
|
JobFactory.build('ResourceRequestJob', { attributes: { clients: this.config.clientRegistry } });
|
|
256
257
|
JobFactory.build('Action', { klass: ActionProcessingJob });
|
|
258
|
+
JobFactory.build('PaginatedAction', { klass: PaginatedActionProcessingJob });
|
|
257
259
|
JobFactory.build('HtmlParse', { klass: HtmlParseJob, attributes: { jobRegistry: JobRegistry, clientRegistry: this.config.clientRegistry } });
|
|
258
260
|
JobFactory.build('AssetDownload', { klass: AssetDownloadJob, attributes: { clientRegistry: this.config.clientRegistry } });
|
|
259
261
|
|
package/lib/services/Client.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { readFileSync } from 'node:fs';
|
|
2
2
|
import YAML from 'yaml';
|
|
3
3
|
import { ConfigParser } from './ConfigParser.js';
|
|
4
|
-
import { ConfigurationFileNotFound } from '../exceptions/ConfigurationFileNotFound.js';
|
|
4
|
+
import { ConfigurationFileNotFound } from '../exceptions/config/ConfigurationFileNotFound.js';
|
|
5
5
|
import { EnvResolver } from '../utils/EnvResolver.js';
|
|
6
6
|
import { Logger } from '../utils/logging/Logger.js';
|
|
7
7
|
|
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
import { Client } from './Client.js';
|
|
2
|
-
import { MissingClientsConfig } from '../exceptions/MissingClientsConfig.js';
|
|
3
|
-
import { MissingResourceConfig } from '../exceptions/MissingResourceConfig.js';
|
|
4
|
-
import { FailureConfig } from '../models/FailureConfig.js';
|
|
5
|
-
import { LogConfig } from '../models/LogConfig.js';
|
|
6
|
-
import {
|
|
7
|
-
import {
|
|
8
|
-
import {
|
|
2
|
+
import { MissingClientsConfig } from '../exceptions/config/MissingClientsConfig.js';
|
|
3
|
+
import { MissingResourceConfig } from '../exceptions/config/MissingResourceConfig.js';
|
|
4
|
+
import { FailureConfig } from '../models/configs/FailureConfig.js';
|
|
5
|
+
import { LogConfig } from '../models/configs/LogConfig.js';
|
|
6
|
+
import { WebConfig } from '../models/configs/WebConfig.js';
|
|
7
|
+
import { WorkersConfig } from '../models/configs/WorkersConfig.js';
|
|
8
|
+
import { Resource } from '../models/request/Resource.js';
|
|
9
9
|
|
|
10
10
|
/**
|
|
11
11
|
* ConfigParser receives a parsed YAML configuration object and maps it to
|
package/lib/services/Engine.js
CHANGED
|
@@ -84,7 +84,7 @@ class Engine {
|
|
|
84
84
|
* Determines whether the engine loop should continue.
|
|
85
85
|
* In keepAlive mode, always returns true. Otherwise, returns true only
|
|
86
86
|
* while there are jobs or busy workers.
|
|
87
|
-
* @returns {boolean}
|
|
87
|
+
* @returns {boolean} True if the loop should continue.
|
|
88
88
|
*/
|
|
89
89
|
#shouldContinue() {
|
|
90
90
|
return this.#keepAlive || this.#continueAllocating();
|
package/lib/utils/HtmlParser.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { parse } from 'node-html-parser';
|
|
2
2
|
import { HtmlElementParser } from './HtmlElementParser.js';
|
|
3
|
-
import { InvalidHtmlResponseBody } from '../exceptions/InvalidHtmlResponseBody.js';
|
|
3
|
+
import { InvalidHtmlResponseBody } from '../exceptions/request/InvalidHtmlResponseBody.js';
|
|
4
4
|
|
|
5
5
|
/**
|
|
6
6
|
* HtmlParser parses a raw HTML string and extracts attribute values using CSS selectors.
|
|
@@ -32,7 +32,7 @@ class BufferedLogger extends BaseLogger {
|
|
|
32
32
|
|
|
33
33
|
/**
|
|
34
34
|
* Returns the most recently added log, or undefined if the buffer is empty.
|
|
35
|
-
* @returns {import('./Log.js').Log|undefined}
|
|
35
|
+
* @returns {import('./Log.js').Log|undefined} The most recently added log entry, or undefined.
|
|
36
36
|
*/
|
|
37
37
|
get latestLog() {
|
|
38
38
|
return this.#buffer.latestLog;
|
|
@@ -80,7 +80,7 @@ class LogBuffer {
|
|
|
80
80
|
|
|
81
81
|
/**
|
|
82
82
|
* Returns the most recently added log, or undefined if the buffer is empty.
|
|
83
|
-
* @returns {import('./Log.js').Log|undefined}
|
|
83
|
+
* @returns {import('./Log.js').Log|undefined} The most recently added log entry, or undefined.
|
|
84
84
|
*/
|
|
85
85
|
get latestLog() {
|
|
86
86
|
return this.#logs[0];
|
|
@@ -37,7 +37,7 @@ class LogBufferCollection {
|
|
|
37
37
|
* Returns all logs for the given key in chronological order (oldest first).
|
|
38
38
|
* Returns an empty array if no buffer exists for that key.
|
|
39
39
|
* @param {string|number} key - The key to look up.
|
|
40
|
-
* @returns {Array<import('./Log.js').Log>}
|
|
40
|
+
* @returns {Array<import('./Log.js').Log>} Array of log entries for the given key, oldest first.
|
|
41
41
|
*/
|
|
42
42
|
getLogs(key) {
|
|
43
43
|
if (!this.#buffers.has(key)) return [];
|
|
@@ -14,10 +14,10 @@ class LogFilter {
|
|
|
14
14
|
|
|
15
15
|
/**
|
|
16
16
|
* Returns the filtered log list.
|
|
17
|
-
* @param {object} [options={}]
|
|
17
|
+
* @param {object} [options={}] - Filter options.
|
|
18
18
|
* @param {number|string} [options.lastId] - When provided, returns only logs newer than this ID.
|
|
19
19
|
* Returns an empty array if the ID is not found.
|
|
20
|
-
* @returns {Array<import('./Log.js').Log>}
|
|
20
|
+
* @returns {Array<import('./Log.js').Log>} Filtered array of log entries.
|
|
21
21
|
*/
|
|
22
22
|
filter({ lastId } = {}) {
|
|
23
23
|
if (lastId === undefined) return this.#logs;
|