navi-hey 1.1.0 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +38 -4
- package/lib/enqueuers/ActionEnqueuer.js +18 -2
- package/lib/enqueuers/ActionsEnqueuer.js +6 -3
- package/lib/enqueuers/PaginatedActionEnqueuer.js +53 -0
- package/lib/enqueuers/PaginatedActionsEnqueuer.js +47 -0
- package/lib/exceptions/{ConfigurationFileNotFound.js → config/ConfigurationFileNotFound.js} +1 -1
- package/lib/exceptions/{ConfigurationFileNotProvided.js → config/ConfigurationFileNotProvided.js} +1 -1
- package/lib/exceptions/{MissingTopLevelConfgKey.js → config/MissingTopLevelConfgKey.js} +1 -1
- package/lib/exceptions/{ConflictError.js → http/ConflictError.js} +1 -1
- package/lib/exceptions/{ForbiddenError.js → http/ForbiddenError.js} +1 -1
- package/lib/exceptions/{NotFoundError.js → http/NotFoundError.js} +1 -1
- package/lib/exceptions/{ItemNotFound.js → registry/ItemNotFound.js} +1 -1
- package/lib/exceptions/{MissingActionResource.js → registry/MissingActionResource.js} +1 -1
- package/lib/exceptions/{MissingMappingVariable.js → registry/MissingMappingVariable.js} +1 -1
- package/lib/exceptions/{InvalidHtmlResponseBody.js → request/InvalidHtmlResponseBody.js} +1 -1
- package/lib/exceptions/{InvalidResponseBody.js → request/InvalidResponseBody.js} +1 -1
- package/lib/exceptions/{NullResponse.js → request/NullResponse.js} +1 -1
- package/lib/exceptions/{RequestFailed.js → request/RequestFailed.js} +1 -1
- package/lib/jobs/ActionProcessingJob.js +17 -3
- package/lib/jobs/HtmlParseJob.js +17 -4
- package/lib/jobs/PaginatedActionProcessingJob.js +77 -0
- package/lib/jobs/ResourceRequestJob.js +8 -5
- package/lib/models/{Config.js → configs/Config.js} +3 -3
- package/lib/models/configs/PaginationConfig.js +63 -0
- package/lib/models/{ResourceRequest.js → request/ResourceRequest.js} +30 -8
- package/lib/models/{ResourceRequestAction.js → request/ResourceRequestAction.js} +6 -6
- package/lib/models/request/ResourceRequestPaginatedAction.js +81 -0
- package/lib/models/{PathSegmentTraverser.js → response/PathSegmentTraverser.js} +1 -1
- package/lib/models/{ResponseParser.js → response/ResponseParser.js} +1 -1
- package/lib/registry/ClientRegistry.js +1 -1
- package/lib/registry/LogRegistry.js +13 -13
- package/lib/registry/LogRegistryInstance.js +16 -16
- package/lib/registry/NamedRegistry.js +1 -1
- package/lib/registry/ResourceRegistry.js +1 -1
- package/lib/serializers/JobIndexSerializer.js +3 -0
- package/lib/serializers/LogSerializer.js +1 -1
- package/lib/server/PathValidator.js +1 -1
- package/lib/server/RouteRegister.js +3 -3
- package/lib/server/Router.js +18 -18
- package/lib/server/{AssetsRequestHandler.js → handlers/AssetsRequestHandler.js} +3 -3
- package/lib/server/{BaseUrlsRequestHandler.js → handlers/BaseUrlsRequestHandler.js} +3 -3
- package/lib/server/{IndexRequestHandler.js → handlers/IndexRequestHandler.js} +2 -2
- package/lib/server/{JobsFilter.js → handlers/JobsFilter.js} +1 -1
- package/lib/server/{LogsRequestHandler.js → handlers/LogsRequestHandler.js} +4 -4
- package/lib/server/{SettingsRequestHandler.js → handlers/SettingsRequestHandler.js} +4 -3
- package/lib/server/{StatsRequestHandler.js → handlers/StatsRequestHandler.js} +3 -3
- package/lib/server/{EngineContinueRequestHandler.js → handlers/engine/EngineContinueRequestHandler.js} +3 -3
- package/lib/server/{EnginePauseRequestHandler.js → handlers/engine/EnginePauseRequestHandler.js} +3 -3
- package/lib/server/{EngineRestartRequestHandler.js → handlers/engine/EngineRestartRequestHandler.js} +3 -3
- package/lib/server/{EngineShutdownRequestHandler.js → handlers/engine/EngineShutdownRequestHandler.js} +2 -2
- package/lib/server/{EngineStartRequestHandler.js → handlers/engine/EngineStartRequestHandler.js} +3 -3
- package/lib/server/{EngineStatusRequestHandler.js → handlers/engine/EngineStatusRequestHandler.js} +2 -2
- package/lib/server/{EngineStopRequestHandler.js → handlers/engine/EngineStopRequestHandler.js} +3 -3
- package/lib/server/{JobLogsRequestHandler.js → handlers/jobs/JobLogsRequestHandler.js} +4 -4
- package/lib/server/{JobRequestHandler.js → handlers/jobs/JobRequestHandler.js} +4 -4
- package/lib/server/{JobRetryRequestHandler.js → handlers/jobs/JobRetryRequestHandler.js} +4 -4
- package/lib/server/{JobsRequestHandler.js → handlers/jobs/JobsRequestHandler.js} +4 -4
- package/lib/services/ApplicationInstance.js +4 -2
- package/lib/services/Client.js +1 -1
- package/lib/services/ConfigLoader.js +1 -1
- package/lib/services/ConfigParser.js +7 -7
- package/lib/services/Engine.js +1 -1
- package/lib/utils/HtmlParser.js +1 -1
- package/lib/utils/logging/BufferedLogger.js +1 -1
- package/lib/utils/logging/LogBuffer.js +1 -1
- package/lib/utils/logging/LogBufferCollection.js +1 -1
- package/lib/utils/logging/LogFilter.js +2 -2
- package/package.json +1 -1
- package/static/assets/index-D5tOOLpl.js +11 -0
- package/static/index.html +1 -1
- package/lib/exceptions/LockedByOtherWorker.js +0 -14
- package/static/assets/index-BfggoMvd.js +0 -11
- /package/lib/exceptions/{MissingClientsConfig.js → config/MissingClientsConfig.js} +0 -0
- /package/lib/exceptions/{MissingResourceConfig.js → config/MissingResourceConfig.js} +0 -0
- /package/lib/exceptions/{ClientNotFound.js → registry/ClientNotFound.js} +0 -0
- /package/lib/exceptions/{ResourceNotFound.js → registry/ResourceNotFound.js} +0 -0
- /package/lib/models/{FailureConfig.js → configs/FailureConfig.js} +0 -0
- /package/lib/models/{LogConfig.js → configs/LogConfig.js} +0 -0
- /package/lib/models/{WebConfig.js → configs/WebConfig.js} +0 -0
- /package/lib/models/{WorkersConfig.js → configs/WorkersConfig.js} +0 -0
- /package/lib/models/{AssetRequest.js → request/AssetRequest.js} +0 -0
- /package/lib/models/{Resource.js → request/Resource.js} +0 -0
- /package/lib/models/{ParametersMapper.js → response/ParametersMapper.js} +0 -0
- /package/lib/models/{PathResolver.js → response/PathResolver.js} +0 -0
- /package/lib/models/{ResponseWrapper.js → response/ResponseWrapper.js} +0 -0
package/README.md
CHANGED
|
@@ -16,6 +16,7 @@ Key features:
|
|
|
16
16
|
- Concurrent HTTP request execution via a worker pool.
|
|
17
17
|
- URL templates with placeholder parameters (e.g. `{:id}`).
|
|
18
18
|
- Response-driven actions: after each successful request, configurable actions extract variables from the response and trigger follow-up processing.
|
|
19
|
+
- Paginated resource support: `paginated_actions` fan out one request per page based on a page-count expression evaluated against the response.
|
|
19
20
|
- Automatic retry of failed requests after the main queue is exhausted.
|
|
20
21
|
|
|
21
22
|
---
|
|
@@ -92,8 +93,13 @@ resources:
|
|
|
92
93
|
- resource: category_information
|
|
93
94
|
- resource: products
|
|
94
95
|
parameters:
|
|
95
|
-
category_id:
|
|
96
|
-
|
|
96
|
+
category_id: parsedBody.id
|
|
97
|
+
paginated_actions:
|
|
98
|
+
- resource: products_page
|
|
99
|
+
pagination:
|
|
100
|
+
- pages: parsedBody.pagination.pages
|
|
101
|
+
- page_key: page
|
|
102
|
+
- zero_indexed: false
|
|
97
103
|
- url: /categories
|
|
98
104
|
status: 302
|
|
99
105
|
- url: /#/categories
|
|
@@ -134,7 +140,13 @@ When the web server is enabled, the following screens are available:
|
|
|
134
140
|
| `client` | Name of the client to use for this request. Defaults to `default`. |
|
|
135
141
|
| `actions` | Optional list of actions to execute after a successful response. |
|
|
136
142
|
| `actions[].resource` | Name of the resource to act upon. Required. |
|
|
137
|
-
| `actions[].parameters` | Optional key-value map. Each key is the destination variable name and each value is a path expression resolved against the response (e.g. `
|
|
143
|
+
| `actions[].parameters` | Optional key-value map. Each key is the destination variable name and each value is a path expression resolved against the response (e.g. `parsedBody.id`, `headers['page']`). When absent, the parsed body item is passed through unchanged. |
|
|
144
|
+
| `paginated_actions` | Optional list of paginated actions. Each entry fans out one `ResourceRequestJob` per page. |
|
|
145
|
+
| `paginated_actions[].resource` | Name of the resource to enqueue per page. Required. |
|
|
146
|
+
| `paginated_actions[].pagination` | List of pagination config entries. Required. |
|
|
147
|
+
| `paginated_actions[].pagination[].pages` | Path expression (e.g. `parsedBody.pagination.pages`) resolving to the total page count. |
|
|
148
|
+
| `paginated_actions[].pagination[].page_key` | Parameter name injected as the current page number into each downstream request. |
|
|
149
|
+
| `paginated_actions[].pagination[].zero_indexed` | Boolean. Pages start at `0` when `true`, at `1` when `false` (default). |
|
|
138
150
|
| `assets` | Optional list of asset extraction rules. When present on an HTML resource, Navi parses the response body and enqueues a download job for each matched URL. |
|
|
139
151
|
| `assets[].selector` | CSS selector used to find asset elements in the HTML response (e.g. `script[src]`, `link[rel="stylesheet"]`). |
|
|
140
152
|
| `assets[].attribute` | Attribute on the matched element that holds the asset URL (e.g. `src`, `href`). |
|
|
@@ -147,7 +159,29 @@ When the web server is enabled, the following screens are available:
|
|
|
147
159
|
|
|
148
160
|
Navi supports multi-level resource chaining. After a successful response, each configured action uses `parameters` path expressions to extract variables from the response body or headers and enqueues new jobs for the target resource. The extracted variables resolve `{:placeholder}` tokens in the target URL templates.
|
|
149
161
|
|
|
150
|
-
For example, requesting `/categories.json` might return `[{ "id": 1 }, { "id": 2 }]`. With an action targeting `category_information` and `parameters: { id:
|
|
162
|
+
For example, requesting `/categories.json` might return `[{ "id": 1 }, { "id": 2 }]`. With an action targeting `category_information` and `parameters: { id: parsedBody.id }`, Navi automatically enqueues requests for `/categories/1.json` and `/categories/2.json`. Header values can also be extracted, e.g. `page: headers['x-next-page']`.
|
|
163
|
+
|
|
164
|
+
> **Note:** HTTP response header names are always lowercase after Node.js normalization. Use lowercase keys in path expressions (e.g. `headers['x-total-pages']`), regardless of how the server set them.
|
|
165
|
+
|
|
166
|
+
## Paginated Actions
|
|
167
|
+
|
|
168
|
+
`paginated_actions` complement `actions` when the response indicates multiple pages. Instead of iterating over array items, Navi evaluates a `pages` expression against the whole response, then enqueues one `ResourceRequestJob` per page, injecting the page number under the configured `page_key`.
|
|
169
|
+
|
|
170
|
+
```yaml
|
|
171
|
+
resources:
|
|
172
|
+
categories:
|
|
173
|
+
- url: /categories.json
|
|
174
|
+
status: 200
|
|
175
|
+
paginated_actions:
|
|
176
|
+
- resource: products_page
|
|
177
|
+
pagination:
|
|
178
|
+
- pages: parsedBody.pagination.pages
|
|
179
|
+
- page_key: page
|
|
180
|
+
- zero_indexed: false
|
|
181
|
+
products_page:
|
|
182
|
+
- url: /products/{:page}.json
|
|
183
|
+
status: 200
|
|
184
|
+
```
|
|
151
185
|
|
|
152
186
|
---
|
|
153
187
|
|
|
@@ -9,16 +9,19 @@ class ActionEnqueuer {
|
|
|
9
9
|
#action;
|
|
10
10
|
#items;
|
|
11
11
|
#jobRegistry;
|
|
12
|
+
#originUrl;
|
|
12
13
|
|
|
13
14
|
/**
|
|
14
15
|
* @param {ResourceRequestAction} action The action to enqueue for each item.
|
|
15
16
|
* @param {Array} items List of parsed response items.
|
|
16
17
|
* @param {object} [jobRegistry=JobRegistry] The job registry to enqueue jobs to.
|
|
18
|
+
* @param {string|null} [originUrl=null] The URL of the ResourceRequestJob that triggered this enqueue.
|
|
17
19
|
*/
|
|
18
|
-
constructor(action, items, jobRegistry = DefaultJobRegistry) {
|
|
20
|
+
constructor(action, items, jobRegistry = DefaultJobRegistry, originUrl = null) {
|
|
19
21
|
this.#action = action;
|
|
20
22
|
this.#items = items;
|
|
21
23
|
this.#jobRegistry = jobRegistry;
|
|
24
|
+
this.#originUrl = originUrl;
|
|
22
25
|
}
|
|
23
26
|
|
|
24
27
|
/**
|
|
@@ -29,9 +32,22 @@ class ActionEnqueuer {
|
|
|
29
32
|
enqueue() {
|
|
30
33
|
if (Application.isStopped()) return;
|
|
31
34
|
for (const item of this.#items) {
|
|
32
|
-
this.#jobRegistry.enqueue('Action',
|
|
35
|
+
this.#jobRegistry.enqueue('Action', this.#buildParams(item));
|
|
33
36
|
}
|
|
34
37
|
}
|
|
38
|
+
|
|
39
|
+
/**
|
|
40
|
+
* Builds the job params object for a single item, including originUrl when present.
|
|
41
|
+
* @param {object} item The parsed response item.
|
|
42
|
+
* @returns {object} The params object for the job registry.
|
|
43
|
+
* @private
|
|
44
|
+
*/
|
|
45
|
+
#buildParams(item) {
|
|
46
|
+
const params = { action: this.#action, item };
|
|
47
|
+
if (this.#originUrl !== null) params.originUrl = this.#originUrl;
|
|
48
|
+
return params;
|
|
49
|
+
}
|
|
50
|
+
|
|
35
51
|
}
|
|
36
52
|
|
|
37
53
|
export { ActionEnqueuer };
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { ActionEnqueuer } from './ActionEnqueuer.js';
|
|
2
|
-
import { NullResponse } from '../exceptions/NullResponse.js';
|
|
2
|
+
import { NullResponse } from '../exceptions/request/NullResponse.js';
|
|
3
3
|
|
|
4
4
|
/**
|
|
5
5
|
* ActionsEnqueuer enqueues one ActionProcessingJob per (item × action) pair.
|
|
@@ -12,16 +12,19 @@ class ActionsEnqueuer {
|
|
|
12
12
|
#actions;
|
|
13
13
|
#items;
|
|
14
14
|
#jobRegistry;
|
|
15
|
+
#originUrl;
|
|
15
16
|
|
|
16
17
|
/**
|
|
17
18
|
* @param {Array} actions List of ResourceRequestAction instances.
|
|
18
19
|
* @param {Array} items List of ResponseWrapper instances (one per response item).
|
|
19
20
|
* @param {object} [jobRegistry] The job registry to enqueue jobs to. Defaults to global JobRegistry.
|
|
21
|
+
* @param {string|null} [originUrl=null] The URL of the ResourceRequestJob that triggered this enqueue.
|
|
20
22
|
*/
|
|
21
|
-
constructor(actions, items, jobRegistry) {
|
|
23
|
+
constructor(actions, items, jobRegistry, originUrl = null) {
|
|
22
24
|
this.#actions = actions;
|
|
23
25
|
this.#items = items;
|
|
24
26
|
this.#jobRegistry = jobRegistry;
|
|
27
|
+
this.#originUrl = originUrl;
|
|
25
28
|
}
|
|
26
29
|
|
|
27
30
|
/**
|
|
@@ -34,7 +37,7 @@ class ActionsEnqueuer {
|
|
|
34
37
|
if (this.#items === null) throw new NullResponse();
|
|
35
38
|
|
|
36
39
|
for (const action of this.#actions) {
|
|
37
|
-
new ActionEnqueuer(action, this.#items, this.#jobRegistry).enqueue();
|
|
40
|
+
new ActionEnqueuer(action, this.#items, this.#jobRegistry, this.#originUrl).enqueue();
|
|
38
41
|
}
|
|
39
42
|
}
|
|
40
43
|
}
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
import { JobRegistry as DefaultJobRegistry } from '../background/JobRegistry.js';
|
|
2
|
+
import { Application } from '../services/Application.js';
|
|
3
|
+
|
|
4
|
+
/**
|
|
5
|
+
* Enqueues one PaginatedActionProcessingJob for a single paginated action.
|
|
6
|
+
* @author darthjee
|
|
7
|
+
*/
|
|
8
|
+
class PaginatedActionEnqueuer {
|
|
9
|
+
#paginatedAction;
|
|
10
|
+
#responseWrapper;
|
|
11
|
+
#parameters;
|
|
12
|
+
#jobRegistry;
|
|
13
|
+
#originUrl;
|
|
14
|
+
|
|
15
|
+
/**
|
|
16
|
+
* @param {ResourceRequestPaginatedAction} paginatedAction The paginated action to enqueue.
|
|
17
|
+
* @param {ResponseWrapper} responseWrapper The response wrapper carrying response data.
|
|
18
|
+
* @param {object} [parameters={}] The original request parameters (key-value map for URL template resolution).
|
|
19
|
+
* @param {object} [jobRegistry=JobRegistry] The job registry to enqueue jobs to.
|
|
20
|
+
* @param {string|null} [originUrl=null] The URL of the ResourceRequestJob that triggered this enqueue.
|
|
21
|
+
*/
|
|
22
|
+
constructor(paginatedAction, responseWrapper, parameters = {}, jobRegistry = DefaultJobRegistry, originUrl = null) {
|
|
23
|
+
this.#paginatedAction = paginatedAction;
|
|
24
|
+
this.#responseWrapper = responseWrapper;
|
|
25
|
+
this.#parameters = parameters;
|
|
26
|
+
this.#jobRegistry = jobRegistry;
|
|
27
|
+
this.#originUrl = originUrl;
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
/**
|
|
31
|
+
* Enqueues one PaginatedActionProcessingJob for the configured paginated action.
|
|
32
|
+
* Does nothing if the application is in 'stopped' status.
|
|
33
|
+
* @returns {void}
|
|
34
|
+
*/
|
|
35
|
+
enqueue() {
|
|
36
|
+
if (Application.isStopped()) return;
|
|
37
|
+
this.#jobRegistry.enqueue('PaginatedAction', this.#buildParams());
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
/**
|
|
41
|
+
* Builds the job params object, including originUrl when present.
|
|
42
|
+
* @returns {object} The params object for the job registry.
|
|
43
|
+
* @private
|
|
44
|
+
*/
|
|
45
|
+
#buildParams() {
|
|
46
|
+
const params = { paginatedAction: this.#paginatedAction, responseWrapper: this.#responseWrapper, parameters: this.#parameters };
|
|
47
|
+
if (this.#originUrl !== null) params.originUrl = this.#originUrl;
|
|
48
|
+
return params;
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
export { PaginatedActionEnqueuer };
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
import { PaginatedActionEnqueuer } from './PaginatedActionEnqueuer.js';
|
|
2
|
+
import { NullResponse } from '../exceptions/request/NullResponse.js';
|
|
3
|
+
|
|
4
|
+
/**
|
|
5
|
+
* Enqueues one PaginatedActionProcessingJob per paginated action for the given parameters.
|
|
6
|
+
*
|
|
7
|
+
* Delegates execution to the job queue instead of running paginated actions inline.
|
|
8
|
+
* Each paginated action is handled by a dedicated PaginatedActionEnqueuer.
|
|
9
|
+
* @author darthjee
|
|
10
|
+
*/
|
|
11
|
+
class PaginatedActionsEnqueuer {
|
|
12
|
+
#paginatedActions;
|
|
13
|
+
#responseWrapper;
|
|
14
|
+
#parameters;
|
|
15
|
+
#jobRegistry;
|
|
16
|
+
#originUrl;
|
|
17
|
+
|
|
18
|
+
/**
|
|
19
|
+
* @param {Array<ResourceRequestPaginatedAction>} paginatedActions List of paginated action instances.
|
|
20
|
+
* @param {ResponseWrapper} responseWrapper The response wrapper carrying response data.
|
|
21
|
+
* @param {object} [parameters={}] The original request parameters (key-value map for URL template resolution).
|
|
22
|
+
* @param {object} [jobRegistry] The job registry to enqueue jobs to. Defaults to global JobRegistry.
|
|
23
|
+
* @param {string|null} [originUrl=null] The URL of the ResourceRequestJob that triggered this enqueue.
|
|
24
|
+
*/
|
|
25
|
+
constructor(paginatedActions, responseWrapper, parameters = {}, jobRegistry, originUrl = null) {
|
|
26
|
+
this.#paginatedActions = paginatedActions;
|
|
27
|
+
this.#responseWrapper = responseWrapper;
|
|
28
|
+
this.#parameters = parameters;
|
|
29
|
+
this.#jobRegistry = jobRegistry;
|
|
30
|
+
this.#originUrl = originUrl;
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
/**
|
|
34
|
+
* Enqueues one PaginatedActionProcessingJob per paginated action.
|
|
35
|
+
* @returns {void}
|
|
36
|
+
* @throws {NullResponse} If the responseWrapper is null.
|
|
37
|
+
*/
|
|
38
|
+
enqueue() {
|
|
39
|
+
if (this.#responseWrapper === null) throw new NullResponse();
|
|
40
|
+
|
|
41
|
+
for (const paginatedAction of this.#paginatedActions) {
|
|
42
|
+
new PaginatedActionEnqueuer(paginatedAction, this.#responseWrapper, this.#parameters, this.#jobRegistry, this.#originUrl).enqueue();
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
export { PaginatedActionsEnqueuer };
|
|
@@ -9,6 +9,7 @@ import { Job } from '../background/Job.js';
|
|
|
9
9
|
class ActionProcessingJob extends Job {
|
|
10
10
|
#action;
|
|
11
11
|
#item;
|
|
12
|
+
#originUrl;
|
|
12
13
|
|
|
13
14
|
/**
|
|
14
15
|
* Creates a new ActionProcessingJob instance.
|
|
@@ -16,19 +17,21 @@ class ActionProcessingJob extends Job {
|
|
|
16
17
|
* @param {string} params.id - The unique identifier for this job.
|
|
17
18
|
* @param {ResourceRequestAction} params.action - The action to execute.
|
|
18
19
|
* @param {object} params.item - The parsed response item to pass to the action.
|
|
20
|
+
* @param {string|null} [params.originUrl=null] - The URL of the ResourceRequestJob that triggered this job.
|
|
19
21
|
*/
|
|
20
|
-
constructor({ id, action, item }) {
|
|
22
|
+
constructor({ id, action, item, originUrl = null }) {
|
|
21
23
|
super({ id });
|
|
22
24
|
this.#action = action;
|
|
23
25
|
this.#item = item;
|
|
26
|
+
this.#originUrl = originUrl;
|
|
24
27
|
}
|
|
25
28
|
|
|
26
29
|
/**
|
|
27
30
|
* Returns the job-specific arguments for serialization.
|
|
28
|
-
* @returns {{ item: object }} The job arguments.
|
|
31
|
+
* @returns {{ item: object, originUrl?: string }} The job arguments.
|
|
29
32
|
*/
|
|
30
33
|
get arguments() {
|
|
31
|
-
return { item: this.#item };
|
|
34
|
+
return { item: this.#item, ...this.#originUrlField() };
|
|
32
35
|
}
|
|
33
36
|
|
|
34
37
|
/**
|
|
@@ -55,6 +58,17 @@ class ActionProcessingJob extends Job {
|
|
|
55
58
|
this._fail(error);
|
|
56
59
|
}
|
|
57
60
|
}
|
|
61
|
+
|
|
62
|
+
/**
|
|
63
|
+
* Returns an object containing the originUrl field when an origin URL is set,
|
|
64
|
+
* or an empty object otherwise.
|
|
65
|
+
* @returns {{ originUrl: string }|{}} The origin URL field or empty object.
|
|
66
|
+
* @private
|
|
67
|
+
*/
|
|
68
|
+
#originUrlField() {
|
|
69
|
+
return this.#originUrl !== null ? { originUrl: this.#originUrl } : {};
|
|
70
|
+
}
|
|
71
|
+
|
|
58
72
|
}
|
|
59
73
|
|
|
60
74
|
export { ActionProcessingJob };
|
package/lib/jobs/HtmlParseJob.js
CHANGED
|
@@ -14,6 +14,7 @@ class HtmlParseJob extends Job {
|
|
|
14
14
|
#assetRequests;
|
|
15
15
|
#jobRegistry;
|
|
16
16
|
#clientRegistry;
|
|
17
|
+
#originUrl;
|
|
17
18
|
|
|
18
19
|
/**
|
|
19
20
|
* Creates a new HtmlParseJob instance.
|
|
@@ -23,21 +24,23 @@ class HtmlParseJob extends Job {
|
|
|
23
24
|
* @param {Array} params.assetRequests - List of AssetRequest instances.
|
|
24
25
|
* @param {object} params.jobRegistry - The job registry used to enqueue AssetDownloadJobs.
|
|
25
26
|
* @param {object} params.clientRegistry - The client registry used for URL resolution.
|
|
27
|
+
* @param {string|null} [params.originUrl=null] - The URL of the ResourceRequestJob that triggered this job.
|
|
26
28
|
*/
|
|
27
|
-
constructor({ id, rawHtml, assetRequests, jobRegistry, clientRegistry }) {
|
|
29
|
+
constructor({ id, rawHtml, assetRequests, jobRegistry, clientRegistry, originUrl = null }) {
|
|
28
30
|
super({ id });
|
|
29
31
|
this.#rawHtml = rawHtml;
|
|
30
32
|
this.#assetRequests = assetRequests;
|
|
31
33
|
this.#jobRegistry = jobRegistry;
|
|
32
34
|
this.#clientRegistry = clientRegistry;
|
|
35
|
+
this.#originUrl = originUrl;
|
|
33
36
|
}
|
|
34
37
|
|
|
35
38
|
/**
|
|
36
39
|
* Returns the job-specific arguments for serialization.
|
|
37
|
-
* @returns {{ assetCount: number }} The job arguments.
|
|
40
|
+
* @returns {{ assetCount: number, originUrl?: string }} The job arguments.
|
|
38
41
|
*/
|
|
39
42
|
get arguments() {
|
|
40
|
-
return { assetCount: this.#assetRequests.length };
|
|
43
|
+
return { assetCount: this.#assetRequests.length, ...this.#originUrlField() };
|
|
41
44
|
}
|
|
42
45
|
|
|
43
46
|
/**
|
|
@@ -49,7 +52,6 @@ class HtmlParseJob extends Job {
|
|
|
49
52
|
get maxRetries() {
|
|
50
53
|
return 1;
|
|
51
54
|
}
|
|
52
|
-
|
|
53
55
|
/**
|
|
54
56
|
* Parses the HTML body, resolves asset URLs, and enqueues one AssetDownloadJob per URL.
|
|
55
57
|
* @param {LogContext} logContext - Context carrying workerId/jobId for log entries.
|
|
@@ -66,6 +68,17 @@ class HtmlParseJob extends Job {
|
|
|
66
68
|
this._fail(error);
|
|
67
69
|
}
|
|
68
70
|
}
|
|
71
|
+
|
|
72
|
+
/**
|
|
73
|
+
* Returns an object containing the originUrl field when an origin URL is set,
|
|
74
|
+
* or an empty object otherwise.
|
|
75
|
+
* @returns {{ originUrl: string }|{}} The origin URL field or empty object.
|
|
76
|
+
* @private
|
|
77
|
+
*/
|
|
78
|
+
#originUrlField() {
|
|
79
|
+
return this.#originUrl !== null ? { originUrl: this.#originUrl } : {};
|
|
80
|
+
}
|
|
81
|
+
|
|
69
82
|
}
|
|
70
83
|
|
|
71
84
|
export { HtmlParseJob };
|
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
import { Job } from '../background/Job.js';
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Processes a single paginated action for a given response wrapper.
|
|
5
|
+
* Exhausted after the first failure — no retry rights.
|
|
6
|
+
* @author darthjee
|
|
7
|
+
* @augments Job
|
|
8
|
+
*/
|
|
9
|
+
class PaginatedActionProcessingJob extends Job {
|
|
10
|
+
#paginatedAction;
|
|
11
|
+
#responseWrapper;
|
|
12
|
+
#parameters;
|
|
13
|
+
#originUrl;
|
|
14
|
+
|
|
15
|
+
/**
|
|
16
|
+
* Creates a new PaginatedActionProcessingJob instance.
|
|
17
|
+
* @param {object} params The parameters for creating a PaginatedActionProcessingJob instance.
|
|
18
|
+
* @param {string} params.id Unique job identifier.
|
|
19
|
+
* @param {ResourceRequestPaginatedAction} params.paginatedAction The paginated action to execute.
|
|
20
|
+
* @param {ResponseWrapper} params.responseWrapper The response wrapper carrying response data.
|
|
21
|
+
* @param {object} [params.parameters={}] The original request parameters (key-value map for URL template resolution).
|
|
22
|
+
* @param {string|null} [params.originUrl=null] The URL of the ResourceRequestJob that triggered this job.
|
|
23
|
+
*/
|
|
24
|
+
constructor({ id, paginatedAction, responseWrapper, parameters = {}, originUrl = null }) {
|
|
25
|
+
super({ id });
|
|
26
|
+
this.#paginatedAction = paginatedAction;
|
|
27
|
+
this.#responseWrapper = responseWrapper;
|
|
28
|
+
this.#parameters = parameters;
|
|
29
|
+
this.#originUrl = originUrl;
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
/**
|
|
33
|
+
* Returns the job-specific arguments for serialization.
|
|
34
|
+
* @returns {{ responseWrapper: ResponseWrapper, parameters: object, originUrl?: string }} The job arguments.
|
|
35
|
+
*/
|
|
36
|
+
get arguments() {
|
|
37
|
+
return { responseWrapper: this.#responseWrapper, parameters: this.#parameters, ...this.#originUrlField() };
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
/**
|
|
41
|
+
* Returns the maximum number of retries for this job type.
|
|
42
|
+
* PaginatedActionProcessingJob is exhausted after the first failure.
|
|
43
|
+
* @returns {number} Always 1.
|
|
44
|
+
* @override
|
|
45
|
+
*/
|
|
46
|
+
get maxRetries() {
|
|
47
|
+
return 1;
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
/**
|
|
51
|
+
* Performs the paginated action for the given response wrapper and parameters.
|
|
52
|
+
* @param {LogContext} logContext - Context carrying workerId/jobId for log entries.
|
|
53
|
+
* @returns {Promise<void>}
|
|
54
|
+
*/
|
|
55
|
+
async perform(logContext) {
|
|
56
|
+
logContext.debug(`PaginatedActionProcessingJob #${this.id} performing`);
|
|
57
|
+
try {
|
|
58
|
+
this.lastError = undefined;
|
|
59
|
+
await this.#paginatedAction.execute(this.#responseWrapper, this.#parameters);
|
|
60
|
+
} catch (error) {
|
|
61
|
+
this._fail(error);
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
/**
|
|
66
|
+
* Returns an object containing the originUrl field when an origin URL is set,
|
|
67
|
+
* or an empty object otherwise.
|
|
68
|
+
* @returns {{ originUrl: string }|{}} The origin URL field or empty object.
|
|
69
|
+
* @private
|
|
70
|
+
*/
|
|
71
|
+
#originUrlField() {
|
|
72
|
+
return this.#originUrl !== null ? { originUrl: this.#originUrl } : {};
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
export { PaginatedActionProcessingJob };
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { Job } from '../background/Job.js';
|
|
2
2
|
import { JobRegistry } from '../background/JobRegistry.js';
|
|
3
|
-
import { ResponseWrapper } from '../models/ResponseWrapper.js';
|
|
3
|
+
import { ResponseWrapper } from '../models/response/ResponseWrapper.js';
|
|
4
4
|
|
|
5
5
|
/**
|
|
6
6
|
* ResourceRequestJob is a Job that performs an HTTP request for a ResourceRequest.
|
|
@@ -60,20 +60,23 @@ class ResourceRequestJob extends Job {
|
|
|
60
60
|
* @private
|
|
61
61
|
*/
|
|
62
62
|
#handleResponse(response) {
|
|
63
|
-
this.#
|
|
63
|
+
const originUrl = this.#resourceRequest.resolveUrl(this.#parameters);
|
|
64
|
+
this.#enqueueAssets(response, originUrl);
|
|
64
65
|
const wrapper = new ResponseWrapper(response, this.#parameters);
|
|
65
|
-
this.#resourceRequest.enqueueActions(wrapper);
|
|
66
|
+
this.#resourceRequest.enqueueActions(wrapper, originUrl);
|
|
67
|
+
this.#resourceRequest.enqueuePaginatedActions(wrapper, this.#parameters, originUrl);
|
|
66
68
|
return response;
|
|
67
69
|
}
|
|
68
70
|
|
|
69
71
|
/**
|
|
70
72
|
* Enqueues asset download jobs when the resource request declares asset rules.
|
|
71
73
|
* @param {object} response The HTTP response object.
|
|
74
|
+
* @param {string} originUrl The resolved URL of this job, forwarded as origin to child jobs.
|
|
72
75
|
* @private
|
|
73
76
|
*/
|
|
74
|
-
#enqueueAssets(response) {
|
|
77
|
+
#enqueueAssets(response, originUrl) {
|
|
75
78
|
if (this.#resourceRequest.hasAssets()) {
|
|
76
|
-
this.#resourceRequest.enqueueAssets(response.data, JobRegistry, this.#clients);
|
|
79
|
+
this.#resourceRequest.enqueueAssets(response.data, JobRegistry, this.#clients, originUrl);
|
|
77
80
|
}
|
|
78
81
|
}
|
|
79
82
|
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { LogConfig } from './LogConfig.js';
|
|
2
|
-
import { ClientRegistry } from '
|
|
3
|
-
import { ResourceRegistry } from '
|
|
4
|
-
import { ConfigLoader } from '
|
|
2
|
+
import { ClientRegistry } from '../../registry/ClientRegistry.js';
|
|
3
|
+
import { ResourceRegistry } from '../../registry/ResourceRegistry.js';
|
|
4
|
+
import { ConfigLoader } from '../../services/ConfigLoader.js';
|
|
5
5
|
|
|
6
6
|
/**
|
|
7
7
|
* Config is a class that represents the configuration for the application.
|
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
import { PathResolver } from '../response/PathResolver.js';
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Parses the pagination block from YAML and resolves the total page count
|
|
5
|
+
* against a response wrapper.
|
|
6
|
+
* @author darthjee
|
|
7
|
+
*/
|
|
8
|
+
class PaginationConfig {
|
|
9
|
+
#pagesResolver;
|
|
10
|
+
#pageKey;
|
|
11
|
+
#zeroIndexed;
|
|
12
|
+
|
|
13
|
+
/**
|
|
14
|
+
* @param {object} params Pagination configuration attributes.
|
|
15
|
+
* @param {string} params.pages Path expression to resolve the total page count.
|
|
16
|
+
* @param {string} params.page_key The parameter name to inject the page number.
|
|
17
|
+
* @param {boolean} [params.zero_indexed=false] Whether pages are zero-indexed.
|
|
18
|
+
*/
|
|
19
|
+
constructor({ pages, page_key, zero_indexed = false }) {
|
|
20
|
+
this.#pagesResolver = PathResolver.fromExpression(pages);
|
|
21
|
+
this.#pageKey = page_key;
|
|
22
|
+
this.#zeroIndexed = zero_indexed;
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
/**
|
|
26
|
+
* @returns {string} The parameter name used to inject the page number.
|
|
27
|
+
*/
|
|
28
|
+
get pageKey() {
|
|
29
|
+
return this.#pageKey;
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
/**
|
|
33
|
+
* Evaluates the `pages` expression against the response wrapper.
|
|
34
|
+
* @param {ResponseWrapper} responseWrapper The response wrapper to resolve against.
|
|
35
|
+
* @returns {number} Total number of pages.
|
|
36
|
+
*/
|
|
37
|
+
resolvePages(responseWrapper) {
|
|
38
|
+
return this.#pagesResolver.resolve(responseWrapper);
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
/**
|
|
42
|
+
* Returns an array of page numbers to iterate over.
|
|
43
|
+
* @param {number} count Total number of pages.
|
|
44
|
+
* @returns {Array<number>} Array of page numbers.
|
|
45
|
+
*/
|
|
46
|
+
pageNumbers(count) {
|
|
47
|
+
const start = this.#zeroIndexed ? 0 : 1;
|
|
48
|
+
const end = this.#zeroIndexed ? count - 1 : count;
|
|
49
|
+
return Array.from({ length: end - start + 1 }, (_, i) => start + i);
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
/**
|
|
53
|
+
* Creates a PaginationConfig from a raw YAML list entry.
|
|
54
|
+
* @param {Array<object>} list List of single-key maps from YAML.
|
|
55
|
+
* @returns {PaginationConfig} A new PaginationConfig instance.
|
|
56
|
+
*/
|
|
57
|
+
static fromList(list) {
|
|
58
|
+
const attrs = Object.assign({}, ...list);
|
|
59
|
+
return new PaginationConfig(attrs);
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
export { PaginationConfig };
|