navi-hey 1.1.0 → 1.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (85) hide show
  1. package/README.md +38 -4
  2. package/lib/enqueuers/ActionEnqueuer.js +18 -2
  3. package/lib/enqueuers/ActionsEnqueuer.js +6 -3
  4. package/lib/enqueuers/PaginatedActionEnqueuer.js +53 -0
  5. package/lib/enqueuers/PaginatedActionsEnqueuer.js +47 -0
  6. package/lib/exceptions/{ConfigurationFileNotFound.js → config/ConfigurationFileNotFound.js} +1 -1
  7. package/lib/exceptions/{ConfigurationFileNotProvided.js → config/ConfigurationFileNotProvided.js} +1 -1
  8. package/lib/exceptions/{MissingTopLevelConfgKey.js → config/MissingTopLevelConfgKey.js} +1 -1
  9. package/lib/exceptions/{ConflictError.js → http/ConflictError.js} +1 -1
  10. package/lib/exceptions/{ForbiddenError.js → http/ForbiddenError.js} +1 -1
  11. package/lib/exceptions/{NotFoundError.js → http/NotFoundError.js} +1 -1
  12. package/lib/exceptions/{ItemNotFound.js → registry/ItemNotFound.js} +1 -1
  13. package/lib/exceptions/{MissingActionResource.js → registry/MissingActionResource.js} +1 -1
  14. package/lib/exceptions/{MissingMappingVariable.js → registry/MissingMappingVariable.js} +1 -1
  15. package/lib/exceptions/{InvalidHtmlResponseBody.js → request/InvalidHtmlResponseBody.js} +1 -1
  16. package/lib/exceptions/{InvalidResponseBody.js → request/InvalidResponseBody.js} +1 -1
  17. package/lib/exceptions/{NullResponse.js → request/NullResponse.js} +1 -1
  18. package/lib/exceptions/{RequestFailed.js → request/RequestFailed.js} +1 -1
  19. package/lib/jobs/ActionProcessingJob.js +17 -3
  20. package/lib/jobs/HtmlParseJob.js +17 -4
  21. package/lib/jobs/PaginatedActionProcessingJob.js +77 -0
  22. package/lib/jobs/ResourceRequestJob.js +8 -5
  23. package/lib/models/{Config.js → configs/Config.js} +3 -3
  24. package/lib/models/configs/PaginationConfig.js +63 -0
  25. package/lib/models/{ResourceRequest.js → request/ResourceRequest.js} +30 -8
  26. package/lib/models/{ResourceRequestAction.js → request/ResourceRequestAction.js} +6 -6
  27. package/lib/models/request/ResourceRequestPaginatedAction.js +81 -0
  28. package/lib/models/{PathSegmentTraverser.js → response/PathSegmentTraverser.js} +1 -1
  29. package/lib/models/{ResponseParser.js → response/ResponseParser.js} +1 -1
  30. package/lib/registry/ClientRegistry.js +1 -1
  31. package/lib/registry/LogRegistry.js +13 -13
  32. package/lib/registry/LogRegistryInstance.js +16 -16
  33. package/lib/registry/NamedRegistry.js +1 -1
  34. package/lib/registry/ResourceRegistry.js +1 -1
  35. package/lib/serializers/JobIndexSerializer.js +3 -0
  36. package/lib/serializers/LogSerializer.js +1 -1
  37. package/lib/server/PathValidator.js +1 -1
  38. package/lib/server/RouteRegister.js +3 -3
  39. package/lib/server/Router.js +18 -18
  40. package/lib/server/{AssetsRequestHandler.js → handlers/AssetsRequestHandler.js} +3 -3
  41. package/lib/server/{BaseUrlsRequestHandler.js → handlers/BaseUrlsRequestHandler.js} +3 -3
  42. package/lib/server/{IndexRequestHandler.js → handlers/IndexRequestHandler.js} +2 -2
  43. package/lib/server/{JobsFilter.js → handlers/JobsFilter.js} +1 -1
  44. package/lib/server/{LogsRequestHandler.js → handlers/LogsRequestHandler.js} +4 -4
  45. package/lib/server/{SettingsRequestHandler.js → handlers/SettingsRequestHandler.js} +4 -3
  46. package/lib/server/{StatsRequestHandler.js → handlers/StatsRequestHandler.js} +3 -3
  47. package/lib/server/{EngineContinueRequestHandler.js → handlers/engine/EngineContinueRequestHandler.js} +3 -3
  48. package/lib/server/{EnginePauseRequestHandler.js → handlers/engine/EnginePauseRequestHandler.js} +3 -3
  49. package/lib/server/{EngineRestartRequestHandler.js → handlers/engine/EngineRestartRequestHandler.js} +3 -3
  50. package/lib/server/{EngineShutdownRequestHandler.js → handlers/engine/EngineShutdownRequestHandler.js} +2 -2
  51. package/lib/server/{EngineStartRequestHandler.js → handlers/engine/EngineStartRequestHandler.js} +3 -3
  52. package/lib/server/{EngineStatusRequestHandler.js → handlers/engine/EngineStatusRequestHandler.js} +2 -2
  53. package/lib/server/{EngineStopRequestHandler.js → handlers/engine/EngineStopRequestHandler.js} +3 -3
  54. package/lib/server/{JobLogsRequestHandler.js → handlers/jobs/JobLogsRequestHandler.js} +4 -4
  55. package/lib/server/{JobRequestHandler.js → handlers/jobs/JobRequestHandler.js} +4 -4
  56. package/lib/server/{JobRetryRequestHandler.js → handlers/jobs/JobRetryRequestHandler.js} +4 -4
  57. package/lib/server/{JobsRequestHandler.js → handlers/jobs/JobsRequestHandler.js} +4 -4
  58. package/lib/services/ApplicationInstance.js +4 -2
  59. package/lib/services/Client.js +1 -1
  60. package/lib/services/ConfigLoader.js +1 -1
  61. package/lib/services/ConfigParser.js +7 -7
  62. package/lib/services/Engine.js +1 -1
  63. package/lib/utils/HtmlParser.js +1 -1
  64. package/lib/utils/logging/BufferedLogger.js +1 -1
  65. package/lib/utils/logging/LogBuffer.js +1 -1
  66. package/lib/utils/logging/LogBufferCollection.js +1 -1
  67. package/lib/utils/logging/LogFilter.js +2 -2
  68. package/package.json +1 -1
  69. package/static/assets/index-D5tOOLpl.js +11 -0
  70. package/static/index.html +1 -1
  71. package/lib/exceptions/LockedByOtherWorker.js +0 -14
  72. package/static/assets/index-BfggoMvd.js +0 -11
  73. /package/lib/exceptions/{MissingClientsConfig.js → config/MissingClientsConfig.js} +0 -0
  74. /package/lib/exceptions/{MissingResourceConfig.js → config/MissingResourceConfig.js} +0 -0
  75. /package/lib/exceptions/{ClientNotFound.js → registry/ClientNotFound.js} +0 -0
  76. /package/lib/exceptions/{ResourceNotFound.js → registry/ResourceNotFound.js} +0 -0
  77. /package/lib/models/{FailureConfig.js → configs/FailureConfig.js} +0 -0
  78. /package/lib/models/{LogConfig.js → configs/LogConfig.js} +0 -0
  79. /package/lib/models/{WebConfig.js → configs/WebConfig.js} +0 -0
  80. /package/lib/models/{WorkersConfig.js → configs/WorkersConfig.js} +0 -0
  81. /package/lib/models/{AssetRequest.js → request/AssetRequest.js} +0 -0
  82. /package/lib/models/{Resource.js → request/Resource.js} +0 -0
  83. /package/lib/models/{ParametersMapper.js → response/ParametersMapper.js} +0 -0
  84. /package/lib/models/{PathResolver.js → response/PathResolver.js} +0 -0
  85. /package/lib/models/{ResponseWrapper.js → response/ResponseWrapper.js} +0 -0
package/README.md CHANGED
@@ -16,6 +16,7 @@ Key features:
16
16
  - Concurrent HTTP request execution via a worker pool.
17
17
  - URL templates with placeholder parameters (e.g. `{:id}`).
18
18
  - Response-driven actions: after each successful request, configurable actions extract variables from the response and trigger follow-up processing.
19
+ - Paginated resource support: `paginated_actions` fan out one request per page based on a page-count expression evaluated against the response.
19
20
  - Automatic retry of failed requests after the main queue is exhausted.
20
21
 
21
22
  ---
@@ -92,8 +93,13 @@ resources:
92
93
  - resource: category_information
93
94
  - resource: products
94
95
  parameters:
95
- category_id: parsed_body.id
96
- page: headers['x-next-page']
96
+ category_id: parsedBody.id
97
+ paginated_actions:
98
+ - resource: products_page
99
+ pagination:
100
+ - pages: parsedBody.pagination.pages
101
+ - page_key: page
102
+ - zero_indexed: false
97
103
  - url: /categories
98
104
  status: 302
99
105
  - url: /#/categories
@@ -134,7 +140,13 @@ When the web server is enabled, the following screens are available:
134
140
  | `client` | Name of the client to use for this request. Defaults to `default`. |
135
141
  | `actions` | Optional list of actions to execute after a successful response. |
136
142
  | `actions[].resource` | Name of the resource to act upon. Required. |
137
- | `actions[].parameters` | Optional key-value map. Each key is the destination variable name and each value is a path expression resolved against the response (e.g. `parsed_body.id`, `headers['page']`). When absent, the parsed body item is passed through unchanged. |
143
+ | `actions[].parameters` | Optional key-value map. Each key is the destination variable name and each value is a path expression resolved against the response (e.g. `parsedBody.id`, `headers['page']`). When absent, the parsed body item is passed through unchanged. |
144
+ | `paginated_actions` | Optional list of paginated actions. Each entry fans out one `ResourceRequestJob` per page. |
145
+ | `paginated_actions[].resource` | Name of the resource to enqueue per page. Required. |
146
+ | `paginated_actions[].pagination` | List of pagination config entries. Required. |
147
+ | `paginated_actions[].pagination[].pages` | Path expression (e.g. `parsedBody.pagination.pages`) resolving to the total page count. |
148
+ | `paginated_actions[].pagination[].page_key` | Parameter name injected as the current page number into each downstream request. |
149
+ | `paginated_actions[].pagination[].zero_indexed` | Boolean. Pages start at `0` when `true`, at `1` when `false` (default). |
138
150
  | `assets` | Optional list of asset extraction rules. When present on an HTML resource, Navi parses the response body and enqueues a download job for each matched URL. |
139
151
  | `assets[].selector` | CSS selector used to find asset elements in the HTML response (e.g. `script[src]`, `link[rel="stylesheet"]`). |
140
152
  | `assets[].attribute` | Attribute on the matched element that holds the asset URL (e.g. `src`, `href`). |
@@ -147,7 +159,29 @@ When the web server is enabled, the following screens are available:
147
159
 
148
160
  Navi supports multi-level resource chaining. After a successful response, each configured action uses `parameters` path expressions to extract variables from the response body or headers and enqueues new jobs for the target resource. The extracted variables resolve `{:placeholder}` tokens in the target URL templates.
149
161
 
150
- For example, requesting `/categories.json` might return `[{ "id": 1 }, { "id": 2 }]`. With an action targeting `category_information` and `parameters: { id: parsed_body.id }`, Navi automatically enqueues requests for `/categories/1.json` and `/categories/2.json`.
162
+ For example, requesting `/categories.json` might return `[{ "id": 1 }, { "id": 2 }]`. With an action targeting `category_information` and `parameters: { id: parsedBody.id }`, Navi automatically enqueues requests for `/categories/1.json` and `/categories/2.json`. Header values can also be extracted, e.g. `page: headers['x-next-page']`.
163
+
164
+ > **Note:** HTTP response header names are always lowercase after Node.js normalization. Use lowercase keys in path expressions (e.g. `headers['x-total-pages']`), regardless of how the server set them.
165
+
166
+ ## Paginated Actions
167
+
168
+ `paginated_actions` complement `actions` when the response indicates multiple pages. Instead of iterating over array items, Navi evaluates a `pages` expression against the whole response, then enqueues one `ResourceRequestJob` per page, injecting the page number under the configured `page_key`.
169
+
170
+ ```yaml
171
+ resources:
172
+ categories:
173
+ - url: /categories.json
174
+ status: 200
175
+ paginated_actions:
176
+ - resource: products_page
177
+ pagination:
178
+ - pages: parsedBody.pagination.pages
179
+ - page_key: page
180
+ - zero_indexed: false
181
+ products_page:
182
+ - url: /products/{:page}.json
183
+ status: 200
184
+ ```
151
185
 
152
186
  ---
153
187
 
@@ -9,16 +9,19 @@ class ActionEnqueuer {
9
9
  #action;
10
10
  #items;
11
11
  #jobRegistry;
12
+ #originUrl;
12
13
 
13
14
  /**
14
15
  * @param {ResourceRequestAction} action The action to enqueue for each item.
15
16
  * @param {Array} items List of parsed response items.
16
17
  * @param {object} [jobRegistry=JobRegistry] The job registry to enqueue jobs to.
18
+ * @param {string|null} [originUrl=null] The URL of the ResourceRequestJob that triggered this enqueue.
17
19
  */
18
- constructor(action, items, jobRegistry = DefaultJobRegistry) {
20
+ constructor(action, items, jobRegistry = DefaultJobRegistry, originUrl = null) {
19
21
  this.#action = action;
20
22
  this.#items = items;
21
23
  this.#jobRegistry = jobRegistry;
24
+ this.#originUrl = originUrl;
22
25
  }
23
26
 
24
27
  /**
@@ -29,9 +32,22 @@ class ActionEnqueuer {
29
32
  enqueue() {
30
33
  if (Application.isStopped()) return;
31
34
  for (const item of this.#items) {
32
- this.#jobRegistry.enqueue('Action', { action: this.#action, item });
35
+ this.#jobRegistry.enqueue('Action', this.#buildParams(item));
33
36
  }
34
37
  }
38
+
39
+ /**
40
+ * Builds the job params object for a single item, including originUrl when present.
41
+ * @param {object} item The parsed response item.
42
+ * @returns {object} The params object for the job registry.
43
+ * @private
44
+ */
45
+ #buildParams(item) {
46
+ const params = { action: this.#action, item };
47
+ if (this.#originUrl !== null) params.originUrl = this.#originUrl;
48
+ return params;
49
+ }
50
+
35
51
  }
36
52
 
37
53
  export { ActionEnqueuer };
@@ -1,5 +1,5 @@
1
1
  import { ActionEnqueuer } from './ActionEnqueuer.js';
2
- import { NullResponse } from '../exceptions/NullResponse.js';
2
+ import { NullResponse } from '../exceptions/request/NullResponse.js';
3
3
 
4
4
  /**
5
5
  * ActionsEnqueuer enqueues one ActionProcessingJob per (item × action) pair.
@@ -12,16 +12,19 @@ class ActionsEnqueuer {
12
12
  #actions;
13
13
  #items;
14
14
  #jobRegistry;
15
+ #originUrl;
15
16
 
16
17
  /**
17
18
  * @param {Array} actions List of ResourceRequestAction instances.
18
19
  * @param {Array} items List of ResponseWrapper instances (one per response item).
19
20
  * @param {object} [jobRegistry] The job registry to enqueue jobs to. Defaults to global JobRegistry.
21
+ * @param {string|null} [originUrl=null] The URL of the ResourceRequestJob that triggered this enqueue.
20
22
  */
21
- constructor(actions, items, jobRegistry) {
23
+ constructor(actions, items, jobRegistry, originUrl = null) {
22
24
  this.#actions = actions;
23
25
  this.#items = items;
24
26
  this.#jobRegistry = jobRegistry;
27
+ this.#originUrl = originUrl;
25
28
  }
26
29
 
27
30
  /**
@@ -34,7 +37,7 @@ class ActionsEnqueuer {
34
37
  if (this.#items === null) throw new NullResponse();
35
38
 
36
39
  for (const action of this.#actions) {
37
- new ActionEnqueuer(action, this.#items, this.#jobRegistry).enqueue();
40
+ new ActionEnqueuer(action, this.#items, this.#jobRegistry, this.#originUrl).enqueue();
38
41
  }
39
42
  }
40
43
  }
@@ -0,0 +1,53 @@
1
+ import { JobRegistry as DefaultJobRegistry } from '../background/JobRegistry.js';
2
+ import { Application } from '../services/Application.js';
3
+
4
+ /**
5
+ * Enqueues one PaginatedActionProcessingJob for a single paginated action.
6
+ * @author darthjee
7
+ */
8
+ class PaginatedActionEnqueuer {
9
+ #paginatedAction;
10
+ #responseWrapper;
11
+ #parameters;
12
+ #jobRegistry;
13
+ #originUrl;
14
+
15
+ /**
16
+ * @param {ResourceRequestPaginatedAction} paginatedAction The paginated action to enqueue.
17
+ * @param {ResponseWrapper} responseWrapper The response wrapper carrying response data.
18
+ * @param {object} [parameters={}] The original request parameters (key-value map for URL template resolution).
19
+ * @param {object} [jobRegistry=JobRegistry] The job registry to enqueue jobs to.
20
+ * @param {string|null} [originUrl=null] The URL of the ResourceRequestJob that triggered this enqueue.
21
+ */
22
+ constructor(paginatedAction, responseWrapper, parameters = {}, jobRegistry = DefaultJobRegistry, originUrl = null) {
23
+ this.#paginatedAction = paginatedAction;
24
+ this.#responseWrapper = responseWrapper;
25
+ this.#parameters = parameters;
26
+ this.#jobRegistry = jobRegistry;
27
+ this.#originUrl = originUrl;
28
+ }
29
+
30
+ /**
31
+ * Enqueues one PaginatedActionProcessingJob for the configured paginated action.
32
+ * Does nothing if the application is in 'stopped' status.
33
+ * @returns {void}
34
+ */
35
+ enqueue() {
36
+ if (Application.isStopped()) return;
37
+ this.#jobRegistry.enqueue('PaginatedAction', this.#buildParams());
38
+ }
39
+
40
+ /**
41
+ * Builds the job params object, including originUrl when present.
42
+ * @returns {object} The params object for the job registry.
43
+ * @private
44
+ */
45
+ #buildParams() {
46
+ const params = { paginatedAction: this.#paginatedAction, responseWrapper: this.#responseWrapper, parameters: this.#parameters };
47
+ if (this.#originUrl !== null) params.originUrl = this.#originUrl;
48
+ return params;
49
+ }
50
+
51
+ }
52
+
53
+ export { PaginatedActionEnqueuer };
@@ -0,0 +1,47 @@
1
+ import { PaginatedActionEnqueuer } from './PaginatedActionEnqueuer.js';
2
+ import { NullResponse } from '../exceptions/request/NullResponse.js';
3
+
4
+ /**
5
+ * Enqueues one PaginatedActionProcessingJob per paginated action for the given parameters.
6
+ *
7
+ * Delegates execution to the job queue instead of running paginated actions inline.
8
+ * Each paginated action is handled by a dedicated PaginatedActionEnqueuer.
9
+ * @author darthjee
10
+ */
11
+ class PaginatedActionsEnqueuer {
12
+ #paginatedActions;
13
+ #responseWrapper;
14
+ #parameters;
15
+ #jobRegistry;
16
+ #originUrl;
17
+
18
+ /**
19
+ * @param {Array<ResourceRequestPaginatedAction>} paginatedActions List of paginated action instances.
20
+ * @param {ResponseWrapper} responseWrapper The response wrapper carrying response data.
21
+ * @param {object} [parameters={}] The original request parameters (key-value map for URL template resolution).
22
+ * @param {object} [jobRegistry] The job registry to enqueue jobs to. Defaults to global JobRegistry.
23
+ * @param {string|null} [originUrl=null] The URL of the ResourceRequestJob that triggered this enqueue.
24
+ */
25
+ constructor(paginatedActions, responseWrapper, parameters = {}, jobRegistry, originUrl = null) {
26
+ this.#paginatedActions = paginatedActions;
27
+ this.#responseWrapper = responseWrapper;
28
+ this.#parameters = parameters;
29
+ this.#jobRegistry = jobRegistry;
30
+ this.#originUrl = originUrl;
31
+ }
32
+
33
+ /**
34
+ * Enqueues one PaginatedActionProcessingJob per paginated action.
35
+ * @returns {void}
36
+ * @throws {NullResponse} If the responseWrapper is null.
37
+ */
38
+ enqueue() {
39
+ if (this.#responseWrapper === null) throw new NullResponse();
40
+
41
+ for (const paginatedAction of this.#paginatedActions) {
42
+ new PaginatedActionEnqueuer(paginatedAction, this.#responseWrapper, this.#parameters, this.#jobRegistry, this.#originUrl).enqueue();
43
+ }
44
+ }
45
+ }
46
+
47
+ export { PaginatedActionsEnqueuer };
@@ -1,4 +1,4 @@
1
- import { AppError } from './AppError.js';
1
+ import { AppError } from '../AppError.js';
2
2
 
3
3
  class ConfigurationFileNotFound extends AppError {
4
4
  constructor(file) {
@@ -1,4 +1,4 @@
1
- import { AppError } from './AppError.js';
1
+ import { AppError } from '../AppError.js';
2
2
 
3
3
  class ConfigurationFileNotProvided extends AppError {
4
4
  constructor() {
@@ -1,4 +1,4 @@
1
- import { AppError } from './AppError.js';
1
+ import { AppError } from '../AppError.js';
2
2
 
3
3
  /**
4
4
  * ItemNotFound is a custom error class thrown when a requested item
@@ -1,4 +1,4 @@
1
- import { AppError } from './AppError.js';
1
+ import { AppError } from '../AppError.js';
2
2
 
3
3
  /**
4
4
  * ConflictError is thrown when a requested state transition is not valid
@@ -1,4 +1,4 @@
1
- import { AppError } from './AppError.js';
1
+ import { AppError } from '../AppError.js';
2
2
 
3
3
  /**
4
4
  * ForbiddenError is thrown when a request is rejected due to a security
@@ -1,4 +1,4 @@
1
- import { AppError } from './AppError.js';
1
+ import { AppError } from '../AppError.js';
2
2
 
3
3
  /**
4
4
  * NotFoundError is thrown when a requested resource cannot be found.
@@ -1,4 +1,4 @@
1
- import { AppError } from './AppError.js';
1
+ import { AppError } from '../AppError.js';
2
2
 
3
3
  /**
4
4
  * ItemNotFound is a custom error class thrown when a requested item
@@ -1,4 +1,4 @@
1
- import { AppError } from './AppError.js';
1
+ import { AppError } from '../AppError.js';
2
2
 
3
3
  /**
4
4
  * Thrown when an action config entry is missing the required "resource" field.
@@ -1,4 +1,4 @@
1
- import { AppError } from './AppError.js';
1
+ import { AppError } from '../AppError.js';
2
2
 
3
3
  /**
4
4
  * Thrown when a parameters path expression cannot be resolved against the response wrapper.
@@ -1,4 +1,4 @@
1
- import { AppError } from './AppError.js';
1
+ import { AppError } from '../AppError.js';
2
2
 
3
3
  /**
4
4
  * Thrown when an HTML response body cannot be parsed.
@@ -1,4 +1,4 @@
1
- import { AppError } from './AppError.js';
1
+ import { AppError } from '../AppError.js';
2
2
 
3
3
  /**
4
4
  * Thrown when a response body cannot be parsed as JSON.
@@ -1,4 +1,4 @@
1
- import { AppError } from './AppError.js';
1
+ import { AppError } from '../AppError.js';
2
2
 
3
3
  /**
4
4
  * Thrown when a parsed response body is null.
@@ -1,4 +1,4 @@
1
- import { AppError } from './AppError.js';
1
+ import { AppError } from '../AppError.js';
2
2
 
3
3
  /**
4
4
  * RequestFailed is a custom error class that represents a failed HTTP request.
@@ -9,6 +9,7 @@ import { Job } from '../background/Job.js';
9
9
  class ActionProcessingJob extends Job {
10
10
  #action;
11
11
  #item;
12
+ #originUrl;
12
13
 
13
14
  /**
14
15
  * Creates a new ActionProcessingJob instance.
@@ -16,19 +17,21 @@ class ActionProcessingJob extends Job {
16
17
  * @param {string} params.id - The unique identifier for this job.
17
18
  * @param {ResourceRequestAction} params.action - The action to execute.
18
19
  * @param {object} params.item - The parsed response item to pass to the action.
20
+ * @param {string|null} [params.originUrl=null] - The URL of the ResourceRequestJob that triggered this job.
19
21
  */
20
- constructor({ id, action, item }) {
22
+ constructor({ id, action, item, originUrl = null }) {
21
23
  super({ id });
22
24
  this.#action = action;
23
25
  this.#item = item;
26
+ this.#originUrl = originUrl;
24
27
  }
25
28
 
26
29
  /**
27
30
  * Returns the job-specific arguments for serialization.
28
- * @returns {{ item: object }} The job arguments.
31
+ * @returns {{ item: object, originUrl?: string }} The job arguments.
29
32
  */
30
33
  get arguments() {
31
- return { item: this.#item };
34
+ return { item: this.#item, ...this.#originUrlField() };
32
35
  }
33
36
 
34
37
  /**
@@ -55,6 +58,17 @@ class ActionProcessingJob extends Job {
55
58
  this._fail(error);
56
59
  }
57
60
  }
61
+
62
+ /**
63
+ * Returns an object containing the originUrl field when an origin URL is set,
64
+ * or an empty object otherwise.
65
+ * @returns {{ originUrl: string }|{}} The origin URL field or empty object.
66
+ * @private
67
+ */
68
+ #originUrlField() {
69
+ return this.#originUrl !== null ? { originUrl: this.#originUrl } : {};
70
+ }
71
+
58
72
  }
59
73
 
60
74
  export { ActionProcessingJob };
@@ -14,6 +14,7 @@ class HtmlParseJob extends Job {
14
14
  #assetRequests;
15
15
  #jobRegistry;
16
16
  #clientRegistry;
17
+ #originUrl;
17
18
 
18
19
  /**
19
20
  * Creates a new HtmlParseJob instance.
@@ -23,21 +24,23 @@ class HtmlParseJob extends Job {
23
24
  * @param {Array} params.assetRequests - List of AssetRequest instances.
24
25
  * @param {object} params.jobRegistry - The job registry used to enqueue AssetDownloadJobs.
25
26
  * @param {object} params.clientRegistry - The client registry used for URL resolution.
27
+ * @param {string|null} [params.originUrl=null] - The URL of the ResourceRequestJob that triggered this job.
26
28
  */
27
- constructor({ id, rawHtml, assetRequests, jobRegistry, clientRegistry }) {
29
+ constructor({ id, rawHtml, assetRequests, jobRegistry, clientRegistry, originUrl = null }) {
28
30
  super({ id });
29
31
  this.#rawHtml = rawHtml;
30
32
  this.#assetRequests = assetRequests;
31
33
  this.#jobRegistry = jobRegistry;
32
34
  this.#clientRegistry = clientRegistry;
35
+ this.#originUrl = originUrl;
33
36
  }
34
37
 
35
38
  /**
36
39
  * Returns the job-specific arguments for serialization.
37
- * @returns {{ assetCount: number }} The job arguments.
40
+ * @returns {{ assetCount: number, originUrl?: string }} The job arguments.
38
41
  */
39
42
  get arguments() {
40
- return { assetCount: this.#assetRequests.length };
43
+ return { assetCount: this.#assetRequests.length, ...this.#originUrlField() };
41
44
  }
42
45
 
43
46
  /**
@@ -49,7 +52,6 @@ class HtmlParseJob extends Job {
49
52
  get maxRetries() {
50
53
  return 1;
51
54
  }
52
-
53
55
  /**
54
56
  * Parses the HTML body, resolves asset URLs, and enqueues one AssetDownloadJob per URL.
55
57
  * @param {LogContext} logContext - Context carrying workerId/jobId for log entries.
@@ -66,6 +68,17 @@ class HtmlParseJob extends Job {
66
68
  this._fail(error);
67
69
  }
68
70
  }
71
+
72
+ /**
73
+ * Returns an object containing the originUrl field when an origin URL is set,
74
+ * or an empty object otherwise.
75
+ * @returns {{ originUrl: string }|{}} The origin URL field or empty object.
76
+ * @private
77
+ */
78
+ #originUrlField() {
79
+ return this.#originUrl !== null ? { originUrl: this.#originUrl } : {};
80
+ }
81
+
69
82
  }
70
83
 
71
84
  export { HtmlParseJob };
@@ -0,0 +1,77 @@
1
+ import { Job } from '../background/Job.js';
2
+
3
+ /**
4
+ * Processes a single paginated action for a given response wrapper.
5
+ * Exhausted after the first failure — no retry rights.
6
+ * @author darthjee
7
+ * @augments Job
8
+ */
9
+ class PaginatedActionProcessingJob extends Job {
10
+ #paginatedAction;
11
+ #responseWrapper;
12
+ #parameters;
13
+ #originUrl;
14
+
15
+ /**
16
+ * Creates a new PaginatedActionProcessingJob instance.
17
+ * @param {object} params The parameters for creating a PaginatedActionProcessingJob instance.
18
+ * @param {string} params.id Unique job identifier.
19
+ * @param {ResourceRequestPaginatedAction} params.paginatedAction The paginated action to execute.
20
+ * @param {ResponseWrapper} params.responseWrapper The response wrapper carrying response data.
21
+ * @param {object} [params.parameters={}] The original request parameters (key-value map for URL template resolution).
22
+ * @param {string|null} [params.originUrl=null] The URL of the ResourceRequestJob that triggered this job.
23
+ */
24
+ constructor({ id, paginatedAction, responseWrapper, parameters = {}, originUrl = null }) {
25
+ super({ id });
26
+ this.#paginatedAction = paginatedAction;
27
+ this.#responseWrapper = responseWrapper;
28
+ this.#parameters = parameters;
29
+ this.#originUrl = originUrl;
30
+ }
31
+
32
+ /**
33
+ * Returns the job-specific arguments for serialization.
34
+ * @returns {{ responseWrapper: ResponseWrapper, parameters: object, originUrl?: string }} The job arguments.
35
+ */
36
+ get arguments() {
37
+ return { responseWrapper: this.#responseWrapper, parameters: this.#parameters, ...this.#originUrlField() };
38
+ }
39
+
40
+ /**
41
+ * Returns the maximum number of retries for this job type.
42
+ * PaginatedActionProcessingJob is exhausted after the first failure.
43
+ * @returns {number} Always 1.
44
+ * @override
45
+ */
46
+ get maxRetries() {
47
+ return 1;
48
+ }
49
+
50
+ /**
51
+ * Performs the paginated action for the given response wrapper and parameters.
52
+ * @param {LogContext} logContext - Context carrying workerId/jobId for log entries.
53
+ * @returns {Promise<void>}
54
+ */
55
+ async perform(logContext) {
56
+ logContext.debug(`PaginatedActionProcessingJob #${this.id} performing`);
57
+ try {
58
+ this.lastError = undefined;
59
+ await this.#paginatedAction.execute(this.#responseWrapper, this.#parameters);
60
+ } catch (error) {
61
+ this._fail(error);
62
+ }
63
+ }
64
+
65
+ /**
66
+ * Returns an object containing the originUrl field when an origin URL is set,
67
+ * or an empty object otherwise.
68
+ * @returns {{ originUrl: string }|{}} The origin URL field or empty object.
69
+ * @private
70
+ */
71
+ #originUrlField() {
72
+ return this.#originUrl !== null ? { originUrl: this.#originUrl } : {};
73
+ }
74
+
75
+ }
76
+
77
+ export { PaginatedActionProcessingJob };
@@ -1,6 +1,6 @@
1
1
  import { Job } from '../background/Job.js';
2
2
  import { JobRegistry } from '../background/JobRegistry.js';
3
- import { ResponseWrapper } from '../models/ResponseWrapper.js';
3
+ import { ResponseWrapper } from '../models/response/ResponseWrapper.js';
4
4
 
5
5
  /**
6
6
  * ResourceRequestJob is a Job that performs an HTTP request for a ResourceRequest.
@@ -60,20 +60,23 @@ class ResourceRequestJob extends Job {
60
60
  * @private
61
61
  */
62
62
  #handleResponse(response) {
63
- this.#enqueueAssets(response);
63
+ const originUrl = this.#resourceRequest.resolveUrl(this.#parameters);
64
+ this.#enqueueAssets(response, originUrl);
64
65
  const wrapper = new ResponseWrapper(response, this.#parameters);
65
- this.#resourceRequest.enqueueActions(wrapper);
66
+ this.#resourceRequest.enqueueActions(wrapper, originUrl);
67
+ this.#resourceRequest.enqueuePaginatedActions(wrapper, this.#parameters, originUrl);
66
68
  return response;
67
69
  }
68
70
 
69
71
  /**
70
72
  * Enqueues asset download jobs when the resource request declares asset rules.
71
73
  * @param {object} response The HTTP response object.
74
+ * @param {string} originUrl The resolved URL of this job, forwarded as origin to child jobs.
72
75
  * @private
73
76
  */
74
- #enqueueAssets(response) {
77
+ #enqueueAssets(response, originUrl) {
75
78
  if (this.#resourceRequest.hasAssets()) {
76
- this.#resourceRequest.enqueueAssets(response.data, JobRegistry, this.#clients);
79
+ this.#resourceRequest.enqueueAssets(response.data, JobRegistry, this.#clients, originUrl);
77
80
  }
78
81
  }
79
82
 
@@ -1,7 +1,7 @@
1
1
  import { LogConfig } from './LogConfig.js';
2
- import { ClientRegistry } from '../registry/ClientRegistry.js';
3
- import { ResourceRegistry } from '../registry/ResourceRegistry.js';
4
- import { ConfigLoader } from '../services/ConfigLoader.js';
2
+ import { ClientRegistry } from '../../registry/ClientRegistry.js';
3
+ import { ResourceRegistry } from '../../registry/ResourceRegistry.js';
4
+ import { ConfigLoader } from '../../services/ConfigLoader.js';
5
5
 
6
6
  /**
7
7
  * Config is a class that represents the configuration for the application.
@@ -0,0 +1,63 @@
1
+ import { PathResolver } from '../response/PathResolver.js';
2
+
3
+ /**
4
+ * Parses the pagination block from YAML and resolves the total page count
5
+ * against a response wrapper.
6
+ * @author darthjee
7
+ */
8
+ class PaginationConfig {
9
+ #pagesResolver;
10
+ #pageKey;
11
+ #zeroIndexed;
12
+
13
+ /**
14
+ * @param {object} params Pagination configuration attributes.
15
+ * @param {string} params.pages Path expression to resolve the total page count.
16
+ * @param {string} params.page_key The parameter name to inject the page number.
17
+ * @param {boolean} [params.zero_indexed=false] Whether pages are zero-indexed.
18
+ */
19
+ constructor({ pages, page_key, zero_indexed = false }) {
20
+ this.#pagesResolver = PathResolver.fromExpression(pages);
21
+ this.#pageKey = page_key;
22
+ this.#zeroIndexed = zero_indexed;
23
+ }
24
+
25
+ /**
26
+ * @returns {string} The parameter name used to inject the page number.
27
+ */
28
+ get pageKey() {
29
+ return this.#pageKey;
30
+ }
31
+
32
+ /**
33
+ * Evaluates the `pages` expression against the response wrapper.
34
+ * @param {ResponseWrapper} responseWrapper The response wrapper to resolve against.
35
+ * @returns {number} Total number of pages.
36
+ */
37
+ resolvePages(responseWrapper) {
38
+ return this.#pagesResolver.resolve(responseWrapper);
39
+ }
40
+
41
+ /**
42
+ * Returns an array of page numbers to iterate over.
43
+ * @param {number} count Total number of pages.
44
+ * @returns {Array<number>} Array of page numbers.
45
+ */
46
+ pageNumbers(count) {
47
+ const start = this.#zeroIndexed ? 0 : 1;
48
+ const end = this.#zeroIndexed ? count - 1 : count;
49
+ return Array.from({ length: end - start + 1 }, (_, i) => start + i);
50
+ }
51
+
52
+ /**
53
+ * Creates a PaginationConfig from a raw YAML list entry.
54
+ * @param {Array<object>} list List of single-key maps from YAML.
55
+ * @returns {PaginationConfig} A new PaginationConfig instance.
56
+ */
57
+ static fromList(list) {
58
+ const attrs = Object.assign({}, ...list);
59
+ return new PaginationConfig(attrs);
60
+ }
61
+ }
62
+
63
+ export { PaginationConfig };