navi-hey 1.0.0 → 1.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (107) hide show
  1. package/README.md +38 -4
  2. package/bin/navi.js +3 -3
  3. package/lib/{factories → background}/JobFactory.js +2 -2
  4. package/lib/{registry → background}/JobRegistry.js +17 -0
  5. package/lib/{registry → background}/JobRegistryInstance.js +42 -1
  6. package/lib/{models → background}/Worker.js +5 -3
  7. package/lib/{factories → background}/WorkerFactory.js +2 -2
  8. package/lib/{registry → background}/WorkersRegistryInstance.js +1 -1
  9. package/lib/enqueuers/ActionEnqueuer.js +53 -0
  10. package/lib/{models → enqueuers}/ActionsEnqueuer.js +6 -3
  11. package/lib/{models → enqueuers}/AssetRequestEnqueuer.js +6 -2
  12. package/lib/enqueuers/PaginatedActionEnqueuer.js +53 -0
  13. package/lib/enqueuers/PaginatedActionsEnqueuer.js +47 -0
  14. package/lib/exceptions/{ConfigurationFileNotFound.js → config/ConfigurationFileNotFound.js} +1 -1
  15. package/lib/exceptions/{ConfigurationFileNotProvided.js → config/ConfigurationFileNotProvided.js} +1 -1
  16. package/lib/exceptions/{MissingTopLevelConfgKey.js → config/MissingTopLevelConfgKey.js} +1 -1
  17. package/lib/exceptions/http/ConflictError.js +14 -0
  18. package/lib/exceptions/{ForbiddenError.js → http/ForbiddenError.js} +1 -1
  19. package/lib/exceptions/{NotFoundError.js → http/NotFoundError.js} +1 -1
  20. package/lib/exceptions/{ItemNotFound.js → registry/ItemNotFound.js} +1 -1
  21. package/lib/exceptions/{MissingActionResource.js → registry/MissingActionResource.js} +1 -1
  22. package/lib/exceptions/{MissingMappingVariable.js → registry/MissingMappingVariable.js} +1 -1
  23. package/lib/exceptions/{InvalidHtmlResponseBody.js → request/InvalidHtmlResponseBody.js} +1 -1
  24. package/lib/exceptions/{InvalidResponseBody.js → request/InvalidResponseBody.js} +1 -1
  25. package/lib/exceptions/{NullResponse.js → request/NullResponse.js} +1 -1
  26. package/lib/exceptions/{RequestFailed.js → request/RequestFailed.js} +1 -1
  27. package/lib/{models → jobs}/ActionProcessingJob.js +21 -7
  28. package/lib/{models → jobs}/AssetDownloadJob.js +6 -6
  29. package/lib/{models → jobs}/HtmlParseJob.js +23 -10
  30. package/lib/jobs/PaginatedActionProcessingJob.js +77 -0
  31. package/lib/{models → jobs}/ResourceRequestJob.js +18 -15
  32. package/lib/models/{Config.js → configs/Config.js} +3 -3
  33. package/lib/models/configs/PaginationConfig.js +63 -0
  34. package/lib/models/configs/WebConfig.js +19 -0
  35. package/lib/models/{ResourceRequest.js → request/ResourceRequest.js} +32 -7
  36. package/lib/models/{ResourceRequestAction.js → request/ResourceRequestAction.js} +9 -6
  37. package/lib/models/request/ResourceRequestPaginatedAction.js +81 -0
  38. package/lib/models/{PathSegmentTraverser.js → response/PathSegmentTraverser.js} +1 -1
  39. package/lib/models/{ResponseParser.js → response/ResponseParser.js} +1 -1
  40. package/lib/registry/ClientRegistry.js +1 -1
  41. package/lib/registry/LogRegistry.js +146 -0
  42. package/lib/registry/LogRegistryInstance.js +156 -0
  43. package/lib/registry/NamedRegistry.js +1 -1
  44. package/lib/registry/ResourceRegistry.js +1 -1
  45. package/lib/serializers/JobIndexSerializer.js +13 -3
  46. package/lib/serializers/LogSerializer.js +26 -0
  47. package/lib/server/PathValidator.js +1 -1
  48. package/lib/server/RouteRegister.js +42 -9
  49. package/lib/server/Router.js +49 -31
  50. package/lib/server/WebServer.js +28 -8
  51. package/lib/server/{AssetsRequestHandler.js → handlers/AssetsRequestHandler.js} +3 -3
  52. package/lib/server/{BaseUrlsRequestHandler.js → handlers/BaseUrlsRequestHandler.js} +3 -3
  53. package/lib/server/{IndexRequestHandler.js → handlers/IndexRequestHandler.js} +2 -2
  54. package/lib/server/handlers/JobsFilter.js +55 -0
  55. package/lib/server/handlers/LogsRequestHandler.js +55 -0
  56. package/lib/server/handlers/SettingsRequestHandler.js +35 -0
  57. package/lib/server/{StatsRequestHandler.js → handlers/StatsRequestHandler.js} +3 -3
  58. package/lib/server/handlers/engine/EngineContinueRequestHandler.js +34 -0
  59. package/lib/server/handlers/engine/EnginePauseRequestHandler.js +34 -0
  60. package/lib/server/handlers/engine/EngineRestartRequestHandler.js +34 -0
  61. package/lib/server/handlers/engine/EngineShutdownRequestHandler.js +29 -0
  62. package/lib/server/handlers/engine/EngineStartRequestHandler.js +34 -0
  63. package/lib/server/handlers/engine/EngineStatusRequestHandler.js +27 -0
  64. package/lib/server/handlers/engine/EngineStopRequestHandler.js +34 -0
  65. package/lib/server/handlers/jobs/JobLogsRequestHandler.js +39 -0
  66. package/lib/server/{JobRequestHandler.js → handlers/jobs/JobRequestHandler.js} +4 -4
  67. package/lib/server/handlers/jobs/JobRetryRequestHandler.js +40 -0
  68. package/lib/server/{JobsRequestHandler.js → handlers/jobs/JobsRequestHandler.js} +7 -4
  69. package/lib/services/Application.js +138 -84
  70. package/lib/services/ApplicationInstance.js +274 -0
  71. package/lib/services/Client.js +23 -19
  72. package/lib/services/ConfigLoader.js +1 -1
  73. package/lib/services/ConfigParser.js +7 -7
  74. package/lib/services/Engine.js +51 -6
  75. package/lib/services/EngineEvents.js +42 -0
  76. package/lib/services/FailureChecker.js +4 -4
  77. package/lib/services/WorkersAllocator.js +2 -2
  78. package/lib/utils/HtmlElementParser.js +5 -4
  79. package/lib/utils/HtmlParser.js +5 -5
  80. package/lib/utils/PromiseAggregator.js +11 -5
  81. package/lib/utils/logging/BufferedLogger.js +8 -0
  82. package/lib/utils/logging/LogBuffer.js +25 -5
  83. package/lib/utils/logging/LogBufferCollection.js +56 -0
  84. package/lib/utils/logging/LogContext.js +51 -0
  85. package/lib/utils/logging/LogFilter.js +34 -0
  86. package/package.json +1 -1
  87. package/static/assets/{index-DBkz-poX.css → index-CuWjW0sT.css} +1 -1
  88. package/static/assets/index-D5tOOLpl.js +11 -0
  89. package/static/index.html +2 -2
  90. package/lib/exceptions/LockedByOtherWorker.js +0 -14
  91. package/lib/models/ActionEnqueuer.js +0 -34
  92. package/lib/models/WebConfig.js +0 -15
  93. /package/lib/{models → background}/Job.js +0 -0
  94. /package/lib/{registry → background}/WorkersRegistry.js +0 -0
  95. /package/lib/exceptions/{MissingClientsConfig.js → config/MissingClientsConfig.js} +0 -0
  96. /package/lib/exceptions/{MissingResourceConfig.js → config/MissingResourceConfig.js} +0 -0
  97. /package/lib/exceptions/{ClientNotFound.js → registry/ClientNotFound.js} +0 -0
  98. /package/lib/exceptions/{ResourceNotFound.js → registry/ResourceNotFound.js} +0 -0
  99. /package/lib/{factories → factory}/Factory.js +0 -0
  100. /package/lib/models/{FailureConfig.js → configs/FailureConfig.js} +0 -0
  101. /package/lib/models/{LogConfig.js → configs/LogConfig.js} +0 -0
  102. /package/lib/models/{WorkersConfig.js → configs/WorkersConfig.js} +0 -0
  103. /package/lib/models/{AssetRequest.js → request/AssetRequest.js} +0 -0
  104. /package/lib/models/{Resource.js → request/Resource.js} +0 -0
  105. /package/lib/models/{ParametersMapper.js → response/ParametersMapper.js} +0 -0
  106. /package/lib/models/{PathResolver.js → response/PathResolver.js} +0 -0
  107. /package/lib/models/{ResponseWrapper.js → response/ResponseWrapper.js} +0 -0
package/README.md CHANGED
@@ -16,6 +16,7 @@ Key features:
16
16
  - Concurrent HTTP request execution via a worker pool.
17
17
  - URL templates with placeholder parameters (e.g. `{:id}`).
18
18
  - Response-driven actions: after each successful request, configurable actions extract variables from the response and trigger follow-up processing.
19
+ - Paginated resource support: `paginated_actions` fan out one request per page based on a page-count expression evaluated against the response.
19
20
  - Automatic retry of failed requests after the main queue is exhausted.
20
21
 
21
22
  ---
@@ -92,8 +93,13 @@ resources:
92
93
  - resource: category_information
93
94
  - resource: products
94
95
  parameters:
95
- category_id: parsed_body.id
96
- page: headers['x-next-page']
96
+ category_id: parsedBody.id
97
+ paginated_actions:
98
+ - resource: products_page
99
+ pagination:
100
+ - pages: parsedBody.pagination.pages
101
+ - page_key: page
102
+ - zero_indexed: false
97
103
  - url: /categories
98
104
  status: 302
99
105
  - url: /#/categories
@@ -134,7 +140,13 @@ When the web server is enabled, the following screens are available:
134
140
  | `client` | Name of the client to use for this request. Defaults to `default`. |
135
141
  | `actions` | Optional list of actions to execute after a successful response. |
136
142
  | `actions[].resource` | Name of the resource to act upon. Required. |
137
- | `actions[].parameters` | Optional key-value map. Each key is the destination variable name and each value is a path expression resolved against the response (e.g. `parsed_body.id`, `headers['page']`). When absent, the parsed body item is passed through unchanged. |
143
+ | `actions[].parameters` | Optional key-value map. Each key is the destination variable name and each value is a path expression resolved against the response (e.g. `parsedBody.id`, `headers['page']`). When absent, the parsed body item is passed through unchanged. |
144
+ | `paginated_actions` | Optional list of paginated actions. Each entry fans out one `ResourceRequestJob` per page. |
145
+ | `paginated_actions[].resource` | Name of the resource to enqueue per page. Required. |
146
+ | `paginated_actions[].pagination` | List of pagination config entries. Required. |
147
+ | `paginated_actions[].pagination[].pages` | Path expression (e.g. `parsedBody.pagination.pages`) resolving to the total page count. |
148
+ | `paginated_actions[].pagination[].page_key` | Parameter name injected as the current page number into each downstream request. |
149
+ | `paginated_actions[].pagination[].zero_indexed` | Boolean. Pages start at `0` when `true`, at `1` when `false` (default). |
138
150
  | `assets` | Optional list of asset extraction rules. When present on an HTML resource, Navi parses the response body and enqueues a download job for each matched URL. |
139
151
  | `assets[].selector` | CSS selector used to find asset elements in the HTML response (e.g. `script[src]`, `link[rel="stylesheet"]`). |
140
152
  | `assets[].attribute` | Attribute on the matched element that holds the asset URL (e.g. `src`, `href`). |
@@ -147,7 +159,29 @@ When the web server is enabled, the following screens are available:
147
159
 
148
160
  Navi supports multi-level resource chaining. After a successful response, each configured action uses `parameters` path expressions to extract variables from the response body or headers and enqueues new jobs for the target resource. The extracted variables resolve `{:placeholder}` tokens in the target URL templates.
149
161
 
150
- For example, requesting `/categories.json` might return `[{ "id": 1 }, { "id": 2 }]`. With an action targeting `category_information` and `parameters: { id: parsed_body.id }`, Navi automatically enqueues requests for `/categories/1.json` and `/categories/2.json`.
162
+ For example, requesting `/categories.json` might return `[{ "id": 1 }, { "id": 2 }]`. With an action targeting `category_information` and `parameters: { id: parsedBody.id }`, Navi automatically enqueues requests for `/categories/1.json` and `/categories/2.json`. Header values can also be extracted, e.g. `page: headers['x-next-page']`.
163
+
164
+ > **Note:** HTTP response header names are always lowercase after Node.js normalization. Use lowercase keys in path expressions (e.g. `headers['x-total-pages']`), regardless of how the server set them.
165
+
166
+ ## Paginated Actions
167
+
168
+ `paginated_actions` complement `actions` when the response indicates multiple pages. Instead of iterating over array items, Navi evaluates a `pages` expression against the whole response, then enqueues one `ResourceRequestJob` per page, injecting the page number under the configured `page_key`.
169
+
170
+ ```yaml
171
+ resources:
172
+ categories:
173
+ - url: /categories.json
174
+ status: 200
175
+ paginated_actions:
176
+ - resource: products_page
177
+ pagination:
178
+ - pages: parsedBody.pagination.pages
179
+ - page_key: page
180
+ - zero_indexed: false
181
+ products_page:
182
+ - url: /products/{:page}.json
183
+ status: 200
184
+ ```
151
185
 
152
186
  ---
153
187
 
package/bin/navi.js CHANGED
@@ -4,6 +4,6 @@ import { Application } from '../lib/services/Application.js';
4
4
  import { ArgumentsParser } from '../lib/services/ArgumentsParser.js';
5
5
 
6
6
  const { config } = ArgumentsParser.parse(process.argv.slice(2));
7
- const app = new Application();
8
- app.loadConfig(config);
9
- app.run();
7
+ Application.build();
8
+ Application.loadConfig(config);
9
+ Application.run();
@@ -1,5 +1,5 @@
1
- import { Factory } from './Factory.js';
2
- import { ResourceRequestJob } from '../models/ResourceRequestJob.js';
1
+ import { Factory } from '../factory/Factory.js';
2
+ import { ResourceRequestJob } from '../jobs/ResourceRequestJob.js';
3
3
  import { IdGenerator } from '../utils/generators/IdGenerator.js';
4
4
 
5
5
  /**
@@ -32,6 +32,14 @@ class JobRegistry {
32
32
  JobRegistry.#instance = null;
33
33
  }
34
34
 
35
+ /**
36
+ * Clears all pending queues (enqueued, retryQueue, and failed).
37
+ * @returns {void}
38
+ */
39
+ static clearQueues() {
40
+ return JobRegistry.#getInstance().clearQueues();
41
+ }
42
+
35
43
  /**
36
44
  * Enqueues a new job using the factory registered under the given key.
37
45
  * @param {string} factoryKey - The factory key to use.
@@ -77,6 +85,15 @@ class JobRegistry {
77
85
  return JobRegistry.#getInstance().requeue(job);
78
86
  }
79
87
 
88
+ /**
89
+ * Moves a job directly to the retry queue, removing it from the failed or dead collection.
90
+ * @param {string} id - The ID of the job to retry.
91
+ * @returns {Job|null} The job if found and moved, or null if not in a retryable state.
92
+ */
93
+ static retryJob(id) {
94
+ return JobRegistry.#getInstance().retryJob(id);
95
+ }
96
+
80
97
  /**
81
98
  * Promotes cooling-down failed jobs that are ready to retry.
82
99
  * @returns {void}
@@ -1,4 +1,4 @@
1
- import { JobFactory } from '../factories/JobFactory.js';
1
+ import { JobFactory } from './JobFactory.js';
2
2
  import { IdentifyableCollection } from '../utils/collections/IdentifyableCollection.js';
3
3
  import { Queue } from '../utils/collections/Queue.js';
4
4
  import { SortedCollection } from '../utils/collections/SortedCollection.js';
@@ -104,6 +104,34 @@ class JobRegistryInstance {
104
104
  this.#enqueued.push(job);
105
105
  }
106
106
 
107
+ /**
108
+ * Moves a job directly to the retry queue, removing it from the failed or dead collection.
109
+ * Returns the job if found and moved, or null if the job is not in a retryable state.
110
+ * @param {string} id - The ID of the job to retry.
111
+ * @returns {Job|null} The job instance, or null if not found in failed or dead.
112
+ */
113
+ retryJob(id) {
114
+ const allFailed = this.#failed.list();
115
+ const failedJob = allFailed.find(j => j.id === id);
116
+ if (failedJob) {
117
+ this.#failed = new SortedCollection(
118
+ allFailed.filter(j => j.id !== id),
119
+ { sortBy: FAILED_SORT_BY }
120
+ );
121
+ this.#retryQueue.push(failedJob);
122
+ return failedJob;
123
+ }
124
+
125
+ const deadJob = this.#dead.get(id);
126
+ if (deadJob) {
127
+ this.#dead.remove(id);
128
+ this.#retryQueue.push(deadJob);
129
+ return deadJob;
130
+ }
131
+
132
+ return null;
133
+ }
134
+
107
135
  /**
108
136
  * Promotes jobs from the failed queue to the retryQueue once their cooldown has elapsed.
109
137
  * @returns {void}
@@ -117,6 +145,19 @@ class JobRegistryInstance {
117
145
  ready.forEach(job => this.#retryQueue.push(job));
118
146
  }
119
147
 
148
+ /**
149
+ * Clears all queues (enqueued, retryQueue, failed, finished, and dead), leaving
150
+ * only the processing collection intact.
151
+ * @returns {void}
152
+ */
153
+ clearQueues() {
154
+ this.#enqueued = new Queue();
155
+ this.#retryQueue = new Queue();
156
+ this.#failed = new SortedCollection([], { sortBy: FAILED_SORT_BY });
157
+ this.#finished = new IdentifyableCollection();
158
+ this.#dead = new IdentifyableCollection();
159
+ }
160
+
120
161
  /**
121
162
  * Returns whether the registry has any jobs pending (including those in cooldown).
122
163
  * @returns {boolean} True if any of enqueued, failed, or retryQueue is non-empty.
@@ -1,4 +1,4 @@
1
- import { Logger } from '../utils/logging/Logger.js';
1
+ import { LogContext } from '../utils/logging/LogContext.js';
2
2
 
3
3
  /**
4
4
  * Worker processes jobs pulled from a JobRegistry.
@@ -41,11 +41,13 @@ class Worker {
41
41
  throw new Error('No job assigned to worker');
42
42
  }
43
43
 
44
+ const logContext = new LogContext({ workerId: this.id, jobId: this.job.id });
45
+
44
46
  try {
45
- await this.job.perform();
47
+ await this.job.perform(logContext);
46
48
  this.#jobRegistry.finish(this.job);
47
49
  } catch (error) {
48
- Logger.error(`Error occurred while performing job: #${this.job.id} - ${error}`);
50
+ logContext.error(`Error occurred while performing job: #${this.job.id} - ${error}`);
49
51
  this.#jobRegistry.fail(this.job);
50
52
  } finally {
51
53
  this.job = undefined;
@@ -1,5 +1,5 @@
1
- import { Factory } from './Factory.js';
2
- import { Worker } from '../models/Worker.js';
1
+ import { Worker } from './Worker.js';
2
+ import { Factory } from '../factory/Factory.js';
3
3
  import { IdGenerator } from '../utils/generators/IdGenerator.js';
4
4
 
5
5
  /**
@@ -1,4 +1,4 @@
1
- import { WorkerFactory } from '../factories/WorkerFactory.js';
1
+ import { WorkerFactory } from './WorkerFactory.js';
2
2
  import { IdentifyableCollection } from '../utils/collections/IdentifyableCollection.js';
3
3
 
4
4
  /**
@@ -0,0 +1,53 @@
1
+ import { JobRegistry as DefaultJobRegistry } from '../background/JobRegistry.js';
2
+ import { Application } from '../services/Application.js';
3
+
4
+ /**
5
+ * ActionEnqueuer enqueues one ActionProcessingJob per item for a single Action.
6
+ * @author darthjee
7
+ */
8
+ class ActionEnqueuer {
9
+ #action;
10
+ #items;
11
+ #jobRegistry;
12
+ #originUrl;
13
+
14
+ /**
15
+ * @param {ResourceRequestAction} action The action to enqueue for each item.
16
+ * @param {Array} items List of parsed response items.
17
+ * @param {object} [jobRegistry=JobRegistry] The job registry to enqueue jobs to.
18
+ * @param {string|null} [originUrl=null] The URL of the ResourceRequestJob that triggered this enqueue.
19
+ */
20
+ constructor(action, items, jobRegistry = DefaultJobRegistry, originUrl = null) {
21
+ this.#action = action;
22
+ this.#items = items;
23
+ this.#jobRegistry = jobRegistry;
24
+ this.#originUrl = originUrl;
25
+ }
26
+
27
+ /**
28
+ * Enqueues one ActionProcessingJob per item for the configured action.
29
+ * Does nothing if the application is in 'stopped' status.
30
+ * @returns {void}
31
+ */
32
+ enqueue() {
33
+ if (Application.isStopped()) return;
34
+ for (const item of this.#items) {
35
+ this.#jobRegistry.enqueue('Action', this.#buildParams(item));
36
+ }
37
+ }
38
+
39
+ /**
40
+ * Builds the job params object for a single item, including originUrl when present.
41
+ * @param {object} item The parsed response item.
42
+ * @returns {object} The params object for the job registry.
43
+ * @private
44
+ */
45
+ #buildParams(item) {
46
+ const params = { action: this.#action, item };
47
+ if (this.#originUrl !== null) params.originUrl = this.#originUrl;
48
+ return params;
49
+ }
50
+
51
+ }
52
+
53
+ export { ActionEnqueuer };
@@ -1,5 +1,5 @@
1
1
  import { ActionEnqueuer } from './ActionEnqueuer.js';
2
- import { NullResponse } from '../exceptions/NullResponse.js';
2
+ import { NullResponse } from '../exceptions/request/NullResponse.js';
3
3
 
4
4
  /**
5
5
  * ActionsEnqueuer enqueues one ActionProcessingJob per (item × action) pair.
@@ -12,16 +12,19 @@ class ActionsEnqueuer {
12
12
  #actions;
13
13
  #items;
14
14
  #jobRegistry;
15
+ #originUrl;
15
16
 
16
17
  /**
17
18
  * @param {Array} actions List of ResourceRequestAction instances.
18
19
  * @param {Array} items List of ResponseWrapper instances (one per response item).
19
20
  * @param {object} [jobRegistry] The job registry to enqueue jobs to. Defaults to global JobRegistry.
21
+ * @param {string|null} [originUrl=null] The URL of the ResourceRequestJob that triggered this enqueue.
20
22
  */
21
- constructor(actions, items, jobRegistry) {
23
+ constructor(actions, items, jobRegistry, originUrl = null) {
22
24
  this.#actions = actions;
23
25
  this.#items = items;
24
26
  this.#jobRegistry = jobRegistry;
27
+ this.#originUrl = originUrl;
25
28
  }
26
29
 
27
30
  /**
@@ -34,7 +37,7 @@ class ActionsEnqueuer {
34
37
  if (this.#items === null) throw new NullResponse();
35
38
 
36
39
  for (const action of this.#actions) {
37
- new ActionEnqueuer(action, this.#items, this.#jobRegistry).enqueue();
40
+ new ActionEnqueuer(action, this.#items, this.#jobRegistry, this.#originUrl).enqueue();
38
41
  }
39
42
  }
40
43
  }
@@ -1,3 +1,4 @@
1
+ import { Application } from '../services/Application.js';
1
2
  import { HtmlParser } from '../utils/HtmlParser.js';
2
3
 
3
4
  /**
@@ -28,10 +29,13 @@ class AssetRequestEnqueuer {
28
29
  /**
29
30
  * Parses the HTML body for URLs matching the asset request rule, resolves each URL
30
31
  * to absolute form, and enqueues one AssetDownloadJob per URL.
32
+ * Does nothing if the application is in 'stopped' status.
33
+ * @param {LogContext} logContext Context carrying workerId/jobId for log entries.
31
34
  * @returns {void}
32
35
  */
33
- enqueue() {
34
- const urls = HtmlParser.parse(this.#rawHtml, this.#assetRequest.selector, this.#assetRequest.attribute);
36
+ enqueue(logContext) {
37
+ if (Application.isStopped()) return;
38
+ const urls = HtmlParser.parse(this.#rawHtml, this.#assetRequest.selector, this.#assetRequest.attribute, logContext);
35
39
  for (const url of urls) {
36
40
  this.#jobRegistry.enqueue('AssetDownload', {
37
41
  url: this.#resolveUrl(url),
@@ -0,0 +1,53 @@
1
+ import { JobRegistry as DefaultJobRegistry } from '../background/JobRegistry.js';
2
+ import { Application } from '../services/Application.js';
3
+
4
+ /**
5
+ * Enqueues one PaginatedActionProcessingJob for a single paginated action.
6
+ * @author darthjee
7
+ */
8
+ class PaginatedActionEnqueuer {
9
+ #paginatedAction;
10
+ #responseWrapper;
11
+ #parameters;
12
+ #jobRegistry;
13
+ #originUrl;
14
+
15
+ /**
16
+ * @param {ResourceRequestPaginatedAction} paginatedAction The paginated action to enqueue.
17
+ * @param {ResponseWrapper} responseWrapper The response wrapper carrying response data.
18
+ * @param {object} [parameters={}] The original request parameters (key-value map for URL template resolution).
19
+ * @param {object} [jobRegistry=JobRegistry] The job registry to enqueue jobs to.
20
+ * @param {string|null} [originUrl=null] The URL of the ResourceRequestJob that triggered this enqueue.
21
+ */
22
+ constructor(paginatedAction, responseWrapper, parameters = {}, jobRegistry = DefaultJobRegistry, originUrl = null) {
23
+ this.#paginatedAction = paginatedAction;
24
+ this.#responseWrapper = responseWrapper;
25
+ this.#parameters = parameters;
26
+ this.#jobRegistry = jobRegistry;
27
+ this.#originUrl = originUrl;
28
+ }
29
+
30
+ /**
31
+ * Enqueues one PaginatedActionProcessingJob for the configured paginated action.
32
+ * Does nothing if the application is in 'stopped' status.
33
+ * @returns {void}
34
+ */
35
+ enqueue() {
36
+ if (Application.isStopped()) return;
37
+ this.#jobRegistry.enqueue('PaginatedAction', this.#buildParams());
38
+ }
39
+
40
+ /**
41
+ * Builds the job params object, including originUrl when present.
42
+ * @returns {object} The params object for the job registry.
43
+ * @private
44
+ */
45
+ #buildParams() {
46
+ const params = { paginatedAction: this.#paginatedAction, responseWrapper: this.#responseWrapper, parameters: this.#parameters };
47
+ if (this.#originUrl !== null) params.originUrl = this.#originUrl;
48
+ return params;
49
+ }
50
+
51
+ }
52
+
53
+ export { PaginatedActionEnqueuer };
@@ -0,0 +1,47 @@
1
+ import { PaginatedActionEnqueuer } from './PaginatedActionEnqueuer.js';
2
+ import { NullResponse } from '../exceptions/request/NullResponse.js';
3
+
4
+ /**
5
+ * Enqueues one PaginatedActionProcessingJob per paginated action for the given parameters.
6
+ *
7
+ * Delegates execution to the job queue instead of running paginated actions inline.
8
+ * Each paginated action is handled by a dedicated PaginatedActionEnqueuer.
9
+ * @author darthjee
10
+ */
11
+ class PaginatedActionsEnqueuer {
12
+ #paginatedActions;
13
+ #responseWrapper;
14
+ #parameters;
15
+ #jobRegistry;
16
+ #originUrl;
17
+
18
+ /**
19
+ * @param {Array<ResourceRequestPaginatedAction>} paginatedActions List of paginated action instances.
20
+ * @param {ResponseWrapper} responseWrapper The response wrapper carrying response data.
21
+ * @param {object} [parameters={}] The original request parameters (key-value map for URL template resolution).
22
+ * @param {object} [jobRegistry] The job registry to enqueue jobs to. Defaults to global JobRegistry.
23
+ * @param {string|null} [originUrl=null] The URL of the ResourceRequestJob that triggered this enqueue.
24
+ */
25
+ constructor(paginatedActions, responseWrapper, parameters = {}, jobRegistry, originUrl = null) {
26
+ this.#paginatedActions = paginatedActions;
27
+ this.#responseWrapper = responseWrapper;
28
+ this.#parameters = parameters;
29
+ this.#jobRegistry = jobRegistry;
30
+ this.#originUrl = originUrl;
31
+ }
32
+
33
+ /**
34
+ * Enqueues one PaginatedActionProcessingJob per paginated action.
35
+ * @returns {void}
36
+ * @throws {NullResponse} If the responseWrapper is null.
37
+ */
38
+ enqueue() {
39
+ if (this.#responseWrapper === null) throw new NullResponse();
40
+
41
+ for (const paginatedAction of this.#paginatedActions) {
42
+ new PaginatedActionEnqueuer(paginatedAction, this.#responseWrapper, this.#parameters, this.#jobRegistry, this.#originUrl).enqueue();
43
+ }
44
+ }
45
+ }
46
+
47
+ export { PaginatedActionsEnqueuer };
@@ -1,4 +1,4 @@
1
- import { AppError } from './AppError.js';
1
+ import { AppError } from '../AppError.js';
2
2
 
3
3
  class ConfigurationFileNotFound extends AppError {
4
4
  constructor(file) {
@@ -1,4 +1,4 @@
1
- import { AppError } from './AppError.js';
1
+ import { AppError } from '../AppError.js';
2
2
 
3
3
  class ConfigurationFileNotProvided extends AppError {
4
4
  constructor() {
@@ -1,4 +1,4 @@
1
- import { AppError } from './AppError.js';
1
+ import { AppError } from '../AppError.js';
2
2
 
3
3
  /**
4
4
  * ItemNotFound is a custom error class thrown when a requested item
@@ -0,0 +1,14 @@
1
+ import { AppError } from '../AppError.js';
2
+
3
+ /**
4
+ * ConflictError is thrown when a requested state transition is not valid
5
+ * given the current application state (e.g. pausing an already-paused engine).
6
+ * @author darthjee
7
+ */
8
+ class ConflictError extends AppError {
9
+ constructor() {
10
+ super('Conflict');
11
+ }
12
+ }
13
+
14
+ export { ConflictError };
@@ -1,4 +1,4 @@
1
- import { AppError } from './AppError.js';
1
+ import { AppError } from '../AppError.js';
2
2
 
3
3
  /**
4
4
  * ForbiddenError is thrown when a request is rejected due to a security
@@ -1,4 +1,4 @@
1
- import { AppError } from './AppError.js';
1
+ import { AppError } from '../AppError.js';
2
2
 
3
3
  /**
4
4
  * NotFoundError is thrown when a requested resource cannot be found.
@@ -1,4 +1,4 @@
1
- import { AppError } from './AppError.js';
1
+ import { AppError } from '../AppError.js';
2
2
 
3
3
  /**
4
4
  * ItemNotFound is a custom error class thrown when a requested item
@@ -1,4 +1,4 @@
1
- import { AppError } from './AppError.js';
1
+ import { AppError } from '../AppError.js';
2
2
 
3
3
  /**
4
4
  * Thrown when an action config entry is missing the required "resource" field.
@@ -1,4 +1,4 @@
1
- import { AppError } from './AppError.js';
1
+ import { AppError } from '../AppError.js';
2
2
 
3
3
  /**
4
4
  * Thrown when a parameters path expression cannot be resolved against the response wrapper.
@@ -1,4 +1,4 @@
1
- import { AppError } from './AppError.js';
1
+ import { AppError } from '../AppError.js';
2
2
 
3
3
  /**
4
4
  * Thrown when an HTML response body cannot be parsed.
@@ -1,4 +1,4 @@
1
- import { AppError } from './AppError.js';
1
+ import { AppError } from '../AppError.js';
2
2
 
3
3
  /**
4
4
  * Thrown when a response body cannot be parsed as JSON.
@@ -1,4 +1,4 @@
1
- import { AppError } from './AppError.js';
1
+ import { AppError } from '../AppError.js';
2
2
 
3
3
  /**
4
4
  * Thrown when a parsed response body is null.
@@ -1,4 +1,4 @@
1
- import { AppError } from './AppError.js';
1
+ import { AppError } from '../AppError.js';
2
2
 
3
3
  /**
4
4
  * RequestFailed is a custom error class that represents a failed HTTP request.
@@ -1,5 +1,4 @@
1
- import { Job } from './Job.js';
2
- import { Logger } from '../utils/logging/Logger.js';
1
+ import { Job } from '../background/Job.js';
3
2
 
4
3
  /**
5
4
  * ActionProcessingJob is a Job that processes a single Action for a given item.
@@ -10,6 +9,7 @@ import { Logger } from '../utils/logging/Logger.js';
10
9
  class ActionProcessingJob extends Job {
11
10
  #action;
12
11
  #item;
12
+ #originUrl;
13
13
 
14
14
  /**
15
15
  * Creates a new ActionProcessingJob instance.
@@ -17,19 +17,21 @@ class ActionProcessingJob extends Job {
17
17
  * @param {string} params.id - The unique identifier for this job.
18
18
  * @param {ResourceRequestAction} params.action - The action to execute.
19
19
  * @param {object} params.item - The parsed response item to pass to the action.
20
+ * @param {string|null} [params.originUrl=null] - The URL of the ResourceRequestJob that triggered this job.
20
21
  */
21
- constructor({ id, action, item }) {
22
+ constructor({ id, action, item, originUrl = null }) {
22
23
  super({ id });
23
24
  this.#action = action;
24
25
  this.#item = item;
26
+ this.#originUrl = originUrl;
25
27
  }
26
28
 
27
29
  /**
28
30
  * Returns the job-specific arguments for serialization.
29
- * @returns {{ item: object }} The job arguments.
31
+ * @returns {{ item: object, originUrl?: string }} The job arguments.
30
32
  */
31
33
  get arguments() {
32
- return { item: this.#item };
34
+ return { item: this.#item, ...this.#originUrlField() };
33
35
  }
34
36
 
35
37
  /**
@@ -44,10 +46,11 @@ class ActionProcessingJob extends Job {
44
46
 
45
47
  /**
46
48
  * Performs the action for the given item.
49
+ * @param {LogContext} logContext - Context carrying workerId/jobId for log entries.
47
50
  * @returns {Promise<void>}
48
51
  */
49
- async perform() {
50
- Logger.debug(`ActionProcessingJob #${this.id} performing`);
52
+ async perform(logContext) {
53
+ logContext.debug(`ActionProcessingJob #${this.id} performing`);
51
54
  try {
52
55
  this.lastError = undefined;
53
56
  await this.#action.execute(this.#item);
@@ -55,6 +58,17 @@ class ActionProcessingJob extends Job {
55
58
  this._fail(error);
56
59
  }
57
60
  }
61
+
62
+ /**
63
+ * Returns an object containing the originUrl field when an origin URL is set,
64
+ * or an empty object otherwise.
65
+ * @returns {{ originUrl: string }|{}} The origin URL field or empty object.
66
+ * @private
67
+ */
68
+ #originUrlField() {
69
+ return this.#originUrl !== null ? { originUrl: this.#originUrl } : {};
70
+ }
71
+
58
72
  }
59
73
 
60
74
  export { ActionProcessingJob };
@@ -1,5 +1,4 @@
1
- import { Job } from './Job.js';
2
- import { Logger } from '../utils/logging/Logger.js';
1
+ import { Job } from '../background/Job.js';
3
2
 
4
3
  /**
5
4
  * AssetDownloadJob is a Job that fetches a single fully-resolved asset URL and validates
@@ -40,16 +39,17 @@ class AssetDownloadJob extends Job {
40
39
 
41
40
  /**
42
41
  * Fetches the asset URL using the named client and validates the HTTP status.
42
+ * @param {LogContext} logContext - Context carrying workerId/jobId for log entries.
43
43
  * @returns {Promise<object>} The HTTP response.
44
44
  */
45
- async perform() {
46
- Logger.debug(`AssetDownloadJob #${this.id} performing`);
45
+ async perform(logContext) {
46
+ logContext.debug(`AssetDownloadJob #${this.id} performing`);
47
47
  try {
48
48
  this.lastError = undefined;
49
49
  const client = this.#clientRegistry.getClient(this.#clientName);
50
- return await client.performUrl(this.#url, this.#status);
50
+ return await client.performUrl(this.#url, this.#status, logContext);
51
51
  } catch (error) {
52
- Logger.error(`AssetDownloadJob #${this.id} failed: ${error}`);
52
+ logContext.error(`AssetDownloadJob #${this.id} failed: ${error}`);
53
53
  this._fail(error);
54
54
  }
55
55
  }