navi-hey 1.0.0 → 1.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (107) hide show
  1. package/README.md +38 -4
  2. package/bin/navi.js +3 -3
  3. package/lib/{factories → background}/JobFactory.js +2 -2
  4. package/lib/{registry → background}/JobRegistry.js +17 -0
  5. package/lib/{registry → background}/JobRegistryInstance.js +42 -1
  6. package/lib/{models → background}/Worker.js +5 -3
  7. package/lib/{factories → background}/WorkerFactory.js +2 -2
  8. package/lib/{registry → background}/WorkersRegistryInstance.js +1 -1
  9. package/lib/enqueuers/ActionEnqueuer.js +53 -0
  10. package/lib/{models → enqueuers}/ActionsEnqueuer.js +6 -3
  11. package/lib/{models → enqueuers}/AssetRequestEnqueuer.js +6 -2
  12. package/lib/enqueuers/PaginatedActionEnqueuer.js +53 -0
  13. package/lib/enqueuers/PaginatedActionsEnqueuer.js +47 -0
  14. package/lib/exceptions/{ConfigurationFileNotFound.js → config/ConfigurationFileNotFound.js} +1 -1
  15. package/lib/exceptions/{ConfigurationFileNotProvided.js → config/ConfigurationFileNotProvided.js} +1 -1
  16. package/lib/exceptions/{MissingTopLevelConfgKey.js → config/MissingTopLevelConfgKey.js} +1 -1
  17. package/lib/exceptions/http/ConflictError.js +14 -0
  18. package/lib/exceptions/{ForbiddenError.js → http/ForbiddenError.js} +1 -1
  19. package/lib/exceptions/{NotFoundError.js → http/NotFoundError.js} +1 -1
  20. package/lib/exceptions/{ItemNotFound.js → registry/ItemNotFound.js} +1 -1
  21. package/lib/exceptions/{MissingActionResource.js → registry/MissingActionResource.js} +1 -1
  22. package/lib/exceptions/{MissingMappingVariable.js → registry/MissingMappingVariable.js} +1 -1
  23. package/lib/exceptions/{InvalidHtmlResponseBody.js → request/InvalidHtmlResponseBody.js} +1 -1
  24. package/lib/exceptions/{InvalidResponseBody.js → request/InvalidResponseBody.js} +1 -1
  25. package/lib/exceptions/{NullResponse.js → request/NullResponse.js} +1 -1
  26. package/lib/exceptions/{RequestFailed.js → request/RequestFailed.js} +1 -1
  27. package/lib/{models → jobs}/ActionProcessingJob.js +21 -7
  28. package/lib/{models → jobs}/AssetDownloadJob.js +6 -6
  29. package/lib/{models → jobs}/HtmlParseJob.js +23 -10
  30. package/lib/jobs/PaginatedActionProcessingJob.js +77 -0
  31. package/lib/{models → jobs}/ResourceRequestJob.js +18 -15
  32. package/lib/models/{Config.js → configs/Config.js} +3 -3
  33. package/lib/models/configs/PaginationConfig.js +63 -0
  34. package/lib/models/configs/WebConfig.js +19 -0
  35. package/lib/models/{ResourceRequest.js → request/ResourceRequest.js} +32 -7
  36. package/lib/models/{ResourceRequestAction.js → request/ResourceRequestAction.js} +9 -6
  37. package/lib/models/request/ResourceRequestPaginatedAction.js +81 -0
  38. package/lib/models/{PathSegmentTraverser.js → response/PathSegmentTraverser.js} +1 -1
  39. package/lib/models/{ResponseParser.js → response/ResponseParser.js} +1 -1
  40. package/lib/registry/ClientRegistry.js +1 -1
  41. package/lib/registry/LogRegistry.js +146 -0
  42. package/lib/registry/LogRegistryInstance.js +156 -0
  43. package/lib/registry/NamedRegistry.js +1 -1
  44. package/lib/registry/ResourceRegistry.js +1 -1
  45. package/lib/serializers/JobIndexSerializer.js +13 -3
  46. package/lib/serializers/LogSerializer.js +26 -0
  47. package/lib/server/PathValidator.js +1 -1
  48. package/lib/server/RouteRegister.js +42 -9
  49. package/lib/server/Router.js +49 -31
  50. package/lib/server/WebServer.js +28 -8
  51. package/lib/server/{AssetsRequestHandler.js → handlers/AssetsRequestHandler.js} +3 -3
  52. package/lib/server/{BaseUrlsRequestHandler.js → handlers/BaseUrlsRequestHandler.js} +3 -3
  53. package/lib/server/{IndexRequestHandler.js → handlers/IndexRequestHandler.js} +2 -2
  54. package/lib/server/handlers/JobsFilter.js +55 -0
  55. package/lib/server/handlers/LogsRequestHandler.js +55 -0
  56. package/lib/server/handlers/SettingsRequestHandler.js +35 -0
  57. package/lib/server/{StatsRequestHandler.js → handlers/StatsRequestHandler.js} +3 -3
  58. package/lib/server/handlers/engine/EngineContinueRequestHandler.js +34 -0
  59. package/lib/server/handlers/engine/EnginePauseRequestHandler.js +34 -0
  60. package/lib/server/handlers/engine/EngineRestartRequestHandler.js +34 -0
  61. package/lib/server/handlers/engine/EngineShutdownRequestHandler.js +29 -0
  62. package/lib/server/handlers/engine/EngineStartRequestHandler.js +34 -0
  63. package/lib/server/handlers/engine/EngineStatusRequestHandler.js +27 -0
  64. package/lib/server/handlers/engine/EngineStopRequestHandler.js +34 -0
  65. package/lib/server/handlers/jobs/JobLogsRequestHandler.js +39 -0
  66. package/lib/server/{JobRequestHandler.js → handlers/jobs/JobRequestHandler.js} +4 -4
  67. package/lib/server/handlers/jobs/JobRetryRequestHandler.js +40 -0
  68. package/lib/server/{JobsRequestHandler.js → handlers/jobs/JobsRequestHandler.js} +7 -4
  69. package/lib/services/Application.js +138 -84
  70. package/lib/services/ApplicationInstance.js +274 -0
  71. package/lib/services/Client.js +23 -19
  72. package/lib/services/ConfigLoader.js +1 -1
  73. package/lib/services/ConfigParser.js +7 -7
  74. package/lib/services/Engine.js +51 -6
  75. package/lib/services/EngineEvents.js +42 -0
  76. package/lib/services/FailureChecker.js +4 -4
  77. package/lib/services/WorkersAllocator.js +2 -2
  78. package/lib/utils/HtmlElementParser.js +5 -4
  79. package/lib/utils/HtmlParser.js +5 -5
  80. package/lib/utils/PromiseAggregator.js +11 -5
  81. package/lib/utils/logging/BufferedLogger.js +8 -0
  82. package/lib/utils/logging/LogBuffer.js +25 -5
  83. package/lib/utils/logging/LogBufferCollection.js +56 -0
  84. package/lib/utils/logging/LogContext.js +51 -0
  85. package/lib/utils/logging/LogFilter.js +34 -0
  86. package/package.json +1 -1
  87. package/static/assets/{index-DBkz-poX.css → index-CuWjW0sT.css} +1 -1
  88. package/static/assets/index-D5tOOLpl.js +11 -0
  89. package/static/index.html +2 -2
  90. package/lib/exceptions/LockedByOtherWorker.js +0 -14
  91. package/lib/models/ActionEnqueuer.js +0 -34
  92. package/lib/models/WebConfig.js +0 -15
  93. /package/lib/{models → background}/Job.js +0 -0
  94. /package/lib/{registry → background}/WorkersRegistry.js +0 -0
  95. /package/lib/exceptions/{MissingClientsConfig.js → config/MissingClientsConfig.js} +0 -0
  96. /package/lib/exceptions/{MissingResourceConfig.js → config/MissingResourceConfig.js} +0 -0
  97. /package/lib/exceptions/{ClientNotFound.js → registry/ClientNotFound.js} +0 -0
  98. /package/lib/exceptions/{ResourceNotFound.js → registry/ResourceNotFound.js} +0 -0
  99. /package/lib/{factories → factory}/Factory.js +0 -0
  100. /package/lib/models/{FailureConfig.js → configs/FailureConfig.js} +0 -0
  101. /package/lib/models/{LogConfig.js → configs/LogConfig.js} +0 -0
  102. /package/lib/models/{WorkersConfig.js → configs/WorkersConfig.js} +0 -0
  103. /package/lib/models/{AssetRequest.js → request/AssetRequest.js} +0 -0
  104. /package/lib/models/{Resource.js → request/Resource.js} +0 -0
  105. /package/lib/models/{ParametersMapper.js → response/ParametersMapper.js} +0 -0
  106. /package/lib/models/{PathResolver.js → response/PathResolver.js} +0 -0
  107. /package/lib/models/{ResponseWrapper.js → response/ResponseWrapper.js} +0 -0
@@ -1,6 +1,5 @@
1
- import { AssetRequestEnqueuer } from './AssetRequestEnqueuer.js';
2
- import { Job } from './Job.js';
3
- import { Logger } from '../utils/logging/Logger.js';
1
+ import { Job } from '../background/Job.js';
2
+ import { AssetRequestEnqueuer } from '../enqueuers/AssetRequestEnqueuer.js';
4
3
 
5
4
  /**
6
5
  * HtmlParseJob is a Job that parses an HTML response body, extracts asset URLs using
@@ -15,6 +14,7 @@ class HtmlParseJob extends Job {
15
14
  #assetRequests;
16
15
  #jobRegistry;
17
16
  #clientRegistry;
17
+ #originUrl;
18
18
 
19
19
  /**
20
20
  * Creates a new HtmlParseJob instance.
@@ -24,21 +24,23 @@ class HtmlParseJob extends Job {
24
24
  * @param {Array} params.assetRequests - List of AssetRequest instances.
25
25
  * @param {object} params.jobRegistry - The job registry used to enqueue AssetDownloadJobs.
26
26
  * @param {object} params.clientRegistry - The client registry used for URL resolution.
27
+ * @param {string|null} [params.originUrl=null] - The URL of the ResourceRequestJob that triggered this job.
27
28
  */
28
- constructor({ id, rawHtml, assetRequests, jobRegistry, clientRegistry }) {
29
+ constructor({ id, rawHtml, assetRequests, jobRegistry, clientRegistry, originUrl = null }) {
29
30
  super({ id });
30
31
  this.#rawHtml = rawHtml;
31
32
  this.#assetRequests = assetRequests;
32
33
  this.#jobRegistry = jobRegistry;
33
34
  this.#clientRegistry = clientRegistry;
35
+ this.#originUrl = originUrl;
34
36
  }
35
37
 
36
38
  /**
37
39
  * Returns the job-specific arguments for serialization.
38
- * @returns {{ assetCount: number }} The job arguments.
40
+ * @returns {{ assetCount: number, originUrl?: string }} The job arguments.
39
41
  */
40
42
  get arguments() {
41
- return { assetCount: this.#assetRequests.length };
43
+ return { assetCount: this.#assetRequests.length, ...this.#originUrlField() };
42
44
  }
43
45
 
44
46
  /**
@@ -50,22 +52,33 @@ class HtmlParseJob extends Job {
50
52
  get maxRetries() {
51
53
  return 1;
52
54
  }
53
-
54
55
  /**
55
56
  * Parses the HTML body, resolves asset URLs, and enqueues one AssetDownloadJob per URL.
57
+ * @param {LogContext} logContext - Context carrying workerId/jobId for log entries.
56
58
  * @returns {Promise<void>}
57
59
  */
58
- async perform() {
59
- Logger.debug(`HtmlParseJob #${this.id} performing`);
60
+ async perform(logContext) {
61
+ logContext.debug(`HtmlParseJob #${this.id} performing`);
60
62
  try {
61
63
  this.lastError = undefined;
62
64
  for (const assetRequest of this.#assetRequests) {
63
- new AssetRequestEnqueuer(this.#rawHtml, assetRequest, this.#jobRegistry, this.#clientRegistry).enqueue();
65
+ new AssetRequestEnqueuer(this.#rawHtml, assetRequest, this.#jobRegistry, this.#clientRegistry).enqueue(logContext);
64
66
  }
65
67
  } catch (error) {
66
68
  this._fail(error);
67
69
  }
68
70
  }
71
+
72
+ /**
73
+ * Returns an object containing the originUrl field when an origin URL is set,
74
+ * or an empty object otherwise.
75
+ * @returns {{ originUrl: string }|{}} The origin URL field or empty object.
76
+ * @private
77
+ */
78
+ #originUrlField() {
79
+ return this.#originUrl !== null ? { originUrl: this.#originUrl } : {};
80
+ }
81
+
69
82
  }
70
83
 
71
84
  export { HtmlParseJob };
@@ -0,0 +1,77 @@
1
+ import { Job } from '../background/Job.js';
2
+
3
+ /**
4
+ * Processes a single paginated action for a given response wrapper.
5
+ * Exhausted after the first failure — no retry rights.
6
+ * @author darthjee
7
+ * @augments Job
8
+ */
9
+ class PaginatedActionProcessingJob extends Job {
10
+ #paginatedAction;
11
+ #responseWrapper;
12
+ #parameters;
13
+ #originUrl;
14
+
15
+ /**
16
+ * Creates a new PaginatedActionProcessingJob instance.
17
+ * @param {object} params The parameters for creating a PaginatedActionProcessingJob instance.
18
+ * @param {string} params.id Unique job identifier.
19
+ * @param {ResourceRequestPaginatedAction} params.paginatedAction The paginated action to execute.
20
+ * @param {ResponseWrapper} params.responseWrapper The response wrapper carrying response data.
21
+ * @param {object} [params.parameters={}] The original request parameters (key-value map for URL template resolution).
22
+ * @param {string|null} [params.originUrl=null] The URL of the ResourceRequestJob that triggered this job.
23
+ */
24
+ constructor({ id, paginatedAction, responseWrapper, parameters = {}, originUrl = null }) {
25
+ super({ id });
26
+ this.#paginatedAction = paginatedAction;
27
+ this.#responseWrapper = responseWrapper;
28
+ this.#parameters = parameters;
29
+ this.#originUrl = originUrl;
30
+ }
31
+
32
+ /**
33
+ * Returns the job-specific arguments for serialization.
34
+ * @returns {{ responseWrapper: ResponseWrapper, parameters: object, originUrl?: string }} The job arguments.
35
+ */
36
+ get arguments() {
37
+ return { responseWrapper: this.#responseWrapper, parameters: this.#parameters, ...this.#originUrlField() };
38
+ }
39
+
40
+ /**
41
+ * Returns the maximum number of retries for this job type.
42
+ * PaginatedActionProcessingJob is exhausted after the first failure.
43
+ * @returns {number} Always 1.
44
+ * @override
45
+ */
46
+ get maxRetries() {
47
+ return 1;
48
+ }
49
+
50
+ /**
51
+ * Performs the paginated action for the given response wrapper and parameters.
52
+ * @param {LogContext} logContext - Context carrying workerId/jobId for log entries.
53
+ * @returns {Promise<void>}
54
+ */
55
+ async perform(logContext) {
56
+ logContext.debug(`PaginatedActionProcessingJob #${this.id} performing`);
57
+ try {
58
+ this.lastError = undefined;
59
+ await this.#paginatedAction.execute(this.#responseWrapper, this.#parameters);
60
+ } catch (error) {
61
+ this._fail(error);
62
+ }
63
+ }
64
+
65
+ /**
66
+ * Returns an object containing the originUrl field when an origin URL is set,
67
+ * or an empty object otherwise.
68
+ * @returns {{ originUrl: string }|{}} The origin URL field or empty object.
69
+ * @private
70
+ */
71
+ #originUrlField() {
72
+ return this.#originUrl !== null ? { originUrl: this.#originUrl } : {};
73
+ }
74
+
75
+ }
76
+
77
+ export { PaginatedActionProcessingJob };
@@ -1,7 +1,6 @@
1
- import { Job } from './Job.js';
2
- import { ResponseWrapper } from './ResponseWrapper.js';
3
- import { JobRegistry } from '../registry/JobRegistry.js';
4
- import { Logger } from '../utils/logging/Logger.js';
1
+ import { Job } from '../background/Job.js';
2
+ import { JobRegistry } from '../background/JobRegistry.js';
3
+ import { ResponseWrapper } from '../models/response/ResponseWrapper.js';
5
4
 
6
5
  /**
7
6
  * ResourceRequestJob is a Job that performs an HTTP request for a ResourceRequest.
@@ -30,24 +29,26 @@ class ResourceRequestJob extends Job {
30
29
 
31
30
  /**
32
31
  * Returns the job-specific arguments for serialization.
33
- * @returns {{ url: string, parameters: object }} The job arguments.
32
+ * The URL is fully resolved with all parameter placeholders substituted.
33
+ * @returns {{ url: string }} The job arguments.
34
34
  */
35
35
  get arguments() {
36
- return { url: this.#resourceRequest.url, parameters: this.#parameters };
36
+ return { url: this.#resourceRequest.resolveUrl(this.#parameters) };
37
37
  }
38
38
 
39
39
  /**
40
40
  * Performs the HTTP request for the resource request.
41
+ * @param {LogContext} logContext - Context carrying workerId/jobId for log entries.
41
42
  * @returns {Promise} A promise that resolves with the HTTP response.
42
43
  */
43
- async perform() {
44
- Logger.debug(`ResourceRequestJob #${this.id} performing`);
44
+ async perform(logContext) {
45
+ logContext.debug(`ResourceRequestJob #${this.id} performing`);
45
46
  try {
46
47
  this.lastError = undefined;
47
- const response = await this.#getClient().perform(this.#resourceRequest, this.#parameters);
48
+ const response = await this.#getClient().perform(this.#resourceRequest, this.#parameters, logContext);
48
49
  return this.#handleResponse(response);
49
50
  } catch (error) {
50
- Logger.error(`Job #${this.id} failed: ${error}`);
51
+ logContext.error(`Job #${this.id} failed: ${error}`);
51
52
  this._fail(error);
52
53
  }
53
54
  }
@@ -59,20 +60,23 @@ class ResourceRequestJob extends Job {
59
60
  * @private
60
61
  */
61
62
  #handleResponse(response) {
62
- this.#enqueueAssets(response);
63
+ const originUrl = this.#resourceRequest.resolveUrl(this.#parameters);
64
+ this.#enqueueAssets(response, originUrl);
63
65
  const wrapper = new ResponseWrapper(response, this.#parameters);
64
- this.#resourceRequest.enqueueActions(wrapper);
66
+ this.#resourceRequest.enqueueActions(wrapper, originUrl);
67
+ this.#resourceRequest.enqueuePaginatedActions(wrapper, this.#parameters, originUrl);
65
68
  return response;
66
69
  }
67
70
 
68
71
  /**
69
72
  * Enqueues asset download jobs when the resource request declares asset rules.
70
73
  * @param {object} response The HTTP response object.
74
+ * @param {string} originUrl The resolved URL of this job, forwarded as origin to child jobs.
71
75
  * @private
72
76
  */
73
- #enqueueAssets(response) {
77
+ #enqueueAssets(response, originUrl) {
74
78
  if (this.#resourceRequest.hasAssets()) {
75
- this.#resourceRequest.enqueueAssets(response.data, JobRegistry, this.#clients);
79
+ this.#resourceRequest.enqueueAssets(response.data, JobRegistry, this.#clients, originUrl);
76
80
  }
77
81
  }
78
82
 
@@ -90,4 +94,3 @@ class ResourceRequestJob extends Job {
90
94
  }
91
95
 
92
96
  export { ResourceRequestJob };
93
-
@@ -1,7 +1,7 @@
1
1
  import { LogConfig } from './LogConfig.js';
2
- import { ClientRegistry } from '../registry/ClientRegistry.js';
3
- import { ResourceRegistry } from '../registry/ResourceRegistry.js';
4
- import { ConfigLoader } from '../services/ConfigLoader.js';
2
+ import { ClientRegistry } from '../../registry/ClientRegistry.js';
3
+ import { ResourceRegistry } from '../../registry/ResourceRegistry.js';
4
+ import { ConfigLoader } from '../../services/ConfigLoader.js';
5
5
 
6
6
  /**
7
7
  * Config is a class that represents the configuration for the application.
@@ -0,0 +1,63 @@
1
+ import { PathResolver } from '../response/PathResolver.js';
2
+
3
+ /**
4
+ * Parses the pagination block from YAML and resolves the total page count
5
+ * against a response wrapper.
6
+ * @author darthjee
7
+ */
8
+ class PaginationConfig {
9
+ #pagesResolver;
10
+ #pageKey;
11
+ #zeroIndexed;
12
+
13
+ /**
14
+ * @param {object} params Pagination configuration attributes.
15
+ * @param {string} params.pages Path expression to resolve the total page count.
16
+ * @param {string} params.page_key The parameter name to inject the page number.
17
+ * @param {boolean} [params.zero_indexed=false] Whether pages are zero-indexed.
18
+ */
19
+ constructor({ pages, page_key, zero_indexed = false }) {
20
+ this.#pagesResolver = PathResolver.fromExpression(pages);
21
+ this.#pageKey = page_key;
22
+ this.#zeroIndexed = zero_indexed;
23
+ }
24
+
25
+ /**
26
+ * @returns {string} The parameter name used to inject the page number.
27
+ */
28
+ get pageKey() {
29
+ return this.#pageKey;
30
+ }
31
+
32
+ /**
33
+ * Evaluates the `pages` expression against the response wrapper.
34
+ * @param {ResponseWrapper} responseWrapper The response wrapper to resolve against.
35
+ * @returns {number} Total number of pages.
36
+ */
37
+ resolvePages(responseWrapper) {
38
+ return this.#pagesResolver.resolve(responseWrapper);
39
+ }
40
+
41
+ /**
42
+ * Returns an array of page numbers to iterate over.
43
+ * @param {number} count Total number of pages.
44
+ * @returns {Array<number>} Array of page numbers.
45
+ */
46
+ pageNumbers(count) {
47
+ const start = this.#zeroIndexed ? 0 : 1;
48
+ const end = this.#zeroIndexed ? count - 1 : count;
49
+ return Array.from({ length: end - start + 1 }, (_, i) => start + i);
50
+ }
51
+
52
+ /**
53
+ * Creates a PaginationConfig from a raw YAML list entry.
54
+ * @param {Array<object>} list List of single-key maps from YAML.
55
+ * @returns {PaginationConfig} A new PaginationConfig instance.
56
+ */
57
+ static fromList(list) {
58
+ const attrs = Object.assign({}, ...list);
59
+ return new PaginationConfig(attrs);
60
+ }
61
+ }
62
+
63
+ export { PaginationConfig };
@@ -0,0 +1,19 @@
1
+ /**
2
+ * Represents the web server configuration.
3
+ * @author darthjee
4
+ */
5
+ class WebConfig {
6
+ /**
7
+ * @param {object} config - Configuration object.
8
+ * @param {number} config.port - The port number to listen on.
9
+ * @param {number} [config.logs_page_size=20] - Maximum number of log entries per page.
10
+ * @param {boolean} [config.enable_shutdown=true] - Whether the shutdown button is enabled.
11
+ */
12
+ constructor({ port, logs_page_size: logsPageSize = 20, enable_shutdown: enableShutdown = true }) {
13
+ this.port = port;
14
+ this.logsPageSize = logsPageSize;
15
+ this.enableShutdown = enableShutdown;
16
+ }
17
+ }
18
+
19
+ export { WebConfig };
@@ -1,7 +1,10 @@
1
- import { ActionsEnqueuer } from './ActionsEnqueuer.js';
2
1
  import { AssetRequest } from './AssetRequest.js';
3
2
  import { ResourceRequestAction } from './ResourceRequestAction.js';
4
- import { JobRegistry as DefaultJobRegistry } from '../registry/JobRegistry.js';
3
+ import { ResourceRequestPaginatedAction } from './ResourceRequestPaginatedAction.js';
4
+ import { JobRegistry as DefaultJobRegistry } from '../../background/JobRegistry.js';
5
+ import { ActionsEnqueuer } from '../../enqueuers/ActionsEnqueuer.js';
6
+ import { PaginatedActionsEnqueuer } from '../../enqueuers/PaginatedActionsEnqueuer.js';
7
+ import { Application } from '../../services/Application.js';
5
8
 
6
9
  /**
7
10
  * ResourceRequest represents a request to a specific URL with an expected status code.
@@ -17,13 +20,15 @@ class ResourceRequest {
17
20
  * @param {string} [attributes.clientName] The name of the client to use for this request.
18
21
  * @param {Array} [attributes.actions=[]] List of raw action config objects.
19
22
  * @param {Array} [attributes.assets=[]] List of raw asset extraction rule objects.
23
+ * @param {Array} [attributes.paginated_actions=[]] List of raw paginated action config objects.
20
24
  */
21
- constructor({ url, status, clientName, actions = [], assets = [] }) {
25
+ constructor({ url, status, clientName, actions = [], assets = [], paginated_actions = [] }) {
22
26
  this.url = url;
23
27
  this.status = status;
24
28
  this.#clientName = clientName;
25
29
  this.actions = ResourceRequestAction.fromList(actions);
26
30
  this.assets = AssetRequest.fromListObject(assets);
31
+ this.paginatedActions = ResourceRequestPaginatedAction.fromList(paginated_actions);
27
32
  }
28
33
 
29
34
  /**
@@ -39,24 +44,44 @@ class ResourceRequest {
39
44
  * Enqueues one ActionProcessingJob per action associated with the resource request.
40
45
  * Returns immediately if there are no actions.
41
46
  * @param {ResponseWrapper} responseWrapper The ResponseWrapper for the HTTP response.
47
+ * @param {string|null} [originUrl=null] The URL of the ResourceRequestJob that triggered this enqueue.
42
48
  * @returns {void}
43
49
  */
44
- enqueueActions(responseWrapper) {
50
+ enqueueActions(responseWrapper, originUrl = null) {
45
51
  if (this.actions.length === 0) return;
46
52
 
47
53
  const itemWrappers = responseWrapper.toItemWrappers();
48
- new ActionsEnqueuer(this.actions, itemWrappers).enqueue();
54
+ new ActionsEnqueuer(this.actions, itemWrappers, undefined, originUrl).enqueue();
55
+ }
56
+
57
+ /**
58
+ * Enqueues one PaginatedActionProcessingJob per paginated action.
59
+ * Returns immediately if there are no paginated actions.
60
+ * @param {ResponseWrapper} responseWrapper The ResponseWrapper for the HTTP response.
61
+ * @param {object} [parameters={}] The original request parameters (key-value map for URL template resolution).
62
+ * @param {string|null} [originUrl=null] The URL of the ResourceRequestJob that triggered this enqueue.
63
+ * @returns {void}
64
+ */
65
+ enqueuePaginatedActions(responseWrapper, parameters = {}, originUrl = null) {
66
+ if (this.paginatedActions.length === 0) return;
67
+
68
+ new PaginatedActionsEnqueuer(this.paginatedActions, responseWrapper, parameters, undefined, originUrl).enqueue();
49
69
  }
50
70
 
51
71
  /**
52
72
  * Enqueues one HtmlParseJob for this resource request's asset extraction rules.
73
+ * Does nothing if the application is in 'stopped' status.
53
74
  * @param {string} rawHtml The raw HTML response body string.
54
75
  * @param {object} [jobRegistry=JobRegistry] The job registry used to enqueue the HtmlParseJob.
55
76
  * @param {object} clientRegistry The client registry for URL resolution inside HtmlParseJob.
77
+ * @param {string|null} [originUrl=null] The URL of the ResourceRequestJob that triggered this enqueue.
56
78
  * @returns {void}
57
79
  */
58
- enqueueAssets(rawHtml, jobRegistry = DefaultJobRegistry, clientRegistry) {
59
- jobRegistry.enqueue('HtmlParse', { rawHtml, assetRequests: this.assets, clientRegistry });
80
+ enqueueAssets(rawHtml, jobRegistry = DefaultJobRegistry, clientRegistry, originUrl = null) {
81
+ if (Application.isStopped()) return;
82
+ const params = { rawHtml, assetRequests: this.assets, clientRegistry };
83
+ if (originUrl !== null) params.originUrl = originUrl;
84
+ jobRegistry.enqueue('HtmlParse', params);
60
85
  }
61
86
 
62
87
  /**
@@ -1,8 +1,9 @@
1
- import { ParametersMapper } from './ParametersMapper.js';
2
- import { MissingActionResource } from '../exceptions/MissingActionResource.js';
3
- import { JobRegistry as DefaultJobRegistry } from '../registry/JobRegistry.js';
4
- import { ResourceRegistry as DefaultResourceRegistry } from '../registry/ResourceRegistry.js';
5
- import { Logger } from '../utils/logging/Logger.js';
1
+ import { JobRegistry as DefaultJobRegistry } from '../../background/JobRegistry.js';
2
+ import { MissingActionResource } from '../../exceptions/registry/MissingActionResource.js';
3
+ import { LogRegistry } from '../../registry/LogRegistry.js';
4
+ import { ResourceRegistry as DefaultResourceRegistry } from '../../registry/ResourceRegistry.js';
5
+ import { Application } from '../../services/Application.js';
6
+ import { ParametersMapper } from '../response/ParametersMapper.js';
6
7
 
7
8
  /**
8
9
  * Represents a single action to execute after a successful resource request response.
@@ -39,11 +40,13 @@ class ResourceRequestAction {
39
40
  /**
40
41
  * Maps the response wrapper to variables, looks up the target resource, and
41
42
  * enqueues one ResourceRequestJob per ResourceRequest in that resource.
43
+ * Does nothing if the application is in 'stopped' status.
42
44
  * @param {ResponseWrapper} responseWrapper A ResponseWrapper instance exposing
43
45
  * parsedBody and headers.
44
46
  * @returns {void}
45
47
  */
46
48
  execute(responseWrapper) {
49
+ if (Application.isStopped()) return;
47
50
  const vars = this.#mapper.map(responseWrapper);
48
51
  const resource = this.#resourceRegistry.getItem(this.resource);
49
52
 
@@ -63,7 +66,7 @@ class ResourceRequestAction {
63
66
  try {
64
67
  return [new ResourceRequestAction(attrs)];
65
68
  } catch (error) {
66
- Logger.error(`Skipping action: ${error}`);
69
+ LogRegistry.error(`Skipping action: ${error}`);
67
70
  return [];
68
71
  }
69
72
  });
@@ -0,0 +1,81 @@
1
+ import { JobRegistry as DefaultJobRegistry } from '../../background/JobRegistry.js';
2
+ import { MissingActionResource } from '../../exceptions/registry/MissingActionResource.js';
3
+ import { LogRegistry } from '../../registry/LogRegistry.js';
4
+ import { ResourceRegistry as DefaultResourceRegistry } from '../../registry/ResourceRegistry.js';
5
+ import { Application } from '../../services/Application.js';
6
+ import { PaginationConfig } from '../configs/PaginationConfig.js';
7
+
8
+ /**
9
+ * Represents a single paginated action to execute after a successful resource request response.
10
+ *
11
+ * Iterates over pages and enqueues one ResourceRequestJob per page, merging the page number
12
+ * into the item's existing parameters.
13
+ * @author darthjee
14
+ */
15
+ class ResourceRequestPaginatedAction {
16
+ #resource;
17
+ #pagination;
18
+ #jobRegistry;
19
+ #resourceRegistry;
20
+
21
+ /**
22
+ * @param {object} attributes Action attributes.
23
+ * @param {string} attributes.resource Name of the resource to act upon.
24
+ * @param {Array<object>} attributes.pagination Pagination config list.
25
+ * @param {object} [attributes.jobRegistry=JobRegistry] The job registry to enqueue jobs to.
26
+ * @param {object} [attributes.resourceRegistry=ResourceRegistry] The resource registry to look up resources.
27
+ * @throws {MissingActionResource} If resource is absent or falsy.
28
+ */
29
+ constructor({
30
+ resource,
31
+ pagination,
32
+ jobRegistry = DefaultJobRegistry,
33
+ resourceRegistry = DefaultResourceRegistry,
34
+ }) {
35
+ if (!resource) throw new MissingActionResource();
36
+ this.#resource = resource;
37
+ this.#pagination = PaginationConfig.fromList(pagination);
38
+ this.#jobRegistry = jobRegistry;
39
+ this.#resourceRegistry = resourceRegistry;
40
+ }
41
+
42
+ /**
43
+ * Evaluates the page count from the response, then enqueues one
44
+ * ResourceRequestJob per page for the target resource.
45
+ * @param {ResponseWrapper} responseWrapper A ResponseWrapper instance exposing parsedBody and headers.
46
+ * @param {object} [parameters={}] The original request parameters to merge with the page number.
47
+ * @returns {void}
48
+ */
49
+ execute(responseWrapper, parameters = {}) {
50
+ if (Application.isStopped()) return;
51
+
52
+ const count = this.#pagination.resolvePages(responseWrapper);
53
+ const pages = this.#pagination.pageNumbers(count);
54
+ const resource = this.#resourceRegistry.getItem(this.#resource);
55
+
56
+ for (const page of pages) {
57
+ const pageParameters = { ...parameters, [this.#pagination.pageKey]: page };
58
+ for (const resourceRequest of resource.resourceRequests) {
59
+ this.#jobRegistry.enqueue('ResourceRequestJob', { resourceRequest, parameters: pageParameters });
60
+ }
61
+ }
62
+ }
63
+
64
+ /**
65
+ * Creates instances from a raw YAML list, skipping invalid entries.
66
+ * @param {Array<object>} [array=[]] Raw paginated action config entries.
67
+ * @returns {Array<ResourceRequestPaginatedAction>} Valid paginated action instances.
68
+ */
69
+ static fromList(array = []) {
70
+ return array.flatMap((attrs) => {
71
+ try {
72
+ return [new ResourceRequestPaginatedAction(attrs)];
73
+ } catch (error) {
74
+ LogRegistry.error(`Skipping paginated_action: ${error}`);
75
+ return [];
76
+ }
77
+ });
78
+ }
79
+ }
80
+
81
+ export { ResourceRequestPaginatedAction };
@@ -1,4 +1,4 @@
1
- import { MissingMappingVariable } from '../exceptions/MissingMappingVariable.js';
1
+ import { MissingMappingVariable } from '../../exceptions/registry/MissingMappingVariable.js';
2
2
 
3
3
  /**
4
4
  * Traverses an object one path segment at a time, throwing when
@@ -1,4 +1,4 @@
1
- import { InvalidResponseBody } from '../exceptions/InvalidResponseBody.js';
1
+ import { InvalidResponseBody } from '../../exceptions/request/InvalidResponseBody.js';
2
2
 
3
3
  /**
4
4
  * Parses a raw JSON response body string into a JavaScript value.
@@ -1,5 +1,5 @@
1
1
  import { NamedRegistry } from './NamedRegistry.js';
2
- import { ClientNotFound } from '../exceptions/ClientNotFound.js';
2
+ import { ClientNotFound } from '../exceptions/registry/ClientNotFound.js';
3
3
 
4
4
  /**
5
5
  * ClientRegistry manages client retrieval logic and exposes a static singleton facade.