navi-hey 0.4.0 → 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (40) hide show
  1. package/README.md +167 -0
  2. package/lib/exceptions/ForbiddenError.js +14 -0
  3. package/lib/exceptions/InvalidHtmlResponseBody.js +19 -0
  4. package/lib/exceptions/NotFoundError.js +13 -0
  5. package/lib/models/ActionProcessingJob.js +18 -9
  6. package/lib/models/AssetDownloadJob.js +58 -0
  7. package/lib/models/AssetRequest.js +39 -0
  8. package/lib/models/AssetRequestEnqueuer.js +68 -0
  9. package/lib/models/Config.js +4 -1
  10. package/lib/models/HtmlParseJob.js +71 -0
  11. package/lib/models/Job.js +12 -2
  12. package/lib/models/LogConfig.js +35 -0
  13. package/lib/models/ResourceRequest.js +24 -1
  14. package/lib/models/ResourceRequestJob.js +35 -3
  15. package/lib/models/WorkersConfig.js +5 -1
  16. package/lib/registry/JobRegistry.js +18 -0
  17. package/lib/registry/JobRegistryInstance.js +55 -2
  18. package/lib/serializers/JobIndexSerializer.js +26 -0
  19. package/lib/serializers/JobSerializer.js +26 -0
  20. package/lib/serializers/JobShowSerializer.js +29 -0
  21. package/lib/serializers/Serializer.js +36 -0
  22. package/lib/server/AssetsRequestHandler.js +50 -0
  23. package/lib/server/IndexRequestHandler.js +31 -0
  24. package/lib/server/JobRequestHandler.js +33 -0
  25. package/lib/server/JobsRequestHandler.js +30 -0
  26. package/lib/server/PathValidator.js +41 -0
  27. package/lib/server/RouteRegister.js +27 -1
  28. package/lib/server/Router.js +27 -3
  29. package/lib/services/Application.js +19 -2
  30. package/lib/services/Client.js +45 -6
  31. package/lib/services/ConfigParser.js +13 -2
  32. package/lib/utils/HtmlElementParser.js +39 -0
  33. package/lib/utils/HtmlParser.js +49 -0
  34. package/lib/utils/collections/Collection.js +9 -0
  35. package/lib/utils/collections/IdentifyableCollection.js +9 -0
  36. package/lib/utils/collections/Queue.js +8 -0
  37. package/package.json +10 -4
  38. package/static/assets/index-DBkz-poX.css +1 -0
  39. package/static/assets/index-sYKRvQgv.js +11 -0
  40. package/static/index.html +13 -0
package/README.md ADDED
@@ -0,0 +1,167 @@
1
+ # Navi
2
+
3
+ [![Build Status](https://circleci.com/gh/darthjee/navi.svg?style=shield)](https://circleci.com/gh/darthjee/navi)
4
+
5
+ Cache Warmer Tool
6
+
7
+ ---
8
+
9
+ ## Overview
10
+
11
+ Navi is a queue-based cache-warmer written in Node.js, distributed as both a Docker image and an npm CLI package.
12
+ It reads a YAML configuration file, enqueues HTTP requests as jobs, and processes them concurrently using a configurable pool of workers.
13
+
14
+ Key features:
15
+
16
+ - Concurrent HTTP request execution via a worker pool.
17
+ - URL templates with placeholder parameters (e.g. `{:id}`).
18
+ - Response-driven actions: after each successful request, configurable actions extract variables from the response and trigger follow-up processing.
19
+ - Automatic retry of failed requests after the main queue is exhausted.
20
+
21
+ ---
22
+
23
+ ## Installation
24
+
25
+ ### Via npx (no install required)
26
+
27
+ ```bash
28
+ npx navi-hey --config /path/to/your/config.yml
29
+ ```
30
+
31
+ ### Global install
32
+
33
+ ```bash
34
+ # npm
35
+ npm install -g navi-hey
36
+
37
+ # yarn
38
+ yarn global add navi-hey
39
+ ```
40
+
41
+ Then run:
42
+
43
+ ```bash
44
+ navi-hey --config /path/to/your/config.yml
45
+ ```
46
+
47
+ ---
48
+
49
+ ## Configuration File
50
+
51
+ Navi is configured via a YAML file that defines HTTP clients, resources, and the worker pool size.
52
+
53
+ ### Structure
54
+
55
+ ```yaml
56
+ workers:
57
+ quantity: 5 # number of concurrent workers (default: 1)
58
+ retry_cooldown: 2000 # ms before a failed job is retried (default: 2000)
59
+ sleep: 500 # ms the engine waits between allocation ticks (default: 500)
60
+ max-retries: 3 # max number of retries before a job is marked dead (default: 3)
61
+
62
+ log:
63
+ size: 100 # max number of log entries kept in memory (default: 100)
64
+
65
+ web:
66
+ port: 3000 # port for the monitoring web UI (omit to disable)
67
+
68
+ clients:
69
+ default:
70
+ base_url: https://example.com
71
+ auth_api:
72
+ base_url: https://api.example.com
73
+ headers:
74
+ Authorization: Bearer $API_TOKEN
75
+
76
+ resources:
77
+ home:
78
+ - url: /
79
+ status: 200
80
+ assets:
81
+ - selector: script[src]
82
+ attribute: src
83
+ - selector: link[rel="stylesheet"]
84
+ attribute: href
85
+ categories:
86
+ - url: /categories.json
87
+ status: 200
88
+ actions:
89
+ - resource: category_information
90
+ - resource: products
91
+ parameters:
92
+ category_id: parsed_body.id
93
+ page: headers['x-next-page']
94
+ - url: /categories
95
+ status: 302
96
+ - url: /#/categories
97
+ status: 200
98
+ category_information:
99
+ - url: /categories/{:id}.json
100
+ status: 200
101
+ client: auth_api
102
+ products:
103
+ - url: /categories/{:category_id}/products/{:page}.json
104
+ status: 200
105
+ ```
106
+
107
+ ### Fields
108
+
109
+ | Field | Description |
110
+ |-------|-------------|
111
+ | `workers.quantity` | Number of concurrent workers. Defaults to `1`. |
112
+ | `workers.retry_cooldown` | Milliseconds a failed job waits before being re-queued for retry. Defaults to `2000`. |
113
+ | `workers.sleep` | Milliseconds the engine waits between allocation ticks. Defaults to `500`. |
114
+ | `workers.max-retries` | Maximum number of times a job is retried before being moved to the dead queue. Defaults to `3`. |
115
+ | `log.size` | Maximum number of log entries kept in the in-memory log buffer. Defaults to `100`. |
116
+ | `web.port` | Port for the local monitoring web UI. Omit the `web` key entirely to run Navi without the web server. The web UI frontend is bundled with the package and served from `source/static/`. After frontend code changes, run `yarn build` in the `navi_frontend` service to update the bundled assets. |
117
+
118
+ When the web server is enabled, the following screens are available:
119
+
120
+ | Screen | URL | Description |
121
+ |--------|-----|-------------|
122
+ | Dashboard | `/#/` | Real-time job queue stats (counts per status). |
123
+ | Jobs list | `/#/jobs` | Table of all jobs across every status, with links to individual job pages. |
124
+ | Job detail | `/#/job/:id` | Full details for a specific job (ID, status, attempt count). |
125
+ | `clients.<name>.base_url` | Base URL for the named HTTP client. |
126
+ | `clients.<name>.headers` | Optional HTTP headers sent with every request of this client. Header values support environment variable references (`$VAR` or `${VAR}`), resolved at configuration load time. |
127
+ | `resources.<name>` | A named group of URL requests to warm. |
128
+ | `url` | URL path (appended to the client's `base_url`). Supports `{:placeholder}` tokens. |
129
+ | `status` | Expected HTTP response status code. Navi marks a request as failed if the actual status differs. |
130
+ | `client` | Name of the client to use for this request. Defaults to `default`. |
131
+ | `actions` | Optional list of actions to execute after a successful response. |
132
+ | `actions[].resource` | Name of the resource to act upon. Required. |
133
+ | `actions[].parameters` | Optional key-value map. Each key is the destination variable name and each value is a path expression resolved against the response (e.g. `parsed_body.id`, `headers['page']`). When absent, the parsed body item is passed through unchanged. |
134
+ | `assets` | Optional list of asset extraction rules. When present on an HTML resource, Navi parses the response body and enqueues a download job for each matched URL. |
135
+ | `assets[].selector` | CSS selector used to find asset elements in the HTML response (e.g. `script[src]`, `link[rel="stylesheet"]`). |
136
+ | `assets[].attribute` | Attribute on the matched element that holds the asset URL (e.g. `src`, `href`). |
137
+ | `assets[].client` | Named client to use when fetching the asset. Defaults to `default`. |
138
+ | `assets[].status` | Expected HTTP status code for asset fetches. Defaults to `200`. |
139
+
140
+ ---
141
+
142
+ ## Resource Chaining
143
+
144
+ Navi supports multi-level resource chaining. After a successful response, each configured action uses `parameters` path expressions to extract variables from the response body or headers and enqueues new jobs for the target resource. The extracted variables resolve `{:placeholder}` tokens in the target URL templates.
145
+
146
+ For example, requesting `/categories.json` might return `[{ "id": 1 }, { "id": 2 }]`. With an action targeting `category_information` and `parameters: { id: parsed_body.id }`, Navi automatically enqueues requests for `/categories/1.json` and `/categories/2.json`.
147
+
148
+ ---
149
+
150
+ ## Docker
151
+
152
+ Navi is also distributed as a Docker image for environments where Docker is preferred:
153
+
154
+ ```bash
155
+ docker run --rm \
156
+ -v /path/to/your/config.yml:/home/node/app/config/navi_config.yml \
157
+ darthjee/navi-hey:latest \
158
+ node navi.js config/navi_config.yml
159
+ ```
160
+
161
+ ---
162
+
163
+ ## Source & Documentation
164
+
165
+ GitHub repository: [darthjee/navi](https://github.com/darthjee/navi)
166
+
167
+ Integration guide for developers and AI agents: [How to Use Navi in Your Project](https://github.com/darthjee/navi/blob/main/docs/HOW_TO_USE_NAVI.md)
@@ -0,0 +1,14 @@
1
+ import { AppError } from './AppError.js';
2
+
3
+ /**
4
+ * ForbiddenError is thrown when a request is rejected due to a security
5
+ * violation, such as a path traversal attempt.
6
+ * @author darthjee
7
+ */
8
+ class ForbiddenError extends AppError {
9
+ constructor() {
10
+ super('Forbidden');
11
+ }
12
+ }
13
+
14
+ export { ForbiddenError };
@@ -0,0 +1,19 @@
1
+ import { AppError } from './AppError.js';
2
+
3
+ /**
4
+ * Thrown when an HTML response body cannot be parsed.
5
+ * @author darthjee
6
+ */
7
+ class InvalidHtmlResponseBody extends AppError {
8
+ /**
9
+ * @param {string} raw The raw HTML response body string that failed to parse.
10
+ * @param {Error} cause The original parse error.
11
+ */
12
+ constructor(raw, cause) {
13
+ super(`Invalid HTML response body: ${cause.message}`);
14
+ this.raw = raw;
15
+ this.cause = cause;
16
+ }
17
+ }
18
+
19
+ export { InvalidHtmlResponseBody };
@@ -0,0 +1,13 @@
1
+ import { AppError } from './AppError.js';
2
+
3
+ /**
4
+ * NotFoundError is thrown when a requested resource cannot be found.
5
+ * @author darthjee
6
+ */
7
+ class NotFoundError extends AppError {
8
+ constructor(message = 'Not Found') {
9
+ super(message);
10
+ }
11
+ }
12
+
13
+ export { NotFoundError };
@@ -24,6 +24,24 @@ class ActionProcessingJob extends Job {
24
24
  this.#item = item;
25
25
  }
26
26
 
27
+ /**
28
+ * Returns the job-specific arguments for serialization.
29
+ * @returns {{ item: object }} The job arguments.
30
+ */
31
+ get arguments() {
32
+ return { item: this.#item };
33
+ }
34
+
35
+ /**
36
+ * Returns the maximum number of retries for this job type.
37
+ * ActionProcessingJob is exhausted after the first failure.
38
+ * @returns {number} Always 1.
39
+ * @override
40
+ */
41
+ get maxRetries() {
42
+ return 1;
43
+ }
44
+
27
45
  /**
28
46
  * Performs the action for the given item.
29
47
  * @returns {Promise<void>}
@@ -37,15 +55,6 @@ class ActionProcessingJob extends Job {
37
55
  this._fail(error);
38
56
  }
39
57
  }
40
-
41
- /**
42
- * Returns true after the first failed attempt — action jobs have no retry rights.
43
- * @returns {boolean} True if the job has failed at least once.
44
- * @override
45
- */
46
- exhausted() {
47
- return this._attempts >= 1;
48
- }
49
58
  }
50
59
 
51
60
  export { ActionProcessingJob };
@@ -0,0 +1,58 @@
1
+ import { Job } from './Job.js';
2
+ import { Logger } from '../utils/logging/Logger.js';
3
+
4
+ /**
5
+ * AssetDownloadJob is a Job that fetches a single fully-resolved asset URL and validates
6
+ * the expected HTTP status. It is a leaf node — no further chaining occurs after a
7
+ * successful fetch. Follows the standard retry/dead path on failure.
8
+ * @author darthjee
9
+ */
10
+ class AssetDownloadJob extends Job {
11
+ #url;
12
+ #clientName;
13
+ #status;
14
+ #clientRegistry;
15
+
16
+ /**
17
+ * Creates a new AssetDownloadJob instance.
18
+ * @param {object} params - The parameters for creating an AssetDownloadJob instance.
19
+ * @param {string} params.id - The unique identifier for this job.
20
+ * @param {string} params.url - The fully-resolved asset URL to fetch.
21
+ * @param {string} [params.client] - Named client to use (defaults to `'default'`).
22
+ * @param {number} params.status - Expected HTTP status code.
23
+ * @param {object} params.clientRegistry - The client registry used to look up the client.
24
+ */
25
+ constructor({ id, url, client, status, clientRegistry }) {
26
+ super({ id });
27
+ this.#url = url;
28
+ this.#clientName = client;
29
+ this.#status = status;
30
+ this.#clientRegistry = clientRegistry;
31
+ }
32
+
33
+ /**
34
+ * Returns the job-specific arguments for serialization.
35
+ * @returns {{ url: string, clientName: string }} The job arguments.
36
+ */
37
+ get arguments() {
38
+ return { url: this.#url, clientName: this.#clientName };
39
+ }
40
+
41
+ /**
42
+ * Fetches the asset URL using the named client and validates the HTTP status.
43
+ * @returns {Promise<object>} The HTTP response.
44
+ */
45
+ async perform() {
46
+ Logger.info(`AssetDownloadJob #${this.id} performing`);
47
+ try {
48
+ this.lastError = undefined;
49
+ const client = this.#clientRegistry.getClient(this.#clientName);
50
+ return await client.performUrl(this.#url, this.#status);
51
+ } catch (error) {
52
+ Logger.error(`AssetDownloadJob #${this.id} failed: ${error}`);
53
+ this._fail(error);
54
+ }
55
+ }
56
+ }
57
+
58
+ export { AssetDownloadJob };
@@ -0,0 +1,39 @@
1
+ /**
2
+ * AssetRequest represents a CSS-selector-based extraction rule for asset URLs from an HTML response.
3
+ * @author darthjee
4
+ */
5
+ class AssetRequest {
6
+ /**
7
+ * @param {object} attributes AssetRequest attributes.
8
+ * @param {string} attributes.selector CSS selector used to find asset elements.
9
+ * @param {string} attributes.attribute Attribute on the matched element holding the asset URL.
10
+ * @param {string} [attributes.client] Named client to use when fetching the asset.
11
+ * @param {number} [attributes.status=200] Expected HTTP status code for asset fetches.
12
+ */
13
+ constructor({ selector, attribute, client, status = 200 }) {
14
+ this.selector = selector;
15
+ this.attribute = attribute;
16
+ this.client = client;
17
+ this.status = status;
18
+ }
19
+
20
+ /**
21
+ * Creates an AssetRequest instance from a plain config object.
22
+ * @param {object} obj Raw config object.
23
+ * @returns {AssetRequest} A new AssetRequest instance.
24
+ */
25
+ static fromObject(obj) {
26
+ return new AssetRequest(obj);
27
+ }
28
+
29
+ /**
30
+ * Creates a list of AssetRequest instances from an array of plain config objects.
31
+ * @param {Array<object>} [list=[]] Array of raw config objects.
32
+ * @returns {Array<AssetRequest>} List of AssetRequest instances.
33
+ */
34
+ static fromListObject(list = []) {
35
+ return list.map((obj) => AssetRequest.fromObject(obj));
36
+ }
37
+ }
38
+
39
+ export { AssetRequest };
@@ -0,0 +1,68 @@
1
+ import { HtmlParser } from '../utils/HtmlParser.js';
2
+
3
+ /**
4
+ * AssetRequestEnqueuer processes a single AssetRequest against a raw HTML body:
5
+ * it parses matching URLs, resolves them to absolute form, and enqueues one
6
+ * AssetDownloadJob per discovered URL.
7
+ * @author darthjee
8
+ */
9
+ class AssetRequestEnqueuer {
10
+ #rawHtml;
11
+ #assetRequest;
12
+ #jobRegistry;
13
+ #clientRegistry;
14
+
15
+ /**
16
+ * @param {string} rawHtml The raw HTML response body.
17
+ * @param {AssetRequest} assetRequest The asset extraction rule.
18
+ * @param {object} jobRegistry The job registry used to enqueue AssetDownloadJobs.
19
+ * @param {object} clientRegistry The client registry used for URL resolution.
20
+ */
21
+ constructor(rawHtml, assetRequest, jobRegistry, clientRegistry) {
22
+ this.#rawHtml = rawHtml;
23
+ this.#assetRequest = assetRequest;
24
+ this.#jobRegistry = jobRegistry;
25
+ this.#clientRegistry = clientRegistry;
26
+ }
27
+
28
+ /**
29
+ * Parses the HTML body for URLs matching the asset request rule, resolves each URL
30
+ * to absolute form, and enqueues one AssetDownloadJob per URL.
31
+ * @returns {void}
32
+ */
33
+ enqueue() {
34
+ const urls = HtmlParser.parse(this.#rawHtml, this.#assetRequest.selector, this.#assetRequest.attribute);
35
+ for (const url of urls) {
36
+ this.#jobRegistry.enqueue('AssetDownload', {
37
+ url: this.#resolveUrl(url),
38
+ client: this.#assetRequest.client,
39
+ status: this.#assetRequest.status,
40
+ });
41
+ }
42
+ }
43
+
44
+ /**
45
+ * Resolves a discovered URL to an absolute URL.
46
+ *
47
+ * | Form | Resolution |
48
+ * |------|------------|
49
+ * | Absolute (`https://…` or `http://…`) | Used as-is. |
50
+ * | Protocol-relative (`//…`) | Prepended with `https:`. |
51
+ * | Root-relative (`/…`) | Concatenated with the named client's `baseUrl`. |
52
+ * @param {string} url The raw URL extracted from the HTML element attribute.
53
+ * @returns {string} The fully-resolved absolute URL.
54
+ * @private
55
+ */
56
+ #resolveUrl(url) {
57
+ if (url.startsWith('https://') || url.startsWith('http://')) {
58
+ return url;
59
+ }
60
+ if (url.startsWith('//')) {
61
+ return `https:${url}`;
62
+ }
63
+ const client = this.#clientRegistry.getClient(this.#assetRequest.client);
64
+ return `${client.baseUrl}${url}`;
65
+ }
66
+ }
67
+
68
+ export { AssetRequestEnqueuer };
@@ -1,3 +1,4 @@
1
+ import { LogConfig } from './LogConfig.js';
1
2
  import { ClientRegistry } from '../registry/ClientRegistry.js';
2
3
  import { ResourceRegistry } from '../registry/ResourceRegistry.js';
3
4
  import { ConfigLoader } from '../services/ConfigLoader.js';
@@ -14,12 +15,14 @@ class Config {
14
15
  * @param {Record<string, Client>} params.clients - A mapping of client names to Client instances.
15
16
  * @param {WorkersConfig} params.workersConfig - The configuration for worker instances.
16
17
  * @param {WebConfig|null} [params.webConfig] - Optional web server configuration.
18
+ * @param {LogConfig} [params.logConfig] - Optional log configuration.
17
19
  */
18
- constructor({ resources, clients, workersConfig, webConfig }) {
20
+ constructor({ resources, clients, workersConfig, webConfig, logConfig }) {
19
21
  this.resourceRegistry = ResourceRegistry.build(resources);
20
22
  this.clientRegistry = new ClientRegistry(clients);
21
23
  this.workersConfig = workersConfig;
22
24
  this.webConfig = webConfig ?? null;
25
+ this.logConfig = logConfig ?? new LogConfig();
23
26
  }
24
27
 
25
28
  /**
@@ -0,0 +1,71 @@
1
+ import { AssetRequestEnqueuer } from './AssetRequestEnqueuer.js';
2
+ import { Job } from './Job.js';
3
+ import { Logger } from '../utils/logging/Logger.js';
4
+
5
+ /**
6
+ * HtmlParseJob is a Job that parses an HTML response body, extracts asset URLs using
7
+ * CSS selectors, and enqueues one AssetDownloadJob per discovered URL.
8
+ *
9
+ * Unlike ResourceRequestJob, this job is exhausted after the first failure — it has no
10
+ * retry rights, analogous to ActionProcessingJob.
11
+ * @author darthjee
12
+ */
13
+ class HtmlParseJob extends Job {
14
+ #rawHtml;
15
+ #assetRequests;
16
+ #jobRegistry;
17
+ #clientRegistry;
18
+
19
+ /**
20
+ * Creates a new HtmlParseJob instance.
21
+ * @param {object} params - The parameters for creating an HtmlParseJob instance.
22
+ * @param {string} params.id - The unique identifier for this job.
23
+ * @param {string} params.rawHtml - The raw HTML response body string.
24
+ * @param {Array} params.assetRequests - List of AssetRequest instances.
25
+ * @param {object} params.jobRegistry - The job registry used to enqueue AssetDownloadJobs.
26
+ * @param {object} params.clientRegistry - The client registry used for URL resolution.
27
+ */
28
+ constructor({ id, rawHtml, assetRequests, jobRegistry, clientRegistry }) {
29
+ super({ id });
30
+ this.#rawHtml = rawHtml;
31
+ this.#assetRequests = assetRequests;
32
+ this.#jobRegistry = jobRegistry;
33
+ this.#clientRegistry = clientRegistry;
34
+ }
35
+
36
+ /**
37
+ * Returns the job-specific arguments for serialization.
38
+ * @returns {{ assetCount: number }} The job arguments.
39
+ */
40
+ get arguments() {
41
+ return { assetCount: this.#assetRequests.length };
42
+ }
43
+
44
+ /**
45
+ * Returns the maximum number of retries for this job type.
46
+ * HtmlParseJob is exhausted after the first failure.
47
+ * @returns {number} Always 1.
48
+ * @override
49
+ */
50
+ get maxRetries() {
51
+ return 1;
52
+ }
53
+
54
+ /**
55
+ * Parses the HTML body, resolves asset URLs, and enqueues one AssetDownloadJob per URL.
56
+ * @returns {Promise<void>}
57
+ */
58
+ async perform() {
59
+ Logger.info(`HtmlParseJob #${this.id} performing`);
60
+ try {
61
+ this.lastError = undefined;
62
+ for (const assetRequest of this.#assetRequests) {
63
+ new AssetRequestEnqueuer(this.#rawHtml, assetRequest, this.#jobRegistry, this.#clientRegistry).enqueue();
64
+ }
65
+ } catch (error) {
66
+ this._fail(error);
67
+ }
68
+ }
69
+ }
70
+
71
+ export { HtmlParseJob };
package/lib/models/Job.js CHANGED
@@ -54,12 +54,22 @@ class Job {
54
54
  return currentTime >= this.#readyBy;
55
55
  }
56
56
 
57
+ /**
58
+ * Returns the maximum number of retries allowed for this job.
59
+ * Subclasses may override this to return a different value.
60
+ * @returns {number} The maximum number of retries.
61
+ */
62
+ get maxRetries() {
63
+ return 3;
64
+ }
65
+
57
66
  /**
58
67
  * Checks if the job has been exhausted (i.e., has reached the maximum number of attempts).
68
+ * @param {number} [maxRetries] - Maximum number of retries before the job is considered dead.
59
69
  * @returns {boolean} True if the job is exhausted, false otherwise.
60
70
  */
61
- exhausted() {
62
- return this.#attempts >= 3;
71
+ exhausted(maxRetries = this.maxRetries) {
72
+ return this.#attempts >= maxRetries;
63
73
  }
64
74
 
65
75
  /**
@@ -0,0 +1,35 @@
1
+ /**
2
+ * Represents the log configuration for the application.
3
+ * @author darthjee
4
+ */
5
+ class LogConfig {
6
+ #size;
7
+
8
+ /**
9
+ * Creates an instance of LogConfig.
10
+ * @param {object} [config={}] - The configuration object.
11
+ * @param {number} [config.size=100] - The maximum number of log entries to retain.
12
+ */
13
+ constructor({ size = 100 } = {}) {
14
+ this.#size = size;
15
+ }
16
+
17
+ /**
18
+ * Gets the maximum number of log entries to retain.
19
+ * @returns {number} The log buffer size.
20
+ */
21
+ get size() {
22
+ return this.#size;
23
+ }
24
+
25
+ /**
26
+ * Creates a LogConfig instance from a raw YAML object.
27
+ * @param {object|null|undefined} obj - The raw log config object from YAML.
28
+ * @returns {LogConfig} A new LogConfig instance.
29
+ */
30
+ static fromObject(obj) {
31
+ return new LogConfig(obj ?? {});
32
+ }
33
+ }
34
+
35
+ export { LogConfig };
@@ -1,5 +1,7 @@
1
1
  import { ActionsEnqueuer } from './ActionsEnqueuer.js';
2
+ import { AssetRequest } from './AssetRequest.js';
2
3
  import { ResourceRequestAction } from './ResourceRequestAction.js';
4
+ import { JobRegistry as DefaultJobRegistry } from '../registry/JobRegistry.js';
3
5
 
4
6
  /**
5
7
  * ResourceRequest represents a request to a specific URL with an expected status code.
@@ -14,12 +16,14 @@ class ResourceRequest {
14
16
  * @param {number} attributes.status The expected status code of the response.
15
17
  * @param {string} [attributes.clientName] The name of the client to use for this request.
16
18
  * @param {Array} [attributes.actions=[]] List of raw action config objects.
19
+ * @param {Array} [attributes.assets=[]] List of raw asset extraction rule objects.
17
20
  */
18
- constructor({ url, status, clientName, actions = [] }) {
21
+ constructor({ url, status, clientName, actions = [], assets = [] }) {
19
22
  this.url = url;
20
23
  this.status = status;
21
24
  this.#clientName = clientName;
22
25
  this.actions = ResourceRequestAction.fromList(actions);
26
+ this.assets = AssetRequest.fromListObject(assets);
23
27
  }
24
28
 
25
29
  /**
@@ -44,6 +48,25 @@ class ResourceRequest {
44
48
  new ActionsEnqueuer(this.actions, itemWrappers).enqueue();
45
49
  }
46
50
 
51
+ /**
52
+ * Enqueues one HtmlParseJob for this resource request's asset extraction rules.
53
+ * @param {string} rawHtml The raw HTML response body string.
54
+ * @param {object} [jobRegistry=JobRegistry] The job registry used to enqueue the HtmlParseJob.
55
+ * @param {object} clientRegistry The client registry for URL resolution inside HtmlParseJob.
56
+ * @returns {void}
57
+ */
58
+ enqueueAssets(rawHtml, jobRegistry = DefaultJobRegistry, clientRegistry) {
59
+ jobRegistry.enqueue('HtmlParse', { rawHtml, assetRequests: this.assets, clientRegistry });
60
+ }
61
+
62
+ /**
63
+ * Returns true when the resource request has at least one asset extraction rule.
64
+ * @returns {boolean} True if assets are configured.
65
+ */
66
+ hasAssets() {
67
+ return this.assets.length > 0;
68
+ }
69
+
47
70
  /**
48
71
  * Returns the URL with every {:placeholder} token replaced by the
49
72
  * corresponding value from the parameters object.