navi-hey 0.4.0 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +167 -0
- package/lib/exceptions/ForbiddenError.js +14 -0
- package/lib/exceptions/InvalidHtmlResponseBody.js +19 -0
- package/lib/exceptions/NotFoundError.js +13 -0
- package/lib/models/ActionProcessingJob.js +18 -9
- package/lib/models/AssetDownloadJob.js +58 -0
- package/lib/models/AssetRequest.js +39 -0
- package/lib/models/AssetRequestEnqueuer.js +68 -0
- package/lib/models/Config.js +4 -1
- package/lib/models/HtmlParseJob.js +71 -0
- package/lib/models/Job.js +12 -2
- package/lib/models/LogConfig.js +35 -0
- package/lib/models/ResourceRequest.js +24 -1
- package/lib/models/ResourceRequestJob.js +35 -3
- package/lib/models/WorkersConfig.js +5 -1
- package/lib/registry/JobRegistry.js +18 -0
- package/lib/registry/JobRegistryInstance.js +55 -2
- package/lib/serializers/JobIndexSerializer.js +26 -0
- package/lib/serializers/JobSerializer.js +26 -0
- package/lib/serializers/JobShowSerializer.js +29 -0
- package/lib/serializers/Serializer.js +36 -0
- package/lib/server/AssetsRequestHandler.js +50 -0
- package/lib/server/IndexRequestHandler.js +31 -0
- package/lib/server/JobRequestHandler.js +33 -0
- package/lib/server/JobsRequestHandler.js +30 -0
- package/lib/server/PathValidator.js +41 -0
- package/lib/server/RouteRegister.js +27 -1
- package/lib/server/Router.js +27 -3
- package/lib/services/Application.js +19 -2
- package/lib/services/Client.js +45 -6
- package/lib/services/ConfigParser.js +13 -2
- package/lib/utils/HtmlElementParser.js +39 -0
- package/lib/utils/HtmlParser.js +49 -0
- package/lib/utils/collections/Collection.js +9 -0
- package/lib/utils/collections/IdentifyableCollection.js +9 -0
- package/lib/utils/collections/Queue.js +8 -0
- package/package.json +10 -4
- package/static/assets/index-DBkz-poX.css +1 -0
- package/static/assets/index-sYKRvQgv.js +11 -0
- package/static/index.html +13 -0
package/README.md
ADDED
|
@@ -0,0 +1,167 @@
|
|
|
1
|
+
# Navi
|
|
2
|
+
|
|
3
|
+
[](https://circleci.com/gh/darthjee/navi)
|
|
4
|
+
|
|
5
|
+
Cache Warmer Tool
|
|
6
|
+
|
|
7
|
+
---
|
|
8
|
+
|
|
9
|
+
## Overview
|
|
10
|
+
|
|
11
|
+
Navi is a queue-based cache-warmer written in Node.js, distributed as both a Docker image and an npm CLI package.
|
|
12
|
+
It reads a YAML configuration file, enqueues HTTP requests as jobs, and processes them concurrently using a configurable pool of workers.
|
|
13
|
+
|
|
14
|
+
Key features:
|
|
15
|
+
|
|
16
|
+
- Concurrent HTTP request execution via a worker pool.
|
|
17
|
+
- URL templates with placeholder parameters (e.g. `{:id}`).
|
|
18
|
+
- Response-driven actions: after each successful request, configurable actions extract variables from the response and trigger follow-up processing.
|
|
19
|
+
- Automatic retry of failed requests after the main queue is exhausted.
|
|
20
|
+
|
|
21
|
+
---
|
|
22
|
+
|
|
23
|
+
## Installation
|
|
24
|
+
|
|
25
|
+
### Via npx (no install required)
|
|
26
|
+
|
|
27
|
+
```bash
|
|
28
|
+
npx navi-hey --config /path/to/your/config.yml
|
|
29
|
+
```
|
|
30
|
+
|
|
31
|
+
### Global install
|
|
32
|
+
|
|
33
|
+
```bash
|
|
34
|
+
# npm
|
|
35
|
+
npm install -g navi-hey
|
|
36
|
+
|
|
37
|
+
# yarn
|
|
38
|
+
yarn global add navi-hey
|
|
39
|
+
```
|
|
40
|
+
|
|
41
|
+
Then run:
|
|
42
|
+
|
|
43
|
+
```bash
|
|
44
|
+
navi-hey --config /path/to/your/config.yml
|
|
45
|
+
```
|
|
46
|
+
|
|
47
|
+
---
|
|
48
|
+
|
|
49
|
+
## Configuration File
|
|
50
|
+
|
|
51
|
+
Navi is configured via a YAML file that defines HTTP clients, resources, and the worker pool size.
|
|
52
|
+
|
|
53
|
+
### Structure
|
|
54
|
+
|
|
55
|
+
```yaml
|
|
56
|
+
workers:
|
|
57
|
+
quantity: 5 # number of concurrent workers (default: 1)
|
|
58
|
+
retry_cooldown: 2000 # ms before a failed job is retried (default: 2000)
|
|
59
|
+
sleep: 500 # ms the engine waits between allocation ticks (default: 500)
|
|
60
|
+
max-retries: 3 # max number of retries before a job is marked dead (default: 3)
|
|
61
|
+
|
|
62
|
+
log:
|
|
63
|
+
size: 100 # max number of log entries kept in memory (default: 100)
|
|
64
|
+
|
|
65
|
+
web:
|
|
66
|
+
port: 3000 # port for the monitoring web UI (omit to disable)
|
|
67
|
+
|
|
68
|
+
clients:
|
|
69
|
+
default:
|
|
70
|
+
base_url: https://example.com
|
|
71
|
+
auth_api:
|
|
72
|
+
base_url: https://api.example.com
|
|
73
|
+
headers:
|
|
74
|
+
Authorization: Bearer $API_TOKEN
|
|
75
|
+
|
|
76
|
+
resources:
|
|
77
|
+
home:
|
|
78
|
+
- url: /
|
|
79
|
+
status: 200
|
|
80
|
+
assets:
|
|
81
|
+
- selector: script[src]
|
|
82
|
+
attribute: src
|
|
83
|
+
- selector: link[rel="stylesheet"]
|
|
84
|
+
attribute: href
|
|
85
|
+
categories:
|
|
86
|
+
- url: /categories.json
|
|
87
|
+
status: 200
|
|
88
|
+
actions:
|
|
89
|
+
- resource: category_information
|
|
90
|
+
- resource: products
|
|
91
|
+
parameters:
|
|
92
|
+
category_id: parsed_body.id
|
|
93
|
+
page: headers['x-next-page']
|
|
94
|
+
- url: /categories
|
|
95
|
+
status: 302
|
|
96
|
+
- url: /#/categories
|
|
97
|
+
status: 200
|
|
98
|
+
category_information:
|
|
99
|
+
- url: /categories/{:id}.json
|
|
100
|
+
status: 200
|
|
101
|
+
client: auth_api
|
|
102
|
+
products:
|
|
103
|
+
- url: /categories/{:category_id}/products/{:page}.json
|
|
104
|
+
status: 200
|
|
105
|
+
```
|
|
106
|
+
|
|
107
|
+
### Fields
|
|
108
|
+
|
|
109
|
+
| Field | Description |
|
|
110
|
+
|-------|-------------|
|
|
111
|
+
| `workers.quantity` | Number of concurrent workers. Defaults to `1`. |
|
|
112
|
+
| `workers.retry_cooldown` | Milliseconds a failed job waits before being re-queued for retry. Defaults to `2000`. |
|
|
113
|
+
| `workers.sleep` | Milliseconds the engine waits between allocation ticks. Defaults to `500`. |
|
|
114
|
+
| `workers.max-retries` | Maximum number of times a job is retried before being moved to the dead queue. Defaults to `3`. |
|
|
115
|
+
| `log.size` | Maximum number of log entries kept in the in-memory log buffer. Defaults to `100`. |
|
|
116
|
+
| `web.port` | Port for the local monitoring web UI. Omit the `web` key entirely to run Navi without the web server. The web UI frontend is bundled with the package and served from `source/static/`. After frontend code changes, run `yarn build` in the `navi_frontend` service to update the bundled assets. |
|
|
117
|
+
|
|
118
|
+
When the web server is enabled, the following screens are available:
|
|
119
|
+
|
|
120
|
+
| Screen | URL | Description |
|
|
121
|
+
|--------|-----|-------------|
|
|
122
|
+
| Dashboard | `/#/` | Real-time job queue stats (counts per status). |
|
|
123
|
+
| Jobs list | `/#/jobs` | Table of all jobs across every status, with links to individual job pages. |
|
|
124
|
+
| Job detail | `/#/job/:id` | Full details for a specific job (ID, status, attempt count). |
|
|
125
|
+
| `clients.<name>.base_url` | Base URL for the named HTTP client. |
|
|
126
|
+
| `clients.<name>.headers` | Optional HTTP headers sent with every request of this client. Header values support environment variable references (`$VAR` or `${VAR}`), resolved at configuration load time. |
|
|
127
|
+
| `resources.<name>` | A named group of URL requests to warm. |
|
|
128
|
+
| `url` | URL path (appended to the client's `base_url`). Supports `{:placeholder}` tokens. |
|
|
129
|
+
| `status` | Expected HTTP response status code. Navi marks a request as failed if the actual status differs. |
|
|
130
|
+
| `client` | Name of the client to use for this request. Defaults to `default`. |
|
|
131
|
+
| `actions` | Optional list of actions to execute after a successful response. |
|
|
132
|
+
| `actions[].resource` | Name of the resource to act upon. Required. |
|
|
133
|
+
| `actions[].parameters` | Optional key-value map. Each key is the destination variable name and each value is a path expression resolved against the response (e.g. `parsed_body.id`, `headers['page']`). When absent, the parsed body item is passed through unchanged. |
|
|
134
|
+
| `assets` | Optional list of asset extraction rules. When present on an HTML resource, Navi parses the response body and enqueues a download job for each matched URL. |
|
|
135
|
+
| `assets[].selector` | CSS selector used to find asset elements in the HTML response (e.g. `script[src]`, `link[rel="stylesheet"]`). |
|
|
136
|
+
| `assets[].attribute` | Attribute on the matched element that holds the asset URL (e.g. `src`, `href`). |
|
|
137
|
+
| `assets[].client` | Named client to use when fetching the asset. Defaults to `default`. |
|
|
138
|
+
| `assets[].status` | Expected HTTP status code for asset fetches. Defaults to `200`. |
|
|
139
|
+
|
|
140
|
+
---
|
|
141
|
+
|
|
142
|
+
## Resource Chaining
|
|
143
|
+
|
|
144
|
+
Navi supports multi-level resource chaining. After a successful response, each configured action uses `parameters` path expressions to extract variables from the response body or headers and enqueues new jobs for the target resource. The extracted variables resolve `{:placeholder}` tokens in the target URL templates.
|
|
145
|
+
|
|
146
|
+
For example, requesting `/categories.json` might return `[{ "id": 1 }, { "id": 2 }]`. With an action targeting `category_information` and `parameters: { id: parsed_body.id }`, Navi automatically enqueues requests for `/categories/1.json` and `/categories/2.json`.
|
|
147
|
+
|
|
148
|
+
---
|
|
149
|
+
|
|
150
|
+
## Docker
|
|
151
|
+
|
|
152
|
+
Navi is also distributed as a Docker image for environments where Docker is preferred:
|
|
153
|
+
|
|
154
|
+
```bash
|
|
155
|
+
docker run --rm \
|
|
156
|
+
-v /path/to/your/config.yml:/home/node/app/config/navi_config.yml \
|
|
157
|
+
darthjee/navi-hey:latest \
|
|
158
|
+
node navi.js config/navi_config.yml
|
|
159
|
+
```
|
|
160
|
+
|
|
161
|
+
---
|
|
162
|
+
|
|
163
|
+
## Source & Documentation
|
|
164
|
+
|
|
165
|
+
GitHub repository: [darthjee/navi](https://github.com/darthjee/navi)
|
|
166
|
+
|
|
167
|
+
Integration guide for developers and AI agents: [How to Use Navi in Your Project](https://github.com/darthjee/navi/blob/main/docs/HOW_TO_USE_NAVI.md)
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
import { AppError } from './AppError.js';
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* ForbiddenError is thrown when a request is rejected due to a security
|
|
5
|
+
* violation, such as a path traversal attempt.
|
|
6
|
+
* @author darthjee
|
|
7
|
+
*/
|
|
8
|
+
class ForbiddenError extends AppError {
|
|
9
|
+
constructor() {
|
|
10
|
+
super('Forbidden');
|
|
11
|
+
}
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
export { ForbiddenError };
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
import { AppError } from './AppError.js';
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Thrown when an HTML response body cannot be parsed.
|
|
5
|
+
* @author darthjee
|
|
6
|
+
*/
|
|
7
|
+
class InvalidHtmlResponseBody extends AppError {
|
|
8
|
+
/**
|
|
9
|
+
* @param {string} raw The raw HTML response body string that failed to parse.
|
|
10
|
+
* @param {Error} cause The original parse error.
|
|
11
|
+
*/
|
|
12
|
+
constructor(raw, cause) {
|
|
13
|
+
super(`Invalid HTML response body: ${cause.message}`);
|
|
14
|
+
this.raw = raw;
|
|
15
|
+
this.cause = cause;
|
|
16
|
+
}
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
export { InvalidHtmlResponseBody };
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
import { AppError } from './AppError.js';
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* NotFoundError is thrown when a requested resource cannot be found.
|
|
5
|
+
* @author darthjee
|
|
6
|
+
*/
|
|
7
|
+
class NotFoundError extends AppError {
|
|
8
|
+
constructor(message = 'Not Found') {
|
|
9
|
+
super(message);
|
|
10
|
+
}
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
export { NotFoundError };
|
|
@@ -24,6 +24,24 @@ class ActionProcessingJob extends Job {
|
|
|
24
24
|
this.#item = item;
|
|
25
25
|
}
|
|
26
26
|
|
|
27
|
+
/**
|
|
28
|
+
* Returns the job-specific arguments for serialization.
|
|
29
|
+
* @returns {{ item: object }} The job arguments.
|
|
30
|
+
*/
|
|
31
|
+
get arguments() {
|
|
32
|
+
return { item: this.#item };
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
/**
|
|
36
|
+
* Returns the maximum number of retries for this job type.
|
|
37
|
+
* ActionProcessingJob is exhausted after the first failure.
|
|
38
|
+
* @returns {number} Always 1.
|
|
39
|
+
* @override
|
|
40
|
+
*/
|
|
41
|
+
get maxRetries() {
|
|
42
|
+
return 1;
|
|
43
|
+
}
|
|
44
|
+
|
|
27
45
|
/**
|
|
28
46
|
* Performs the action for the given item.
|
|
29
47
|
* @returns {Promise<void>}
|
|
@@ -37,15 +55,6 @@ class ActionProcessingJob extends Job {
|
|
|
37
55
|
this._fail(error);
|
|
38
56
|
}
|
|
39
57
|
}
|
|
40
|
-
|
|
41
|
-
/**
|
|
42
|
-
* Returns true after the first failed attempt — action jobs have no retry rights.
|
|
43
|
-
* @returns {boolean} True if the job has failed at least once.
|
|
44
|
-
* @override
|
|
45
|
-
*/
|
|
46
|
-
exhausted() {
|
|
47
|
-
return this._attempts >= 1;
|
|
48
|
-
}
|
|
49
58
|
}
|
|
50
59
|
|
|
51
60
|
export { ActionProcessingJob };
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
import { Job } from './Job.js';
|
|
2
|
+
import { Logger } from '../utils/logging/Logger.js';
|
|
3
|
+
|
|
4
|
+
/**
|
|
5
|
+
* AssetDownloadJob is a Job that fetches a single fully-resolved asset URL and validates
|
|
6
|
+
* the expected HTTP status. It is a leaf node — no further chaining occurs after a
|
|
7
|
+
* successful fetch. Follows the standard retry/dead path on failure.
|
|
8
|
+
* @author darthjee
|
|
9
|
+
*/
|
|
10
|
+
class AssetDownloadJob extends Job {
|
|
11
|
+
#url;
|
|
12
|
+
#clientName;
|
|
13
|
+
#status;
|
|
14
|
+
#clientRegistry;
|
|
15
|
+
|
|
16
|
+
/**
|
|
17
|
+
* Creates a new AssetDownloadJob instance.
|
|
18
|
+
* @param {object} params - The parameters for creating an AssetDownloadJob instance.
|
|
19
|
+
* @param {string} params.id - The unique identifier for this job.
|
|
20
|
+
* @param {string} params.url - The fully-resolved asset URL to fetch.
|
|
21
|
+
* @param {string} [params.client] - Named client to use (defaults to `'default'`).
|
|
22
|
+
* @param {number} params.status - Expected HTTP status code.
|
|
23
|
+
* @param {object} params.clientRegistry - The client registry used to look up the client.
|
|
24
|
+
*/
|
|
25
|
+
constructor({ id, url, client, status, clientRegistry }) {
|
|
26
|
+
super({ id });
|
|
27
|
+
this.#url = url;
|
|
28
|
+
this.#clientName = client;
|
|
29
|
+
this.#status = status;
|
|
30
|
+
this.#clientRegistry = clientRegistry;
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
/**
|
|
34
|
+
* Returns the job-specific arguments for serialization.
|
|
35
|
+
* @returns {{ url: string, clientName: string }} The job arguments.
|
|
36
|
+
*/
|
|
37
|
+
get arguments() {
|
|
38
|
+
return { url: this.#url, clientName: this.#clientName };
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
/**
|
|
42
|
+
* Fetches the asset URL using the named client and validates the HTTP status.
|
|
43
|
+
* @returns {Promise<object>} The HTTP response.
|
|
44
|
+
*/
|
|
45
|
+
async perform() {
|
|
46
|
+
Logger.info(`AssetDownloadJob #${this.id} performing`);
|
|
47
|
+
try {
|
|
48
|
+
this.lastError = undefined;
|
|
49
|
+
const client = this.#clientRegistry.getClient(this.#clientName);
|
|
50
|
+
return await client.performUrl(this.#url, this.#status);
|
|
51
|
+
} catch (error) {
|
|
52
|
+
Logger.error(`AssetDownloadJob #${this.id} failed: ${error}`);
|
|
53
|
+
this._fail(error);
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
export { AssetDownloadJob };
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* AssetRequest represents a CSS-selector-based extraction rule for asset URLs from an HTML response.
|
|
3
|
+
* @author darthjee
|
|
4
|
+
*/
|
|
5
|
+
class AssetRequest {
|
|
6
|
+
/**
|
|
7
|
+
* @param {object} attributes AssetRequest attributes.
|
|
8
|
+
* @param {string} attributes.selector CSS selector used to find asset elements.
|
|
9
|
+
* @param {string} attributes.attribute Attribute on the matched element holding the asset URL.
|
|
10
|
+
* @param {string} [attributes.client] Named client to use when fetching the asset.
|
|
11
|
+
* @param {number} [attributes.status=200] Expected HTTP status code for asset fetches.
|
|
12
|
+
*/
|
|
13
|
+
constructor({ selector, attribute, client, status = 200 }) {
|
|
14
|
+
this.selector = selector;
|
|
15
|
+
this.attribute = attribute;
|
|
16
|
+
this.client = client;
|
|
17
|
+
this.status = status;
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
/**
|
|
21
|
+
* Creates an AssetRequest instance from a plain config object.
|
|
22
|
+
* @param {object} obj Raw config object.
|
|
23
|
+
* @returns {AssetRequest} A new AssetRequest instance.
|
|
24
|
+
*/
|
|
25
|
+
static fromObject(obj) {
|
|
26
|
+
return new AssetRequest(obj);
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
/**
|
|
30
|
+
* Creates a list of AssetRequest instances from an array of plain config objects.
|
|
31
|
+
* @param {Array<object>} [list=[]] Array of raw config objects.
|
|
32
|
+
* @returns {Array<AssetRequest>} List of AssetRequest instances.
|
|
33
|
+
*/
|
|
34
|
+
static fromListObject(list = []) {
|
|
35
|
+
return list.map((obj) => AssetRequest.fromObject(obj));
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
export { AssetRequest };
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
import { HtmlParser } from '../utils/HtmlParser.js';
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* AssetRequestEnqueuer processes a single AssetRequest against a raw HTML body:
|
|
5
|
+
* it parses matching URLs, resolves them to absolute form, and enqueues one
|
|
6
|
+
* AssetDownloadJob per discovered URL.
|
|
7
|
+
* @author darthjee
|
|
8
|
+
*/
|
|
9
|
+
class AssetRequestEnqueuer {
|
|
10
|
+
#rawHtml;
|
|
11
|
+
#assetRequest;
|
|
12
|
+
#jobRegistry;
|
|
13
|
+
#clientRegistry;
|
|
14
|
+
|
|
15
|
+
/**
|
|
16
|
+
* @param {string} rawHtml The raw HTML response body.
|
|
17
|
+
* @param {AssetRequest} assetRequest The asset extraction rule.
|
|
18
|
+
* @param {object} jobRegistry The job registry used to enqueue AssetDownloadJobs.
|
|
19
|
+
* @param {object} clientRegistry The client registry used for URL resolution.
|
|
20
|
+
*/
|
|
21
|
+
constructor(rawHtml, assetRequest, jobRegistry, clientRegistry) {
|
|
22
|
+
this.#rawHtml = rawHtml;
|
|
23
|
+
this.#assetRequest = assetRequest;
|
|
24
|
+
this.#jobRegistry = jobRegistry;
|
|
25
|
+
this.#clientRegistry = clientRegistry;
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
/**
|
|
29
|
+
* Parses the HTML body for URLs matching the asset request rule, resolves each URL
|
|
30
|
+
* to absolute form, and enqueues one AssetDownloadJob per URL.
|
|
31
|
+
* @returns {void}
|
|
32
|
+
*/
|
|
33
|
+
enqueue() {
|
|
34
|
+
const urls = HtmlParser.parse(this.#rawHtml, this.#assetRequest.selector, this.#assetRequest.attribute);
|
|
35
|
+
for (const url of urls) {
|
|
36
|
+
this.#jobRegistry.enqueue('AssetDownload', {
|
|
37
|
+
url: this.#resolveUrl(url),
|
|
38
|
+
client: this.#assetRequest.client,
|
|
39
|
+
status: this.#assetRequest.status,
|
|
40
|
+
});
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
/**
|
|
45
|
+
* Resolves a discovered URL to an absolute URL.
|
|
46
|
+
*
|
|
47
|
+
* | Form | Resolution |
|
|
48
|
+
* |------|------------|
|
|
49
|
+
* | Absolute (`https://…` or `http://…`) | Used as-is. |
|
|
50
|
+
* | Protocol-relative (`//…`) | Prepended with `https:`. |
|
|
51
|
+
* | Root-relative (`/…`) | Concatenated with the named client's `baseUrl`. |
|
|
52
|
+
* @param {string} url The raw URL extracted from the HTML element attribute.
|
|
53
|
+
* @returns {string} The fully-resolved absolute URL.
|
|
54
|
+
* @private
|
|
55
|
+
*/
|
|
56
|
+
#resolveUrl(url) {
|
|
57
|
+
if (url.startsWith('https://') || url.startsWith('http://')) {
|
|
58
|
+
return url;
|
|
59
|
+
}
|
|
60
|
+
if (url.startsWith('//')) {
|
|
61
|
+
return `https:${url}`;
|
|
62
|
+
}
|
|
63
|
+
const client = this.#clientRegistry.getClient(this.#assetRequest.client);
|
|
64
|
+
return `${client.baseUrl}${url}`;
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
export { AssetRequestEnqueuer };
|
package/lib/models/Config.js
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { LogConfig } from './LogConfig.js';
|
|
1
2
|
import { ClientRegistry } from '../registry/ClientRegistry.js';
|
|
2
3
|
import { ResourceRegistry } from '../registry/ResourceRegistry.js';
|
|
3
4
|
import { ConfigLoader } from '../services/ConfigLoader.js';
|
|
@@ -14,12 +15,14 @@ class Config {
|
|
|
14
15
|
* @param {Record<string, Client>} params.clients - A mapping of client names to Client instances.
|
|
15
16
|
* @param {WorkersConfig} params.workersConfig - The configuration for worker instances.
|
|
16
17
|
* @param {WebConfig|null} [params.webConfig] - Optional web server configuration.
|
|
18
|
+
* @param {LogConfig} [params.logConfig] - Optional log configuration.
|
|
17
19
|
*/
|
|
18
|
-
constructor({ resources, clients, workersConfig, webConfig }) {
|
|
20
|
+
constructor({ resources, clients, workersConfig, webConfig, logConfig }) {
|
|
19
21
|
this.resourceRegistry = ResourceRegistry.build(resources);
|
|
20
22
|
this.clientRegistry = new ClientRegistry(clients);
|
|
21
23
|
this.workersConfig = workersConfig;
|
|
22
24
|
this.webConfig = webConfig ?? null;
|
|
25
|
+
this.logConfig = logConfig ?? new LogConfig();
|
|
23
26
|
}
|
|
24
27
|
|
|
25
28
|
/**
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
import { AssetRequestEnqueuer } from './AssetRequestEnqueuer.js';
|
|
2
|
+
import { Job } from './Job.js';
|
|
3
|
+
import { Logger } from '../utils/logging/Logger.js';
|
|
4
|
+
|
|
5
|
+
/**
|
|
6
|
+
* HtmlParseJob is a Job that parses an HTML response body, extracts asset URLs using
|
|
7
|
+
* CSS selectors, and enqueues one AssetDownloadJob per discovered URL.
|
|
8
|
+
*
|
|
9
|
+
* Unlike ResourceRequestJob, this job is exhausted after the first failure — it has no
|
|
10
|
+
* retry rights, analogous to ActionProcessingJob.
|
|
11
|
+
* @author darthjee
|
|
12
|
+
*/
|
|
13
|
+
class HtmlParseJob extends Job {
|
|
14
|
+
#rawHtml;
|
|
15
|
+
#assetRequests;
|
|
16
|
+
#jobRegistry;
|
|
17
|
+
#clientRegistry;
|
|
18
|
+
|
|
19
|
+
/**
|
|
20
|
+
* Creates a new HtmlParseJob instance.
|
|
21
|
+
* @param {object} params - The parameters for creating an HtmlParseJob instance.
|
|
22
|
+
* @param {string} params.id - The unique identifier for this job.
|
|
23
|
+
* @param {string} params.rawHtml - The raw HTML response body string.
|
|
24
|
+
* @param {Array} params.assetRequests - List of AssetRequest instances.
|
|
25
|
+
* @param {object} params.jobRegistry - The job registry used to enqueue AssetDownloadJobs.
|
|
26
|
+
* @param {object} params.clientRegistry - The client registry used for URL resolution.
|
|
27
|
+
*/
|
|
28
|
+
constructor({ id, rawHtml, assetRequests, jobRegistry, clientRegistry }) {
|
|
29
|
+
super({ id });
|
|
30
|
+
this.#rawHtml = rawHtml;
|
|
31
|
+
this.#assetRequests = assetRequests;
|
|
32
|
+
this.#jobRegistry = jobRegistry;
|
|
33
|
+
this.#clientRegistry = clientRegistry;
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
/**
|
|
37
|
+
* Returns the job-specific arguments for serialization.
|
|
38
|
+
* @returns {{ assetCount: number }} The job arguments.
|
|
39
|
+
*/
|
|
40
|
+
get arguments() {
|
|
41
|
+
return { assetCount: this.#assetRequests.length };
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
/**
|
|
45
|
+
* Returns the maximum number of retries for this job type.
|
|
46
|
+
* HtmlParseJob is exhausted after the first failure.
|
|
47
|
+
* @returns {number} Always 1.
|
|
48
|
+
* @override
|
|
49
|
+
*/
|
|
50
|
+
get maxRetries() {
|
|
51
|
+
return 1;
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
/**
|
|
55
|
+
* Parses the HTML body, resolves asset URLs, and enqueues one AssetDownloadJob per URL.
|
|
56
|
+
* @returns {Promise<void>}
|
|
57
|
+
*/
|
|
58
|
+
async perform() {
|
|
59
|
+
Logger.info(`HtmlParseJob #${this.id} performing`);
|
|
60
|
+
try {
|
|
61
|
+
this.lastError = undefined;
|
|
62
|
+
for (const assetRequest of this.#assetRequests) {
|
|
63
|
+
new AssetRequestEnqueuer(this.#rawHtml, assetRequest, this.#jobRegistry, this.#clientRegistry).enqueue();
|
|
64
|
+
}
|
|
65
|
+
} catch (error) {
|
|
66
|
+
this._fail(error);
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
export { HtmlParseJob };
|
package/lib/models/Job.js
CHANGED
|
@@ -54,12 +54,22 @@ class Job {
|
|
|
54
54
|
return currentTime >= this.#readyBy;
|
|
55
55
|
}
|
|
56
56
|
|
|
57
|
+
/**
|
|
58
|
+
* Returns the maximum number of retries allowed for this job.
|
|
59
|
+
* Subclasses may override this to return a different value.
|
|
60
|
+
* @returns {number} The maximum number of retries.
|
|
61
|
+
*/
|
|
62
|
+
get maxRetries() {
|
|
63
|
+
return 3;
|
|
64
|
+
}
|
|
65
|
+
|
|
57
66
|
/**
|
|
58
67
|
* Checks if the job has been exhausted (i.e., has reached the maximum number of attempts).
|
|
68
|
+
* @param {number} [maxRetries] - Maximum number of retries before the job is considered dead.
|
|
59
69
|
* @returns {boolean} True if the job is exhausted, false otherwise.
|
|
60
70
|
*/
|
|
61
|
-
exhausted() {
|
|
62
|
-
return this.#attempts >=
|
|
71
|
+
exhausted(maxRetries = this.maxRetries) {
|
|
72
|
+
return this.#attempts >= maxRetries;
|
|
63
73
|
}
|
|
64
74
|
|
|
65
75
|
/**
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Represents the log configuration for the application.
|
|
3
|
+
* @author darthjee
|
|
4
|
+
*/
|
|
5
|
+
class LogConfig {
|
|
6
|
+
#size;
|
|
7
|
+
|
|
8
|
+
/**
|
|
9
|
+
* Creates an instance of LogConfig.
|
|
10
|
+
* @param {object} [config={}] - The configuration object.
|
|
11
|
+
* @param {number} [config.size=100] - The maximum number of log entries to retain.
|
|
12
|
+
*/
|
|
13
|
+
constructor({ size = 100 } = {}) {
|
|
14
|
+
this.#size = size;
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
/**
|
|
18
|
+
* Gets the maximum number of log entries to retain.
|
|
19
|
+
* @returns {number} The log buffer size.
|
|
20
|
+
*/
|
|
21
|
+
get size() {
|
|
22
|
+
return this.#size;
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
/**
|
|
26
|
+
* Creates a LogConfig instance from a raw YAML object.
|
|
27
|
+
* @param {object|null|undefined} obj - The raw log config object from YAML.
|
|
28
|
+
* @returns {LogConfig} A new LogConfig instance.
|
|
29
|
+
*/
|
|
30
|
+
static fromObject(obj) {
|
|
31
|
+
return new LogConfig(obj ?? {});
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
export { LogConfig };
|
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
import { ActionsEnqueuer } from './ActionsEnqueuer.js';
|
|
2
|
+
import { AssetRequest } from './AssetRequest.js';
|
|
2
3
|
import { ResourceRequestAction } from './ResourceRequestAction.js';
|
|
4
|
+
import { JobRegistry as DefaultJobRegistry } from '../registry/JobRegistry.js';
|
|
3
5
|
|
|
4
6
|
/**
|
|
5
7
|
* ResourceRequest represents a request to a specific URL with an expected status code.
|
|
@@ -14,12 +16,14 @@ class ResourceRequest {
|
|
|
14
16
|
* @param {number} attributes.status The expected status code of the response.
|
|
15
17
|
* @param {string} [attributes.clientName] The name of the client to use for this request.
|
|
16
18
|
* @param {Array} [attributes.actions=[]] List of raw action config objects.
|
|
19
|
+
* @param {Array} [attributes.assets=[]] List of raw asset extraction rule objects.
|
|
17
20
|
*/
|
|
18
|
-
constructor({ url, status, clientName, actions = [] }) {
|
|
21
|
+
constructor({ url, status, clientName, actions = [], assets = [] }) {
|
|
19
22
|
this.url = url;
|
|
20
23
|
this.status = status;
|
|
21
24
|
this.#clientName = clientName;
|
|
22
25
|
this.actions = ResourceRequestAction.fromList(actions);
|
|
26
|
+
this.assets = AssetRequest.fromListObject(assets);
|
|
23
27
|
}
|
|
24
28
|
|
|
25
29
|
/**
|
|
@@ -44,6 +48,25 @@ class ResourceRequest {
|
|
|
44
48
|
new ActionsEnqueuer(this.actions, itemWrappers).enqueue();
|
|
45
49
|
}
|
|
46
50
|
|
|
51
|
+
/**
|
|
52
|
+
* Enqueues one HtmlParseJob for this resource request's asset extraction rules.
|
|
53
|
+
* @param {string} rawHtml The raw HTML response body string.
|
|
54
|
+
* @param {object} [jobRegistry=JobRegistry] The job registry used to enqueue the HtmlParseJob.
|
|
55
|
+
* @param {object} clientRegistry The client registry for URL resolution inside HtmlParseJob.
|
|
56
|
+
* @returns {void}
|
|
57
|
+
*/
|
|
58
|
+
enqueueAssets(rawHtml, jobRegistry = DefaultJobRegistry, clientRegistry) {
|
|
59
|
+
jobRegistry.enqueue('HtmlParse', { rawHtml, assetRequests: this.assets, clientRegistry });
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
/**
|
|
63
|
+
* Returns true when the resource request has at least one asset extraction rule.
|
|
64
|
+
* @returns {boolean} True if assets are configured.
|
|
65
|
+
*/
|
|
66
|
+
hasAssets() {
|
|
67
|
+
return this.assets.length > 0;
|
|
68
|
+
}
|
|
69
|
+
|
|
47
70
|
/**
|
|
48
71
|
* Returns the URL with every {:placeholder} token replaced by the
|
|
49
72
|
* corresponding value from the parameters object.
|