@maschinenlesbar.org/pegel-online-cli 0.0.8 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +26 -14
- package/dist/src/cli/commands/stations.d.ts +0 -1
- package/dist/src/cli/commands/stations.js +6 -6
- package/dist/src/cli/commands/timeseries.d.ts +0 -1
- package/dist/src/cli/commands/timeseries.js +10 -12
- package/dist/src/cli/index.d.ts +0 -1
- package/dist/src/cli/index.js +2 -1
- package/dist/src/cli/io.d.ts +17 -1
- package/dist/src/cli/io.js +22 -1
- package/dist/src/cli/program.d.ts +0 -1
- package/dist/src/cli/program.js +4 -4
- package/dist/src/cli/run.d.ts +0 -1
- package/dist/src/cli/run.js +7 -2
- package/dist/src/cli/shared.d.ts +25 -14
- package/dist/src/cli/shared.js +60 -34
- package/dist/src/client/client.d.ts +6 -1
- package/dist/src/client/client.js +71 -17
- package/dist/src/client/engine.d.ts +60 -6
- package/dist/src/client/engine.js +157 -54
- package/dist/src/client/errors.d.ts +22 -1
- package/dist/src/client/errors.js +49 -4
- package/dist/src/client/http.d.ts +0 -1
- package/dist/src/client/http.js +3 -4
- package/dist/src/client/index.d.ts +4 -3
- package/dist/src/client/index.js +3 -3
- package/dist/src/client/query.d.ts +0 -1
- package/dist/src/client/query.js +0 -1
- package/dist/src/client/types.d.ts +36 -6
- package/dist/src/client/types.js +0 -1
- package/dist/src/client/validate.d.ts +51 -0
- package/dist/src/client/validate.js +108 -0
- package/dist/src/index.d.ts +0 -1
- package/dist/src/index.js +0 -1
- package/package.json +2 -1
- package/dist/src/cli/commands/stations.d.ts.map +0 -1
- package/dist/src/cli/commands/stations.js.map +0 -1
- package/dist/src/cli/commands/timeseries.d.ts.map +0 -1
- package/dist/src/cli/commands/timeseries.js.map +0 -1
- package/dist/src/cli/index.d.ts.map +0 -1
- package/dist/src/cli/index.js.map +0 -1
- package/dist/src/cli/io.d.ts.map +0 -1
- package/dist/src/cli/io.js.map +0 -1
- package/dist/src/cli/program.d.ts.map +0 -1
- package/dist/src/cli/program.js.map +0 -1
- package/dist/src/cli/run.d.ts.map +0 -1
- package/dist/src/cli/run.js.map +0 -1
- package/dist/src/cli/shared.d.ts.map +0 -1
- package/dist/src/cli/shared.js.map +0 -1
- package/dist/src/client/client.d.ts.map +0 -1
- package/dist/src/client/client.js.map +0 -1
- package/dist/src/client/engine.d.ts.map +0 -1
- package/dist/src/client/engine.js.map +0 -1
- package/dist/src/client/errors.d.ts.map +0 -1
- package/dist/src/client/errors.js.map +0 -1
- package/dist/src/client/http.d.ts.map +0 -1
- package/dist/src/client/http.js.map +0 -1
- package/dist/src/client/index.d.ts.map +0 -1
- package/dist/src/client/index.js.map +0 -1
- package/dist/src/client/query.d.ts.map +0 -1
- package/dist/src/client/query.js.map +0 -1
- package/dist/src/client/types.d.ts.map +0 -1
- package/dist/src/client/types.js.map +0 -1
- package/dist/src/index.d.ts.map +0 -1
- package/dist/src/index.js.map +0 -1
|
@@ -6,8 +6,45 @@
|
|
|
6
6
|
// client.timeseries.currentMeasurement("BONN", "W")
|
|
7
7
|
// client.timeseries.measurements("BONN", "W", { start: "P3D" })
|
|
8
8
|
import { RequestEngine } from "./engine.js";
|
|
9
|
+
import { PegelError } from "./errors.js";
|
|
10
|
+
import { assertValid, idListProblem, nonEmptyProblem } from "./validate.js";
|
|
9
11
|
const API = "/webservices/rest-api/v2";
|
|
10
|
-
|
|
12
|
+
/**
|
|
13
|
+
* Station names, waters and ids are matched exactly by the API, which stores them
|
|
14
|
+
* composed (NFC): a decomposed umlaut ("KO" + U+0308 + "LN", as pasted from macOS
|
|
15
|
+
* file names or some PDFs) is a 404 / an empty list. Compose every such input.
|
|
16
|
+
* NFC, not NFKC: an id lookup must not rewrite compatibility characters.
|
|
17
|
+
*/
|
|
18
|
+
function nfc(value) {
|
|
19
|
+
return value.normalize("NFC");
|
|
20
|
+
}
|
|
21
|
+
/**
|
|
22
|
+
* One URL path segment from a caller-supplied station or timeseries id. A blank
|
|
23
|
+
* value would build a different path (`stations/.json`, `stations//W.json`), and
|
|
24
|
+
* "." / ".." pass encodeURIComponent unchanged and are resolved by URL parsing
|
|
25
|
+
* (the engine's `buildUrl` refuses them as path segments too), so all are refused.
|
|
26
|
+
*/
|
|
27
|
+
function enc(name, value) {
|
|
28
|
+
if (typeof value !== "string" || value.trim() === "") {
|
|
29
|
+
throw new PegelError(`Invalid ${name}: expected a non-empty string, got ${JSON.stringify(value)}.`);
|
|
30
|
+
}
|
|
31
|
+
if (value === "." || value === "..") {
|
|
32
|
+
throw new PegelError(`Invalid ${name} "${value}": "." and ".." cannot be used as an id.`);
|
|
33
|
+
}
|
|
34
|
+
return encodeURIComponent(nfc(value));
|
|
35
|
+
}
|
|
36
|
+
/**
|
|
37
|
+
* An optional query value: `undefined` means omitted; anything else must be a
|
|
38
|
+
* non-blank string (nonEmptyProblem), or PegelValidationError is thrown.
|
|
39
|
+
*/
|
|
40
|
+
function optionalValue(name, value) {
|
|
41
|
+
return value === undefined ? undefined : assertValid(name, value, nonEmptyProblem);
|
|
42
|
+
}
|
|
43
|
+
/** An optional filter value (optionalValue), composed to NFC like every name the API matches. */
|
|
44
|
+
function optionalFilter(name, value) {
|
|
45
|
+
const checked = optionalValue(name, value);
|
|
46
|
+
return checked === undefined ? undefined : nfc(checked);
|
|
47
|
+
}
|
|
11
48
|
/** Drop undefined values so only the parameters the caller set are sent. */
|
|
12
49
|
function prune(params) {
|
|
13
50
|
const out = {};
|
|
@@ -17,6 +54,20 @@ function prune(params) {
|
|
|
17
54
|
}
|
|
18
55
|
return out;
|
|
19
56
|
}
|
|
57
|
+
/**
|
|
58
|
+
* The station include parameters. The API nests the current measurement and the
|
|
59
|
+
* gauge marks *inside* each timeseries, so without `includeTimeseries=true` it
|
|
60
|
+
* silently drops both. Asking for either therefore implies `includeTimeseries`
|
|
61
|
+
* unless the caller set it explicitly.
|
|
62
|
+
*/
|
|
63
|
+
function stationIncludes(p) {
|
|
64
|
+
const nested = p.includeCurrentMeasurement === true || p.includeCharacteristicValues === true;
|
|
65
|
+
return prune({
|
|
66
|
+
includeTimeseries: p.includeTimeseries ?? (nested ? true : undefined),
|
|
67
|
+
includeCurrentMeasurement: p.includeCurrentMeasurement,
|
|
68
|
+
includeCharacteristicValues: p.includeCharacteristicValues,
|
|
69
|
+
});
|
|
70
|
+
}
|
|
20
71
|
function includeQuery(p) {
|
|
21
72
|
return prune({
|
|
22
73
|
includeTimeseries: p.includeTimeseries,
|
|
@@ -30,19 +81,22 @@ class StationsResource {
|
|
|
30
81
|
constructor(e) {
|
|
31
82
|
this.e = e;
|
|
32
83
|
}
|
|
33
|
-
|
|
84
|
+
/**
|
|
85
|
+
* Rejects (PegelValidationError, no request) a blank `waters` or `fuzzyId`, a
|
|
86
|
+
* blank `ids` entry and an empty `ids` array: the API reads an empty parameter as
|
|
87
|
+
* no filter and would answer with every station.
|
|
88
|
+
*/
|
|
89
|
+
async list(params = {}) {
|
|
34
90
|
const query = prune({
|
|
35
|
-
ids: params.ids
|
|
36
|
-
waters: params.waters,
|
|
37
|
-
fuzzyId: params.fuzzyId,
|
|
38
|
-
|
|
39
|
-
includeCurrentMeasurement: params.includeCurrentMeasurement,
|
|
40
|
-
includeCharacteristicValues: params.includeCharacteristicValues,
|
|
91
|
+
ids: params.ids === undefined ? undefined : assertValid("ids", params.ids, idListProblem).map(nfc).join(","),
|
|
92
|
+
waters: optionalFilter("waters", params.waters),
|
|
93
|
+
fuzzyId: optionalFilter("fuzzyId", params.fuzzyId),
|
|
94
|
+
...stationIncludes(params),
|
|
41
95
|
});
|
|
42
96
|
return this.e.getJson(`${API}/stations.json`, query);
|
|
43
97
|
}
|
|
44
|
-
get(station, params = {}) {
|
|
45
|
-
return this.e.getJson(`${API}/stations/${enc(station)}.json`,
|
|
98
|
+
async get(station, params = {}) {
|
|
99
|
+
return this.e.getJson(`${API}/stations/${enc("station", station)}.json`, stationIncludes(params));
|
|
46
100
|
}
|
|
47
101
|
}
|
|
48
102
|
/** Timeseries: metadata, the current measurement, a window of measurements, gauge marks. */
|
|
@@ -52,14 +106,15 @@ class TimeseriesResource {
|
|
|
52
106
|
this.e = e;
|
|
53
107
|
}
|
|
54
108
|
/** Timeseries metadata (e.g. "W" = water level, "Q" = flow). */
|
|
55
|
-
get(station, timeseries = "W", params = {}) {
|
|
56
|
-
return this.e.getJson(`${API}/stations/${enc(station)}/${enc(timeseries)}.json`, includeQuery(params));
|
|
109
|
+
async get(station, timeseries = "W", params = {}) {
|
|
110
|
+
return this.e.getJson(`${API}/stations/${enc("station", station)}/${enc("timeseries", timeseries)}.json`, includeQuery(params));
|
|
57
111
|
}
|
|
58
|
-
currentMeasurement(station, timeseries = "W") {
|
|
59
|
-
return this.e.getJson(`${API}/stations/${enc(station)}/${enc(timeseries)}/currentmeasurement.json`);
|
|
112
|
+
async currentMeasurement(station, timeseries = "W") {
|
|
113
|
+
return this.e.getJson(`${API}/stations/${enc("station", station)}/${enc("timeseries", timeseries)}/currentmeasurement.json`);
|
|
60
114
|
}
|
|
61
|
-
|
|
62
|
-
|
|
115
|
+
/** Rejects (PegelValidationError, no request) a blank `start` or `end`. */
|
|
116
|
+
async measurements(station, timeseries = "W", params = {}) {
|
|
117
|
+
return this.e.getJson(`${API}/stations/${enc("station", station)}/${enc("timeseries", timeseries)}/measurements.json`, prune({ start: optionalValue("start", params.start), end: optionalValue("end", params.end) }));
|
|
63
118
|
}
|
|
64
119
|
}
|
|
65
120
|
export class PegelOnlineClient {
|
|
@@ -76,4 +131,3 @@ export class PegelOnlineClient {
|
|
|
76
131
|
return this.engine.getJson(`${API}/waters.json`);
|
|
77
132
|
}
|
|
78
133
|
}
|
|
79
|
-
//# sourceMappingURL=client.js.map
|
|
@@ -6,6 +6,11 @@ export interface RawResponse {
|
|
|
6
6
|
contentType: string;
|
|
7
7
|
status: number;
|
|
8
8
|
}
|
|
9
|
+
/**
|
|
10
|
+
* Options for {@link RequestEngine} and the client. The numeric options must be
|
|
11
|
+
* integers within their documented range; anything else (negative, fractional,
|
|
12
|
+
* NaN, Infinity, too large) makes the constructor throw a PegelError.
|
|
13
|
+
*/
|
|
9
14
|
export interface EngineOptions {
|
|
10
15
|
/** Base URL of the API. Defaults to https://www.pegelonline.wsv.de */
|
|
11
16
|
baseUrl?: string;
|
|
@@ -23,14 +28,26 @@ export interface EngineOptions {
|
|
|
23
28
|
headers?: Record<string, string>;
|
|
24
29
|
/**
|
|
25
30
|
* Time limit per request in milliseconds, covering the whole response body, not
|
|
26
|
-
* only idle gaps (0 disables;
|
|
31
|
+
* only idle gaps (0 disables; at most `MAX_TIMEOUT_MS`, 2^31 - 1 ms).
|
|
27
32
|
*/
|
|
28
33
|
timeoutMs?: number;
|
|
29
|
-
/**
|
|
34
|
+
/**
|
|
35
|
+
* Number of automatic retries for transient (429/503) responses, 0..`MAX_RETRIES`
|
|
36
|
+
* (10). Each waits the
|
|
37
|
+
* response's `Retry-After` (up to `MAX_RETRY_AFTER_MS`; a longer one is not
|
|
38
|
+
* retried), or else `retryDelayMs * attempt`.
|
|
39
|
+
*/
|
|
30
40
|
maxRetries?: number;
|
|
31
|
-
/**
|
|
41
|
+
/**
|
|
42
|
+
* Base backoff between retries in milliseconds (grows linearly); used without a
|
|
43
|
+
* Retry-After. At most `MAX_RETRY_AFTER_MS`.
|
|
44
|
+
*/
|
|
32
45
|
retryDelayMs?: number;
|
|
33
|
-
/**
|
|
46
|
+
/**
|
|
47
|
+
* Number of HTTP redirects (301/302/303/307/308) to follow, 0..20. Defaults to 5. Any
|
|
48
|
+
* other 3xx, one with a missing or malformed Location, and one past this limit
|
|
49
|
+
* surface as a PegelApiError naming the target.
|
|
50
|
+
*/
|
|
34
51
|
maxRedirects?: number;
|
|
35
52
|
/**
|
|
36
53
|
* Hard cap on response body size in bytes (defends against memory exhaustion
|
|
@@ -40,6 +57,35 @@ export interface EngineOptions {
|
|
|
40
57
|
/** Injectable sleep, primarily for deterministic tests. */
|
|
41
58
|
sleep?: (ms: number) => Promise<void>;
|
|
42
59
|
}
|
|
60
|
+
/** Most automatic retries a caller may ask for (the CLI's --max-retries shares it). */
|
|
61
|
+
export declare const MAX_RETRIES = 10;
|
|
62
|
+
/**
|
|
63
|
+
* Longest `Retry-After` the engine waits out before retrying a 429/503. When the
|
|
64
|
+
* server asks for longer, the engine does not retry at all and surfaces the error at
|
|
65
|
+
* once: retrying early would only land inside the window the server asked us to wait
|
|
66
|
+
* out, and a hostile value must not stall the CLI.
|
|
67
|
+
*/
|
|
68
|
+
export declare const MAX_RETRY_AFTER_MS = 30000;
|
|
69
|
+
/**
|
|
70
|
+
* Parse a `Retry-After` header into a delay in milliseconds (RFC 9110 §10.2.3):
|
|
71
|
+
* either delay-seconds (`"120"`) or an HTTP-date (`"Wed, 21 Oct 2026 07:28:00 GMT"`,
|
|
72
|
+
* turned into the time left from `now`; a date in the past gives 0).
|
|
73
|
+
*
|
|
74
|
+
* Returns `undefined` when the header is absent or malformed — negative (`"-1"`),
|
|
75
|
+
* fractional (`"1.5"`), padded inside, any other date format — so the caller falls
|
|
76
|
+
* back to its own backoff. The strict patterns matter: `Date.parse` alone would
|
|
77
|
+
* read `"1.5"` as a date in 2001 and retry at once.
|
|
78
|
+
*/
|
|
79
|
+
export declare function parseRetryAfter(header: string | string[] | undefined, now?: number): number | undefined;
|
|
80
|
+
/**
|
|
81
|
+
* Check a base URL against the library's rules (baseUrlProblem: no whitespace or
|
|
82
|
+
* control characters, an absolute http(s) URL, no query or fragment) and return it
|
|
83
|
+
* without trailing slashes. Throws PegelValidationError `Invalid baseUrl: …`: a
|
|
84
|
+
* configuration mistake, not a PegelNetworkError. The RequestEngine constructor
|
|
85
|
+
* calls it on the raw value, so a custom transport never sees a bad base URL; the
|
|
86
|
+
* default transport still re-checks the scheme on every hop.
|
|
87
|
+
*/
|
|
88
|
+
export declare function validateBaseUrl(raw: string): string;
|
|
43
89
|
export declare class RequestEngine {
|
|
44
90
|
private readonly baseUrl;
|
|
45
91
|
private readonly transport;
|
|
@@ -52,7 +98,16 @@ export declare class RequestEngine {
|
|
|
52
98
|
private readonly maxResponseBytes;
|
|
53
99
|
private readonly sleep;
|
|
54
100
|
constructor(options?: EngineOptions);
|
|
55
|
-
/**
|
|
101
|
+
/**
|
|
102
|
+
* Build a fully-qualified URL from a path and optional query parameters.
|
|
103
|
+
*
|
|
104
|
+
* Throws a PegelError for a path with a "." or ".." segment. The resource methods
|
|
105
|
+
* put ids into the path with `encodeURIComponent`, which leaves those two
|
|
106
|
+
* unchanged, and URL parsing then resolves them: `currentMeasurement("BONN", "..")`
|
|
107
|
+
* would request `/stations/currentmeasurement.json` (a station of that name).
|
|
108
|
+
* Neither can name a resource. (Percent-encoded forms such as "%2e%2e" are safe:
|
|
109
|
+
* encodeURIComponent turns their "%" into "%25".)
|
|
110
|
+
*/
|
|
56
111
|
buildUrl(path: string, query?: QueryParams): string;
|
|
57
112
|
/** Perform a request with Accept negotiation and transient-error retries. */
|
|
58
113
|
request(method: string, path: string, options?: {
|
|
@@ -63,4 +118,3 @@ export declare class RequestEngine {
|
|
|
63
118
|
getJson<T>(path: string, query?: QueryParams): Promise<T>;
|
|
64
119
|
private toApiError;
|
|
65
120
|
}
|
|
66
|
-
//# sourceMappingURL=engine.d.ts.map
|
|
@@ -1,12 +1,66 @@
|
|
|
1
1
|
// The request engine: turns logical (method, path, query) calls into HTTP
|
|
2
2
|
// requests via a Transport, applies retry/backoff for transient statuses
|
|
3
3
|
// (429, 503), and decodes responses.
|
|
4
|
-
import { nodeHttpTransport } from "./http.js";
|
|
4
|
+
import { MAX_TIMEOUT_MS, nodeHttpTransport } from "./http.js";
|
|
5
5
|
import { buildQueryString } from "./query.js";
|
|
6
|
-
import { PegelApiError, PegelError,
|
|
6
|
+
import { PegelApiError, PegelError, PegelParseError, redactUrl } from "./errors.js";
|
|
7
|
+
import { assertValid, baseUrlProblem, headerValueProblem } from "./validate.js";
|
|
7
8
|
export const DEFAULT_BASE_URL = "https://www.pegelonline.wsv.de";
|
|
8
9
|
const DEFAULT_USER_AGENT = "pegel-online-cli";
|
|
9
10
|
const DEFAULT_MAX_RESPONSE_BYTES = 100 * 1024 * 1024;
|
|
11
|
+
/** Most automatic retries a caller may ask for (the CLI's --max-retries shares it). */
|
|
12
|
+
export const MAX_RETRIES = 10;
|
|
13
|
+
/** Most redirects a caller may let the engine follow (the Fetch standard's limit). */
|
|
14
|
+
const MAX_REDIRECTS = 20;
|
|
15
|
+
/**
|
|
16
|
+
* Read a numeric engine option: `undefined` gives the default; anything but an
|
|
17
|
+
* integer in [0, max] throws. Without this a negative or NaN `timeoutMs` silently
|
|
18
|
+
* disabled the timeout, and `maxRetries: NaN` silently meant no retries.
|
|
19
|
+
*/
|
|
20
|
+
function intOption(name, value, fallback, max) {
|
|
21
|
+
if (value === undefined)
|
|
22
|
+
return fallback;
|
|
23
|
+
if (!Number.isSafeInteger(value) || value < 0 || value > max) {
|
|
24
|
+
throw new PegelError(`Invalid option ${name}: expected an integer from 0 to ${max}, got ${String(value)}.`);
|
|
25
|
+
}
|
|
26
|
+
return value;
|
|
27
|
+
}
|
|
28
|
+
/**
|
|
29
|
+
* The redirect statuses the engine follows. 300 (a choice for the user), 304 (a
|
|
30
|
+
* cache answer to a conditional request this client never sends) and 305/306
|
|
31
|
+
* (deprecated) are not redirects to follow; they surface as a PegelApiError.
|
|
32
|
+
*/
|
|
33
|
+
const FOLLOWED_REDIRECTS = new Set([301, 302, 303, 307, 308]);
|
|
34
|
+
/**
|
|
35
|
+
* Longest `Retry-After` the engine waits out before retrying a 429/503. When the
|
|
36
|
+
* server asks for longer, the engine does not retry at all and surfaces the error at
|
|
37
|
+
* once: retrying early would only land inside the window the server asked us to wait
|
|
38
|
+
* out, and a hostile value must not stall the CLI.
|
|
39
|
+
*/
|
|
40
|
+
export const MAX_RETRY_AFTER_MS = 30_000;
|
|
41
|
+
/** An IMF-fixdate (RFC 9110 §5.6.7), the one HTTP-date form senders must generate. */
|
|
42
|
+
const IMF_FIXDATE = /^(Mon|Tue|Wed|Thu|Fri|Sat|Sun), \d{2} (Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec) \d{4} \d{2}:\d{2}:\d{2} GMT$/;
|
|
43
|
+
/**
|
|
44
|
+
* Parse a `Retry-After` header into a delay in milliseconds (RFC 9110 §10.2.3):
|
|
45
|
+
* either delay-seconds (`"120"`) or an HTTP-date (`"Wed, 21 Oct 2026 07:28:00 GMT"`,
|
|
46
|
+
* turned into the time left from `now`; a date in the past gives 0).
|
|
47
|
+
*
|
|
48
|
+
* Returns `undefined` when the header is absent or malformed — negative (`"-1"`),
|
|
49
|
+
* fractional (`"1.5"`), padded inside, any other date format — so the caller falls
|
|
50
|
+
* back to its own backoff. The strict patterns matter: `Date.parse` alone would
|
|
51
|
+
* read `"1.5"` as a date in 2001 and retry at once.
|
|
52
|
+
*/
|
|
53
|
+
export function parseRetryAfter(header, now = Date.now()) {
|
|
54
|
+
const value = (Array.isArray(header) ? header[0] : header)?.trim();
|
|
55
|
+
if (value === undefined || value === "")
|
|
56
|
+
return undefined;
|
|
57
|
+
if (/^\d+$/.test(value))
|
|
58
|
+
return Number(value) * 1000;
|
|
59
|
+
if (!IMF_FIXDATE.test(value))
|
|
60
|
+
return undefined;
|
|
61
|
+
const when = Date.parse(value);
|
|
62
|
+
return Number.isNaN(when) ? undefined : Math.max(0, when - now);
|
|
63
|
+
}
|
|
10
64
|
/**
|
|
11
65
|
* Credential-bearing headers that must never be carried across an origin boundary
|
|
12
66
|
* on a redirect. Stored lower-cased and compared case-insensitively so a header
|
|
@@ -51,22 +105,15 @@ function sanitizeServerText(text) {
|
|
|
51
105
|
return out;
|
|
52
106
|
}
|
|
53
107
|
/**
|
|
54
|
-
*
|
|
55
|
-
*
|
|
56
|
-
*
|
|
57
|
-
*
|
|
108
|
+
* Check a base URL against the library's rules (baseUrlProblem: no whitespace or
|
|
109
|
+
* control characters, an absolute http(s) URL, no query or fragment) and return it
|
|
110
|
+
* without trailing slashes. Throws PegelValidationError `Invalid baseUrl: …`: a
|
|
111
|
+
* configuration mistake, not a PegelNetworkError. The RequestEngine constructor
|
|
112
|
+
* calls it on the raw value, so a custom transport never sees a bad base URL; the
|
|
113
|
+
* default transport still re-checks the scheme on every hop.
|
|
58
114
|
*/
|
|
59
|
-
function
|
|
60
|
-
|
|
61
|
-
try {
|
|
62
|
-
url = new URL(baseUrl);
|
|
63
|
-
}
|
|
64
|
-
catch {
|
|
65
|
-
throw new PegelNetworkError(`Invalid base URL: ${baseUrl}`);
|
|
66
|
-
}
|
|
67
|
-
if (url.protocol !== "http:" && url.protocol !== "https:") {
|
|
68
|
-
throw new PegelNetworkError(`Unsupported protocol "${url.protocol}" in base URL: ${baseUrl}`);
|
|
69
|
-
}
|
|
115
|
+
export function validateBaseUrl(raw) {
|
|
116
|
+
return assertValid("baseUrl", raw, baseUrlProblem).replace(/\/+$/, "");
|
|
70
117
|
}
|
|
71
118
|
const realSleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
|
|
72
119
|
export class RequestEngine {
|
|
@@ -81,27 +128,43 @@ export class RequestEngine {
|
|
|
81
128
|
maxResponseBytes;
|
|
82
129
|
sleep;
|
|
83
130
|
constructor(options = {}) {
|
|
84
|
-
|
|
85
|
-
|
|
131
|
+
// The raw value, before the slash strip: the engine glues it into every URL, so
|
|
132
|
+
// "https://h/ " must not lose its slash first and slip past the check.
|
|
133
|
+
this.baseUrl = validateBaseUrl(options.baseUrl ?? DEFAULT_BASE_URL);
|
|
86
134
|
this.transport = options.transport ?? nodeHttpTransport;
|
|
87
|
-
|
|
88
|
-
//
|
|
89
|
-
//
|
|
90
|
-
// which
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
135
|
+
// Only `undefined` selects the default. An explicit value must be one an HTTP
|
|
136
|
+
// header can carry (headerValueProblem): not blank, which would replace the
|
|
137
|
+
// default with an empty header, and no control character (CR/LF in particular,
|
|
138
|
+
// which also closes header injection; tab is allowed) or character above U+00FF,
|
|
139
|
+
// which Node would refuse late with a raw TypeError.
|
|
140
|
+
this.userAgent =
|
|
141
|
+
options.userAgent === undefined
|
|
142
|
+
? DEFAULT_USER_AGENT
|
|
143
|
+
: assertValid("User-Agent", options.userAgent, headerValueProblem);
|
|
94
144
|
this.extraHeaders = options.headers ?? {};
|
|
95
|
-
this.timeoutMs = options.timeoutMs
|
|
96
|
-
this.maxRetries = options.maxRetries
|
|
97
|
-
this.retryDelayMs = options.retryDelayMs
|
|
98
|
-
this.maxRedirects = options.maxRedirects
|
|
99
|
-
this.maxResponseBytes = options.maxResponseBytes
|
|
145
|
+
this.timeoutMs = intOption("timeoutMs", options.timeoutMs, 30_000, MAX_TIMEOUT_MS);
|
|
146
|
+
this.maxRetries = intOption("maxRetries", options.maxRetries, 2, MAX_RETRIES);
|
|
147
|
+
this.retryDelayMs = intOption("retryDelayMs", options.retryDelayMs, 200, MAX_RETRY_AFTER_MS);
|
|
148
|
+
this.maxRedirects = intOption("maxRedirects", options.maxRedirects, 5, MAX_REDIRECTS);
|
|
149
|
+
this.maxResponseBytes = intOption("maxResponseBytes", options.maxResponseBytes, DEFAULT_MAX_RESPONSE_BYTES, Number.MAX_SAFE_INTEGER);
|
|
100
150
|
this.sleep = options.sleep ?? realSleep;
|
|
101
151
|
}
|
|
102
|
-
/**
|
|
152
|
+
/**
|
|
153
|
+
* Build a fully-qualified URL from a path and optional query parameters.
|
|
154
|
+
*
|
|
155
|
+
* Throws a PegelError for a path with a "." or ".." segment. The resource methods
|
|
156
|
+
* put ids into the path with `encodeURIComponent`, which leaves those two
|
|
157
|
+
* unchanged, and URL parsing then resolves them: `currentMeasurement("BONN", "..")`
|
|
158
|
+
* would request `/stations/currentmeasurement.json` (a station of that name).
|
|
159
|
+
* Neither can name a resource. (Percent-encoded forms such as "%2e%2e" are safe:
|
|
160
|
+
* encodeURIComponent turns their "%" into "%25".)
|
|
161
|
+
*/
|
|
103
162
|
buildUrl(path, query) {
|
|
104
163
|
const normalizedPath = path.startsWith("/") ? path : `/${path}`;
|
|
164
|
+
const dotSegment = normalizedPath.split("/").find((s) => s === "." || s === "..");
|
|
165
|
+
if (dotSegment !== undefined) {
|
|
166
|
+
throw new PegelError(`Invalid path segment "${dotSegment}" in ${normalizedPath}: "." and ".." cannot be used as an id.`);
|
|
167
|
+
}
|
|
105
168
|
const qs = query ? buildQueryString(query) : "";
|
|
106
169
|
return `${this.baseUrl}${normalizedPath}${qs ? `?${qs}` : ""}`;
|
|
107
170
|
}
|
|
@@ -127,31 +190,41 @@ export class RequestEngine {
|
|
|
127
190
|
const status = response.status;
|
|
128
191
|
const retryable = status === 429 || status === 503;
|
|
129
192
|
if (retryable && attempt < this.maxRetries) {
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
193
|
+
// Honour Retry-After; without a usable one, back off linearly. A Retry-After
|
|
194
|
+
// beyond MAX_RETRY_AFTER_MS is not retried: the error below surfaces at once.
|
|
195
|
+
const retryAfter = parseRetryAfter(response.headers["retry-after"]);
|
|
196
|
+
if (retryAfter === undefined || retryAfter <= MAX_RETRY_AFTER_MS) {
|
|
197
|
+
attempt += 1;
|
|
198
|
+
await this.sleep(retryAfter ?? this.retryDelayMs * attempt);
|
|
199
|
+
continue;
|
|
200
|
+
}
|
|
133
201
|
}
|
|
134
202
|
// Follow redirects, resolving the Location relative to the current URL.
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
203
|
+
const location = response.headers["location"];
|
|
204
|
+
const next = FOLLOWED_REDIRECTS.has(status) ? resolveLocation(location, url) : undefined;
|
|
205
|
+
if (next !== undefined && redirects >= this.maxRedirects) {
|
|
206
|
+
// A loop (or a long chain): say how far it got rather than a bare 3xx.
|
|
207
|
+
// (With maxRedirects 0 nothing was followed; the plain text says enough.)
|
|
208
|
+
throw this.toApiError(method, url, status, response.body, location, redirects || undefined);
|
|
209
|
+
}
|
|
210
|
+
if (next !== undefined) {
|
|
211
|
+
// Security: never carry credential-bearing headers across an origin
|
|
212
|
+
// boundary. The CLI sends none today, but this guards a future
|
|
213
|
+
// Authorization/Cookie/X-Api-Key header from leaking to an
|
|
214
|
+
// attacker-controlled redirect target. Comparing full origin (scheme +
|
|
215
|
+
// host + port) also strips on a same-host https->http downgrade.
|
|
216
|
+
if (next.origin !== new URL(url).origin) {
|
|
217
|
+
stripSensitiveHeaders(headers);
|
|
150
218
|
}
|
|
219
|
+
url = next.toString();
|
|
220
|
+
redirects += 1;
|
|
221
|
+
continue;
|
|
151
222
|
}
|
|
223
|
+
// Any other 3xx — not a followed status, or no usable Location — falls
|
|
224
|
+
// through and surfaces as a PegelApiError naming the target.
|
|
152
225
|
const contentType = String(response.headers["content-type"] ?? "");
|
|
153
226
|
if (status < 200 || status >= 300) {
|
|
154
|
-
throw this.toApiError(method, url, status, response.body);
|
|
227
|
+
throw this.toApiError(method, url, status, response.body, location);
|
|
155
228
|
}
|
|
156
229
|
return { data: response.body, contentType, status };
|
|
157
230
|
}
|
|
@@ -167,7 +240,7 @@ export class RequestEngine {
|
|
|
167
240
|
throw new PegelParseError(`Failed to parse JSON response from ${path}`, { cause });
|
|
168
241
|
}
|
|
169
242
|
}
|
|
170
|
-
toApiError(method, url, status, body) {
|
|
243
|
+
toApiError(method, url, status, body, locationHeader, redirectsFollowed) {
|
|
171
244
|
const text = body.toString("utf8");
|
|
172
245
|
let detail;
|
|
173
246
|
try {
|
|
@@ -184,7 +257,37 @@ export class RequestEngine {
|
|
|
184
257
|
// endpoint cannot inject terminal escape sequences via the stderr error message.
|
|
185
258
|
if (detail !== undefined)
|
|
186
259
|
detail = sanitizeServerText(detail);
|
|
187
|
-
|
|
260
|
+
// Name the target of a redirect that was not followed.
|
|
261
|
+
const location = status >= 300 && status < 400 && locationHeader ? redirectTarget(url, locationHeader) : undefined;
|
|
262
|
+
return new PegelApiError({
|
|
263
|
+
status,
|
|
264
|
+
url,
|
|
265
|
+
method,
|
|
266
|
+
body: text,
|
|
267
|
+
detail,
|
|
268
|
+
...(location !== undefined ? { location } : {}),
|
|
269
|
+
...(redirectsFollowed !== undefined ? { redirectsFollowed } : {}),
|
|
270
|
+
});
|
|
188
271
|
}
|
|
189
272
|
}
|
|
190
|
-
|
|
273
|
+
/** Resolve a Location header against the current URL; undefined if missing or malformed. */
|
|
274
|
+
function resolveLocation(location, base) {
|
|
275
|
+
if (location === undefined || location === "")
|
|
276
|
+
return undefined;
|
|
277
|
+
try {
|
|
278
|
+
return new URL(location, base);
|
|
279
|
+
}
|
|
280
|
+
catch {
|
|
281
|
+
return undefined;
|
|
282
|
+
}
|
|
283
|
+
}
|
|
284
|
+
/**
|
|
285
|
+
* The absolute, printable form of a `Location` header: resolved against the request
|
|
286
|
+
* URL, userinfo redacted, control characters stripped (it is server text bound for
|
|
287
|
+
* stderr). An unparseable value is shown sanitised as it came.
|
|
288
|
+
*/
|
|
289
|
+
function redirectTarget(requestUrl, location) {
|
|
290
|
+
const resolved = resolveLocation(location, requestUrl);
|
|
291
|
+
const clean = sanitizeServerText(resolved ? redactUrl(resolved.href) : location).trim();
|
|
292
|
+
return clean === "" ? undefined : clean;
|
|
293
|
+
}
|
|
@@ -4,6 +4,12 @@ export declare class PegelError extends Error {
|
|
|
4
4
|
cause?: unknown;
|
|
5
5
|
});
|
|
6
6
|
}
|
|
7
|
+
/**
|
|
8
|
+
* Replace the userinfo of a URL (`https://user:secret@host/...`) with `***`, so a
|
|
9
|
+
* credential in a base URL never reaches an error message, a log or CI output.
|
|
10
|
+
* A URL without userinfo, or one that does not parse, is returned unchanged.
|
|
11
|
+
*/
|
|
12
|
+
export declare function redactUrl(url: string): string;
|
|
7
13
|
/**
|
|
8
14
|
* The API responded with a non-2xx status code. `detail` holds a human-readable
|
|
9
15
|
* message extracted from the response body when one is present.
|
|
@@ -14,20 +20,35 @@ export declare class PegelApiError extends PegelError {
|
|
|
14
20
|
readonly url: string;
|
|
15
21
|
readonly method: string;
|
|
16
22
|
readonly body: string;
|
|
23
|
+
/**
|
|
24
|
+
* For a 3xx that was not followed (not a followed status, a malformed Location,
|
|
25
|
+
* or past `maxRedirects`): the redirect target, absolute, sanitised, userinfo
|
|
26
|
+
* redacted. The message names it.
|
|
27
|
+
*/
|
|
28
|
+
readonly location: string | undefined;
|
|
17
29
|
constructor(args: {
|
|
18
30
|
status: number;
|
|
19
31
|
url: string;
|
|
20
32
|
method: string;
|
|
21
33
|
body: string;
|
|
22
34
|
detail?: string;
|
|
35
|
+
location?: string;
|
|
36
|
+
/** Redirects already followed when the limit stopped this one (> 0 only). */
|
|
37
|
+
redirectsFollowed?: number;
|
|
23
38
|
});
|
|
24
39
|
/** True for statuses the API documents as transient and retry-able. */
|
|
25
40
|
get isRetryable(): boolean;
|
|
26
41
|
}
|
|
42
|
+
/**
|
|
43
|
+
* An input the library rejects before sending any request: a client option or a
|
|
44
|
+
* method argument that breaks one of the rules in `validate.ts`. The message reads
|
|
45
|
+
* `Invalid <name>: <reason>`. The CLI reports it as a usage error (exit 2).
|
|
46
|
+
*/
|
|
47
|
+
export declare class PegelValidationError extends PegelError {
|
|
48
|
+
}
|
|
27
49
|
/** A transport-level failure (DNS, connection reset, timeout, ...). */
|
|
28
50
|
export declare class PegelNetworkError extends PegelError {
|
|
29
51
|
}
|
|
30
52
|
/** The response body could not be parsed as the expected JSON shape. */
|
|
31
53
|
export declare class PegelParseError extends PegelError {
|
|
32
54
|
}
|
|
33
|
-
//# sourceMappingURL=errors.d.ts.map
|
|
@@ -7,6 +7,25 @@ export class PegelError extends Error {
|
|
|
7
7
|
this.name = new.target.name;
|
|
8
8
|
}
|
|
9
9
|
}
|
|
10
|
+
/**
|
|
11
|
+
* Replace the userinfo of a URL (`https://user:secret@host/...`) with `***`, so a
|
|
12
|
+
* credential in a base URL never reaches an error message, a log or CI output.
|
|
13
|
+
* A URL without userinfo, or one that does not parse, is returned unchanged.
|
|
14
|
+
*/
|
|
15
|
+
export function redactUrl(url) {
|
|
16
|
+
let parsed;
|
|
17
|
+
try {
|
|
18
|
+
parsed = new URL(url);
|
|
19
|
+
}
|
|
20
|
+
catch {
|
|
21
|
+
return url;
|
|
22
|
+
}
|
|
23
|
+
if (parsed.username === "" && parsed.password === "")
|
|
24
|
+
return url;
|
|
25
|
+
parsed.username = "***";
|
|
26
|
+
parsed.password = "";
|
|
27
|
+
return parsed.href;
|
|
28
|
+
}
|
|
10
29
|
/**
|
|
11
30
|
* The API responded with a non-2xx status code. `detail` holds a human-readable
|
|
12
31
|
* message extracted from the response body when one is present.
|
|
@@ -17,24 +36,50 @@ export class PegelApiError extends PegelError {
|
|
|
17
36
|
url;
|
|
18
37
|
method;
|
|
19
38
|
body;
|
|
39
|
+
/**
|
|
40
|
+
* For a 3xx that was not followed (not a followed status, a malformed Location,
|
|
41
|
+
* or past `maxRedirects`): the redirect target, absolute, sanitised, userinfo
|
|
42
|
+
* redacted. The message names it.
|
|
43
|
+
*/
|
|
44
|
+
location;
|
|
20
45
|
constructor(args) {
|
|
21
|
-
const
|
|
22
|
-
|
|
46
|
+
const parts = [];
|
|
47
|
+
if (args.detail)
|
|
48
|
+
parts.push(args.detail);
|
|
49
|
+
if (args.status >= 300 && args.status < 400) {
|
|
50
|
+
const limit = args.redirectsFollowed !== undefined && args.redirectsFollowed > 0
|
|
51
|
+
? ` (stopped after ${args.redirectsFollowed} redirects)`
|
|
52
|
+
: "";
|
|
53
|
+
parts.push(args.location
|
|
54
|
+
? `redirect to ${args.location} not followed${limit}`
|
|
55
|
+
: "redirect not followed (no Location header)");
|
|
56
|
+
}
|
|
57
|
+
const detailPart = parts.length > 0 ? `: ${parts.join("; ")}` : "";
|
|
58
|
+
// The URL is shown without userinfo: a credential in --base-url must not leak.
|
|
59
|
+
const url = redactUrl(args.url);
|
|
60
|
+
super(`HTTP ${args.status} for ${args.method} ${url}${detailPart}`);
|
|
23
61
|
this.status = args.status;
|
|
24
|
-
this.url =
|
|
62
|
+
this.url = url;
|
|
25
63
|
this.method = args.method;
|
|
26
64
|
this.body = args.body;
|
|
27
65
|
this.detail = args.detail;
|
|
66
|
+
this.location = args.location;
|
|
28
67
|
}
|
|
29
68
|
/** True for statuses the API documents as transient and retry-able. */
|
|
30
69
|
get isRetryable() {
|
|
31
70
|
return this.status === 429 || this.status === 503;
|
|
32
71
|
}
|
|
33
72
|
}
|
|
73
|
+
/**
|
|
74
|
+
* An input the library rejects before sending any request: a client option or a
|
|
75
|
+
* method argument that breaks one of the rules in `validate.ts`. The message reads
|
|
76
|
+
* `Invalid <name>: <reason>`. The CLI reports it as a usage error (exit 2).
|
|
77
|
+
*/
|
|
78
|
+
export class PegelValidationError extends PegelError {
|
|
79
|
+
}
|
|
34
80
|
/** A transport-level failure (DNS, connection reset, timeout, ...). */
|
|
35
81
|
export class PegelNetworkError extends PegelError {
|
|
36
82
|
}
|
|
37
83
|
/** The response body could not be parsed as the expected JSON shape. */
|
|
38
84
|
export class PegelParseError extends PegelError {
|
|
39
85
|
}
|
|
40
|
-
//# sourceMappingURL=errors.js.map
|