@runeditrun/rer 0.2.66 → 0.2.68
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/cli/business/vendors/README.md +38 -2
- package/cli/business/vendors/calendly.mjs +2 -2
- package/cli/business/vendors/common.mjs +94 -0
- package/cli/business/vendors/credential-names.mjs +124 -0
- package/cli/business/vendors/pagerduty.mjs +6 -15
- package/cli/business/vendors/sentry.mjs +4 -3
- package/cli/business/vendors/uptimerobot.mjs +7 -6
- package/cli/import-summary.mjs +7 -0
- package/cli/output.mjs +2 -0
- package/cli/portability-worker.mjs +10 -4
- package/cli/portability.mjs +1 -0
- package/cli/production-mail.mjs +4 -1
- package/global/CHANGELOG.md +14 -0
- package/modules/deploy-kit/d1-import.mjs +3 -1
- package/modules/deploy-kit/migrations.mjs +1 -0
- package/modules/deploy-kit/owner-cli.mjs +1 -1
- package/modules/deploy-kit/provider.mjs +16 -4
- package/modules/portability/README.md +73 -8
- package/modules/portability/bridge.mjs +44 -29
- package/modules/portability/catalogue.mjs +33 -3
- package/modules/portability/files.mjs +9 -3
- package/modules/portability/import-safety.mjs +80 -0
- package/modules/portability/import.mjs +13 -7
- package/modules/portability/index.d.mts +9 -3
- package/modules/portability/index.mjs +1 -1
- package/modules/portability/ledgers.mjs +83 -0
- package/modules/portability/legacy-files.mjs +7 -4
- package/modules/portability/migration-history.mjs +3 -2
- package/modules/portability/migration-statements.mjs +83 -9
- package/modules/release/README.md +25 -7
- package/modules/release/cleanup.mjs +16 -0
- package/modules/release/produce.mjs +11 -4
- package/modules/release/publication-scan.mjs +18 -8
- package/package.json +1 -1
- package/scripts/publish-check/README.md +5 -2
- package/scripts/publish-check/destination.py +8 -1
- package/scripts/publish-check/git_history.py +7 -6
- package/scripts/publish-check/history_hydration.py +32 -9
|
@@ -57,6 +57,42 @@ emission. HTTP errors contain only a fixed code and numeric status; network,
|
|
|
57
57
|
parsing and unexpected errors never expose response text, URLs or key values.
|
|
58
58
|
429 stops with `RATE_LIMITED`; there is no hidden retry loop.
|
|
59
59
|
|
|
60
|
+
## Credentials in responses
|
|
61
|
+
|
|
62
|
+
Every adapter strips credential values from a parsed response before it emits
|
|
63
|
+
bytes for upload, keeping the record, by the same rules as the seed's import
|
|
64
|
+
door. A field whose name's parts are a credential word or phrase (`apiKey`,
|
|
65
|
+
`httpPassword`, `http_password`, `client_secret`, `X-Api-Key`, `integration_key`,
|
|
66
|
+
`webhook_url`, `secret_key`, `x-functions-key`, `Ocp-Apim-Subscription-Key`,
|
|
67
|
+
`pwd`, `X-Auth` and the like), the `value` of a `{name|key, value}` pair
|
|
68
|
+
naming one (`{name: 'Authorization', value}`), or the value of such a
|
|
69
|
+
`[name, value]` pair in a list of them (Sentry's header tuples), becomes
|
|
70
|
+
`"[removed]"`. A plural name (`apiKeys`, `tokens`) counts only when it holds
|
|
71
|
+
text, so a count such as `tokens: 1500` stays. A URL of any scheme keeps a
|
|
72
|
+
marker where each credential was: its user name and password become `removed@`,
|
|
73
|
+
and the value of each credential query or fragment parameter (`api_key`,
|
|
74
|
+
`access_token`, `sig` and the like) becomes `removed`
|
|
75
|
+
(`?api_key=removed®ion=eu`, `#access_token=removed`), except the adapter's
|
|
76
|
+
own validated pagination keys (`page_token`). The door drops each marker and
|
|
77
|
+
reports it per record as `CREDENTIAL_REMOVED`, so the owner sees which record
|
|
78
|
+
lost a credential. Null, booleans, empty strings and masks such as `****` are
|
|
79
|
+
not credentials and stay as they are. Names are matched by whole parts
|
|
80
|
+
(camelCase, `_`, `-`, digits), so names that merely contain such letters
|
|
81
|
+
(`keyword`, `monkey`, `author`, `signal`, `design`, `primaryKey`) are not
|
|
82
|
+
credential names and a `?keyword=up` URL stays intact. `Signature` and
|
|
83
|
+
`Session-Id` count only as header names (a key under `headers`, a tuple, a
|
|
84
|
+
`{name|key, value}` pair in a header list), so an agent's `signature` field and
|
|
85
|
+
Help Scout's `customFields: [{name: 'Session ID', value}]` stay;
|
|
86
|
+
`auth` and `credentials` count only when they hold text naming no scheme or mode,
|
|
87
|
+
and an object under them loses only its own credential fields
|
|
88
|
+
(`auth: {type: 'oauth2', token: "[removed]"}`) and a `value`, `pass`, `pw` or
|
|
89
|
+
`secret…` member (Keycloak's `credentials: [{type: 'password', value: "[removed]"}]`).
|
|
90
|
+
`passphrase`, `jwt`, `JSESSIONID` and `PHPSESSID` count everywhere. Query and fragment keys also
|
|
91
|
+
count documented API key names such as `token_auth`, `application_key`,
|
|
92
|
+
`user_key` and `hapikey`.
|
|
93
|
+
Pagination is read from the response as received, so a credential-bearing
|
|
94
|
+
pagination link is still refused.
|
|
95
|
+
|
|
60
96
|
## Inputs and native coverage
|
|
61
97
|
|
|
62
98
|
* `sentry`: `organization: {id, slug, region: 'us'|'de'}`, `start`, `end`, and
|
|
@@ -75,8 +111,8 @@ parsing and unexpected errors never expose response text, URLs or key values.
|
|
|
75
111
|
with layers/final coverage, overrides for the window and escalation policies.
|
|
76
112
|
Schedule listing only discovers IDs and is not duplicated in the bundle.
|
|
77
113
|
Overrides retain full boundaries with `overflow=true` and add `schedule_id`,
|
|
78
|
-
as Oncall's documented bundle requires.
|
|
79
|
-
|
|
114
|
+
as Oncall's documented bundle requires. Incident history and paging
|
|
115
|
+
configuration are not fetched.
|
|
80
116
|
* `calendly`: optional `asOf` timestamp (recommended for a repeatable catch-up).
|
|
81
117
|
An omitted timestamp is captured at first invocation and retained on resume.
|
|
82
118
|
Downloads the current user, event types, availability schedules, per-type
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { object, list, id, timestamp, reference, cursor, assertCount, hash, shape, fail } from './common.mjs';
|
|
1
|
+
import { object, list, id, timestamp, reference, cursor, assertCount, hash, shape, fail, withoutCredentials } from './common.mjs';
|
|
2
2
|
|
|
3
3
|
export const origin = 'https://api.calendly.com';
|
|
4
4
|
const queries = ['user','count','event_type','min_start_time','page_token'];
|
|
@@ -66,7 +66,7 @@ export async function* run(ctx) {
|
|
|
66
66
|
const next = s.phase === 0 ? null : nextToken(body, path, s.token);
|
|
67
67
|
if (next !== null && (next === s.token || s.seen.includes(next))) fail('PAGINATION_CYCLE');
|
|
68
68
|
const name = `${resource}-${String(s.files.length + 1).padStart(6,'0')}.json`;
|
|
69
|
-
const bytes = Buffer.from(JSON.stringify(body) + '\n');
|
|
69
|
+
const bytes = Buffer.from(JSON.stringify(withoutCredentials(body, queries)) + '\n');
|
|
70
70
|
if (s.files.length >= 600) fail('FILE_LIMIT');
|
|
71
71
|
s.files.push({ name, resource, page: s.seen.length + 1, size: bytes.length, sha256: hash(bytes), capturedAt: s.asOf });
|
|
72
72
|
s.token = next;
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { createHash } from 'node:crypto';
|
|
2
2
|
import { setTimeout as delay } from 'node:timers/promises';
|
|
3
|
+
import { credentialMember, credentialName, credentialScheme, headerContainer } from './credential-names.mjs';
|
|
3
4
|
|
|
4
5
|
export class VendorExportError extends Error {
|
|
5
6
|
constructor(code, status) {
|
|
@@ -46,6 +47,99 @@ export function reference(value, origin, pathname, allowedQuery = []) {
|
|
|
46
47
|
if ([...url.searchParams.keys()].some(key => !allowedQuery.includes(key))) fail('INVALID_PAGINATION');
|
|
47
48
|
return url;
|
|
48
49
|
}
|
|
50
|
+
// The seed's import door applies the same rules (modules/import-kit, removeCredentials).
|
|
51
|
+
// A field holds a credential when the parts of its name are a credential word or
|
|
52
|
+
// phrase (apiKey, httpPassword, http_password, client_secret, X-Api-Key,
|
|
53
|
+
// integration_key, webhook_url, secret_key, x-functions-key,
|
|
54
|
+
// Ocp-Apim-Subscription-Key, db_pwd, X-Auth), unless its value is empty: null, a
|
|
55
|
+
// boolean (Calendly's `secret` flag), whitespace or a mask such as "****" or
|
|
56
|
+
// "[redacted]". Names that only contain such letters (keyword, monkey, author,
|
|
57
|
+
// signal, design) are not credentials. A plural (apiKeys, tokens) holds
|
|
58
|
+
// credentials only as text, so a count such as `tokens: 1500` stays. `auth` and
|
|
59
|
+
// `credentials` hold one only as text naming no scheme or mode (`include`,
|
|
60
|
+
// `oauth2`); an object under them keeps its fields and loses its own credential
|
|
61
|
+
// fields (`auth: {token}`) and, directly under them, a `value`, `pass`, `pw` or
|
|
62
|
+
// `secret…` member (Keycloak's `credentials: [{type, value}]`, nodemailer's
|
|
63
|
+
// `{user, pass}`). `passphrase`, `jwt`, `JSESSIONID` and `PHPSESSID` count
|
|
64
|
+
// everywhere. `sig`, `signature` and `sessionId` are credentials as header names
|
|
65
|
+
// (a tuple, a key under `headers`, a `{name|key, value}` pair in a header list),
|
|
66
|
+
// never as fields; a pair elsewhere (Help Scout's `customFields`) labels a field.
|
|
67
|
+
const placeholder = /^(?:\s*|[*\u2022\u25CF\u00B7x#_.-]{1,64}|[<[({]?\s*(?:redacted|hidden|masked|removed|omitted|none|null|n\/a|placeholder)\s*[>\])}]?)$/iu;
|
|
68
|
+
/** What a stripped credential value becomes; the seed's import door reports it per record. */
|
|
69
|
+
export const REMOVED_CREDENTIAL = '[removed]';
|
|
70
|
+
/** What a stripped credential query or fragment parameter's value becomes
|
|
71
|
+
* (`?api_key=removed`), as a URL's user name and password become `removed@`; the
|
|
72
|
+
* door drops the pair and reports it per record. */
|
|
73
|
+
export const REMOVED_PARAMETER = 'removed';
|
|
74
|
+
const empty = value => value === null || typeof value === 'boolean' || (typeof value === 'string' && placeholder.test(value));
|
|
75
|
+
// Text as a member (`['…']`, `{github: '…'}`); a list of objects is checked field by field.
|
|
76
|
+
const text = value => typeof value === 'string' ? !placeholder.test(value) : !!value && typeof value === 'object' && Object.values(value).some(child => (typeof child === 'string' || Array.isArray(child)) && text(child));
|
|
77
|
+
/** Every percent-encoding layer of a key, decoded byte by byte, as the door inspects it. */
|
|
78
|
+
function decoded(text) {
|
|
79
|
+
for (let depth = 0; depth < 32 && /%[\da-f]{2}/iu.test(text); depth++) text = text.replace(/%([\da-f]{2})/giu, (_, hex) => String.fromCharCode(parseInt(hex, 16)));
|
|
80
|
+
return text;
|
|
81
|
+
}
|
|
82
|
+
/** Text under `auth` or `credentials` naming no scheme or mode, or a list with such a member. */
|
|
83
|
+
const schemeless = value => typeof value === 'string' ? !placeholder.test(value) && !credentialScheme(value) : Array.isArray(value) && value.some(member => typeof member === 'string' && schemeless(member));
|
|
84
|
+
/** Whether a value under this name, a record field or an HTTP header name, is a credential, by the door's rule. */
|
|
85
|
+
function credential(name, value, place) {
|
|
86
|
+
const found = credentialName(decoded(name), 'remove', place);
|
|
87
|
+
return found !== undefined && !empty(value) && (found.container ? schemeless(value) : !found.plural || text(value));
|
|
88
|
+
}
|
|
89
|
+
/** Whether this key is `auth`, `X-Auth`, `credential` or `credentials`, whose members are searched for a credential under a generic name. */
|
|
90
|
+
const container = key => credentialName(decoded(key), 'remove')?.container === true;
|
|
91
|
+
/** A member of such a container holding a credential under a generic name (`value`, `pass`, `pw`, `secretData`). */
|
|
92
|
+
const member = (key, value) => credentialMember(decoded(key)) !== undefined && !empty(value);
|
|
93
|
+
// The door's credentialQueryKey rule for URL query and fragment parameter names:
|
|
94
|
+
// api_key, access_token, a bare `key`, sig and X-Amz-Signature, never keyword or author.
|
|
95
|
+
const credentialParameter = key => credentialName(decoded(key), 'query') !== undefined;
|
|
96
|
+
const authority = /^(\s*(?:[a-z][a-z\d+.-]*:)?[\\/]{2})[^\\/?#]*@/iu;
|
|
97
|
+
/** A URL of any scheme with its user name and password replaced by `removed@`,
|
|
98
|
+
* and the value of each credential query or fragment parameter by `removed`,
|
|
99
|
+
* except the adapter's own validated pagination keys, spelled exactly. */
|
|
100
|
+
function withoutUrlCredentials(text, pagination) {
|
|
101
|
+
if (!/^\s*(?:[a-z][a-z\d+.-]*:)?[\\/]{2}|^[/?]/iu.test(text)) return text;
|
|
102
|
+
const result = text.replace(authority, '$1removed@');
|
|
103
|
+
const mark = pairs => pairs.split('&').map(pair => {
|
|
104
|
+
const at = pair.indexOf('='), key = at < 0 ? pair : pair.slice(0, at);
|
|
105
|
+
return at < 0 || pagination.includes(key) || !credentialParameter(key) ? pair : `${key}=${REMOVED_PARAMETER}`;
|
|
106
|
+
}).join('&');
|
|
107
|
+
const hashAt = result.indexOf('#'), end = hashAt < 0 ? result.length : hashAt;
|
|
108
|
+
const fragment = hashAt < 0 ? '' : result.slice(hashAt);
|
|
109
|
+
const marked = fragment.includes('=') ? '#' + mark(fragment.slice(1)) : fragment;
|
|
110
|
+
const queryAt = result.indexOf('?');
|
|
111
|
+
if (queryAt < 0 || queryAt > end) return result.slice(0, end) + marked;
|
|
112
|
+
return result.slice(0, queryAt + 1) + mark(result.slice(queryAt + 1, end)) + marked;
|
|
113
|
+
}
|
|
114
|
+
/** The `[name, value]` pairs of a list holding only such pairs (Sentry's header tuples). */
|
|
115
|
+
const tuples = list => list.length > 0 && list.every(item => Array.isArray(item) && item.length === 2 && typeof item[0] === 'string');
|
|
116
|
+
/** A copy of a parsed vendor response without credential values, made before any
|
|
117
|
+
* byte is emitted for upload: a credential-named field's value, the `value` of a
|
|
118
|
+
* `{name|key, value}` pair naming one (a header pair), or the value of a
|
|
119
|
+
* credential-named `[name, value]` pair in a list of them, becomes
|
|
120
|
+
* REMOVED_CREDENTIAL, so the record stays and the seed can say what was removed.
|
|
121
|
+
* URLs lose credentials as withoutUrlCredentials describes; `pagination` lists
|
|
122
|
+
* the adapter's validated query keys, which stay. */
|
|
123
|
+
export function withoutCredentials(value, pagination = []) {
|
|
124
|
+
// `headers`: the keys of this object, or of objects in this list, are HTTP header names.
|
|
125
|
+
// `inside`: this object, or the objects in this list, sit under `auth` or `credential(s)`.
|
|
126
|
+
const strip = (value, headers, inside = false) => {
|
|
127
|
+
if (Array.isArray(value)) {
|
|
128
|
+
const pairs = tuples(value);
|
|
129
|
+
return value.map(child => pairs && credential(child[0], child[1], 'header') ? [child[0], REMOVED_CREDENTIAL] : strip(child, headers, inside));
|
|
130
|
+
}
|
|
131
|
+
if (value && typeof value === 'object') {
|
|
132
|
+
// A `{name|key, value}` pair's name is a header name only under a header container; elsewhere it labels a field.
|
|
133
|
+
const place = headers ? 'header' : 'field';
|
|
134
|
+
const label = Object.entries(value).find(([key, child]) => /^(?:name|key)$/iu.test(key) && typeof child === 'string');
|
|
135
|
+
const pair = !!label && Object.hasOwn(value, 'value') && credentialName(decoded(label[1]), 'remove', place) !== undefined;
|
|
136
|
+
return Object.fromEntries(Object.entries(value).map(([key, child]) => [key, (pair && key === 'value' ? credential(label[1], child, place) : credential(key, child, place) || (inside && member(key, child)))
|
|
137
|
+
? REMOVED_CREDENTIAL : strip(child, headerContainer(key), container(key))]));
|
|
138
|
+
}
|
|
139
|
+
return typeof value === 'string' ? withoutUrlCredentials(value, pagination) : value;
|
|
140
|
+
};
|
|
141
|
+
return strip(value, false);
|
|
142
|
+
}
|
|
49
143
|
export function assertCount(value, max = 1000) {
|
|
50
144
|
if (!Number.isSafeInteger(value) || value < 0 || value > max) fail('INVALID_CHECKPOINT');
|
|
51
145
|
return value;
|
|
@@ -0,0 +1,124 @@
|
|
|
1
|
+
// A JavaScript copy of modules/import-kit/src/provenance/credential-names.ts, which the
|
|
2
|
+
// seed's import door applies; keep the two the same (rer's vendor exports use `remove` and `query`).
|
|
3
|
+
/** Whether a name (an object key, a header name, the name in a `[name, value]`
|
|
4
|
+
* pair, a URL query or fragment key) names a credential, by its whole parts.
|
|
5
|
+
*
|
|
6
|
+
* A name is split into lower-case parts at camelCase and PascalCase boundaries
|
|
7
|
+
* (`apiKey`, `XAPIKey`), at `_`, `-`, `.`, spaces and any other non-letter, and
|
|
8
|
+
* at digits (`token2` is `token`). A name is a credential when its parts are a
|
|
9
|
+
* credential word or phrase, never because a part only contains one: `keyword`,
|
|
10
|
+
* `monkey`, `author`, `signal`, `design` and `tokenizer` hold the letters of a
|
|
11
|
+
* credential word and are not credentials. */
|
|
12
|
+
// credentialName returns the name's letters, lower-cased (`apiKey` and `api_key` are both
|
|
13
|
+
// `apikey`), and whether it is the word's plural (`tokens`, `apiKeys`: it holds credentials
|
|
14
|
+
// only as text, so a count stays), or undefined. Modes: `refuse`: the whole name is a
|
|
15
|
+
// credential phrase, so a pagination cursor such as `next_page_token` is not one. `remove`:
|
|
16
|
+
// the name ends in a credential word (`pushToken`, `webhook_url`), the wider net of the
|
|
17
|
+
// import door, which drops the value and keeps the record. `query`: a URL query or fragment
|
|
18
|
+
// key, as `remove` plus a bare `key` and `X-Amz-Signature`. It also says whether the name is
|
|
19
|
+
// a container (`auth`, `X-Auth`, `credential(s)`): its value is a credential when it is text
|
|
20
|
+
// that names no scheme or mode (`include`, `oauth2`), or a list with such a member; an object
|
|
21
|
+
// under it is searched field by field. Places, in `remove` mode: a record `field`, or an HTTP
|
|
22
|
+
// `header` name (a `[name, value]` tuple, a key of a `headers` object, a `{name|key, value}`
|
|
23
|
+
// pair in a header list). `sig`, `signature` and `sessionId` are credentials only as header names and URL
|
|
24
|
+
// query or fragment keys, never as record fields, so an agent's `signature` or a chat's
|
|
25
|
+
// `session_id` imports.
|
|
26
|
+
const WORDS = new Set(['token', 'secret', 'password', 'passwd', 'pwd', 'cookie', 'authorization', 'bearer', 'credential']);
|
|
27
|
+
/** Words that make a trailing `key` a credential: `api key`, `secret key`, `Ocp-Apim-Subscription-Key`. A bare `key` or `primaryKey` is not. */
|
|
28
|
+
const KEY_QUALIFIERS = new Set(['api', 'access', 'private', 'secret', 'auth', 'client', 'integration', 'routing', 'service', 'signing', 'session',
|
|
29
|
+
'refresh', 'subscription', 'functions', 'consumer', 'app', 'license', 'master', 'encryption', 'admin']);
|
|
30
|
+
/** URL query and fragment keys only: `?application_key=` (Datadog), `?user_key=` (3scale), `?account_key=`, `?deploy_key=`, `?shared_key=`, and `?token_auth=` (Matomo), `?basic_auth=`. */
|
|
31
|
+
const QUERY_KEY_QUALIFIERS = new Set(['application', 'user', 'account', 'deploy', 'shared']);
|
|
32
|
+
const QUERY_AUTH_QUALIFIERS = new Set(['token', 'basic', 'http', 'proxy', 'api']);
|
|
33
|
+
const TOKEN_QUALIFIERS = ['access', 'refresh', 'auth', 'api', 'bearer', 'id', 'session', 'csrf', 'xsrf', 'bot', 'push', 'oauth', 'jwt'];
|
|
34
|
+
const SECRET_QUALIFIERS = ['client', 'app', 'api', 'oauth', 'webhook', 'signing', 'shared'];
|
|
35
|
+
const PASSWORD_QUALIFIERS = ['http', 'db', 'ftp', 'smtp', 'admin', 'user', 'basic', 'proxy', 'root', 'mail', 'login'];
|
|
36
|
+
/** One-word credentials however a name splits them: a `passphrase` (`passPhrase`), a `jwt`, and Java's and PHP's session
|
|
37
|
+
* cookies (`JSESSIONID`, `jsessionId`, `PHPSESSID`). They name nothing else, so they count as fields, header names and URL keys. */
|
|
38
|
+
const SPELLED = /^(?:passphrase|jwt|jsessionid|phpsessid)(s?)$/;
|
|
39
|
+
/** What may stand before one in `refuse` mode (`ssh_passphrase`, `id_jwt`). */
|
|
40
|
+
const SPELLED_QUALIFIERS = new Set([...PASSWORD_QUALIFIERS, ...TOKEN_QUALIFIERS, 'ssh', 'key', 'gpg', 'pgp', 'private']);
|
|
41
|
+
/** What may stand before a credential word in `refuse` mode, which names the whole phrase (`access token`, `http password`). */
|
|
42
|
+
const QUALIFIERS = {
|
|
43
|
+
token: new Set(TOKEN_QUALIFIERS), secret: new Set(SECRET_QUALIFIERS),
|
|
44
|
+
password: new Set(PASSWORD_QUALIFIERS), passwd: new Set(PASSWORD_QUALIFIERS), pwd: new Set(PASSWORD_QUALIFIERS),
|
|
45
|
+
key: KEY_QUALIFIERS,
|
|
46
|
+
};
|
|
47
|
+
/** The same phrases written as one lower-case part (`apikey`, `accesstoken`, `clientsecret`, `httppassword`, `webhookurl`). */
|
|
48
|
+
const COMPOUND = new RegExp('^x?(?:'
|
|
49
|
+
+ `(?:${[...KEY_QUALIFIERS].join('|')})(?:key)|(?:${TOKEN_QUALIFIERS.join('|')})(?:token)|(?:${SECRET_QUALIFIERS.join('|')})(?:secret)`
|
|
50
|
+
+ `|(?:${PASSWORD_QUALIFIERS.join('|')})(?:password|passwd|pwd)|webhook(?:url)|cal(?:url))(s?)$`);
|
|
51
|
+
/** A scheme or mode an `auth` or `credentials` field names, never a credential:
|
|
52
|
+
* fetch's `include`, `same-origin` and `omit`, HTTP and API authentication schemes, and boolean
|
|
53
|
+
* words (Keycloak's `smtpServer.auth: "true"`). */
|
|
54
|
+
const SCHEME = /^(?:omit|sameorigin|include|none|noauth|inherit|anonymous|basic|bearer|digest|negotiate|ntlm|kerberos|oauth(?:1|2|10|20|1a)?|apikey|jwt|hawk|hmac|aws|awsv4|sigv4|awssigv4|edgegrid|mutual|vapid|token|password|cookie|session|header|query|true|false|yes|no|on|off|enabled|disabled)$/;
|
|
55
|
+
|
|
56
|
+
/** Whether text under an `auth` or `credentials` name names a scheme or mode (`include`, `oauth2`, `Bearer`, `same-origin`). */
|
|
57
|
+
export function credentialScheme(text) {
|
|
58
|
+
return SCHEME.test(text.trim().toLowerCase().replace(/[\s_.-]/g, ''));
|
|
59
|
+
}
|
|
60
|
+
/** Whether an object under this key holds HTTP headers by name (`headers`, `customHttpHeaders`, `header`). */
|
|
61
|
+
export function headerContainer(key) {
|
|
62
|
+
const last = nameParts(key).at(-1);
|
|
63
|
+
return last === 'headers' || last === 'header';
|
|
64
|
+
}
|
|
65
|
+
// Under `auth`, `X-Auth`, `credential` or `credentials`, a member holding the credential under a generic name: `value`
|
|
66
|
+
// (Keycloak's `{type: 'password', value}`, `credential: {type: 'apiKey', value}`), `pass` and `pw` (nodemailer's
|
|
67
|
+
// `{user, pass}`), or a name whose first part is `secret` (Keycloak's `secretData`). Its name's letters, or undefined.
|
|
68
|
+
export function credentialMember(name) {
|
|
69
|
+
const parts = nameParts(name);
|
|
70
|
+
return (parts.length === 1 && /^(?:value|pass|pw)$/.test(parts[0])) || parts[0] === 'secret' ? parts.join('') : undefined;
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
/** A name's lower-case parts, without digits. Format characters and accents are folded away first. */
|
|
74
|
+
export function nameParts(name) {
|
|
75
|
+
const text = name.normalize('NFKC').replace(/[\p{Cf}\p{M}]/gu, '');
|
|
76
|
+
return (text.match(/\p{Lu}+(?!\p{Ll})|\p{Lu}?\p{Ll}+|\p{Lo}+/gu) ?? []).map(part => part.toLowerCase());
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
export function credentialName(name, mode, place = 'field') {
|
|
80
|
+
const parts = nameParts(name), last = parts.at(-1), before = parts.at(-2);
|
|
81
|
+
if (last === undefined) return undefined;
|
|
82
|
+
const letters = name.normalize('NFKC').toLowerCase().replace(/[^a-z]/g, '').slice(-64);
|
|
83
|
+
const compound = COMPOUND.exec(last);
|
|
84
|
+
if (compound) return { name: letters, plural: compound[1] === 's', container: false };
|
|
85
|
+
// A header's leading `x-` (`x-api-key`, `X-Auth-Token`) is not part of the phrase; `refuse` mode takes the rest whole.
|
|
86
|
+
const phrase = parts.length > 1 && parts[0] === 'x' ? parts.slice(1) : parts;
|
|
87
|
+
const found = (plural, container = false) => ({ name: letters, plural, container });
|
|
88
|
+
const qualifier = word => word !== undefined && (KEY_QUALIFIERS.has(word) || (word.startsWith('x') && KEY_QUALIFIERS.has(word.slice(1))));
|
|
89
|
+
const strict = mode === 'refuse';
|
|
90
|
+
// A URL key or a header name: where `sig`, `signature` and `sessionId` are credentials.
|
|
91
|
+
const header = mode === 'query' || place === 'header';
|
|
92
|
+
for (let size = 1; size <= Math.min(3, phrase.length); size++) {
|
|
93
|
+
const spelled = SPELLED.exec(phrase.slice(-size).join(''));
|
|
94
|
+
if (spelled && (!strict || phrase.length === size || (phrase.length === size + 1 && SPELLED_QUALIFIERS.has(phrase[0])))) return found(spelled[1] === 's');
|
|
95
|
+
}
|
|
96
|
+
// `credentials` holds a credential object as one value; the other plurals hold a list or map of text.
|
|
97
|
+
const word = last === 'credentials' ? 'credential' : WORDS.has(last) ? last : last.endsWith('s') && WORDS.has(last.slice(0, -1)) ? last.slice(0, -1) : undefined;
|
|
98
|
+
if (word !== undefined) {
|
|
99
|
+
const plural = word !== last && last !== 'credentials';
|
|
100
|
+
// `refuse` mode: the word alone or after one qualifier (`token`, `access_token`), and `pwd` after anything.
|
|
101
|
+
const named = phrase.length === 1 || (phrase.length === 2 && QUALIFIERS[word]?.has(phrase[0]) === true) || word === 'pwd';
|
|
102
|
+
if (!strict || named) return found(plural, word === 'credential' && mode !== 'query');
|
|
103
|
+
}
|
|
104
|
+
if (last === 'key' || last === 'keys') {
|
|
105
|
+
// `refuse` mode: `api_key`, `x-functions-key`, and `secret`/`subscription`/`functions` before the key under any name.
|
|
106
|
+
const named = phrase.length === 2 || (before !== undefined && ['secret', 'subscription', 'functions'].includes(before));
|
|
107
|
+
if (qualifier(before) && (!strict || named)) return found(last === 'keys');
|
|
108
|
+
// A bare `key` is a credential only as a URL query or fragment key (the `?key=` convention).
|
|
109
|
+
if (mode === 'query' && parts.length === 1 && last === 'key') return found(false);
|
|
110
|
+
if (mode === 'query' && before !== undefined && QUERY_KEY_QUALIFIERS.has(before)) return found(last === 'keys');
|
|
111
|
+
}
|
|
112
|
+
// A session id is a credential as a header or URL key (`Session-Id`, `?sessionid=`), never as a record's field (a chat's `session_id`).
|
|
113
|
+
const session = /^x?sessionid(s?)$/.exec(last);
|
|
114
|
+
if (header && session) return found(session[1] === 's');
|
|
115
|
+
if (header && last === 'id' && before === 'session' && (!strict || phrase.length === 2)) return found(false);
|
|
116
|
+
if (!strict && last === 'url' && (before === 'webhook' || before === 'cal')) return found(false);
|
|
117
|
+
// A signature is a credential as a whole header name (`Signature`) and, as a query key, as its last part (an Azure SAS `sig`, `X-Amz-Signature`); an agent's `signature` field is not.
|
|
118
|
+
if ((last === 'sig' || last === 'signature') && (mode === 'query' || (place === 'header' && parts.length === 1))) return found(false);
|
|
119
|
+
if (last === 'auth' && (parts.length === 1 || (parts.length === 2 && before === 'x'))) return found(false, mode !== 'query');
|
|
120
|
+
if (mode === 'query' && last === 'auth' && before !== undefined && QUERY_AUTH_QUALIFIERS.has(before)) return found(false);
|
|
121
|
+
// A query key written as one part that ends in an unambiguous phrase: HubSpot's `?hapikey=`.
|
|
122
|
+
if (mode === 'query' && parts.length === 1 && /(?:apikey|accesstoken|authtoken)$/.test(last)) return found(false);
|
|
123
|
+
return undefined;
|
|
124
|
+
}
|
|
@@ -1,15 +1,7 @@
|
|
|
1
|
-
import { object, list, id, windowInputs, assertCount, shape, fail } from './common.mjs';
|
|
1
|
+
import { object, list, id, windowInputs, assertCount, shape, fail, withoutCredentials } from './common.mjs';
|
|
2
2
|
|
|
3
3
|
export const origin = 'https://api.pagerduty.com';
|
|
4
4
|
export const route = url => /^\/(?:users|escalation_policies|schedules(?:\/[a-zA-Z0-9_-]+(?:\/overrides)?)?)$/u.test(url.pathname) && [...url.searchParams.keys()].every(x => ['limit','offset','since','until','overflow'].includes(x));
|
|
5
|
-
const credential = /^(?:password|passwd|secret|token|credentials?|authorization|cookie)$|(?:api|access|auth|integration|routing|service|signing|private|client|refresh|session)[_-]?(?:key|secret|token)$|^webhook[_-]?(?:url|secret)$|(?:http|web)[_-]?cal[_-]?url$/iu;
|
|
6
|
-
function noCredentials(value) {
|
|
7
|
-
if (Array.isArray(value)) value.forEach(noCredentials);
|
|
8
|
-
else if (value && typeof value === 'object') for (const [key, child] of Object.entries(value)) {
|
|
9
|
-
if (credential.test(key)) fail('CREDENTIAL_IN_RESPONSE');
|
|
10
|
-
noCredentials(child);
|
|
11
|
-
}
|
|
12
|
-
}
|
|
13
5
|
function advance(body, state) {
|
|
14
6
|
if (typeof body.more !== 'boolean') fail();
|
|
15
7
|
if (body.more) {
|
|
@@ -40,8 +32,7 @@ export async function* run(ctx) {
|
|
|
40
32
|
if (s.schedules.length > 500) fail('RECORD_LIMIT');
|
|
41
33
|
bytes = '';
|
|
42
34
|
} else {
|
|
43
|
-
|
|
44
|
-
bytes = (s.first ? (s.phase === 0 ? '{"users":[' : '],"escalation_policies":[') : ',') + JSON.stringify(body);
|
|
35
|
+
bytes = (s.first ? (s.phase === 0 ? '{"users":[' : '],"escalation_policies":[') : ',') + JSON.stringify(withoutCredentials(body));
|
|
45
36
|
s.first = false;
|
|
46
37
|
}
|
|
47
38
|
if (!advance(body, s)) { s.phase++; s.first = true; }
|
|
@@ -49,8 +40,8 @@ export async function* run(ctx) {
|
|
|
49
40
|
if (s.index === s.schedules.length) { s.phase = 3; s.index = 0; s.first = true; bytes = s.schedules.length ? '' : '],"schedules":['; }
|
|
50
41
|
else {
|
|
51
42
|
const { body } = await ctx.get(`/schedules/${id(s.schedules[s.index])}?${window}`);
|
|
52
|
-
object(body); object(body.schedule);
|
|
53
|
-
bytes = (s.first ? '],"schedules":[' : ',') + JSON.stringify(body);
|
|
43
|
+
object(body); object(body.schedule); ctx.records(1);
|
|
44
|
+
bytes = (s.first ? '],"schedules":[' : ',') + JSON.stringify(withoutCredentials(body));
|
|
54
45
|
s.first = false; s.index++;
|
|
55
46
|
}
|
|
56
47
|
} else {
|
|
@@ -58,8 +49,8 @@ export async function* run(ctx) {
|
|
|
58
49
|
else {
|
|
59
50
|
const schedule = id(s.schedules[s.index]);
|
|
60
51
|
const { body } = await ctx.get(`/schedules/${schedule}/overrides?${window}&overflow=true`);
|
|
61
|
-
object(body); ctx.records(list(body.overrides).length);
|
|
62
|
-
bytes = (s.first ? '],"overrides":[' : ',') + JSON.stringify({ ...body, schedule_id: schedule });
|
|
52
|
+
object(body); ctx.records(list(body.overrides).length);
|
|
53
|
+
bytes = (s.first ? '],"overrides":[' : ',') + JSON.stringify(withoutCredentials({ ...body, schedule_id: schedule }));
|
|
63
54
|
s.first = false;
|
|
64
55
|
s.index++;
|
|
65
56
|
}
|
|
@@ -1,7 +1,8 @@
|
|
|
1
|
-
import { object, list, id, windowInputs, cursor, assertCount, shape, fail } from './common.mjs';
|
|
1
|
+
import { object, list, id, windowInputs, cursor, assertCount, shape, fail, withoutCredentials } from './common.mjs';
|
|
2
2
|
|
|
3
3
|
export const origins = Object.freeze({ us: 'https://us.sentry.io', de: 'https://de.sentry.io' });
|
|
4
|
-
|
|
4
|
+
const queries = ['per_page','limit','cursor','query','project','start','end','sort'];
|
|
5
|
+
export const route = url => /^\/api\/0\/(?:organizations\/[a-zA-Z0-9_-]+\/(?:projects|releases|issues)|projects\/[a-zA-Z0-9_-]+\/[a-zA-Z0-9_-]+\/events\/[a-fA-F0-9]{32})\/$/u.test(url.pathname) && [...url.searchParams.keys()].every(x => queries.includes(x));
|
|
5
6
|
function nextCursor(headers) {
|
|
6
7
|
const link = headers.get('link');
|
|
7
8
|
if (!link) return null;
|
|
@@ -73,7 +74,7 @@ export async function* run(ctx) {
|
|
|
73
74
|
if (next !== null && (next === s.cursor || s.seen.includes(next))) fail('PAGINATION_CYCLE');
|
|
74
75
|
if (++s.page > 100) fail('PAGE_LIMIT');
|
|
75
76
|
let bytes = s.cursor === null ? (s.collections ? ',' : '') + JSON.stringify({kind,project}).slice(0,-1) + ',"pages":[' : ',';
|
|
76
|
-
bytes += JSON.stringify({ cursor: s.cursor, nextCursor: next, response });
|
|
77
|
+
bytes += JSON.stringify({ cursor: s.cursor, nextCursor: next, response: withoutCredentials(response, queries) });
|
|
77
78
|
s.cursor = next;
|
|
78
79
|
if (next === null) {
|
|
79
80
|
bytes += ']}'; s.collections++; s.seen = []; s.page = 0; s.event = 0;
|
|
@@ -1,20 +1,21 @@
|
|
|
1
|
-
import { object, list, reference, shape, fail } from './common.mjs';
|
|
1
|
+
import { object, list, reference, shape, fail, withoutCredentials } from './common.mjs';
|
|
2
2
|
|
|
3
3
|
export const origin = 'https://api.uptimerobot.com';
|
|
4
|
-
|
|
4
|
+
const pagination = ['limit','offset','cursor','page','page_token'];
|
|
5
|
+
export const route = url => url.pathname === '/v3/monitors' && [...url.searchParams.keys()].every(x => pagination.includes(x));
|
|
5
6
|
export async function* run(ctx) {
|
|
6
7
|
const state = ctx.state ?? { next: '/v3/monitors?limit=200', seen: [] };
|
|
7
8
|
shape(state, ['next','seen']);
|
|
8
9
|
if (!Array.isArray(state.seen) || state.seen.length > 600) fail('INVALID_CHECKPOINT');
|
|
9
|
-
state.seen.forEach(value => reference(value, origin, /^\/v3\/monitors$/u,
|
|
10
|
+
state.seen.forEach(value => reference(value, origin, /^\/v3\/monitors$/u, pagination));
|
|
10
11
|
while (state.next !== null) {
|
|
11
|
-
const url = reference(state.next, origin, /^\/v3\/monitors$/u,
|
|
12
|
+
const url = reference(state.next, origin, /^\/v3\/monitors$/u, pagination);
|
|
12
13
|
if (state.seen.includes(url.href)) fail('PAGINATION_CYCLE');
|
|
13
14
|
const { body } = await ctx.get(url.href);
|
|
14
15
|
object(body); ctx.records(list(body.data).length);
|
|
15
16
|
if (!Object.hasOwn(body, 'nextLink')) fail();
|
|
16
17
|
state.seen.push(url.href);
|
|
17
|
-
state.next = body.nextLink == null ? null : reference(body.nextLink, origin, /^\/v3\/monitors$/u,
|
|
18
|
-
yield ctx.item(JSON.stringify(body) + '\n', state);
|
|
18
|
+
state.next = body.nextLink == null ? null : reference(body.nextLink, origin, /^\/v3\/monitors$/u, pagination).href;
|
|
19
|
+
yield ctx.item(JSON.stringify(withoutCredentials(body, pagination)) + '\n', state);
|
|
19
20
|
}
|
|
20
21
|
}
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
// Only declared safe replacement values and aggregate counts reach the terminal.
|
|
2
|
+
export function importSafetySummary({safetyChanges}={}) {
|
|
3
|
+
if(!safetyChanges?.length)return undefined;
|
|
4
|
+
return safetyChanges.map(({table,column,value,rows})=>
|
|
5
|
+
`${rows.toLocaleString('en-US')} ${table} imported with ${column}=${JSON.stringify(value)}`
|
|
6
|
+
).join('\n');
|
|
7
|
+
}
|
package/cli/output.mjs
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { handleBrokenPipes } from './stdio.mjs';
|
|
2
2
|
import { exportSummary } from './export-summary.mjs';
|
|
3
|
+
import { importSafetySummary } from './import-summary.mjs';
|
|
3
4
|
export function output(options) {
|
|
4
5
|
handleBrokenPipes();
|
|
5
6
|
let finished = false;
|
|
@@ -19,6 +20,7 @@ export function output(options) {
|
|
|
19
20
|
if (structuredResult) this.event('result', { ok, exitCode, ...details });
|
|
20
21
|
// A finished export ends with one line naming its archive; the receipt is --json data.
|
|
21
22
|
else if (ok && options.verb === 'export' && exportSummary(details)) this.human(exportSummary(details));
|
|
23
|
+
else if (ok && options.verb === 'import' && importSafetySummary(details)) this.human('import: complete (exit 0)\n' + importSafetySummary(details));
|
|
22
24
|
else if (!['describe', 'check', 'dev', 'preview', 'setup'].includes(options.verb) || !ok) this.human(`${options.verb}: ${ok ? 'complete' : 'failed'} (exit ${exitCode})`);
|
|
23
25
|
},
|
|
24
26
|
error(error) {
|
|
@@ -6,6 +6,7 @@ import { legacyTargetState, requireLegacyEmpty } from '../modules/portability/le
|
|
|
6
6
|
import { migrationReplay } from '../modules/portability/migration-history.mjs';
|
|
7
7
|
import { appliedMigrations as appliedMigrationNames } from '../modules/portability/bridge.mjs';
|
|
8
8
|
import { join } from 'node:path';
|
|
9
|
+
import { createHash } from 'node:crypto';
|
|
9
10
|
import { mkdir, readFile, rm, stat, writeFile } from 'node:fs/promises';
|
|
10
11
|
import { createRequire } from 'node:module';
|
|
11
12
|
import { createPortability, PortabilityError, readArchive } from '../modules/portability/index.mjs';
|
|
@@ -14,13 +15,16 @@ import { nativeFailure, stateBytes } from './portability-progress.mjs';
|
|
|
14
15
|
import { parseJsonc } from './jsonc.mjs';
|
|
15
16
|
import { loadPolicy } from './portability-policy.mjs';
|
|
16
17
|
|
|
17
|
-
// The migrations this installation applied, in its ledger's order,
|
|
18
|
-
// restore can place an older archive within them.
|
|
18
|
+
// The migrations this installation applied, in its ledger's order, with each
|
|
19
|
+
// file's hash, so a restore can place an older archive within them.
|
|
19
20
|
async function appliedMigrations(root, db, binding, split) {
|
|
20
21
|
const config = parseJsonc(await readFile(join(root, 'wrangler.jsonc'), 'utf8'));
|
|
21
22
|
const directory = join(root, config.d1_databases.find(entry => entry.binding === binding).migrations_dir ?? 'migrations');
|
|
22
23
|
const names = await appliedMigrationNames(db) ?? [];
|
|
23
|
-
return Promise.all(names.map(async name =>
|
|
24
|
+
return Promise.all(names.map(async name => {
|
|
25
|
+
const bytes = await readFile(join(directory, name));
|
|
26
|
+
return { name, statements: split(bytes.toString('utf8')), sha256: createHash('sha256').update(bytes).digest('hex') };
|
|
27
|
+
}));
|
|
24
28
|
}
|
|
25
29
|
|
|
26
30
|
// Transfers read only the D1 database and R2 buckets. A configuration with just
|
|
@@ -81,8 +85,10 @@ catch (error) {
|
|
|
81
85
|
const migration = error instanceof PortabilityError && error.code.startsWith('schema_mismatch:migration_data:') ? error.code.slice('schema_mismatch:migration_data:'.length) : undefined;
|
|
82
86
|
// A populated destination names its table or bucket: schema names, never row data.
|
|
83
87
|
const occupied = error instanceof PortabilityError && /^(target|bucket)_not_empty:/.test(error.code) ? error.code.slice(error.code.indexOf(':') + 1) : undefined;
|
|
88
|
+
// Safety diagnostics contain only policy schema names, never source values.
|
|
89
|
+
const safety = error instanceof PortabilityError && error.code.startsWith('import_row_') ? error.code.slice(error.code.indexOf(':')+1) : undefined;
|
|
84
90
|
// Anything that is not a portability refusal names its class and code, never its message: that can hold row data.
|
|
85
91
|
const native = error instanceof PortabilityError || error.message === 'archive_json' ? undefined : nativeFailure(error);
|
|
86
92
|
process.exitCode = exitCode;
|
|
87
|
-
await reply({ ok: false, code: exitCode === 2 ? 'invalid_archive' : exitCode === 4 ? 'restore_refused' : 'transfer_failed', reason, ...(migration ? { migration } : {}), ...(occupied ? { occupied } : {}), ...(native ? { native } : {}), exitCode });
|
|
93
|
+
await reply({ ok: false, code: exitCode === 2 ? 'invalid_archive' : exitCode === 4 ? 'restore_refused' : 'transfer_failed', reason, ...(migration ? { migration } : {}), ...(occupied ? { occupied } : {}), ...(safety ? { safety } : {}), ...(native ? { native } : {}), exitCode });
|
|
88
94
|
}
|
package/cli/portability.mjs
CHANGED
|
@@ -82,6 +82,7 @@ async function nativeWorker(store,direction,input,signal) {
|
|
|
82
82
|
if (!receipt || typeof receipt !== 'object') throw new RerError('transfer_failed', `Local portability ended before it finished (${status.signal ? `signal ${status.signal}` : `exit code ${status.code}`}). Run the same command again; it starts over${direction === 'import' ? ' and first cleans up anything this attempt staged' : ''}.`, 1);
|
|
83
83
|
if (status.code !== 0 || !receipt.ok) throw new RerError(receipt.code ?? 'transfer_failed', receipt.reason==='legacy_restored_export_failed'
|
|
84
84
|
? 'Legacy data was restored and audited, but its v2 export failed. Preserve this installation, resolve the output problem, then run rer export --output FILE. Do not reset it to retry recovery.'
|
|
85
|
+
: receipt.safety ? `Restore refused (${receipt.reason}): safety normalization for ${receipt.safety} violates the declared import policy.`
|
|
85
86
|
: receipt.migration ? `Restore refused (schema_mismatch:migration_data): migration ${receipt.migration} changed data after this archive was made, so its rows would skip that change. Restore an archive made after it.`
|
|
86
87
|
: receipt.occupied ? `Restore refused (${receipt.reason}): ${receipt.reason === 'bucket_not_empty' ? `storage bucket ${receipt.occupied} in this installation already holds files` : `table ${receipt.occupied} in this installation already holds rows its migrations did not insert`}, and a restore only goes into an empty installation. To replace this installation's data with the archive, keep a copy with rer export --output exports/before-restore.rer, then run rer reset --yes and rer import --input ${relative(input.root, input.path)} --yes again.`
|
|
87
88
|
: `The portability module refused or failed the archive operation (${receipt.reason ?? 'incomplete'}${receipt.native ? `: ${receipt.native.class}${receipt.native.code && receipt.native.code !== receipt.reason ? `, code ${receipt.native.code}` : ''}` : ''}).${receipt.native && receipt.exitCode === 1 ? ' Run the same command again; it starts over.' : ''}`, receipt.exitCode ?? 1);
|
package/cli/production-mail.mjs
CHANGED
|
@@ -49,11 +49,14 @@ export async function productionMail(root, declaration, env) {
|
|
|
49
49
|
const routes = selected.map(provider => {
|
|
50
50
|
const credentialKeys = deferred && nativeCredential[provider] && !declaredProviders.has(provider) ? [nativeCredential[provider]]
|
|
51
51
|
: ['MAIL_API_KEY', ...(nativeCredential[provider] ? [nativeCredential[provider]] : [])];
|
|
52
|
+
// A seed's direct-provider adapter can declare the contract's shared provider
|
|
53
|
+
// binding. Its explicit environment mapping (or live declared secret) still wins.
|
|
54
|
+
if (nativeCredential[provider] && secrets.has('MAIL_PROVIDER_API_KEY')) credentialKeys.push('MAIL_PROVIDER_API_KEY');
|
|
52
55
|
const declaredCredentials = credentialKeys.filter(key => secrets.has(key));
|
|
53
56
|
// Only resend and sendgrid have a provider key of their own; cloudflare and rer send with MAIL_API_KEY alone.
|
|
54
57
|
const providerKeys = (nativeCredential[provider] ? [nativeCredential[provider]] : []).filter(key => !declaredCredentials.includes(key));
|
|
55
58
|
const credentialText = declaredCredentials.includes('MAIL_API_KEY') && providerKeys.length
|
|
56
|
-
?
|
|
59
|
+
? `${declaredCredentials.join(' or ')} (or ${providerKeys.join(' / ')})`
|
|
57
60
|
: declaredCredentials.join(' or ') || credentialKeys.join(' or ');
|
|
58
61
|
return live => {
|
|
59
62
|
let relaySelected = true;
|
package/global/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,19 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## 0.2.68
|
|
4
|
+
|
|
5
|
+
- **Retained-archive restores (#316, #318).** Placing an older archive treats a CREATE TRIGGER or DROP TRIGGER statement bounded as exactly one trigger as schema-only, and restore drops and recreates the destination's triggers around insertion; a later write that reaches an archived table through a trigger is still refused. Base reads RER Platform's rer_migrations ledger, places an archive without a migration record by the ledger it carries (every archived name and hash must match an applied migration), and admits a later UPDATE or DELETE of a table the archive holds no rows of.
|
|
6
|
+
- **Release scans committed bytes only (#317).** rer release scans the raw committed source in an isolated mirror, so ignored local files such as .dev.vars and dist no longer block a release; a committed credential still does.
|
|
7
|
+
- **Import credential names (#311).** The import door matches credential names by whole name parts (keyword, author and primaryKey are not credentials), catches query-key credentials such as application_key, hapikey and token_auth, and treats signatures and session ids as credentials only as URL keys and header names.
|
|
8
|
+
- **Lobby recovery help on this computer only (#319).** The claimed Lobby shows the project-folder `rer signin` line only on loopback hosts, so hosted owners no longer see it.
|
|
9
|
+
- **Import safety normalization (#322).** A seed's importSafety/importRow hook bounds staged rows to declared safe values (for example connections forced inactive with their tokens cleared), checks determinism and idempotence, and journals per-table, column and value counts into status, receipts and owner CLI output.
|
|
10
|
+
|
|
11
|
+
## 0.2.67
|
|
12
|
+
|
|
13
|
+
- **Import door options (#298 with #308).** Seeds choose credential removal modes and a preserve or strict reversal policy when an import is created. Failed applies can be retried, CLI-stripped query and fragment credentials receive per-record notes, more credential shapes are recognized, and a blocked re-baseline is refused at plan time with a named 409.
|
|
14
|
+
- **Release from partial clones (#313).** Hydrate missing history blobs before release reads and tolerate temporary-directory cleanup failures after a saved result.
|
|
15
|
+
- **Mail credentials and large migration histories (#314).** Deploy honors a declared MAIL_PROVIDER_API_KEY for SendGrid and Resend, and scales the bounded provider request budget to pending migrations.
|
|
16
|
+
|
|
3
17
|
## 0.2.66
|
|
4
18
|
|
|
5
19
|
- **Seed copies pass the pin-time credential scan (#305).** Credential-shaped test fixtures in seed-vendored Base files are constructed at runtime, preserving their values and assertions. Seed copies and exported references no longer trigger the hosted pin-time credential scan, so seeds can refresh Base past 0.2.61 and pin again.
|
|
@@ -2,6 +2,8 @@ import { createHash } from 'node:crypto';
|
|
|
2
2
|
import { assert, uuid } from './validation.mjs';
|
|
3
3
|
import { recordDeployFailure } from './diagnostics.mjs';
|
|
4
4
|
|
|
5
|
+
export const D1_IMPORT_POLL_LIMIT = 100;
|
|
6
|
+
|
|
5
7
|
// Wrangler's remote --file protocol. D1 imports execute the whole file atomically,
|
|
6
8
|
// including trigger bodies; the query endpoint's semicolon splitting cannot.
|
|
7
9
|
// Cloudflare's own reason for an import that failed or did not finish, kept on the
|
|
@@ -31,7 +33,7 @@ export async function importD1({ envelope, upload, redact = String }, id, sql) {
|
|
|
31
33
|
await upload(response.upload_url, bytes, etag);
|
|
32
34
|
response = await post({ action:'ingest', filename:response.filename, etag });
|
|
33
35
|
}
|
|
34
|
-
for (let polls = 0; polls <
|
|
36
|
+
for (let polls = 0; polls < D1_IMPORT_POLL_LIMIT; polls++) {
|
|
35
37
|
// REST places errors on the envelope; validate nested errors when supplied.
|
|
36
38
|
if (!(response?.success === true && response.type === 'import' && response.status !== 'error' && (!Object.hasOwn(response, 'errors') || Array.isArray(response.errors) && !response.errors.length))) throw importFailure('d1-import-failed', id, response, redact);
|
|
37
39
|
if (response.status === 'complete') {
|
|
@@ -58,6 +58,7 @@ export async function migrate(api, id, migrations, { apply = true, upgrade = app
|
|
|
58
58
|
validateMigrations(migrations);
|
|
59
59
|
const { applied, pending, transition, exists, claimed, claimedFrom } = await readLedger(api, id, migrations, query, { rewritten:locked });
|
|
60
60
|
assert(apply || pending.length === 0,'pending-migrations');
|
|
61
|
+
if (apply && pending.length) api.reserveMigrationRequests?.(id, pending.length);
|
|
61
62
|
if (transition && upgrade) {
|
|
62
63
|
await api.importSql(id, transition);
|
|
63
64
|
const verified = await readLedger(api, id, migrations, query);
|
|
@@ -64,7 +64,7 @@ export async function main(argv = process.argv.slice(2), env = process.env, {fet
|
|
|
64
64
|
const deadline = AbortSignal.timeout(Math.max(1, Math.ceil(remaining)));
|
|
65
65
|
return request(input, {...init,signal:init.signal ? AbortSignal.any([init.signal,deadline]) : deadline});
|
|
66
66
|
};
|
|
67
|
-
const api = deploymentDiagnostics(cloudflare({accountId:env.CLOUDFLARE_ACCOUNT_ID,token:env.CLOUDFLARE_API_TOKEN,fetch,...(command === 'remove' ? {maxRequests:Infinity,removal:true} : {})}),{onCreated});
|
|
67
|
+
const api = deploymentDiagnostics(cloudflare({accountId:env.CLOUDFLARE_ACCOUNT_ID,token:env.CLOUDFLARE_API_TOKEN,fetch,budgetMigrations:command === 'deploy',...(command === 'remove' ? {maxRequests:Infinity,removal:true} : {})}),{onCreated});
|
|
68
68
|
// Removal needs the recorded account/name identity, not a workers.dev route.
|
|
69
69
|
const target = command === 'remove'
|
|
70
70
|
? {accountId:api.accountId,project:declaration.project,environment:values.environment,worker:ownerWorkerName(declaration.project,values.environment)}
|
|
@@ -2,8 +2,8 @@ import { providerPages, providerListShape } from './pagination.mjs';
|
|
|
2
2
|
import { deleteR2Object } from './r2-object-transport.mjs';
|
|
3
3
|
import { isR2ObjectDelete } from './r2-objects.mjs';
|
|
4
4
|
import { recordDeployFailure } from './diagnostics.mjs';
|
|
5
|
-
import { importD1, importUploadURL } from './d1-import.mjs';
|
|
6
|
-
import { assert, hexId } from './validation.mjs';
|
|
5
|
+
import { D1_IMPORT_POLL_LIMIT, importD1, importUploadURL } from './d1-import.mjs';
|
|
6
|
+
import { assert, hexId, uuid } from './validation.mjs';
|
|
7
7
|
import { providerResponse, transportOutput, transportReason } from './provider-transport.mjs';
|
|
8
8
|
|
|
9
9
|
// A gateway/server error can arrive after a write. An explicit API refusal cannot.
|
|
@@ -28,10 +28,22 @@ export async function boundedJSON(response, limit) {
|
|
|
28
28
|
}
|
|
29
29
|
const redactValues = (text, values) => values.filter(Boolean).flatMap(value=>[value,encodeURIComponent(value),JSON.stringify(value).slice(1,-1)]).sort((a,b)=>b.length-a.length).reduce((value,secret)=>value.split(secret).join('[REDACTED]'),text);
|
|
30
30
|
/** Only this fixed origin and the explicitly selected account ever receive the token. */
|
|
31
|
-
export function cloudflare({ accountId, token, fetch: request = globalThis.fetch, objectDelete = deleteR2Object, timeoutMs = 30000, maxRequests = 500, removal = false }) {
|
|
31
|
+
export function cloudflare({ accountId, token, fetch: request = globalThis.fetch, objectDelete = deleteR2Object, timeoutMs = 30000, maxRequests = 500, removal = false, budgetMigrations = false }) {
|
|
32
32
|
assert(hexId(accountId), 'invalid-cloudflare-account-id');
|
|
33
33
|
assert(typeof token === 'string' && token.trim() && !/[\r\n]/.test(token), 'invalid-cloudflare-api-token');
|
|
34
34
|
let requests = 0;
|
|
35
|
+
const migrationBudgets = new Set();
|
|
36
|
+
function reserveMigrationRequests(id, pending) {
|
|
37
|
+
if (!budgetMigrations || migrationBudgets.has(id)) return;
|
|
38
|
+
assert(uuid(id) && Number.isSafeInteger(pending) && pending >= 0, 'invalid-provider-request-budget');
|
|
39
|
+
// Each pending migration can claim, initialize, upload and ingest, then use
|
|
40
|
+
// the bounded import poll protocol. Metadata and transport retries share the
|
|
41
|
+
// original allowance. Reserve validated work only once per database/run.
|
|
42
|
+
const limit = maxRequests + pending * (4 + D1_IMPORT_POLL_LIMIT);
|
|
43
|
+
assert(Number.isSafeInteger(limit), 'invalid-provider-request-budget');
|
|
44
|
+
migrationBudgets.add(id);
|
|
45
|
+
maxRequests = limit;
|
|
46
|
+
}
|
|
35
47
|
// `zone` scope: the zone-level API (/zones/...), for a run site's zone lookup and redirect rule.
|
|
36
48
|
async function rawEnvelope(path, { method = 'GET', body, assetToken, zone = false, signal: callerSignal } = {}) {
|
|
37
49
|
assert(typeof path === 'string' && (isR2ObjectDelete(path, method) || /^\/[a-z0-9/_?=&.,%-]+$/i.test(path) && !path.includes('..') && !path.split('?')[0].includes('%')), 'invalid-provider-path');
|
|
@@ -147,6 +159,6 @@ export function cloudflare({ accountId, token, fetch: request = globalThis.fetch
|
|
|
147
159
|
assert(response.status === 200, 'd1-import-upload-failed');
|
|
148
160
|
assert(response.headers.get('etag')?.replace(/^"|"$/g, '') === etag, 'd1-import-upload-etag-mismatch');
|
|
149
161
|
}
|
|
150
|
-
return { accountId, envelope, list, importSql(id, sql) { return importD1({envelope,upload,redact:text => redactValues(text,[accountId,token])}, id, sql); }, request:requestResult, workerContent,
|
|
162
|
+
return { accountId, envelope, list, reserveMigrationRequests, importSql(id, sql) { return importD1({envelope,upload,redact:text => redactValues(text,[accountId,token])}, id, sql); }, request:requestResult, workerContent,
|
|
151
163
|
zoneRequest(path, options = {}) { return requestResult(path, { ...options, zone: true }); } };
|
|
152
164
|
}
|