@runeditrun/rer 0.2.100 → 0.2.102
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +30 -102
- package/cli/README.md +16 -89
- package/cli/app-address.mjs +2 -0
- package/cli/business/api.mjs +2 -1
- package/cli/business/choices.mjs +1 -1
- package/cli/business/command.mjs +36 -30
- package/cli/business/connect.mjs +1 -1
- package/cli/business/lease.mjs +0 -9
- package/cli/business/local-commands.mjs +1 -27
- package/cli/business/outbound.mjs +2 -14
- package/cli/business/questions.mjs +85 -48
- package/cli/business/service.mjs +1 -24
- package/cli/business/setup-command.mjs +33 -26
- package/cli/business/state.mjs +1 -1
- package/cli/business/supervisor.mjs +26 -91
- package/cli/companion/command.mjs +2 -0
- package/cli/demo-native.mjs +1 -1
- package/cli/parse.mjs +2 -1
- package/global/CHANGELOG.md +32 -0
- package/global/README.md +30 -102
- package/global/main.mjs +2 -0
- package/modules/container/adapters/d1-limits.mjs +2 -10
- package/modules/d1-transport/limits.d.mts +10 -0
- package/modules/d1-transport/limits.mjs +10 -0
- package/modules/demo-host/native/prepare.mjs +2 -1
- package/modules/deploy-kit/d1-import.mjs +11 -1
- package/modules/deploy-kit/migrations.mjs +14 -1
- package/modules/import-kit/schema/importers.schema.json +104 -2
- package/modules/import-kit/src/d1/README.md +14 -0
- package/modules/import-kit/src/d1/apply-reads.ts +74 -0
- package/modules/import-kit/src/d1/apply.ts +32 -9
- package/modules/import-kit/src/d1/history.ts +2 -2
- package/modules/import-kit/src/d1/redaction-step.ts +17 -7
- package/modules/import-kit/src/d1/types.ts +11 -0
- package/modules/import-kit/src/door/ADOPTING.md +0 -37
- package/modules/import-kit/src/door/README.md +0 -66
- package/modules/import-kit/src/door/admission.ts +3 -13
- package/modules/import-kit/src/door/authority.ts +0 -1
- package/modules/import-kit/src/door/failures.ts +1 -1
- package/modules/import-kit/src/door/grants.ts +5 -19
- package/modules/import-kit/src/door/handler.ts +1 -12
- package/modules/import-kit/src/door/index.ts +0 -6
- package/modules/import-kit/src/door/planning.ts +19 -44
- package/modules/import-kit/src/door/protocol.ts +2 -4
- package/modules/import-kit/src/door/schema.ts +0 -2
- package/modules/import-kit/src/door/store.ts +2 -2
- package/modules/import-kit/src/door/types.ts +1 -24
- package/modules/import-kit/src/provenance/credentials.ts +14 -5
- package/modules/import-kit/src/status/evidence.ts +3 -1
- package/modules/import-kit/src/status/index.ts +17 -2
- package/modules/import-kit/src/status/schema.ts +8 -0
- package/modules/import-kit/src/status/types.ts +25 -4
- package/package.json +1 -1
- package/cli/business/archive-task.mjs +0 -32
- package/cli/business/bounded-waves.d.mts +0 -23
- package/cli/business/bounded-waves.mjs +0 -57
- package/cli/business/catchup-task.mjs +0 -49
- package/cli/business/export-inputs.mjs +0 -79
- package/cli/business/export-task.mjs +0 -24
- package/cli/business/legacy-package.mjs +0 -117
- package/cli/business/package-transfer.mjs +0 -127
- package/cli/business/setup-demo.mjs +0 -95
- package/cli/business/vendor-door.mjs +0 -303
- package/cli/business/vendors/README.md +0 -215
- package/cli/business/vendors/calendly.mjs +0 -80
- package/cli/business/vendors/common.mjs +0 -280
- package/cli/business/vendors/credential-names.mjs +0 -124
- package/cli/business/vendors/index.mjs +0 -96
- package/cli/business/vendors/pagerduty.mjs +0 -61
- package/cli/business/vendors/sentry.mjs +0 -85
- package/cli/business/vendors/sorry.mjs +0 -106
- package/cli/business/vendors/uptimerobot.mjs +0 -21
- package/cli/business/vendors/zendesk-attachments.mjs +0 -57
- package/cli/business/vendors/zendesk-encode.mjs +0 -133
- package/cli/business/vendors/zendesk-guide.mjs +0 -34
- package/cli/business/vendors/zendesk-package.mjs +0 -28
- package/cli/business/vendors/zendesk-pages.mjs +0 -61
- package/cli/business/vendors/zendesk-people.mjs +0 -7
- package/cli/business/vendors/zendesk-store.mjs +0 -154
- package/cli/business/vendors/zendesk-tickets.mjs +0 -21
- package/cli/business/vendors/zendesk-token.mjs +0 -31
- package/cli/business/vendors/zendesk-transport.mjs +0 -48
- package/cli/business/vendors/zendesk.mjs +0 -61
- package/modules/import-kit/src/door/package.ts +0 -121
- package/modules/import-kit/src/door/waves/README.md +0 -158
- package/modules/import-kit/src/door/waves/digests.ts +0 -38
- package/modules/import-kit/src/door/waves/inventory.ts +0 -84
- package/modules/import-kit/src/door/waves/journal.ts +0 -19
- package/modules/import-kit/src/door/waves/lifecycle.ts +0 -242
- package/modules/import-kit/src/door/waves/preparation.ts +0 -190
- package/modules/import-kit/src/door/waves/schema.ts +0 -73
- package/modules/import-kit/src/door/waves/store.ts +0 -112
- package/modules/import-kit/src/door/waves/types.ts +0 -76
|
@@ -1,280 +0,0 @@
|
|
|
1
|
-
import { createHash } from 'node:crypto';
|
|
2
|
-
import { setTimeout as delay } from 'node:timers/promises';
|
|
3
|
-
import { credentialMember, credentialName, credentialScheme, headerContainer } from './credential-names.mjs';
|
|
4
|
-
|
|
5
|
-
export class VendorExportError extends Error {
|
|
6
|
-
constructor(code, status) {
|
|
7
|
-
super(`Vendor export stopped (${code}).`);
|
|
8
|
-
this.name = 'VendorExportError';
|
|
9
|
-
this.code = code;
|
|
10
|
-
if (status) this.status = status;
|
|
11
|
-
}
|
|
12
|
-
}
|
|
13
|
-
export const fail = (code = 'INVALID_RESPONSE') => { throw new VendorExportError(code); };
|
|
14
|
-
export const hash = value => createHash('sha256').update(value).digest('hex');
|
|
15
|
-
export const json = value => JSON.stringify(value);
|
|
16
|
-
export function object(value) {
|
|
17
|
-
if (!value || typeof value !== 'object' || Array.isArray(value)) fail();
|
|
18
|
-
return value;
|
|
19
|
-
}
|
|
20
|
-
export function list(value, max = 2500) {
|
|
21
|
-
if (!Array.isArray(value) || value.length > max || value.some(x => !x || typeof x !== 'object' || Array.isArray(x))) fail();
|
|
22
|
-
return value;
|
|
23
|
-
}
|
|
24
|
-
export function id(value) {
|
|
25
|
-
if (typeof value !== 'string' || !/^[a-zA-Z0-9_-]{1,128}$/u.test(value)) fail('INVALID_INPUT');
|
|
26
|
-
return value;
|
|
27
|
-
}
|
|
28
|
-
export function timestamp(value) {
|
|
29
|
-
if (typeof value !== 'string' || !/^\d{4}-\d\d-\d\dT/u.test(value) || value.length > 40 || !Number.isFinite(Date.parse(value))) fail('INVALID_INPUT');
|
|
30
|
-
return new Date(value).toISOString();
|
|
31
|
-
}
|
|
32
|
-
export function windowInputs(inputs) {
|
|
33
|
-
const start = timestamp(inputs.start ?? inputs.since), end = timestamp(inputs.end ?? inputs.until);
|
|
34
|
-
if (Date.parse(start) >= Date.parse(end)) fail('INVALID_INPUT');
|
|
35
|
-
return { start, end };
|
|
36
|
-
}
|
|
37
|
-
export function cursor(value) {
|
|
38
|
-
if (value == null) return null;
|
|
39
|
-
if (typeof value !== 'string' || value.length > 2048 || /[\u0000-\u0020\u007f]/u.test(value)) fail('INVALID_PAGINATION');
|
|
40
|
-
return value;
|
|
41
|
-
}
|
|
42
|
-
// References supply pagination/identity only; neither host nor credentials can come from them.
|
|
43
|
-
export function reference(value, origin, pathname, allowedQuery = []) {
|
|
44
|
-
let url;
|
|
45
|
-
try { url = new URL(value, origin); } catch { fail('INVALID_PAGINATION'); }
|
|
46
|
-
if (url.origin !== origin || url.username || url.password || url.hash || !pathname.test(url.pathname)) fail('INVALID_PAGINATION');
|
|
47
|
-
if ([...url.searchParams.keys()].some(key => !allowedQuery.includes(key))) fail('INVALID_PAGINATION');
|
|
48
|
-
return url;
|
|
49
|
-
}
|
|
50
|
-
// The seed's import door applies the same rules (modules/import-kit, removeCredentials).
|
|
51
|
-
// A field holds a credential when the parts of its name are a credential word or
|
|
52
|
-
// phrase (apiKey, httpPassword, http_password, client_secret, X-Api-Key,
|
|
53
|
-
// integration_key, webhook_url, secret_key, x-functions-key,
|
|
54
|
-
// Ocp-Apim-Subscription-Key, db_pwd, X-Auth), unless its value is empty: null, a
|
|
55
|
-
// boolean (Calendly's `secret` flag), whitespace or a mask such as "****" or
|
|
56
|
-
// "[redacted]". Names that only contain such letters (keyword, monkey, author,
|
|
57
|
-
// signal, design) are not credentials. A plural (apiKeys, tokens) holds
|
|
58
|
-
// credentials only as text, so a count such as `tokens: 1500` stays. `auth` and
|
|
59
|
-
// `credentials` hold one only as text naming no scheme or mode (`include`,
|
|
60
|
-
// `oauth2`); an object under them keeps its fields and loses its own credential
|
|
61
|
-
// fields (`auth: {token}`) and, directly under them, a `value`, `pass`, `pw` or
|
|
62
|
-
// `secret…` member (Keycloak's `credentials: [{type, value}]`, nodemailer's
|
|
63
|
-
// `{user, pass}`). `passphrase`, `jwt`, `JSESSIONID` and `PHPSESSID` count
|
|
64
|
-
// everywhere. `sig`, `signature` and `sessionId` are credentials as header names
|
|
65
|
-
// (a tuple, a key under `headers`, a `{name|key, value}` pair in a header list),
|
|
66
|
-
// never as fields; a pair elsewhere (Help Scout's `customFields`) labels a field.
|
|
67
|
-
const placeholder = /^(?:\s*|[*\u2022\u25CF\u00B7x#_.-]{1,64}|[<[({]?\s*(?:redacted|hidden|masked|removed|omitted|none|null|n\/a|placeholder)\s*[>\])}]?)$/iu;
|
|
68
|
-
/** What a stripped credential value becomes; the seed's import door reports it per record. */
|
|
69
|
-
export const REMOVED_CREDENTIAL = '[removed]';
|
|
70
|
-
/** What a stripped credential query or fragment parameter's value becomes
|
|
71
|
-
* (`?api_key=removed`), as a URL's user name and password become `removed@`; the
|
|
72
|
-
* door drops the pair and reports it per record. */
|
|
73
|
-
export const REMOVED_PARAMETER = 'removed';
|
|
74
|
-
const empty = value => value === null || typeof value === 'boolean' || (typeof value === 'string' && placeholder.test(value));
|
|
75
|
-
// Text as a member (`['…']`, `{github: '…'}`); a list of objects is checked field by field.
|
|
76
|
-
const text = value => typeof value === 'string' ? !placeholder.test(value) : !!value && typeof value === 'object' && Object.values(value).some(child => (typeof child === 'string' || Array.isArray(child)) && text(child));
|
|
77
|
-
/** Every percent-encoding layer of a key, decoded byte by byte, as the door inspects it. */
|
|
78
|
-
function decoded(text) {
|
|
79
|
-
for (let depth = 0; depth < 32 && /%[\da-f]{2}/iu.test(text); depth++) text = text.replace(/%([\da-f]{2})/giu, (_, hex) => String.fromCharCode(parseInt(hex, 16)));
|
|
80
|
-
return text;
|
|
81
|
-
}
|
|
82
|
-
/** Text under `auth` or `credentials` naming no scheme or mode, or a list with such a member. */
|
|
83
|
-
const schemeless = value => typeof value === 'string' ? !placeholder.test(value) && !credentialScheme(value) : Array.isArray(value) && value.some(member => typeof member === 'string' && schemeless(member));
|
|
84
|
-
/** Whether a value under this name, a record field or an HTTP header name, is a credential, by the door's rule. */
|
|
85
|
-
function credential(name, value, place) {
|
|
86
|
-
const found = credentialName(decoded(name), 'remove', place);
|
|
87
|
-
return found !== undefined && !empty(value) && (found.container ? schemeless(value) : !found.plural || text(value));
|
|
88
|
-
}
|
|
89
|
-
/** Whether this key is `auth`, `X-Auth`, `credential` or `credentials`, whose members are searched for a credential under a generic name. */
|
|
90
|
-
const container = key => credentialName(decoded(key), 'remove')?.container === true;
|
|
91
|
-
/** A member of such a container holding a credential under a generic name (`value`, `pass`, `pw`, `secretData`). */
|
|
92
|
-
const member = (key, value) => credentialMember(decoded(key)) !== undefined && !empty(value);
|
|
93
|
-
// The door's credentialQueryKey rule for URL query and fragment parameter names:
|
|
94
|
-
// api_key, access_token, a bare `key`, sig and X-Amz-Signature, never keyword or author.
|
|
95
|
-
const credentialParameter = key => credentialName(decoded(key), 'query') !== undefined;
|
|
96
|
-
const authority = /^(\s*(?:[a-z][a-z\d+.-]*:)?[\\/]{2})[^\\/?#]*@/iu;
|
|
97
|
-
/** A URL of any scheme with its user name and password replaced by `removed@`,
|
|
98
|
-
* and the value of each credential query or fragment parameter by `removed`,
|
|
99
|
-
* except the adapter's own validated pagination keys, spelled exactly. */
|
|
100
|
-
function withoutUrlCredentials(text, pagination) {
|
|
101
|
-
if (!/^\s*(?:[a-z][a-z\d+.-]*:)?[\\/]{2}|^[/?]/iu.test(text)) return text;
|
|
102
|
-
const result = text.replace(authority, '$1removed@');
|
|
103
|
-
const mark = pairs => pairs.split('&').map(pair => {
|
|
104
|
-
const at = pair.indexOf('='), key = at < 0 ? pair : pair.slice(0, at);
|
|
105
|
-
return at < 0 || pagination.includes(key) || !credentialParameter(key) ? pair : `${key}=${REMOVED_PARAMETER}`;
|
|
106
|
-
}).join('&');
|
|
107
|
-
const hashAt = result.indexOf('#'), end = hashAt < 0 ? result.length : hashAt;
|
|
108
|
-
const fragment = hashAt < 0 ? '' : result.slice(hashAt);
|
|
109
|
-
const marked = fragment.includes('=') ? '#' + mark(fragment.slice(1)) : fragment;
|
|
110
|
-
const queryAt = result.indexOf('?');
|
|
111
|
-
if (queryAt < 0 || queryAt > end) return result.slice(0, end) + marked;
|
|
112
|
-
return result.slice(0, queryAt + 1) + mark(result.slice(queryAt + 1, end)) + marked;
|
|
113
|
-
}
|
|
114
|
-
/** The `[name, value]` pairs of a list holding only such pairs (Sentry's header tuples). */
|
|
115
|
-
const tuples = list => list.length > 0 && list.every(item => Array.isArray(item) && item.length === 2 && typeof item[0] === 'string');
|
|
116
|
-
/** A copy of a parsed vendor response without credential values, made before any
|
|
117
|
-
* byte is emitted for upload: a credential-named field's value, the `value` of a
|
|
118
|
-
* `{name|key, value}` pair naming one (a header pair), or the value of a
|
|
119
|
-
* credential-named `[name, value]` pair in a list of them, becomes
|
|
120
|
-
* REMOVED_CREDENTIAL, so the record stays and the seed can say what was removed.
|
|
121
|
-
* URLs lose credentials as withoutUrlCredentials describes; `pagination` lists
|
|
122
|
-
* the adapter's validated query keys, which stay. */
|
|
123
|
-
export function withoutCredentials(value, pagination = []) {
|
|
124
|
-
// `headers`: the keys of this object, or of objects in this list, are HTTP header names.
|
|
125
|
-
// `inside`: this object, or the objects in this list, sit under `auth` or `credential(s)`.
|
|
126
|
-
const strip = (value, headers, inside = false) => {
|
|
127
|
-
if (Array.isArray(value)) {
|
|
128
|
-
const pairs = tuples(value);
|
|
129
|
-
return value.map(child => pairs && credential(child[0], child[1], 'header') ? [child[0], REMOVED_CREDENTIAL] : strip(child, headers, inside));
|
|
130
|
-
}
|
|
131
|
-
if (value && typeof value === 'object') {
|
|
132
|
-
// A `{name|key, value}` pair's name is a header name only under a header container; elsewhere it labels a field.
|
|
133
|
-
const place = headers ? 'header' : 'field';
|
|
134
|
-
const label = Object.entries(value).find(([key, child]) => /^(?:name|key)$/iu.test(key) && typeof child === 'string');
|
|
135
|
-
const pair = !!label && Object.hasOwn(value, 'value') && credentialName(decoded(label[1]), 'remove', place) !== undefined;
|
|
136
|
-
return Object.fromEntries(Object.entries(value).map(([key, child]) => [key, (pair && key === 'value' ? credential(label[1], child, place) : credential(key, child, place) || (inside && member(key, child)))
|
|
137
|
-
? REMOVED_CREDENTIAL : strip(child, headerContainer(key), container(key))]));
|
|
138
|
-
}
|
|
139
|
-
return typeof value === 'string' ? withoutUrlCredentials(value, pagination) : value;
|
|
140
|
-
};
|
|
141
|
-
return strip(value, false);
|
|
142
|
-
}
|
|
143
|
-
export function assertCount(value, max = 1000) {
|
|
144
|
-
if (!Number.isSafeInteger(value) || value < 0 || value > max) fail('INVALID_CHECKPOINT');
|
|
145
|
-
return value;
|
|
146
|
-
}
|
|
147
|
-
export function shape(value, fields) {
|
|
148
|
-
object(value);
|
|
149
|
-
if (Object.keys(value).some(key => !fields.includes(key)) || fields.some(key => !Object.hasOwn(value, key))) fail('INVALID_CHECKPOINT');
|
|
150
|
-
return value;
|
|
151
|
-
}
|
|
152
|
-
export function stable(value) {
|
|
153
|
-
if (Array.isArray(value)) return value.map(stable);
|
|
154
|
-
if (value && typeof value === 'object') return Object.fromEntries(Object.keys(value).sort().map(key => [key, stable(value[key])]));
|
|
155
|
-
return value;
|
|
156
|
-
}
|
|
157
|
-
|
|
158
|
-
export function client({ origin, headers, route, signal, limits, counters, key }) {
|
|
159
|
-
let lastRequest = 0;
|
|
160
|
-
return async function get(path) {
|
|
161
|
-
signal?.throwIfAborted();
|
|
162
|
-
const url = new URL(path, origin);
|
|
163
|
-
if (url.origin !== origin || url.username || url.password || url.hash || !route(url)) fail('INVALID_REFERENCE');
|
|
164
|
-
if (++counters.requests > limits.requests) fail('REQUEST_LIMIT');
|
|
165
|
-
// At most one request per `spacing` (a second unless the adapter's limits say
|
|
166
|
-
// otherwise); no automatic retry or rate-limit hammering.
|
|
167
|
-
const pause = (limits.spacing ?? 1000) - (Date.now() - lastRequest);
|
|
168
|
-
if (pause > 0) await delay(pause, undefined, { signal });
|
|
169
|
-
lastRequest = Date.now();
|
|
170
|
-
const timeout = AbortSignal.timeout(30_000);
|
|
171
|
-
const requestSignal = signal ? AbortSignal.any([signal, timeout]) : timeout;
|
|
172
|
-
let response;
|
|
173
|
-
try { response = await fetch(url, { method: 'GET', headers, redirect: 'error', signal: requestSignal }); }
|
|
174
|
-
catch { fail(signal?.aborted ? 'CANCELLED' : 'NETWORK_FAILED'); }
|
|
175
|
-
if (!response.ok) {
|
|
176
|
-
await response.body?.cancel().catch(() => {});
|
|
177
|
-
throw new VendorExportError(response.status === 429 ? 'RATE_LIMITED' : 'HTTP_REFUSED', response.status);
|
|
178
|
-
}
|
|
179
|
-
const chunks = []; let size = 0;
|
|
180
|
-
try {
|
|
181
|
-
for await (const chunk of response.body ?? []) {
|
|
182
|
-
size += chunk.byteLength;
|
|
183
|
-
counters.received += chunk.byteLength;
|
|
184
|
-
if (size > limits.pageBytes || counters.received > limits.bytes) fail('BYTE_LIMIT');
|
|
185
|
-
chunks.push(chunk);
|
|
186
|
-
}
|
|
187
|
-
} catch (error) {
|
|
188
|
-
if (error instanceof VendorExportError) throw error;
|
|
189
|
-
fail(signal?.aborted ? 'CANCELLED' : 'NETWORK_FAILED');
|
|
190
|
-
}
|
|
191
|
-
const bytes = Buffer.concat(chunks);
|
|
192
|
-
// A reflected authentication key must never become a vendor artifact.
|
|
193
|
-
if (key && bytes.includes(Buffer.from(key))) fail('CREDENTIAL_IN_RESPONSE');
|
|
194
|
-
let text, body;
|
|
195
|
-
try { text = new TextDecoder('utf-8', { fatal: true }).decode(bytes); body = JSON.parse(text); } catch { fail('INVALID_JSON'); }
|
|
196
|
-
// `text` is the response exactly as received, for adapters whose bundle keeps original bytes.
|
|
197
|
-
return { body, text, headers: response.headers };
|
|
198
|
-
};
|
|
199
|
-
}
|
|
200
|
-
|
|
201
|
-
/** Opt-in transport for long-lived snapshots. Existing adapters retain client()'s
|
|
202
|
-
* no-retry contract. The caller validates every target before credentials or a
|
|
203
|
-
* request are constructed, including manually followed attachment redirects. */
|
|
204
|
-
export function pacedClient({ origin, signal, validate, secrets = [], limits = {}, beforeRequest }) {
|
|
205
|
-
const maximum = { requests: 250_000, response: 32 * 1024 * 1024, bytes: 32 * 1024 ** 3, ...limits };
|
|
206
|
-
const counters = { requests: 0, bytes: 0, retryMs: 0 };
|
|
207
|
-
let token, last = 0, next = 0, advertisedBefore = false;
|
|
208
|
-
const wait = async milliseconds => { if (milliseconds > 0) await delay(milliseconds, undefined, { signal }); };
|
|
209
|
-
const request = async (path, options = {}) => {
|
|
210
|
-
let url;
|
|
211
|
-
try { url = new URL(path, origin); } catch { fail('INVALID_REFERENCE'); }
|
|
212
|
-
await validate(url, options);
|
|
213
|
-
const method = options.method ?? 'GET';
|
|
214
|
-
if (method !== 'GET' && !(method === 'POST' && url.origin === origin && url.pathname === '/oauth/tokens')) fail('INVALID_REFERENCE');
|
|
215
|
-
for (let retries = 0; ; retries++) {
|
|
216
|
-
signal?.throwIfAborted();
|
|
217
|
-
await wait(Math.max(next, last + (advertisedBefore ? 0 : url.pathname.includes('/incremental/') ? 6000 : 300)) - Date.now());
|
|
218
|
-
if (++counters.requests > maximum.requests) fail('REQUEST_LIMIT');
|
|
219
|
-
await beforeRequest?.();
|
|
220
|
-
last = Date.now();
|
|
221
|
-
const headers = { accept: 'application/json', ...(options.json ? { 'content-type': 'application/json' } : {}),
|
|
222
|
-
...(token && url.origin === origin && options.authenticated !== false ? { authorization: `Bearer ${token}` } : {}) };
|
|
223
|
-
let response;
|
|
224
|
-
try { response = await fetch(url, { method, headers, redirect: options.redirect ?? 'error',
|
|
225
|
-
...(options.json ? { body: JSON.stringify(options.json) } : {}), signal: signal ? AbortSignal.any([signal, AbortSignal.timeout(30_000)]) : AbortSignal.timeout(30_000) }); }
|
|
226
|
-
catch { fail(signal?.aborted ? 'CANCELLED' : 'NETWORK_FAILED'); }
|
|
227
|
-
const waits = [];
|
|
228
|
-
const remaining = response.headers.get('ratelimit-remaining') ?? response.headers.get('x-rate-limit-remaining');
|
|
229
|
-
const reset = response.headers.get('ratelimit-reset') ?? '60';
|
|
230
|
-
const capacity = (remaining, seconds) => {
|
|
231
|
-
if (!/^\d+$/u.test(remaining ?? '') || !/^\d+$/u.test(seconds ?? '') || Number(seconds) > 3600) return;
|
|
232
|
-
waits.push(Math.max(300, Math.ceil(Number(seconds) * 1000 / Math.max(1, Number(remaining)))));
|
|
233
|
-
};
|
|
234
|
-
capacity(remaining, reset);
|
|
235
|
-
for (const [name,value] of response.headers) if (name.toLowerCase().startsWith('zendesk-ratelimit-')) capacity(/remaining=(\d+)/u.exec(value)?.[1], /resets=(\d+)/u.exec(value)?.[1]);
|
|
236
|
-
const interval = response.headers.get('ratelimit-limit') ?? response.headers.get('x-rate-limit');
|
|
237
|
-
if (!waits.length && /^\d+$/u.test(interval ?? '')) waits.push(Math.max(300, Math.ceil(60_000 / Math.max(1, Number(interval)))));
|
|
238
|
-
// Obey the tightest account/endpoint budget. Without valid advertised
|
|
239
|
-
// capacity the next request retains the conservative fallback spacing.
|
|
240
|
-
advertisedBefore = waits.length > 0;
|
|
241
|
-
next = Date.now() + Math.max(0, ...waits);
|
|
242
|
-
if (response.status === 429) {
|
|
243
|
-
const after = response.headers.get('retry-after');
|
|
244
|
-
await response.body?.cancel().catch(() => {});
|
|
245
|
-
const seconds = /^\d+$/u.test(after ?? '') ? Number(after) : 0;
|
|
246
|
-
if (retries >= 3 || seconds < 1 || seconds > 60 || counters.retryMs + seconds * 1000 > 600_000) throw new VendorExportError('RATE_LIMITED', 429);
|
|
247
|
-
counters.retryMs += seconds * 1000;
|
|
248
|
-
await wait(seconds * 1000);
|
|
249
|
-
continue;
|
|
250
|
-
}
|
|
251
|
-
if (options.redirect === 'manual' && [301, 302, 303, 307, 308].includes(response.status)) {
|
|
252
|
-
const location = response.headers.get('location');
|
|
253
|
-
await response.body?.cancel().catch(() => {});
|
|
254
|
-
return { status: response.status, location, headers: response.headers };
|
|
255
|
-
}
|
|
256
|
-
if (!response.ok) {
|
|
257
|
-
await response.body?.cancel().catch(() => {});
|
|
258
|
-
if (options.allow?.includes(response.status)) return { status: response.status };
|
|
259
|
-
throw new VendorExportError('HTTP_REFUSED', response.status);
|
|
260
|
-
}
|
|
261
|
-
const chunks = []; let size = 0;
|
|
262
|
-
try {
|
|
263
|
-
for await (const chunk of response.body ?? []) {
|
|
264
|
-
size += chunk.byteLength; counters.bytes += chunk.byteLength;
|
|
265
|
-
if (size > (options.maxBytes ?? maximum.response) || counters.bytes > maximum.bytes) fail('BYTE_LIMIT');
|
|
266
|
-
chunks.push(chunk);
|
|
267
|
-
}
|
|
268
|
-
} catch (error) { if (error instanceof VendorExportError) throw error; fail(signal?.aborted ? 'CANCELLED' : 'NETWORK_FAILED'); }
|
|
269
|
-
const bytes = Buffer.concat(chunks);
|
|
270
|
-
if (!options.tokenResponse && [...secrets, token].some(secret => secret && bytes.includes(Buffer.from(secret)))) fail('CREDENTIAL_IN_RESPONSE');
|
|
271
|
-
if (options.binary) return { bytes, headers: response.headers, status: response.status };
|
|
272
|
-
let body;
|
|
273
|
-
try { body = JSON.parse(new TextDecoder('utf-8', { fatal: true }).decode(bytes)); } catch { fail('INVALID_JSON'); }
|
|
274
|
-
return { body, bytes, headers: response.headers, status: response.status };
|
|
275
|
-
}
|
|
276
|
-
};
|
|
277
|
-
request.authorize = value => { token = value; };
|
|
278
|
-
request.counters = counters;
|
|
279
|
-
return request;
|
|
280
|
-
}
|
|
@@ -1,124 +0,0 @@
|
|
|
1
|
-
// A JavaScript copy of modules/import-kit/src/provenance/credential-names.ts, which the
|
|
2
|
-
// seed's import door applies; keep the two the same (rer's vendor exports use `remove` and `query`).
|
|
3
|
-
/** Whether a name (an object key, a header name, the name in a `[name, value]`
|
|
4
|
-
* pair, a URL query or fragment key) names a credential, by its whole parts.
|
|
5
|
-
*
|
|
6
|
-
* A name is split into lower-case parts at camelCase and PascalCase boundaries
|
|
7
|
-
* (`apiKey`, `XAPIKey`), at `_`, `-`, `.`, spaces and any other non-letter, and
|
|
8
|
-
* at digits (`token2` is `token`). A name is a credential when its parts are a
|
|
9
|
-
* credential word or phrase, never because a part only contains one: `keyword`,
|
|
10
|
-
* `monkey`, `author`, `signal`, `design` and `tokenizer` hold the letters of a
|
|
11
|
-
* credential word and are not credentials. */
|
|
12
|
-
// credentialName returns the name's letters, lower-cased (`apiKey` and `api_key` are both
|
|
13
|
-
// `apikey`), and whether it is the word's plural (`tokens`, `apiKeys`: it holds credentials
|
|
14
|
-
// only as text, so a count stays), or undefined. Modes: `refuse`: the whole name is a
|
|
15
|
-
// credential phrase, so a pagination cursor such as `next_page_token` is not one. `remove`:
|
|
16
|
-
// the name ends in a credential word (`pushToken`, `webhook_url`), the wider net of the
|
|
17
|
-
// import door, which drops the value and keeps the record. `query`: a URL query or fragment
|
|
18
|
-
// key, as `remove` plus a bare `key` and `X-Amz-Signature`. It also says whether the name is
|
|
19
|
-
// a container (`auth`, `X-Auth`, `credential(s)`): its value is a credential when it is text
|
|
20
|
-
// that names no scheme or mode (`include`, `oauth2`), or a list with such a member; an object
|
|
21
|
-
// under it is searched field by field. Places, in `remove` mode: a record `field`, or an HTTP
|
|
22
|
-
// `header` name (a `[name, value]` tuple, a key of a `headers` object, a `{name|key, value}`
|
|
23
|
-
// pair in a header list). `sig`, `signature` and `sessionId` are credentials only as header names and URL
|
|
24
|
-
// query or fragment keys, never as record fields, so an agent's `signature` or a chat's
|
|
25
|
-
// `session_id` imports.
|
|
26
|
-
const WORDS = new Set(['token', 'secret', 'password', 'passwd', 'pwd', 'cookie', 'authorization', 'bearer', 'credential']);
|
|
27
|
-
/** Words that make a trailing `key` a credential: `api key`, `secret key`, `Ocp-Apim-Subscription-Key`. A bare `key` or `primaryKey` is not. */
|
|
28
|
-
const KEY_QUALIFIERS = new Set(['api', 'access', 'private', 'secret', 'auth', 'client', 'integration', 'routing', 'service', 'signing', 'session',
|
|
29
|
-
'refresh', 'subscription', 'functions', 'consumer', 'app', 'license', 'master', 'encryption', 'admin']);
|
|
30
|
-
/** URL query and fragment keys only: `?application_key=` (Datadog), `?user_key=` (3scale), `?account_key=`, `?deploy_key=`, `?shared_key=`, and `?token_auth=` (Matomo), `?basic_auth=`. */
|
|
31
|
-
const QUERY_KEY_QUALIFIERS = new Set(['application', 'user', 'account', 'deploy', 'shared']);
|
|
32
|
-
const QUERY_AUTH_QUALIFIERS = new Set(['token', 'basic', 'http', 'proxy', 'api']);
|
|
33
|
-
const TOKEN_QUALIFIERS = ['access', 'refresh', 'auth', 'api', 'bearer', 'id', 'session', 'csrf', 'xsrf', 'bot', 'push', 'oauth', 'jwt'];
|
|
34
|
-
const SECRET_QUALIFIERS = ['client', 'app', 'api', 'oauth', 'webhook', 'signing', 'shared'];
|
|
35
|
-
const PASSWORD_QUALIFIERS = ['http', 'db', 'ftp', 'smtp', 'admin', 'user', 'basic', 'proxy', 'root', 'mail', 'login'];
|
|
36
|
-
/** One-word credentials however a name splits them: a `passphrase` (`passPhrase`), a `jwt`, and Java's and PHP's session
|
|
37
|
-
* cookies (`JSESSIONID`, `jsessionId`, `PHPSESSID`). They name nothing else, so they count as fields, header names and URL keys. */
|
|
38
|
-
const SPELLED = /^(?:passphrase|jwt|jsessionid|phpsessid)(s?)$/;
|
|
39
|
-
/** What may stand before one in `refuse` mode (`ssh_passphrase`, `id_jwt`). */
|
|
40
|
-
const SPELLED_QUALIFIERS = new Set([...PASSWORD_QUALIFIERS, ...TOKEN_QUALIFIERS, 'ssh', 'key', 'gpg', 'pgp', 'private']);
|
|
41
|
-
/** What may stand before a credential word in `refuse` mode, which names the whole phrase (`access token`, `http password`). */
|
|
42
|
-
const QUALIFIERS = {
|
|
43
|
-
token: new Set(TOKEN_QUALIFIERS), secret: new Set(SECRET_QUALIFIERS),
|
|
44
|
-
password: new Set(PASSWORD_QUALIFIERS), passwd: new Set(PASSWORD_QUALIFIERS), pwd: new Set(PASSWORD_QUALIFIERS),
|
|
45
|
-
key: KEY_QUALIFIERS,
|
|
46
|
-
};
|
|
47
|
-
/** The same phrases written as one lower-case part (`apikey`, `accesstoken`, `clientsecret`, `httppassword`, `webhookurl`). */
|
|
48
|
-
const COMPOUND = new RegExp('^x?(?:'
|
|
49
|
-
+ `(?:${[...KEY_QUALIFIERS].join('|')})(?:key)|(?:${TOKEN_QUALIFIERS.join('|')})(?:token)|(?:${SECRET_QUALIFIERS.join('|')})(?:secret)`
|
|
50
|
-
+ `|(?:${PASSWORD_QUALIFIERS.join('|')})(?:password|passwd|pwd)|webhook(?:url)|cal(?:url))(s?)$`);
|
|
51
|
-
/** A scheme or mode an `auth` or `credentials` field names, never a credential:
|
|
52
|
-
* fetch's `include`, `same-origin` and `omit`, HTTP and API authentication schemes, and boolean
|
|
53
|
-
* words (Keycloak's `smtpServer.auth: "true"`). */
|
|
54
|
-
const SCHEME = /^(?:omit|sameorigin|include|none|noauth|inherit|anonymous|basic|bearer|digest|negotiate|ntlm|kerberos|oauth(?:1|2|10|20|1a)?|apikey|jwt|hawk|hmac|aws|awsv4|sigv4|awssigv4|edgegrid|mutual|vapid|token|password|cookie|session|header|query|true|false|yes|no|on|off|enabled|disabled)$/;
|
|
55
|
-
|
|
56
|
-
/** Whether text under an `auth` or `credentials` name names a scheme or mode (`include`, `oauth2`, `Bearer`, `same-origin`). */
|
|
57
|
-
export function credentialScheme(text) {
|
|
58
|
-
return SCHEME.test(text.trim().toLowerCase().replace(/[\s_.-]/g, ''));
|
|
59
|
-
}
|
|
60
|
-
/** Whether an object under this key holds HTTP headers by name (`headers`, `customHttpHeaders`, `header`). */
|
|
61
|
-
export function headerContainer(key) {
|
|
62
|
-
const last = nameParts(key).at(-1);
|
|
63
|
-
return last === 'headers' || last === 'header';
|
|
64
|
-
}
|
|
65
|
-
// Under `auth`, `X-Auth`, `credential` or `credentials`, a member holding the credential under a generic name: `value`
|
|
66
|
-
// (Keycloak's `{type: 'password', value}`, `credential: {type: 'apiKey', value}`), `pass` and `pw` (nodemailer's
|
|
67
|
-
// `{user, pass}`), or a name whose first part is `secret` (Keycloak's `secretData`). Its name's letters, or undefined.
|
|
68
|
-
export function credentialMember(name) {
|
|
69
|
-
const parts = nameParts(name);
|
|
70
|
-
return (parts.length === 1 && /^(?:value|pass|pw)$/.test(parts[0])) || parts[0] === 'secret' ? parts.join('') : undefined;
|
|
71
|
-
}
|
|
72
|
-
|
|
73
|
-
/** A name's lower-case parts, without digits. Format characters and accents are folded away first. */
|
|
74
|
-
export function nameParts(name) {
|
|
75
|
-
const text = name.normalize('NFKC').replace(/[\p{Cf}\p{M}]/gu, '');
|
|
76
|
-
return (text.match(/\p{Lu}+(?!\p{Ll})|\p{Lu}?\p{Ll}+|\p{Lo}+/gu) ?? []).map(part => part.toLowerCase());
|
|
77
|
-
}
|
|
78
|
-
|
|
79
|
-
export function credentialName(name, mode, place = 'field') {
|
|
80
|
-
const parts = nameParts(name), last = parts.at(-1), before = parts.at(-2);
|
|
81
|
-
if (last === undefined) return undefined;
|
|
82
|
-
const letters = name.normalize('NFKC').toLowerCase().replace(/[^a-z]/g, '').slice(-64);
|
|
83
|
-
const compound = COMPOUND.exec(last);
|
|
84
|
-
if (compound) return { name: letters, plural: compound[1] === 's', container: false };
|
|
85
|
-
// A header's leading `x-` (`x-api-key`, `X-Auth-Token`) is not part of the phrase; `refuse` mode takes the rest whole.
|
|
86
|
-
const phrase = parts.length > 1 && parts[0] === 'x' ? parts.slice(1) : parts;
|
|
87
|
-
const found = (plural, container = false) => ({ name: letters, plural, container });
|
|
88
|
-
const qualifier = word => word !== undefined && (KEY_QUALIFIERS.has(word) || (word.startsWith('x') && KEY_QUALIFIERS.has(word.slice(1))));
|
|
89
|
-
const strict = mode === 'refuse';
|
|
90
|
-
// A URL key or a header name: where `sig`, `signature` and `sessionId` are credentials.
|
|
91
|
-
const header = mode === 'query' || place === 'header';
|
|
92
|
-
for (let size = 1; size <= Math.min(3, phrase.length); size++) {
|
|
93
|
-
const spelled = SPELLED.exec(phrase.slice(-size).join(''));
|
|
94
|
-
if (spelled && (!strict || phrase.length === size || (phrase.length === size + 1 && SPELLED_QUALIFIERS.has(phrase[0])))) return found(spelled[1] === 's');
|
|
95
|
-
}
|
|
96
|
-
// `credentials` holds a credential object as one value; the other plurals hold a list or map of text.
|
|
97
|
-
const word = last === 'credentials' ? 'credential' : WORDS.has(last) ? last : last.endsWith('s') && WORDS.has(last.slice(0, -1)) ? last.slice(0, -1) : undefined;
|
|
98
|
-
if (word !== undefined) {
|
|
99
|
-
const plural = word !== last && last !== 'credentials';
|
|
100
|
-
// `refuse` mode: the word alone or after one qualifier (`token`, `access_token`), and `pwd` after anything.
|
|
101
|
-
const named = phrase.length === 1 || (phrase.length === 2 && QUALIFIERS[word]?.has(phrase[0]) === true) || word === 'pwd';
|
|
102
|
-
if (!strict || named) return found(plural, word === 'credential' && mode !== 'query');
|
|
103
|
-
}
|
|
104
|
-
if (last === 'key' || last === 'keys') {
|
|
105
|
-
// `refuse` mode: `api_key`, `x-functions-key`, and `secret`/`subscription`/`functions` before the key under any name.
|
|
106
|
-
const named = phrase.length === 2 || (before !== undefined && ['secret', 'subscription', 'functions'].includes(before));
|
|
107
|
-
if (qualifier(before) && (!strict || named)) return found(last === 'keys');
|
|
108
|
-
// A bare `key` is a credential only as a URL query or fragment key (the `?key=` convention).
|
|
109
|
-
if (mode === 'query' && parts.length === 1 && last === 'key') return found(false);
|
|
110
|
-
if (mode === 'query' && before !== undefined && QUERY_KEY_QUALIFIERS.has(before)) return found(last === 'keys');
|
|
111
|
-
}
|
|
112
|
-
// A session id is a credential as a header or URL key (`Session-Id`, `?sessionid=`), never as a record's field (a chat's `session_id`).
|
|
113
|
-
const session = /^x?sessionid(s?)$/.exec(last);
|
|
114
|
-
if (header && session) return found(session[1] === 's');
|
|
115
|
-
if (header && last === 'id' && before === 'session' && (!strict || phrase.length === 2)) return found(false);
|
|
116
|
-
if (!strict && last === 'url' && (before === 'webhook' || before === 'cal')) return found(false);
|
|
117
|
-
// A signature is a credential as a whole header name (`Signature`) and, as a query key, as its last part (an Azure SAS `sig`, `X-Amz-Signature`); an agent's `signature` field is not.
|
|
118
|
-
if ((last === 'sig' || last === 'signature') && (mode === 'query' || (place === 'header' && parts.length === 1))) return found(false);
|
|
119
|
-
if (last === 'auth' && (parts.length === 1 || (parts.length === 2 && before === 'x'))) return found(false, mode !== 'query');
|
|
120
|
-
if (mode === 'query' && last === 'auth' && before !== undefined && QUERY_AUTH_QUALIFIERS.has(before)) return found(false);
|
|
121
|
-
// A query key written as one part that ends in an unambiguous phrase: HubSpot's `?hapikey=`.
|
|
122
|
-
if (mode === 'query' && parts.length === 1 && /(?:apikey|accesstoken|authtoken)$/.test(last)) return found(false);
|
|
123
|
-
return undefined;
|
|
124
|
-
}
|
|
@@ -1,96 +0,0 @@
|
|
|
1
|
-
import { VendorExportError, fail, hash, client, object, assertCount, shape, stable } from './common.mjs';
|
|
2
|
-
import * as sentry from './sentry.mjs';
|
|
3
|
-
import * as uptimerobot from './uptimerobot.mjs';
|
|
4
|
-
import * as pagerduty from './pagerduty.mjs';
|
|
5
|
-
import * as calendly from './calendly.mjs';
|
|
6
|
-
import * as sorry from './sorry.mjs';
|
|
7
|
-
import * as zendesk from './zendesk.mjs';
|
|
8
|
-
|
|
9
|
-
export { VendorExportError } from './common.mjs';
|
|
10
|
-
export const VENDORS = Object.freeze(['sentry','uptimerobot','pagerduty','calendly','sorry','zendesk','happytohelp-legacy']);
|
|
11
|
-
const adapters = { sentry, uptimerobot, pagerduty, calendly, sorry };
|
|
12
|
-
export const VENDOR_CAPABILITIES = Object.freeze({
|
|
13
|
-
'happytohelp-legacy': Object.freeze({ vendor: 'happytohelp-legacy', adapter: 'rer:happytohelp-legacy', capability: null, format: 'happytohelp-legacy-chunks-v1', layout: 'package', output: 'rer-package.json', readOnly: true, resume: true, scopes: [], origins: [] }),
|
|
14
|
-
zendesk: Object.freeze({ vendor: 'zendesk', adapter: 'rer:zendesk', capability: 'vendor_key:zendesk', format: 'happytohelp-zendesk-offline-v1', layout: 'snapshot', output: 'zendesk.ndjson', readOnly: true, resume: true, scopes: ['tickets:read','users:read','organizations:read','hc:read','ticket_attachments:read'], origins: ['https://<subdomain>.zendesk.com'] }),
|
|
15
|
-
sentry: Object.freeze({ vendor: 'sentry', adapter: 'rer:sentry', capability: 'vendor_key:sentry', format: 'spool-sentry-org-export-v1', layout: 'concatenate', output: 'sentry-export.json', readOnly: true, resume: true, scopes: ['org:read','project:read','event:read'], origins: Object.values(sentry.origins) }),
|
|
16
|
-
uptimerobot: Object.freeze({ vendor: 'uptimerobot', adapter: 'rer:uptimerobot', capability: 'vendor_key:uptimerobot', format: 'uptimerobot-v3-monitors', layout: 'concatenate', output: 'uptimerobot-monitors.json', readOnly: true, resume: true, scopes: ['read-only API key'], origins: [uptimerobot.origin] }),
|
|
17
|
-
pagerduty: Object.freeze({ vendor: 'pagerduty', adapter: 'rer:pagerduty', capability: 'vendor_key:pagerduty', format: 'pagerduty-saved-responses', layout: 'concatenate', output: 'pagerduty-export.json', readOnly: true, resume: true, scopes: ['read-only REST API key'], origins: [pagerduty.origin] }),
|
|
18
|
-
calendly: Object.freeze({ vendor: 'calendly', adapter: 'rer:calendly', capability: 'vendor_key:calendly', format: 'dial-calendly-api-folder-v1', layout: 'files', output: 'manifest.json', readOnly: true, resume: true, scopes: ['personal access token with read access'], origins: [calendly.origin] }),
|
|
19
|
-
// A public status page: read anonymously from the page's own address, so it takes no key.
|
|
20
|
-
sorry: Object.freeze({ vendor: 'sorry', adapter: 'rer:sorry', capability: null, format: 'lamp-sorry-public-responses', layout: 'concatenate', output: 'sorry-export.json', readOnly: true, resume: true, scopes: [], origins: ['https://<page host>'] }),
|
|
21
|
-
});
|
|
22
|
-
/** The vendors whose export needs the owner's read-only key. */
|
|
23
|
-
export const KEYED_VENDORS = Object.freeze(VENDORS.filter(vendor => VENDOR_CAPABILITIES[vendor].capability !== null));
|
|
24
|
-
const LIMITS = Object.freeze({
|
|
25
|
-
// Spool's 500-record bound also counts its synthesized manifest record.
|
|
26
|
-
sentry: { bytes: 2 * 1024 * 1024, records: 499 },
|
|
27
|
-
uptimerobot: { bytes: 8 * 1024 * 1024, records: 2500 },
|
|
28
|
-
pagerduty: { bytes: 512 * 1024, records: 500 },
|
|
29
|
-
calendly: { bytes: 8 * 1024 * 1024, records: 2500 },
|
|
30
|
-
// Lamp's Sorry reader: 8 MiB of responses in at most 200 files and 5,000
|
|
31
|
-
// records; its door takes twice that, as the bundle carries each response as
|
|
32
|
-
// JSON text. Lamp reads a public page at most every 125 ms.
|
|
33
|
-
sorry: { bytes: 8 * 1024 * 1024, emitted: 16 * 1024 * 1024, records: 5000, requests: 200, spacing: 125 },
|
|
34
|
-
});
|
|
35
|
-
|
|
36
|
-
/** Persist each checkpoint only after the corresponding bytes have been acknowledged.
|
|
37
|
-
* Bytes are private seed-bound artifacts; checkpoints are metadata, never API bodies.
|
|
38
|
-
*/
|
|
39
|
-
export async function* exportVendor(vendor, { key, inputs = {}, checkpoint, signal, directory, runId } = {}) {
|
|
40
|
-
try {
|
|
41
|
-
if (!VENDORS.includes(vendor)) fail('UNSUPPORTED_VENDOR');
|
|
42
|
-
if (vendor === 'happytohelp-legacy') fail('LOCAL_PACKAGE_REQUIRED');
|
|
43
|
-
if (vendor === 'zendesk') {
|
|
44
|
-
const result = await zendesk.snapshot({ key, inputs, directory, runId, signal });
|
|
45
|
-
if (result.kind !== 'bundle') fail(result.kind === 'package' ? 'SEED_PACKAGE_UNSUPPORTED' : 'INVALID_INPUT');
|
|
46
|
-
yield { name: 'zendesk.ndjson', bytes: result.bytes, checkpoint: { version: 1, vendor, head: result.store.head } };
|
|
47
|
-
return;
|
|
48
|
-
}
|
|
49
|
-
const keyed = KEYED_VENDORS.includes(vendor);
|
|
50
|
-
if (keyed ? typeof key !== 'string' || !key || key.length > 8192 || /[\u0000-\u0020\u007f]/u.test(key) : key != null) fail('INVALID_KEY');
|
|
51
|
-
object(inputs);
|
|
52
|
-
const fingerprint = hash(JSON.stringify(stable({ vendor, inputs })));
|
|
53
|
-
const limits = { requests: 1000, ...LIMITS[vendor], pageBytes: 1024 * 1024 - 16 * 1024 };
|
|
54
|
-
let state, sequence = 0;
|
|
55
|
-
let counters = { requests: 0, received: 0, emitted: 0, records: 0 };
|
|
56
|
-
if (checkpoint != null) {
|
|
57
|
-
const encoded = JSON.stringify(checkpoint);
|
|
58
|
-
if (Buffer.byteLength(encoded) >= 128 * 1024 || key && encoded.includes(key)) fail('INVALID_CHECKPOINT');
|
|
59
|
-
const cp = JSON.parse(encoded);
|
|
60
|
-
shape(cp, ['version','vendor','fingerprint','sequence','counters','state']);
|
|
61
|
-
if (cp.version !== 1 || cp.vendor !== vendor || cp.fingerprint !== fingerprint) fail('INVALID_CHECKPOINT');
|
|
62
|
-
sequence = assertCount(cp.sequence, 2000);
|
|
63
|
-
shape(cp.counters, ['requests','received','emitted','records']); object(cp.state);
|
|
64
|
-
counters = Object.fromEntries(Object.keys(counters).map(name => [name, assertCount(cp.counters[name], name === 'requests' ? limits.requests : name === 'records' ? limits.records : name === 'emitted' ? limits.emitted ?? limits.bytes : limits.bytes)]));
|
|
65
|
-
state = cp.state;
|
|
66
|
-
}
|
|
67
|
-
const adapter = adapters[vendor];
|
|
68
|
-
const origin = vendor === 'sentry' ? adapter.origins[inputs.organization?.region] : vendor === 'sorry' ? adapter.originOf(inputs) : adapter.origin;
|
|
69
|
-
if (!origin) fail('INVALID_INPUT');
|
|
70
|
-
const headers = { Accept: vendor === 'pagerduty' ? 'application/vnd.pagerduty+json;version=2' : 'application/json',
|
|
71
|
-
...(keyed ? { Authorization: vendor === 'pagerduty' ? `Token token=${key}` : `Bearer ${key}` } : {}) };
|
|
72
|
-
const ctx = {
|
|
73
|
-
inputs, state,
|
|
74
|
-
get: client({ origin, headers, route: adapter.route, signal, limits, counters, key }),
|
|
75
|
-
records(count) {
|
|
76
|
-
counters.records += count;
|
|
77
|
-
if (counters.records > limits.records) fail('RECORD_LIMIT');
|
|
78
|
-
},
|
|
79
|
-
item(value, nextState, name) {
|
|
80
|
-
signal?.throwIfAborted();
|
|
81
|
-
const bytes = Buffer.isBuffer(value) ? value : Buffer.from(value);
|
|
82
|
-
if (bytes.length > 1024 * 1024 || (counters.emitted += bytes.length) > (limits.emitted ?? limits.bytes)) fail('BYTE_LIMIT');
|
|
83
|
-
if (key && bytes.includes(Buffer.from(key))) fail('CREDENTIAL_IN_RESPONSE');
|
|
84
|
-
if (++sequence > 2000) fail('ITEM_LIMIT');
|
|
85
|
-
const cp = { version: 1, vendor, fingerprint, sequence, counters: { ...counters }, state: nextState };
|
|
86
|
-
const encoded = JSON.stringify(cp);
|
|
87
|
-
if (Buffer.byteLength(encoded) >= 128 * 1024 || key && encoded.includes(key)) fail('CHECKPOINT_LIMIT');
|
|
88
|
-
return { name: name ?? `${String(sequence).padStart(6,'0')}.part`, bytes, checkpoint: JSON.parse(encoded) };
|
|
89
|
-
},
|
|
90
|
-
};
|
|
91
|
-
yield* adapter.run(ctx);
|
|
92
|
-
} catch (error) {
|
|
93
|
-
if (error instanceof VendorExportError) throw error;
|
|
94
|
-
throw new VendorExportError(signal?.aborted ? 'CANCELLED' : 'INVALID_EXPORT');
|
|
95
|
-
}
|
|
96
|
-
}
|
|
@@ -1,61 +0,0 @@
|
|
|
1
|
-
import { object, list, id, windowInputs, assertCount, shape, fail, withoutCredentials } from './common.mjs';
|
|
2
|
-
|
|
3
|
-
export const origin = 'https://api.pagerduty.com';
|
|
4
|
-
export const route = url => /^\/(?:users|escalation_policies|schedules(?:\/[a-zA-Z0-9_-]+(?:\/overrides)?)?)$/u.test(url.pathname) && [...url.searchParams.keys()].every(x => ['limit','offset','since','until','overflow'].includes(x));
|
|
5
|
-
function advance(body, state) {
|
|
6
|
-
if (typeof body.more !== 'boolean') fail();
|
|
7
|
-
if (body.more) {
|
|
8
|
-
const limit = assertCount(body.limit, 100);
|
|
9
|
-
if (!limit || body.offset !== state.offset) fail('INVALID_PAGINATION');
|
|
10
|
-
state.offset += limit;
|
|
11
|
-
assertCount(state.offset, 100000);
|
|
12
|
-
} else state.offset = 0;
|
|
13
|
-
return body.more;
|
|
14
|
-
}
|
|
15
|
-
export async function* run(ctx) {
|
|
16
|
-
const { start, end } = windowInputs(ctx.inputs);
|
|
17
|
-
const window = new URLSearchParams({ since: start, until: end });
|
|
18
|
-
const s = ctx.state ?? { phase: 0, offset: 0, schedules: [], index: 0, first: true };
|
|
19
|
-
shape(s, ['phase','offset','schedules','index','first']);
|
|
20
|
-
if (typeof s.first !== 'boolean') fail('INVALID_CHECKPOINT');
|
|
21
|
-
assertCount(s.phase, 5); assertCount(s.offset, 100000); assertCount(s.index, 500);
|
|
22
|
-
if (!Array.isArray(s.schedules) || s.schedules.length > 500) fail('INVALID_CHECKPOINT');
|
|
23
|
-
s.schedules.forEach(id);
|
|
24
|
-
while (s.phase < 5) {
|
|
25
|
-
let bytes;
|
|
26
|
-
if (s.phase === 0 || s.phase === 1 || s.phase === 4) {
|
|
27
|
-
const kind = ['users','schedules',null,null,'escalation_policies'][s.phase];
|
|
28
|
-
const { body } = await ctx.get(`/${kind}?limit=100&offset=${s.offset}`);
|
|
29
|
-
object(body); const rows = list(body[kind], 100); ctx.records(s.phase === 1 ? 0 : rows.length);
|
|
30
|
-
if (s.phase === 1) {
|
|
31
|
-
for (const row of rows) { const value = id(row.id); if (s.schedules.includes(value)) fail('DUPLICATE_ID'); s.schedules.push(value); }
|
|
32
|
-
if (s.schedules.length > 500) fail('RECORD_LIMIT');
|
|
33
|
-
bytes = '';
|
|
34
|
-
} else {
|
|
35
|
-
bytes = (s.first ? (s.phase === 0 ? '{"users":[' : '],"escalation_policies":[') : ',') + JSON.stringify(withoutCredentials(body));
|
|
36
|
-
s.first = false;
|
|
37
|
-
}
|
|
38
|
-
if (!advance(body, s)) { s.phase++; s.first = true; }
|
|
39
|
-
} else if (s.phase === 2) {
|
|
40
|
-
if (s.index === s.schedules.length) { s.phase = 3; s.index = 0; s.first = true; bytes = s.schedules.length ? '' : '],"schedules":['; }
|
|
41
|
-
else {
|
|
42
|
-
const { body } = await ctx.get(`/schedules/${id(s.schedules[s.index])}?${window}`);
|
|
43
|
-
object(body); object(body.schedule); ctx.records(1);
|
|
44
|
-
bytes = (s.first ? '],"schedules":[' : ',') + JSON.stringify(withoutCredentials(body));
|
|
45
|
-
s.first = false; s.index++;
|
|
46
|
-
}
|
|
47
|
-
} else {
|
|
48
|
-
if (s.index === s.schedules.length) { s.phase = 4; s.first = true; bytes = s.schedules.length ? '' : '],"overrides":['; }
|
|
49
|
-
else {
|
|
50
|
-
const schedule = id(s.schedules[s.index]);
|
|
51
|
-
const { body } = await ctx.get(`/schedules/${schedule}/overrides?${window}&overflow=true`);
|
|
52
|
-
object(body); ctx.records(list(body.overrides).length);
|
|
53
|
-
bytes = (s.first ? '],"overrides":[' : ',') + JSON.stringify(withoutCredentials({ ...body, schedule_id: schedule }));
|
|
54
|
-
s.first = false;
|
|
55
|
-
s.index++;
|
|
56
|
-
}
|
|
57
|
-
}
|
|
58
|
-
if (s.phase === 5) bytes += ']}\n';
|
|
59
|
-
yield ctx.item(bytes, s);
|
|
60
|
-
}
|
|
61
|
-
}
|