@runeditrun/rer 0.2.100 → 0.2.102

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (93) hide show
  1. package/README.md +30 -102
  2. package/cli/README.md +16 -89
  3. package/cli/app-address.mjs +2 -0
  4. package/cli/business/api.mjs +2 -1
  5. package/cli/business/choices.mjs +1 -1
  6. package/cli/business/command.mjs +36 -30
  7. package/cli/business/connect.mjs +1 -1
  8. package/cli/business/lease.mjs +0 -9
  9. package/cli/business/local-commands.mjs +1 -27
  10. package/cli/business/outbound.mjs +2 -14
  11. package/cli/business/questions.mjs +85 -48
  12. package/cli/business/service.mjs +1 -24
  13. package/cli/business/setup-command.mjs +33 -26
  14. package/cli/business/state.mjs +1 -1
  15. package/cli/business/supervisor.mjs +26 -91
  16. package/cli/companion/command.mjs +2 -0
  17. package/cli/demo-native.mjs +1 -1
  18. package/cli/parse.mjs +2 -1
  19. package/global/CHANGELOG.md +32 -0
  20. package/global/README.md +30 -102
  21. package/global/main.mjs +2 -0
  22. package/modules/container/adapters/d1-limits.mjs +2 -10
  23. package/modules/d1-transport/limits.d.mts +10 -0
  24. package/modules/d1-transport/limits.mjs +10 -0
  25. package/modules/demo-host/native/prepare.mjs +2 -1
  26. package/modules/deploy-kit/d1-import.mjs +11 -1
  27. package/modules/deploy-kit/migrations.mjs +14 -1
  28. package/modules/import-kit/schema/importers.schema.json +104 -2
  29. package/modules/import-kit/src/d1/README.md +14 -0
  30. package/modules/import-kit/src/d1/apply-reads.ts +74 -0
  31. package/modules/import-kit/src/d1/apply.ts +32 -9
  32. package/modules/import-kit/src/d1/history.ts +2 -2
  33. package/modules/import-kit/src/d1/redaction-step.ts +17 -7
  34. package/modules/import-kit/src/d1/types.ts +11 -0
  35. package/modules/import-kit/src/door/ADOPTING.md +0 -37
  36. package/modules/import-kit/src/door/README.md +0 -66
  37. package/modules/import-kit/src/door/admission.ts +3 -13
  38. package/modules/import-kit/src/door/authority.ts +0 -1
  39. package/modules/import-kit/src/door/failures.ts +1 -1
  40. package/modules/import-kit/src/door/grants.ts +5 -19
  41. package/modules/import-kit/src/door/handler.ts +1 -12
  42. package/modules/import-kit/src/door/index.ts +0 -6
  43. package/modules/import-kit/src/door/planning.ts +19 -44
  44. package/modules/import-kit/src/door/protocol.ts +2 -4
  45. package/modules/import-kit/src/door/schema.ts +0 -2
  46. package/modules/import-kit/src/door/store.ts +2 -2
  47. package/modules/import-kit/src/door/types.ts +1 -24
  48. package/modules/import-kit/src/provenance/credentials.ts +14 -5
  49. package/modules/import-kit/src/status/evidence.ts +3 -1
  50. package/modules/import-kit/src/status/index.ts +17 -2
  51. package/modules/import-kit/src/status/schema.ts +8 -0
  52. package/modules/import-kit/src/status/types.ts +25 -4
  53. package/package.json +1 -1
  54. package/cli/business/archive-task.mjs +0 -32
  55. package/cli/business/bounded-waves.d.mts +0 -23
  56. package/cli/business/bounded-waves.mjs +0 -57
  57. package/cli/business/catchup-task.mjs +0 -49
  58. package/cli/business/export-inputs.mjs +0 -79
  59. package/cli/business/export-task.mjs +0 -24
  60. package/cli/business/legacy-package.mjs +0 -117
  61. package/cli/business/package-transfer.mjs +0 -127
  62. package/cli/business/setup-demo.mjs +0 -95
  63. package/cli/business/vendor-door.mjs +0 -303
  64. package/cli/business/vendors/README.md +0 -215
  65. package/cli/business/vendors/calendly.mjs +0 -80
  66. package/cli/business/vendors/common.mjs +0 -280
  67. package/cli/business/vendors/credential-names.mjs +0 -124
  68. package/cli/business/vendors/index.mjs +0 -96
  69. package/cli/business/vendors/pagerduty.mjs +0 -61
  70. package/cli/business/vendors/sentry.mjs +0 -85
  71. package/cli/business/vendors/sorry.mjs +0 -106
  72. package/cli/business/vendors/uptimerobot.mjs +0 -21
  73. package/cli/business/vendors/zendesk-attachments.mjs +0 -57
  74. package/cli/business/vendors/zendesk-encode.mjs +0 -133
  75. package/cli/business/vendors/zendesk-guide.mjs +0 -34
  76. package/cli/business/vendors/zendesk-package.mjs +0 -28
  77. package/cli/business/vendors/zendesk-pages.mjs +0 -61
  78. package/cli/business/vendors/zendesk-people.mjs +0 -7
  79. package/cli/business/vendors/zendesk-store.mjs +0 -154
  80. package/cli/business/vendors/zendesk-tickets.mjs +0 -21
  81. package/cli/business/vendors/zendesk-token.mjs +0 -31
  82. package/cli/business/vendors/zendesk-transport.mjs +0 -48
  83. package/cli/business/vendors/zendesk.mjs +0 -61
  84. package/modules/import-kit/src/door/package.ts +0 -121
  85. package/modules/import-kit/src/door/waves/README.md +0 -158
  86. package/modules/import-kit/src/door/waves/digests.ts +0 -38
  87. package/modules/import-kit/src/door/waves/inventory.ts +0 -84
  88. package/modules/import-kit/src/door/waves/journal.ts +0 -19
  89. package/modules/import-kit/src/door/waves/lifecycle.ts +0 -242
  90. package/modules/import-kit/src/door/waves/preparation.ts +0 -190
  91. package/modules/import-kit/src/door/waves/schema.ts +0 -73
  92. package/modules/import-kit/src/door/waves/store.ts +0 -112
  93. package/modules/import-kit/src/door/waves/types.ts +0 -76
@@ -1,303 +0,0 @@
1
- import { buildPackage, transferPackage } from './package-transfer.mjs';
2
- import { registeredLegacyPackage } from './legacy-package.mjs';
3
- import { prepareZendeskPackage } from './vendors/zendesk-package.mjs';
4
- import { createHash } from 'node:crypto';
5
- import { constants } from 'node:fs';
6
- import { open, unlink } from 'node:fs/promises';
7
- import { join } from 'node:path';
8
- import { loadState, saveState } from './state.mjs';
9
- import { snapshot as zendeskSnapshot, storePath as zendeskStorePath, removeZendeskStores } from './vendors/zendesk.mjs';
10
- import { taskPath } from './api.mjs';
11
- import { assertChunk } from './outbound.mjs';
12
- import { createCredentialStore } from './keychain.mjs';
13
- import { exportVendor, VENDORS, KEYED_VENDORS, VENDOR_CAPABILITIES, VendorExportError } from './vendors/index.mjs';
14
- import { exportInputs, inputNeeds } from './export-inputs.mjs';
15
- import { businessCommand } from './invocation.mjs';
16
-
17
- // What every vendor task that writes into an app's import door shares
18
- // (modules/import-kit/src/door): the claimed grant's door, one RER-signed grant
19
- // per door request, the owner's read-only key from OS credential storage, the
20
- // shipped adapter into a private 0600 bundle, and 64 KiB chunks.
21
-
22
- // The door's upload unit ("Requests"): every chunk is exactly 64 KiB except the last.
23
- const CHUNK_BYTES = 65_536;
24
- const DOOR_RESPONSE_BYTES = 4 * 1024 * 1024;
25
- // The catalogue names the platform builds vendor task titles from.
26
- export const vendorNames = Object.freeze({ sentry: 'Sentry', uptimerobot: 'UptimeRobot', pagerduty: 'PagerDuty', calendly: 'Calendly', sorry: 'Sorry', zendesk: 'Zendesk', 'happytohelp-legacy': 'HappyToHelp legacy' });
27
- /** Whether a vendor's export needs the owner's read-only key (Sorry's public page does not). */
28
- export const keyed = vendor => KEYED_VENDORS.includes(vendor);
29
- export const vendorHandle = vendor => `vendor:${vendor}:read`;
30
- export const hash = bytes => createHash('sha256').update(bytes).digest('hex');
31
- const loopback = hostname => ['localhost', '127.0.0.1', '[::1]'].includes(hostname);
32
- // The door identifiers Business accepts in a task's outputs (platform business/imports.ts).
33
- export const DOOR_ID = /^[0-9A-Za-z][0-9A-Za-z_-]{0,127}$/;
34
- export const count = value => Number.isSafeInteger(value) && value >= 0 ? value : null;
35
- export const plural = (value, word) => `${value} ${word}${value === 1 ? '' : 's'}`;
36
-
37
- /** A vendor task that cannot go on until someone else acts. The supervisor tells
38
- * Business with `question`, hands the lease back as paused and says `message`. */
39
- export class ExportWaiting extends Error {
40
- constructor(message, question, { keyMissing = null } = {}) { super(message); this.question = question; this.keyMissing = keyMissing; }
41
- }
42
-
43
- // The platform's task titles (business/task-titles.ts) for the kinds that name a vendor.
44
- const titles = {
45
- 'export.vendor': name => `Export ${name} history`,
46
- 'catchup.vendor': name => `Bring over recent ${name} changes`,
47
- 'switchoff.archive': name => `Archive ${name} before switching it off`,
48
- 'freeze.vendor': name => `Freeze ${name} before the switch`,
49
- 'switchoff.cancel': name => `Cancel ${name}`,
50
- };
51
- /** The shipped vendor a task names, read from its catalogue title
52
- * (`Export UptimeRobot history`), or null. Run trusts only the grant's namespace. */
53
- export function titledVendor(kind, title) {
54
- if (!Object.hasOwn(titles, kind)) return null;
55
- return VENDORS.find(vendor => title === titles[kind](vendorNames[vendor])) ?? null;
56
- }
57
-
58
- const keyCommand = vendor => businessCommand(`vendor-key add ${vendor ?? '<vendor>'}`);
59
- /** The owner-typed step that keeps a vendor's read-only key on this computer. */
60
- export function keyStep(vendor) {
61
- return `ask the owner to type this in their own terminal, outside any agent sandbox (it asks for the key with hidden input; never put a key in chat, a command or a file):\n ${keyCommand(vendor)}`;
62
- }
63
-
64
- const unavailableWords = {
65
- 'vendor-not-in-catalogue': 'this tool has no shipped export',
66
- 'no-linked-installation': 'Business has not set up the app this tool moves into',
67
- 'ambiguous-target-installation': 'more than one app could receive this export',
68
- 'installation-import-origin-unavailable': "the app's address is not known yet",
69
- 'owner-claim-keys-unconfigured': 'Business cannot sign import grants yet',
70
- 'import-door-off': "the app's import door is off",
71
- 'installation-owner-claim-unconfigured': "the app's import door does not trust Business yet",
72
- 'door-trust-keys-stale': "the app's import door holds old Business keys",
73
- 'owner-not-claimed': 'the owner has not claimed the app yet',
74
- 'connected-seed-import-required': "the app's import door is not ready",
75
- };
76
-
77
- /** The claimed grant's door: its origin and byte quota. */
78
- function destination(task, grant) {
79
- if (grant.available === false || typeof grant.destination === 'string' && grant.destination.startsWith('unavailable:')) {
80
- const reason = Object.hasOwn(unavailableWords, grant.reason) && grant.destination === `unavailable:${grant.reason}` ? grant.reason : 'connected-seed-import-required';
81
- throw new ExportWaiting(`Business vendor export unavailable: ${reason} (${unavailableWords[reason]}). Task released as paused.`, 'connected_runner_required');
82
- }
83
- let url;
84
- try { url = new URL(grant.destination); } catch { throw Error('Business returned an invalid import destination.'); }
85
- if (!(url.protocol === 'https:' || url.protocol === 'http:' && loopback(url.hostname)) || url.username || url.password || url.search || url.hash
86
- || url.pathname !== '/api/rer/import/uploads' || grant.import_grants !== taskPath(task.id, 'import-grants')
87
- || !Number.isSafeInteger(grant.quota_bytes) || grant.quota_bytes < 1 || grant.quota_bytes > 300_000_000) throw Error('Business returned an invalid import destination.');
88
- return { origin: url.origin, quota: grant.quota_bytes };
89
- }
90
-
91
- /** Inputs the shipped adapter needs beyond the key: the leased task's own
92
- * non-secret `export_inputs`. A task that does not carry them yet waits for
93
- * Business (which asks the owner when its evidence is missing). */
94
- function adapterInputs(vendor, task) {
95
- const inputs = exportInputs(vendor, task);
96
- if (inputs) return inputs;
97
- const message = vendor === 'calendly' ? 'Waiting for the owner: this tool needs its folder of files shared with Business.'
98
- : `Waiting for the owner to confirm ${inputNeeds[vendor]} in Business.`;
99
- throw new ExportWaiting(message, 'configuration_required');
100
- }
101
-
102
- /** One RER-signed import grant for one door request. `review` asks for a
103
- * catch-up plan grant without the owner's standing approval; a plan grant
104
- * Business minted under that approval says so with `catchup: 'bounded'`. */
105
- async function doorGrant(lease, action, origin, vendor, { review = false, bounded } = {}) {
106
- const grant = await lease.importGrant(action, { ...(review ? { review: true } : {}), ...(bounded ? { bounded } : {}) });
107
- if (!grant || grant.action !== action || grant.destination !== new URL(`/api/rer/import/${action}`, origin).href
108
- || typeof grant.authorization !== 'string' || grant.authorization.length > 8192 || !/^Bearer [A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+$/.test(grant.authorization)
109
- || !VENDORS.includes(grant.namespace) || vendor && grant.namespace !== vendor || !(Date.parse(grant.expires_at) > Date.now())
110
- || Object.hasOwn(grant, 'catchup') && (grant.catchup !== 'bounded' || !['plan','prepare'].includes(action) || review)) throw Error('Business returned an invalid import grant.');
111
- return grant;
112
- }
113
-
114
- /** POST one request to the app's door. Only the door's error code is ever
115
- * shown: its bodies can hold vendor records. */
116
- export async function door(grant, body, signal) {
117
- const action = grant.action;
118
- let response;
119
- try {
120
- response = await fetch(grant.destination, { method: 'POST', redirect: 'error', body: JSON.stringify(body),
121
- headers: { authorization: grant.authorization, 'content-type': 'application/json' }, signal: AbortSignal.any([signal, AbortSignal.timeout(30_000)]) });
122
- } catch (error) {
123
- if (signal.aborted) throw signal.reason;
124
- const code = error?.cause?.code ?? error?.code;
125
- throw Error(`Could not reach the app's import door at ${new URL(grant.destination).origin} (${error?.name === 'TimeoutError' ? 'no answer within 30 seconds' : typeof code === 'string' && /^[A-Z_]{2,40}$/.test(code) ? code : 'the request failed'}).`);
126
- }
127
- const chunks = []; let size = 0;
128
- for await (const chunk of response.body ?? []) {
129
- size += chunk.byteLength;
130
- if (size > DOOR_RESPONSE_BYTES) { await response.body.cancel().catch(() => {}); throw Error(`The app's import door answered the ${action} request with more than the CLI reads.`); }
131
- chunks.push(chunk);
132
- }
133
- let result = null;
134
- try { result = JSON.parse(Buffer.concat(chunks).toString('utf8')); } catch { /* Judged below. */ }
135
- if (!response.ok) {
136
- // A door code is snake_case, sometimes with one named reason (`catchup_requires_proposal:source_records_gone`).
137
- const doorCode = typeof result?.error === 'string' && /^[a-z][a-z0-9_]{0,79}(?::[a-z][a-z0-9_]{0,79})?$/.test(result.error) ? result.error : null;
138
- throw Object.assign(Error(`The app's import door refused the ${action} request (HTTP ${response.status}${doorCode ? `: ${doorCode}` : ''}).`), { doorStatus: response.status, doorCode, ...(doorCode === 'package_segment_missing' ? { doorDetails: result.details } : {}) });
139
- }
140
- if (!result || typeof result !== 'object' || Array.isArray(result)) throw Error(`The app's import door returned an invalid ${action} answer.`);
141
- return result;
142
- }
143
-
144
- /** Run the shipped adapter into a private 0600 bundle file, checking every
145
- * item with the outbound detector before it is written. */
146
- async function exportBundle({ vendor, key = null, inputs, path, quota, signal, directory, runId, business }) {
147
- await unlink(path).catch(error => { if (error.code !== 'ENOENT') throw error; });
148
- const file = await open(path, constants.O_WRONLY | constants.O_CREAT | constants.O_EXCL | constants.O_NOFOLLOW, 0o600);
149
- const digest = createHash('sha256'); let size = 0, zendeskStore;
150
- try {
151
- let items;
152
- if (vendor === 'zendesk') {
153
- const root = zendeskStorePath(directory, business, `https://${inputs.account.subdomain}.zendesk.com`);
154
- const result = await zendeskSnapshot({ key, inputs, directory: inputs.mode === 'census' ? join(root, 'probe') : root, runId, signal });
155
- if (result.kind === 'census') throw new ExportWaiting(`Zendesk census: ${result.census.tickets.unique} tickets; ${result.census.unswept} unswept. CENSUS_OUTPUTS_UNSUPPORTED: waiting for Business census task outputs.`, 'connected_runner_required');
156
- if (result.kind === 'package') return { package: result, store: result.store, mode: inputs.mode };
157
- zendeskStore = result.store;
158
- items = [{ bytes: result.bytes }];
159
- } else items = exportVendor(vendor, { key, inputs, signal });
160
- for await (const item of items) {
161
- assertChunk(item.bytes, key ? [key] : []);
162
- size += item.bytes.length;
163
- if (size > quota) throw Error(`Business vendor export is larger than the app's import door accepts (${quota} bytes). Nothing was uploaded.`);
164
- digest.update(item.bytes);
165
- await file.write(item.bytes);
166
- }
167
- await file.sync();
168
- } catch (error) {
169
- if (!(error instanceof VendorExportError)) throw error;
170
- if (error.code === 'HTTP_REFUSED' && [401, 403].includes(error.status) && key) {
171
- throw new ExportWaiting(`${vendorNames[vendor]} refused the stored read-only key (HTTP ${error.status}). To replace it, ${keyStep(vendor)}`, 'owner_required');
172
- }
173
- throw Error(`${vendorNames[vendor]} export stopped (${error.code}${error.status ? `, HTTP ${error.status}` : ''}). Nothing was uploaded.`);
174
- } finally { await file.close(); }
175
- if (!size) throw Error(`${vendorNames[vendor]} export produced nothing to upload.`);
176
- return { size, sha256: digest.digest('hex'), ...(zendeskStore ? { store: zendeskStore, mode: inputs.mode } : {}) };
177
- }
178
-
179
- async function readChunk(path, index, size) {
180
- const file = await open(path, constants.O_RDONLY | constants.O_NOFOLLOW);
181
- try {
182
- const length = Math.min(CHUNK_BYTES, size - index * CHUNK_BYTES), bytes = Buffer.alloc(length);
183
- const { bytesRead } = await file.read(bytes, 0, length, index * CHUNK_BYTES);
184
- if (bytesRead !== length) throw Error('Business vendor export bundle changed while uploading.');
185
- return bytes;
186
- } finally { await file.close(); }
187
- }
188
-
189
- /** A door plan's preview of exactly this bundle: its ids and the door's counts. */
190
- export function checkedPreview(preview, bundle) {
191
- const summary = preview.summary ?? {}, [records, created, updated, unchanged, gone] = ['sourceRecords', 'created', 'updated', 'unchanged', 'gone'].map(name => count(summary[name]));
192
- const rejected = count(preview.counts?.rejected);
193
- if (typeof preview.planId !== 'string' || !DOOR_ID.test(preview.planId) || !/^[a-f0-9]{64}$/.test(preview.fingerprint ?? '')
194
- || preview.bundleSha256 !== bundle.sha256 || [records, created, updated, unchanged, gone, rejected].includes(null)) throw Error("The app's import door returned an invalid preview.");
195
- return { records, created, updated, unchanged, gone, rejected };
196
- }
197
-
198
- /**
199
- * The shared start of a claimed vendor door task: the claimed grant's door, the
200
- * vendor its first grant names, the owner's key from the OS store, the shipped
201
- * adapter into a private 0600 bundle, then the bundle in 64 KiB chunks, each with
202
- * a fresh grant. A vendor that needs no key (a public status page) reads none.
203
- * `finish` gets the uploaded bundle and returns `{outputs, said}`:
204
- * the task then completes with those outputs (identifiers only, which Business
205
- * checks against the door itself) and `said` is the line to say. The bundle file
206
- * is removed on every exit. `doing(name, origin, keyed)` is said first.
207
- */
208
- export async function withUploadedBundle({ lease, task, grant: claimedGrant, directory, say, doing }, finish) {
209
- const signal = lease.signal, { origin, quota } = destination(task, claimedGrant);
210
- // The first grant names the vendor (its namespace); it then carries chunk 0.
211
- let spare = await doorGrant(lease, 'uploads', origin, null);
212
- const vendor = spare.namespace, name = vendorNames[vendor], capability = VENDOR_CAPABILITIES[vendor];
213
- const fresh = async (action, options) => {
214
- if (action === 'uploads' && spare && Date.parse(spare.expires_at) - Date.now() > 60_000) { const grant = spare; spare = null; return grant; }
215
- spare = null;
216
- return doorGrant(lease, action, origin, vendor, options ?? (task.bounded_waves && action==='prepare' ? {bounded:{protocol:task.bounded_waves.protocol,migration:task.bounded_waves.migration,revision:task.bounded_waves.revision,policy:task.bounded_waves.policy}} : undefined));
217
- };
218
- const inputs = adapterInputs(vendor, task);
219
- const state = vendor === 'zendesk' ? await loadState() : null;
220
- if (vendor === 'zendesk' && inputs.mode === 'retire') {
221
- // The retire signal is local revocation, never an OAuth/provider write.
222
- await createCredentialStore().delete(vendorHandle(vendor));
223
- await removeZendeskStores(directory);
224
- await saveState({ ...state, vendorHandles: (state.vendorHandles ?? []).filter(handle => handle !== vendorHandle(vendor)), vendorRevocations: { ...state.vendorRevocations, zendesk: true } });
225
- throw new ExportWaiting('Zendesk retired locally (VENDOR_REVOKED). RETIRE_OUTPUTS_UNSUPPORTED: waiting for Business retire task acknowledgement.', 'connected_runner_required');
226
- }
227
- if (vendor === 'zendesk' && state?.vendorRevocations?.zendesk) throw new VendorExportError('VENDOR_REVOKED');
228
- let key = keyed(vendor) ? await createCredentialStore().get(vendorHandle(vendor)) : null;
229
- if (keyed(vendor) && !key) throw new ExportWaiting(`Business vendor export needs the owner's read-only ${name} key on this computer: ${keyStep(vendor)}\nThen run ${businessCommand('run')} again.`, 'owner_required', { keyMissing: vendor });
230
- if (vendor === 'happytohelp-legacy') {
231
- if (!['export.vendor', 'catchup.vendor'].includes(task.kind)) throw new ExportWaiting('Legacy package tasks require a package-aware export or catch-up reader.', 'configuration_required');
232
- let legacy;
233
- try { legacy = await registeredLegacyPackage(); }
234
- catch (error) { throw new ExportWaiting(error.message, 'configuration_required'); }
235
- say(doing(name, origin, false));
236
- const packageDirectory = join(directory, `package-${hash(task.id).slice(0, 32)}`);
237
- let prepared;
238
- try {
239
- prepared = await buildPackage({ files: legacy.files, quota, directory: packageDirectory, signal,
240
- source: { format: legacy.manifest.format, version: 1, origin: 'legacy-database', exportedAt: legacy.manifest.exportedAt ?? legacy.manifest.source.exportedAt ?? null,
241
- synthetic: legacy.manifest.source.synthetic === true, adapter: 'rer:happytohelp-legacy' },
242
- records: { total: null, byKind: {} }, totals: { rowCounts: legacy.rowCounts, objects: legacy.objects } });
243
- } catch { throw new ExportWaiting(`Legacy package changed or could not be prepared: ${legacy.path}. Recheck and register that package again.`, 'configuration_required'); }
244
- let uploaded;
245
- try { uploaded = await transferPackage({ prepared, fresh, request: door, signal, checkpoint: cp => lease.checkpointSaved(cp), catchup: task.kind === 'catchup.vendor', ...(task.bounded_waves ? { planAction:'prepare', planInput:{ordinal:0} } : {}) }); }
246
- catch (error) { if (!error.message.startsWith('SEED_PACKAGE_UNSUPPORTED')) throw error; throw new ExportWaiting(error.message, 'configuration_required'); }
247
- const { outputs, said } = await finish({ vendor, name, origin, ...uploaded, fresh });
248
- await lease.complete(outputs);
249
- return said;
250
- }
251
- const path = join(directory, `export-${hash(task.id).slice(0, 32)}.bundle`);
252
- try {
253
- say(doing(name, origin, keyed(vendor)));
254
- let bundle;
255
- try { bundle = await exportBundle({ vendor, key, inputs, path, quota, signal, directory, runId: task.id, business: state?.businessId }); } finally { key = undefined; }
256
- if (bundle.package) {
257
- const prepared = await prepareZendeskPackage({ result: bundle.package, quota, directory: join(bundle.store.path, `package-${hash(task.id).slice(0,32)}`), signal, mode: inputs.mode });
258
- const uploaded = await transferPackage({ prepared, fresh, request: door, signal, checkpoint: cp => lease.checkpointSaved(cp), catchup: task.kind === 'catchup.vendor' || inputs.mode === 'sweep' });
259
- const finished = await finish({ vendor, name, origin, ...uploaded, fresh });
260
- await lease.complete(finished.outputs);
261
- return finished.said;
262
- }
263
- const total = Math.ceil(bundle.size / CHUNK_BYTES);
264
- let uploadId;
265
- const received = new Set();
266
- const journal = bundle.store?.events.findLast(event => event.kind === 'transfer');
267
- if (journal && journal.origin === origin && journal.bundleSha256 === bundle.sha256 && journal.totalBytes === bundle.size) {
268
- try {
269
- const inventory = await door(await fresh('uploads'), { uploadId: journal.uploadId }, signal);
270
- if (inventory.uploadId !== journal.uploadId || inventory.bundleSha256 !== bundle.sha256 || !Array.isArray(inventory.received)) throw new VendorExportError('INVALID_CHECKPOINT');
271
- for (const item of inventory.received) {
272
- if (!Number.isSafeInteger(item.ordinal) || item.ordinal < 0 || item.ordinal >= total || received.has(item.ordinal)) throw new VendorExportError('INVALID_CHECKPOINT');
273
- const bytes = await readChunk(path, item.ordinal, bundle.size);
274
- if (item.sha256 !== hash(bytes) || item.size !== bytes.length) throw new VendorExportError('INVALID_CHECKPOINT');
275
- received.add(item.ordinal);
276
- }
277
- uploadId = journal.uploadId;
278
- } catch (error) { if (error.doorStatus !== 404) throw error; }
279
- }
280
- for (let index = 0; index < total; index++) {
281
- if (received.has(index)) continue;
282
- signal.throwIfAborted();
283
- const bytes = await readChunk(path, index, bundle.size);
284
- const result = await door(await fresh('uploads'), { ...(uploadId ? { uploadId } : {}), name: capability.output, bundleSha256: bundle.sha256,
285
- totalBytes: bundle.size, totalChunks: total, index, chunkSha256: hash(bytes), base64: bytes.toString('base64') }, signal);
286
- if (typeof result.uploadId !== 'string' || !DOOR_ID.test(result.uploadId) || uploadId && result.uploadId !== uploadId
287
- || result.bundleSha256 !== bundle.sha256 || index === total - 1 && result.complete !== true) throw Error("The app's import door returned an invalid upload answer.");
288
- uploadId = result.uploadId;
289
- if (bundle.store) {
290
- const transfer = { kind: 'transfer', origin, bundleSha256: bundle.sha256, totalBytes: bundle.size, uploadId };
291
- await bundle.store.commit(bundle.store.state, transfer);
292
- await bundle.store.write('transfer.json', transfer);
293
- }
294
- await lease.checkpointSaved({ page: index + 1, total, bytes_done: Math.min((index + 1) * CHUNK_BYTES, bundle.size), progress: Math.floor((index + 1) / total * 90) });
295
- }
296
- const { outputs, said } = await finish({ vendor, name, origin, uploadId, bundle, fresh });
297
- await lease.checkpointSaved({ page: total, total, bytes_done: bundle.size, progress: 100 });
298
- await lease.complete(outputs);
299
- return said;
300
- } finally {
301
- await unlink(path).catch(error => { if (error.code !== 'ENOENT') throw error; });
302
- }
303
- }
@@ -1,215 +0,0 @@
1
- # Shipped vendor exports
2
-
3
- To bring existing data into a seed, sign in as the owner and use its import
4
- screen. First step: open this path on the destination app:
5
-
6
- ```text
7
- /app/import
8
- ```
9
-
10
- Upload a supported source export, review the plan and its differences, then
11
- approve that exact plan to apply it. Reconcile the result against native data;
12
- use Undo from the same history when needed. Base's reference accepts people JSON
13
- here, not installation `.rer` archives. Editions adopt this same door
14
- with their own supported readers and native adapters.
15
-
16
- `exportVendor(vendor, {key, inputs, checkpoint, signal})` is an async generator.
17
- It yields `{name, bytes: Buffer, checkpoint}`. Keys remain in memory and are
18
- attached only to GET requests on the adapter's fixed HTTPS origin. Sorry reads a
19
- public status page and takes no key (`key` must be absent); `KEYED_VENDORS` lists
20
- the vendors that need one. Redirects,
21
- caller origins, arbitrary paths and credential-bearing pagination are refused.
22
- No export runs merely by importing these modules. No account export was run
23
- during implementation.
24
-
25
- `VENDORS` is the closed vendor enum (`sentry`, `uptimerobot`, `pagerduty`,
26
- `calendly`, `sorry`, `zendesk`, `happytohelp-legacy`). `VENDOR_CAPABILITIES` describes the adapter,
27
- required vendor-key capability, native importer format, layout and read scopes.
28
- Except for Zendesk's checked OAuth scopes, adapters cannot verify that a supplied key itself has read-only scope;
29
- the owner must create the documented read-only key where the vendor offers one.
30
-
31
- ## Transport and resume
32
-
33
- Upload bytes only to the task grant's seed destination. They contain private,
34
- untrusted vendor data and must never appear in CLI output, events or model
35
- context. The caller applies its outbound secret detector and computes SHA-256,
36
- enforces the grant quota, and atomically acknowledges the item before persisting
37
- its checkpoint. Checkpoints contain progress, source IDs, cursor strings, file
38
- metadata and hashes, never response records. They are less than 128 KiB.
39
-
40
- An interrupted upload is retried under the same task/fence, deterministic name
41
- and SHA-256. Resume with exactly the acknowledged checkpoint and original
42
- inputs; object key ordering does not affect input identity. An unacknowledged
43
- page may be fetched again, so source edits during interruption can change it:
44
- the seed must reject a conflicting hash under the same name, and the owner must
45
- restart that export. This is an API export, not a vendor snapshot transaction.
46
-
47
- For `layout: 'concatenate'`, concatenate the numbered `000001.part` items in
48
- sequence to the metadata's `output` filename. Some PagerDuty items have empty
49
- bytes: they checkpoint schedule discovery and must still be acknowledged.
50
- Fragments are deliberately not standalone JSON. For `layout: 'files'`, retain
51
- the given filenames; Calendly ends with `manifest.json`, matching Dial's folder
52
- import. Never treat a partial stream as a completed bundle.
53
-
54
- Every item is at most 1 MiB. Exports are bounded to 1,000 requests, one request
55
- per second, 30 seconds per request, 2 MiB/499 API records for Sentry,
56
- 512 KiB/500 records for PagerDuty, and 8 MiB/2,500 records for UptimeRobot and
57
- Calendly. Sorry follows Lamp's reader: at most 200 requests (one saved file
58
- each), one every 125 ms, 8 MiB of responses, 5,000 records and a 16 MiB bundle. The output bound includes JSON framing and Calendly's manifest.
59
- Limits stop rather than truncate. AbortSignal stops waits, HTTP reads and
60
- emission. HTTP errors contain only a fixed code and numeric status; network,
61
- parsing and unexpected errors never expose response text, URLs or key values.
62
- 429 stops with `RATE_LIMITED`; there is no hidden retry loop.
63
-
64
- The Business CLI supervisor currently captures a complete private bundle before
65
- uploading it. After a lease handover, its next run captures and uploads the export
66
- again under a fresh lease; it does not resume a partially uploaded bundle in
67
- place. Only a completed upload and the door's matching plan are reported as a
68
- preview. Calendly's folder layout remains outside this single-bundle path.
69
-
70
- ## Credentials in responses
71
-
72
- Every adapter strips credential values from a parsed response before it emits
73
- bytes for upload, keeping the record, by the same rules as the seed's import
74
- door. A field whose name's parts are a credential word or phrase (`apiKey`,
75
- `httpPassword`, `http_password`, `client_secret`, `X-Api-Key`, `integration_key`,
76
- `webhook_url`, `secret_key`, `x-functions-key`, `Ocp-Apim-Subscription-Key`,
77
- `pwd`, `X-Auth` and the like), the `value` of a `{name|key, value}` pair
78
- naming one (`{name: 'Authorization', value}`), or the value of such a
79
- `[name, value]` pair in a list of them (Sentry's header tuples), becomes
80
- `"[removed]"`. A plural name (`apiKeys`, `tokens`) counts only when it holds
81
- text, so a count such as `tokens: 1500` stays. A URL of any scheme keeps a
82
- marker where each credential was: its user name and password become `removed@`,
83
- and the value of each credential query or fragment parameter (`api_key`,
84
- `access_token`, `sig` and the like) becomes `removed`
85
- (`?api_key=removed&region=eu`, `#access_token=removed`), except the adapter's
86
- own validated pagination keys (`page_token`). The door drops each marker and
87
- reports it per record as `CREDENTIAL_REMOVED`, so the owner sees which record
88
- lost a credential. Null, booleans, empty strings and masks such as `****` are
89
- not credentials and stay as they are. Names are matched by whole parts
90
- (camelCase, `_`, `-`, digits), so names that merely contain such letters
91
- (`keyword`, `monkey`, `author`, `signal`, `design`, `primaryKey`) are not
92
- credential names and a `?keyword=up` URL stays intact. `Signature` and
93
- `Session-Id` count only as header names (a key under `headers`, a tuple, a
94
- `{name|key, value}` pair in a header list), so an agent's `signature` field and
95
- Help Scout's `customFields: [{name: 'Session ID', value}]` stay;
96
- `auth` and `credentials` count only when they hold text naming no scheme or mode,
97
- and an object under them loses only its own credential fields
98
- (`auth: {type: 'oauth2', token: "[removed]"}`) and a `value`, `pass`, `pw` or
99
- `secret…` member (Keycloak's `credentials: [{type: 'password', value: "[removed]"}]`).
100
- `passphrase`, `jwt`, `JSESSIONID` and `PHPSESSID` count everywhere. Query and fragment keys also
101
- count documented API key names such as `token_auth`, `application_key`,
102
- `user_key` and `hapikey`.
103
- Pagination is read from the response as received, so a credential-bearing
104
- pagination link is still refused.
105
-
106
- ## Inputs and native coverage
107
-
108
- * `sentry`: `organization: {id, slug, region: 'us'|'de'}`, `start`, `end`, and
109
- optional `events: [{project, eventId}]`. Self-hosted origins are unsupported.
110
- Projects, releases and all-status issues are paginated; issues use the current
111
- organization issues API filtered by project and window. Event details are
112
- fetched only for the explicitly selected IDs, must fall in the window and
113
- form a complete local cursor chain. A fixed omissions list declares that all
114
- other event history and unsupported Sentry features remain outside the export.
115
- Projects/releases are not time-filtered. Spool counts one manifest record in
116
- addition to the API records. At most 49 projects fit its 100-collection bound.
117
- * `uptimerobot`: no additional inputs. Exports v3 monitor response pages,
118
- including their original `data` and `nextLink` envelopes. History, contacts
119
- and status pages are not fetched.
120
- * `pagerduty`: `start`/`end` (or `since`/`until`). Exports users, schedule details
121
- with layers/final coverage, overrides for the window and escalation policies.
122
- Schedule listing only discovers IDs and is not duplicated in the bundle.
123
- Oncall compares a schedule's final timeline over at most 31 days, so the
124
- Business CLI passes a window of at most 31 whole UTC days ending two weeks
125
- ahead.
126
- Overrides retain full boundaries with `overflow=true` and add `schedule_id`,
127
- as Oncall's documented bundle requires. Incident history and paging
128
- configuration are not fetched.
129
- * `sorry`: `page: {host}`, the public status page's host name (a public DNS
130
- name; no scheme, port, path, IP address or local name). Reads, anonymously,
131
- `/api/v1/status`, every `components` and `notices` page (`meta.next_page`) and
132
- each notice's detail with its components and full update history, like Lamp's
133
- public Move in. The bundle is Lamp's door `sorry` source: `{source: 'sorry',
134
- publicUrl, publicAccessAcknowledged: true, files: [{name, text}]}`, each file
135
- the original response text under the name Lamp's reader requires
136
- (`sorry-status.json`, `sorry-components[-N].json`, `sorry-notices[-N].json`,
137
- `sorry-notice-<id>.json`). Every response came from the page's own public API,
138
- so it is public by construction; Lamp still refuses private or
139
- audience-scoped records, and public responses carry no subscribers.
140
- * `calendly`: optional `asOf` timestamp (recommended for a repeatable catch-up).
141
- An omitted timestamp is captured at first invocation and retained on resume.
142
- Downloads the current user, event types, availability schedules, per-type
143
- availability, future scheduled events including cancellations, and invitees.
144
- Like Dial's downloader, this is the current user's scope, at most 100 event
145
- types and 500 scheduled events. Every original page receives a manifest hash.
146
- No past booking history or arbitrary referenced resource is fetched.
147
-
148
- ## Primary-source provenance
149
-
150
- Implementation preserves the response formats and flow in these seed sources;
151
- it adds streaming, bounded resume and fixed-origin transport rather than running
152
- their file-writing command wrappers.
153
-
154
- * [Dial downloader at 6014c99875dd002545e194ab23a7852d30e94e01](https://github.com/just-every/run-dial/blob/6014c99875dd002545e194ab23a7852d30e94e01/scripts/move-in/download.mjs).
155
- * [Uptime guide at 157686b52ecc652d64250a30ec224f1a47355a6c](https://github.com/just-every/run-uptime/blob/157686b52ecc652d64250a30ec224f1a47355a6c/docs/moving-from-uptimerobot.md).
156
- * [Lamp's Sorry public reader at a0179bfe](https://github.com/just-every/run-lamp/blob/a0179bfe/src/core/features/statuspage-import/sorry/fetch.ts)
157
- and its door source (`door-plan.ts` `doorExport`).
158
- * [Oncall guide at 4f928effc3ce735f3d246bb8a3c4bbb8436cae6b](https://github.com/just-every/run-oncall/blob/4f928effc3ce735f3d246bb8a3c4bbb8436cae6b/docs/move-in.md).
159
- * Spool's proposal-cited `app/docs/SENTRY-SWITCHING.md` and
160
- `app/src/domain/sentry-import/source.ts`, local source commit
161
- `39cbaf67de3bc8174bf941a6922bb92ff31110de`. Spool supplied a bundle contract,
162
- not an existing downloader. This adapter implements that contract against
163
- the official APIs below.
164
- * Sentry: [projects](https://docs.sentry.io/api/organizations/list-an-organizations-projects/),
165
- [releases](https://docs.sentry.io/api/releases/list-an-organizations-releases/),
166
- [organization issues](https://docs.sentry.io/api/events/list-an-organizations-issues/),
167
- [event details](https://docs.sentry.io/api/events/retrieve-an-event-for-a-project/),
168
- [pagination](https://docs.sentry.io/api/pagination/),
169
- [regional domains](https://docs.sentry.io/api/).
170
- * [PagerDuty overrides](https://docs.pagerduty.com/developer/api/reference/rest/schedules/list-schedule-overrides)
171
- is a window-bounded response, not an offset-paginated list.
172
-
173
- Seed documentation and official APIs were inspected on 16 October 2026.
174
- These sources document shapes; no real-account parity or migration success is
175
- claimed. Parent integration must provide seed assembly and transport under the
176
- actual import grant protocol, plus network-edge acceptance coverage.
177
-
178
- ## Zendesk's retained snapshot
179
-
180
- `zendesk` is a keyed vendor with `layout: 'snapshot'`. Its credential is the
181
- hidden-input JSON client credential documented in [the CLI guide](../../README.md#zendesk-offline-snapshots),
182
- rather than a bare API token. `snapshot({key,inputs,directory,runId,signal})` in
183
- `zendesk.mjs` drives phases A–F into the durable store. `exportVendor` accepts
184
- `directory` and `runId` for this vendor; the Business door provides a directory
185
- keyed by Business and account. It does not apply the older vendors' 2,000-item
186
- or one-MiB item limits. The complete deliverable is checked against HH v1 limits
187
- before any upload. Package-required results use the shared package sender with path bindings and
188
- sidecars. It probes the manifest before sending segments, resumes the transfer
189
- journal and plans one complete snapshot; no per-part plan or fallback is used.
190
-
191
- Responsibilities are separated into token/scope checking, transport, pages,
192
- tickets/comments, people, attachments, Guide, store and HH encoding. Requests
193
- stop at 250,000, responses at 32 MiB, records/blobs at 64 MiB, snapshots at
194
- 32 GiB. The opt-in `pacedClient` in `common.mjs` supports bounded 429 retries
195
- (1–60 seconds, at most three per request and ten minutes per run), rate headers,
196
- and manual attachment redirects. Without rate headers it waits at least six
197
- seconds between incremental requests and 300 ms otherwise. Other vendors keep
198
- `client()` and its previous no-retry behavior.
199
-
200
- Only the account origin receives Authorization. The first attachment URL must
201
- be on that origin's attachment path. One redirect may go to a public HTTPS DNS
202
- host, with no userinfo, IP, explicit port or local/private address, and receives
203
- no credential even when it returns to the account origin. Further redirects
204
- are errors. Pagination is independently checked for account origin, exact
205
- collection and allowed query parameters.
206
-
207
- Checkpoints point at immutable SHA-256 journal nodes and raw objects. Each page
208
- is fsynced before its journal and atomic checkpoint; a kill before acknowledgement
209
- leaves only unreachable objects. Resume validates the reachable graph and
210
- rebuilds last-version records and census from it. Repeated attachment bytes
211
- have one blob. The census declares tombstones, inaccessible/deleted/malware/large
212
- attachments and omissions explicitly. Guide 404 records unavailable coverage.
213
- The JSON census and Guide sidecars stay local and in the package; Business
214
- receives only the door's ids and SHA for a successful bundle. See the CLI guide
215
- for the separate Business census/retire output dependencies.
@@ -1,80 +0,0 @@
1
- import { object, list, id, timestamp, reference, cursor, assertCount, hash, shape, fail, withoutCredentials } from './common.mjs';
2
-
3
- export const origin = 'https://api.calendly.com';
4
- const queries = ['user','count','event_type','min_start_time','page_token'];
5
- export const route = url => /^\/(?:users\/me|event_types|user_availability_schedules|event_type_availability_schedules|scheduled_events(?:\/[a-zA-Z0-9_-]+\/invitees)?)$/u.test(url.pathname) && [...url.searchParams.keys()].every(x => queries.includes(x));
6
- function identity(uri, kind) {
7
- const url = reference(uri, origin, new RegExp(`^/${kind}/[a-zA-Z0-9_-]+$`, 'u'));
8
- return id(url.pathname.split('/').at(-1));
9
- }
10
- function nextToken(body, path, previous) {
11
- object(body.pagination);
12
- const next = body.pagination.next_page;
13
- if (next == null) {
14
- if (body.pagination.next_page_token != null) return cursor(body.pagination.next_page_token);
15
- return null;
16
- }
17
- const url = reference(next, origin, new RegExp(`^${path}$`, 'u'), queries);
18
- const token = cursor(url.searchParams.get('page_token'));
19
- if (!token || token === previous) fail('INVALID_PAGINATION');
20
- return token;
21
- }
22
- export async function* run(ctx) {
23
- const s = ctx.state ?? { phase: 0, user: null, types: [], events: [], index: 0, token: null, seen: [], files: [], asOf: timestamp(ctx.inputs.asOf ?? new Date().toISOString()) };
24
- shape(s, ['phase','user','types','events','index','token','seen','files','asOf']);
25
- assertCount(s.phase, 7); assertCount(s.index, 500); timestamp(s.asOf); cursor(s.token);
26
- if (!Array.isArray(s.types) || s.types.length > 100 || !Array.isArray(s.events) || s.events.length > 500 || !Array.isArray(s.files) || s.files.length > 600 || !Array.isArray(s.seen) || s.seen.length > 600) fail('INVALID_CHECKPOINT');
27
- s.types.forEach(id); s.events.forEach(id); if (s.user !== null) id(s.user);
28
- s.seen.forEach(cursor);
29
- for (const file of s.files) {
30
- shape(file, ['name','resource','page','size','sha256','capturedAt']);
31
- if (typeof file.name !== 'string' || !/^(?:user|event_types|schedules|event_availability|scheduled_events|invitees)-\d{6}\.json$/u.test(file.name) || !['user','event_types','schedules','event_availability','scheduled_events','invitees'].includes(file.resource) || !/^[a-f0-9]{64}$/u.test(file.sha256) || file.capturedAt !== s.asOf) fail('INVALID_CHECKPOINT');
32
- assertCount(file.page, 600); assertCount(file.size, 1024 * 1024);
33
- }
34
- while (s.phase < 7) {
35
- if (s.phase === 3 && s.index >= s.types.length) { s.phase = 4; s.index = 0; }
36
- if (s.phase === 5 && s.index >= s.events.length) s.phase = 6;
37
- if (s.phase === 6) {
38
- s.phase = 7;
39
- yield ctx.item(JSON.stringify({ version: 1, capturedAt: s.asOf, files: s.files }) + '\n', s, 'manifest.json');
40
- break;
41
- }
42
- const resource = ['user','event_types','schedules','event_availability','scheduled_events','invitees'][s.phase];
43
- const path = ['/users/me','/event_types','/user_availability_schedules','/event_type_availability_schedules','/scheduled_events',`/scheduled_events/${s.events[s.index]}/invitees`][s.phase];
44
- const query = new URLSearchParams();
45
- if (s.phase !== 0) query.set('count', '100');
46
- if ([1,2,4].includes(s.phase)) query.set('user', `${origin}/users/${id(s.user)}`);
47
- if (s.phase === 3) query.set('event_type', `${origin}/event_types/${id(s.types[s.index])}`);
48
- if (s.phase === 4) query.set('min_start_time', s.asOf);
49
- if (s.token !== null) query.set('page_token', s.token);
50
- const { body } = await ctx.get(path + (query.size ? `?${query}` : ''));
51
- object(body);
52
- const rows = s.phase === 0 ? [object(body.resource)] : list(body.collection);
53
- if (rows.some(row => Buffer.byteLength(JSON.stringify(row)) > 512 * 1024)) fail('RECORD_BYTE_LIMIT');
54
- ctx.records(rows.length);
55
- if (s.phase === 0) s.user = identity(rows[0].uri, 'users');
56
- if (s.phase === 1 || s.phase === 4) {
57
- const values = s.phase === 1 ? s.types : s.events;
58
- const kind = s.phase === 1 ? 'event_types' : 'scheduled_events';
59
- for (const row of rows) {
60
- const value = identity(row.uri, kind);
61
- if (values.includes(value)) fail('DUPLICATE_ID');
62
- values.push(value);
63
- }
64
- if (values.length > (s.phase === 1 ? 100 : 500)) fail('RECORD_LIMIT');
65
- }
66
- const next = s.phase === 0 ? null : nextToken(body, path, s.token);
67
- if (next !== null && (next === s.token || s.seen.includes(next))) fail('PAGINATION_CYCLE');
68
- const name = `${resource}-${String(s.files.length + 1).padStart(6,'0')}.json`;
69
- const bytes = Buffer.from(JSON.stringify(withoutCredentials(body, queries)) + '\n');
70
- if (s.files.length >= 600) fail('FILE_LIMIT');
71
- s.files.push({ name, resource, page: s.seen.length + 1, size: bytes.length, sha256: hash(bytes), capturedAt: s.asOf });
72
- s.token = next;
73
- if (next === null) {
74
- s.seen = [];
75
- if (s.phase === 3 || s.phase === 5) s.index++;
76
- else { s.phase++; s.index = 0; }
77
- } else s.seen.push(next);
78
- yield ctx.item(bytes, s, name);
79
- }
80
- }