@runeditrun/rer 0.2.99 → 0.2.100
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/cli/README.md +88 -2
- package/cli/business/bounded-waves.d.mts +23 -0
- package/cli/business/bounded-waves.mjs +57 -0
- package/cli/business/catchup-task.mjs +9 -2
- package/cli/business/command.mjs +8 -3
- package/cli/business/export-inputs.mjs +12 -2
- package/cli/business/export-task.mjs +9 -2
- package/cli/business/legacy-package.mjs +117 -0
- package/cli/business/local-commands.mjs +10 -1
- package/cli/business/package-transfer.mjs +127 -0
- package/cli/business/state.mjs +3 -3
- package/cli/business/vendor-door.mjs +83 -11
- package/cli/business/vendors/README.md +41 -2
- package/cli/business/vendors/common.mjs +81 -0
- package/cli/business/vendors/index.mjs +12 -2
- package/cli/business/vendors/zendesk-attachments.mjs +57 -0
- package/cli/business/vendors/zendesk-encode.mjs +133 -0
- package/cli/business/vendors/zendesk-guide.mjs +34 -0
- package/cli/business/vendors/zendesk-package.mjs +28 -0
- package/cli/business/vendors/zendesk-pages.mjs +61 -0
- package/cli/business/vendors/zendesk-people.mjs +7 -0
- package/cli/business/vendors/zendesk-store.mjs +154 -0
- package/cli/business/vendors/zendesk-tickets.mjs +21 -0
- package/cli/business/vendors/zendesk-token.mjs +31 -0
- package/cli/business/vendors/zendesk-transport.mjs +48 -0
- package/cli/business/vendors/zendesk.mjs +61 -0
- package/cli/deploy-command.mjs +5 -4
- package/cli/deploy-inputs.mjs +1 -1
- package/cli/deploy.mjs +6 -4
- package/cli/production-auth.mjs +16 -3
- package/cli/production-mail.mjs +18 -1
- package/cli/site.mjs +1 -1
- package/global/CHANGELOG.md +21 -0
- package/modules/deploy-kit/README.md +10 -0
- package/modules/deploy-kit/provider.mjs +18 -2
- package/modules/import-kit/src/d1/jobs.ts +3 -1
- package/modules/import-kit/src/d1/retirement-scope.ts +2 -2
- package/modules/import-kit/src/d1/retirement-tally.ts +1 -0
- package/modules/import-kit/src/d1/retirement.ts +15 -5
- package/modules/import-kit/src/d1/types.ts +3 -0
- package/modules/import-kit/src/d1/wave-boundary.ts +33 -0
- package/modules/import-kit/src/d1/wave-retirement.ts +47 -0
- package/modules/import-kit/src/door/ADOPTING.md +37 -0
- package/modules/import-kit/src/door/README.md +66 -0
- package/modules/import-kit/src/door/admission.ts +13 -3
- package/modules/import-kit/src/door/authority.ts +1 -0
- package/modules/import-kit/src/door/failures.ts +1 -1
- package/modules/import-kit/src/door/grants.ts +19 -5
- package/modules/import-kit/src/door/handler.ts +12 -1
- package/modules/import-kit/src/door/index.ts +6 -0
- package/modules/import-kit/src/door/package.ts +121 -0
- package/modules/import-kit/src/door/planning.ts +44 -19
- package/modules/import-kit/src/door/protocol.ts +4 -2
- package/modules/import-kit/src/door/schema.ts +2 -0
- package/modules/import-kit/src/door/store.ts +2 -2
- package/modules/import-kit/src/door/types.ts +24 -1
- package/modules/import-kit/src/door/waves/README.md +158 -0
- package/modules/import-kit/src/door/waves/digests.ts +38 -0
- package/modules/import-kit/src/door/waves/inventory.ts +84 -0
- package/modules/import-kit/src/door/waves/journal.ts +19 -0
- package/modules/import-kit/src/door/waves/lifecycle.ts +242 -0
- package/modules/import-kit/src/door/waves/preparation.ts +190 -0
- package/modules/import-kit/src/door/waves/schema.ts +73 -0
- package/modules/import-kit/src/door/waves/store.ts +112 -0
- package/modules/import-kit/src/door/waves/types.ts +76 -0
- package/onboarding/adapter.mjs +7 -1
- package/onboarding/operator-installer.mjs +3 -1
- package/package.json +1 -1
- package/scripts/publish-check/README.md +20 -0
- package/scripts/publish-check/diagnostics.mjs +73 -0
- package/scripts/publish-check/slots.py +37 -7
- package/scripts/publish-check.mjs +64 -37
|
@@ -1,7 +1,12 @@
|
|
|
1
|
+
import { buildPackage, transferPackage } from './package-transfer.mjs';
|
|
2
|
+
import { registeredLegacyPackage } from './legacy-package.mjs';
|
|
3
|
+
import { prepareZendeskPackage } from './vendors/zendesk-package.mjs';
|
|
1
4
|
import { createHash } from 'node:crypto';
|
|
2
5
|
import { constants } from 'node:fs';
|
|
3
6
|
import { open, unlink } from 'node:fs/promises';
|
|
4
7
|
import { join } from 'node:path';
|
|
8
|
+
import { loadState, saveState } from './state.mjs';
|
|
9
|
+
import { snapshot as zendeskSnapshot, storePath as zendeskStorePath, removeZendeskStores } from './vendors/zendesk.mjs';
|
|
5
10
|
import { taskPath } from './api.mjs';
|
|
6
11
|
import { assertChunk } from './outbound.mjs';
|
|
7
12
|
import { createCredentialStore } from './keychain.mjs';
|
|
@@ -18,7 +23,7 @@ import { businessCommand } from './invocation.mjs';
|
|
|
18
23
|
const CHUNK_BYTES = 65_536;
|
|
19
24
|
const DOOR_RESPONSE_BYTES = 4 * 1024 * 1024;
|
|
20
25
|
// The catalogue names the platform builds vendor task titles from.
|
|
21
|
-
export const vendorNames = Object.freeze({ sentry: 'Sentry', uptimerobot: 'UptimeRobot', pagerduty: 'PagerDuty', calendly: 'Calendly', sorry: 'Sorry' });
|
|
26
|
+
export const vendorNames = Object.freeze({ sentry: 'Sentry', uptimerobot: 'UptimeRobot', pagerduty: 'PagerDuty', calendly: 'Calendly', sorry: 'Sorry', zendesk: 'Zendesk', 'happytohelp-legacy': 'HappyToHelp legacy' });
|
|
22
27
|
/** Whether a vendor's export needs the owner's read-only key (Sorry's public page does not). */
|
|
23
28
|
export const keyed = vendor => KEYED_VENDORS.includes(vendor);
|
|
24
29
|
export const vendorHandle = vendor => `vendor:${vendor}:read`;
|
|
@@ -97,12 +102,12 @@ function adapterInputs(vendor, task) {
|
|
|
97
102
|
/** One RER-signed import grant for one door request. `review` asks for a
|
|
98
103
|
* catch-up plan grant without the owner's standing approval; a plan grant
|
|
99
104
|
* Business minted under that approval says so with `catchup: 'bounded'`. */
|
|
100
|
-
async function doorGrant(lease, action, origin, vendor, { review = false } = {}) {
|
|
101
|
-
const grant = await lease.importGrant(action, review ? { review: true } : {});
|
|
105
|
+
async function doorGrant(lease, action, origin, vendor, { review = false, bounded } = {}) {
|
|
106
|
+
const grant = await lease.importGrant(action, { ...(review ? { review: true } : {}), ...(bounded ? { bounded } : {}) });
|
|
102
107
|
if (!grant || grant.action !== action || grant.destination !== new URL(`/api/rer/import/${action}`, origin).href
|
|
103
108
|
|| typeof grant.authorization !== 'string' || grant.authorization.length > 8192 || !/^Bearer [A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+$/.test(grant.authorization)
|
|
104
109
|
|| !VENDORS.includes(grant.namespace) || vendor && grant.namespace !== vendor || !(Date.parse(grant.expires_at) > Date.now())
|
|
105
|
-
|| Object.hasOwn(grant, 'catchup') && (grant.catchup !== 'bounded' ||
|
|
110
|
+
|| Object.hasOwn(grant, 'catchup') && (grant.catchup !== 'bounded' || !['plan','prepare'].includes(action) || review)) throw Error('Business returned an invalid import grant.');
|
|
106
111
|
return grant;
|
|
107
112
|
}
|
|
108
113
|
|
|
@@ -130,7 +135,7 @@ export async function door(grant, body, signal) {
|
|
|
130
135
|
if (!response.ok) {
|
|
131
136
|
// A door code is snake_case, sometimes with one named reason (`catchup_requires_proposal:source_records_gone`).
|
|
132
137
|
const doorCode = typeof result?.error === 'string' && /^[a-z][a-z0-9_]{0,79}(?::[a-z][a-z0-9_]{0,79})?$/.test(result.error) ? result.error : null;
|
|
133
|
-
throw Object.assign(Error(`The app's import door refused the ${action} request (HTTP ${response.status}${doorCode ? `: ${doorCode}` : ''}).`), { doorStatus: response.status, doorCode });
|
|
138
|
+
throw Object.assign(Error(`The app's import door refused the ${action} request (HTTP ${response.status}${doorCode ? `: ${doorCode}` : ''}).`), { doorStatus: response.status, doorCode, ...(doorCode === 'package_segment_missing' ? { doorDetails: result.details } : {}) });
|
|
134
139
|
}
|
|
135
140
|
if (!result || typeof result !== 'object' || Array.isArray(result)) throw Error(`The app's import door returned an invalid ${action} answer.`);
|
|
136
141
|
return result;
|
|
@@ -138,12 +143,21 @@ export async function door(grant, body, signal) {
|
|
|
138
143
|
|
|
139
144
|
/** Run the shipped adapter into a private 0600 bundle file, checking every
|
|
140
145
|
* item with the outbound detector before it is written. */
|
|
141
|
-
async function exportBundle({ vendor, key = null, inputs, path, quota, signal }) {
|
|
146
|
+
async function exportBundle({ vendor, key = null, inputs, path, quota, signal, directory, runId, business }) {
|
|
142
147
|
await unlink(path).catch(error => { if (error.code !== 'ENOENT') throw error; });
|
|
143
148
|
const file = await open(path, constants.O_WRONLY | constants.O_CREAT | constants.O_EXCL | constants.O_NOFOLLOW, 0o600);
|
|
144
|
-
const digest = createHash('sha256'); let size = 0;
|
|
149
|
+
const digest = createHash('sha256'); let size = 0, zendeskStore;
|
|
145
150
|
try {
|
|
146
|
-
|
|
151
|
+
let items;
|
|
152
|
+
if (vendor === 'zendesk') {
|
|
153
|
+
const root = zendeskStorePath(directory, business, `https://${inputs.account.subdomain}.zendesk.com`);
|
|
154
|
+
const result = await zendeskSnapshot({ key, inputs, directory: inputs.mode === 'census' ? join(root, 'probe') : root, runId, signal });
|
|
155
|
+
if (result.kind === 'census') throw new ExportWaiting(`Zendesk census: ${result.census.tickets.unique} tickets; ${result.census.unswept} unswept. CENSUS_OUTPUTS_UNSUPPORTED: waiting for Business census task outputs.`, 'connected_runner_required');
|
|
156
|
+
if (result.kind === 'package') return { package: result, store: result.store, mode: inputs.mode };
|
|
157
|
+
zendeskStore = result.store;
|
|
158
|
+
items = [{ bytes: result.bytes }];
|
|
159
|
+
} else items = exportVendor(vendor, { key, inputs, signal });
|
|
160
|
+
for await (const item of items) {
|
|
147
161
|
assertChunk(item.bytes, key ? [key] : []);
|
|
148
162
|
size += item.bytes.length;
|
|
149
163
|
if (size > quota) throw Error(`Business vendor export is larger than the app's import door accepts (${quota} bytes). Nothing was uploaded.`);
|
|
@@ -159,7 +173,7 @@ async function exportBundle({ vendor, key = null, inputs, path, quota, signal })
|
|
|
159
173
|
throw Error(`${vendorNames[vendor]} export stopped (${error.code}${error.status ? `, HTTP ${error.status}` : ''}). Nothing was uploaded.`);
|
|
160
174
|
} finally { await file.close(); }
|
|
161
175
|
if (!size) throw Error(`${vendorNames[vendor]} export produced nothing to upload.`);
|
|
162
|
-
return { size, sha256: digest.digest('hex') };
|
|
176
|
+
return { size, sha256: digest.digest('hex'), ...(zendeskStore ? { store: zendeskStore, mode: inputs.mode } : {}) };
|
|
163
177
|
}
|
|
164
178
|
|
|
165
179
|
async function readChunk(path, index, size) {
|
|
@@ -199,19 +213,72 @@ export async function withUploadedBundle({ lease, task, grant: claimedGrant, dir
|
|
|
199
213
|
const fresh = async (action, options) => {
|
|
200
214
|
if (action === 'uploads' && spare && Date.parse(spare.expires_at) - Date.now() > 60_000) { const grant = spare; spare = null; return grant; }
|
|
201
215
|
spare = null;
|
|
202
|
-
return doorGrant(lease, action, origin, vendor, options);
|
|
216
|
+
return doorGrant(lease, action, origin, vendor, options ?? (task.bounded_waves && action==='prepare' ? {bounded:{protocol:task.bounded_waves.protocol,migration:task.bounded_waves.migration,revision:task.bounded_waves.revision,policy:task.bounded_waves.policy}} : undefined));
|
|
203
217
|
};
|
|
204
218
|
const inputs = adapterInputs(vendor, task);
|
|
219
|
+
const state = vendor === 'zendesk' ? await loadState() : null;
|
|
220
|
+
if (vendor === 'zendesk' && inputs.mode === 'retire') {
|
|
221
|
+
// The retire signal is local revocation, never an OAuth/provider write.
|
|
222
|
+
await createCredentialStore().delete(vendorHandle(vendor));
|
|
223
|
+
await removeZendeskStores(directory);
|
|
224
|
+
await saveState({ ...state, vendorHandles: (state.vendorHandles ?? []).filter(handle => handle !== vendorHandle(vendor)), vendorRevocations: { ...state.vendorRevocations, zendesk: true } });
|
|
225
|
+
throw new ExportWaiting('Zendesk retired locally (VENDOR_REVOKED). RETIRE_OUTPUTS_UNSUPPORTED: waiting for Business retire task acknowledgement.', 'connected_runner_required');
|
|
226
|
+
}
|
|
227
|
+
if (vendor === 'zendesk' && state?.vendorRevocations?.zendesk) throw new VendorExportError('VENDOR_REVOKED');
|
|
205
228
|
let key = keyed(vendor) ? await createCredentialStore().get(vendorHandle(vendor)) : null;
|
|
206
229
|
if (keyed(vendor) && !key) throw new ExportWaiting(`Business vendor export needs the owner's read-only ${name} key on this computer: ${keyStep(vendor)}\nThen run ${businessCommand('run')} again.`, 'owner_required', { keyMissing: vendor });
|
|
230
|
+
if (vendor === 'happytohelp-legacy') {
|
|
231
|
+
if (!['export.vendor', 'catchup.vendor'].includes(task.kind)) throw new ExportWaiting('Legacy package tasks require a package-aware export or catch-up reader.', 'configuration_required');
|
|
232
|
+
let legacy;
|
|
233
|
+
try { legacy = await registeredLegacyPackage(); }
|
|
234
|
+
catch (error) { throw new ExportWaiting(error.message, 'configuration_required'); }
|
|
235
|
+
say(doing(name, origin, false));
|
|
236
|
+
const packageDirectory = join(directory, `package-${hash(task.id).slice(0, 32)}`);
|
|
237
|
+
let prepared;
|
|
238
|
+
try {
|
|
239
|
+
prepared = await buildPackage({ files: legacy.files, quota, directory: packageDirectory, signal,
|
|
240
|
+
source: { format: legacy.manifest.format, version: 1, origin: 'legacy-database', exportedAt: legacy.manifest.exportedAt ?? legacy.manifest.source.exportedAt ?? null,
|
|
241
|
+
synthetic: legacy.manifest.source.synthetic === true, adapter: 'rer:happytohelp-legacy' },
|
|
242
|
+
records: { total: null, byKind: {} }, totals: { rowCounts: legacy.rowCounts, objects: legacy.objects } });
|
|
243
|
+
} catch { throw new ExportWaiting(`Legacy package changed or could not be prepared: ${legacy.path}. Recheck and register that package again.`, 'configuration_required'); }
|
|
244
|
+
let uploaded;
|
|
245
|
+
try { uploaded = await transferPackage({ prepared, fresh, request: door, signal, checkpoint: cp => lease.checkpointSaved(cp), catchup: task.kind === 'catchup.vendor', ...(task.bounded_waves ? { planAction:'prepare', planInput:{ordinal:0} } : {}) }); }
|
|
246
|
+
catch (error) { if (!error.message.startsWith('SEED_PACKAGE_UNSUPPORTED')) throw error; throw new ExportWaiting(error.message, 'configuration_required'); }
|
|
247
|
+
const { outputs, said } = await finish({ vendor, name, origin, ...uploaded, fresh });
|
|
248
|
+
await lease.complete(outputs);
|
|
249
|
+
return said;
|
|
250
|
+
}
|
|
207
251
|
const path = join(directory, `export-${hash(task.id).slice(0, 32)}.bundle`);
|
|
208
252
|
try {
|
|
209
253
|
say(doing(name, origin, keyed(vendor)));
|
|
210
254
|
let bundle;
|
|
211
|
-
try { bundle = await exportBundle({ vendor, key, inputs, path, quota, signal }); } finally { key = undefined; }
|
|
255
|
+
try { bundle = await exportBundle({ vendor, key, inputs, path, quota, signal, directory, runId: task.id, business: state?.businessId }); } finally { key = undefined; }
|
|
256
|
+
if (bundle.package) {
|
|
257
|
+
const prepared = await prepareZendeskPackage({ result: bundle.package, quota, directory: join(bundle.store.path, `package-${hash(task.id).slice(0,32)}`), signal, mode: inputs.mode });
|
|
258
|
+
const uploaded = await transferPackage({ prepared, fresh, request: door, signal, checkpoint: cp => lease.checkpointSaved(cp), catchup: task.kind === 'catchup.vendor' || inputs.mode === 'sweep' });
|
|
259
|
+
const finished = await finish({ vendor, name, origin, ...uploaded, fresh });
|
|
260
|
+
await lease.complete(finished.outputs);
|
|
261
|
+
return finished.said;
|
|
262
|
+
}
|
|
212
263
|
const total = Math.ceil(bundle.size / CHUNK_BYTES);
|
|
213
264
|
let uploadId;
|
|
265
|
+
const received = new Set();
|
|
266
|
+
const journal = bundle.store?.events.findLast(event => event.kind === 'transfer');
|
|
267
|
+
if (journal && journal.origin === origin && journal.bundleSha256 === bundle.sha256 && journal.totalBytes === bundle.size) {
|
|
268
|
+
try {
|
|
269
|
+
const inventory = await door(await fresh('uploads'), { uploadId: journal.uploadId }, signal);
|
|
270
|
+
if (inventory.uploadId !== journal.uploadId || inventory.bundleSha256 !== bundle.sha256 || !Array.isArray(inventory.received)) throw new VendorExportError('INVALID_CHECKPOINT');
|
|
271
|
+
for (const item of inventory.received) {
|
|
272
|
+
if (!Number.isSafeInteger(item.ordinal) || item.ordinal < 0 || item.ordinal >= total || received.has(item.ordinal)) throw new VendorExportError('INVALID_CHECKPOINT');
|
|
273
|
+
const bytes = await readChunk(path, item.ordinal, bundle.size);
|
|
274
|
+
if (item.sha256 !== hash(bytes) || item.size !== bytes.length) throw new VendorExportError('INVALID_CHECKPOINT');
|
|
275
|
+
received.add(item.ordinal);
|
|
276
|
+
}
|
|
277
|
+
uploadId = journal.uploadId;
|
|
278
|
+
} catch (error) { if (error.doorStatus !== 404) throw error; }
|
|
279
|
+
}
|
|
214
280
|
for (let index = 0; index < total; index++) {
|
|
281
|
+
if (received.has(index)) continue;
|
|
215
282
|
signal.throwIfAborted();
|
|
216
283
|
const bytes = await readChunk(path, index, bundle.size);
|
|
217
284
|
const result = await door(await fresh('uploads'), { ...(uploadId ? { uploadId } : {}), name: capability.output, bundleSha256: bundle.sha256,
|
|
@@ -219,6 +286,11 @@ export async function withUploadedBundle({ lease, task, grant: claimedGrant, dir
|
|
|
219
286
|
if (typeof result.uploadId !== 'string' || !DOOR_ID.test(result.uploadId) || uploadId && result.uploadId !== uploadId
|
|
220
287
|
|| result.bundleSha256 !== bundle.sha256 || index === total - 1 && result.complete !== true) throw Error("The app's import door returned an invalid upload answer.");
|
|
221
288
|
uploadId = result.uploadId;
|
|
289
|
+
if (bundle.store) {
|
|
290
|
+
const transfer = { kind: 'transfer', origin, bundleSha256: bundle.sha256, totalBytes: bundle.size, uploadId };
|
|
291
|
+
await bundle.store.commit(bundle.store.state, transfer);
|
|
292
|
+
await bundle.store.write('transfer.json', transfer);
|
|
293
|
+
}
|
|
222
294
|
await lease.checkpointSaved({ page: index + 1, total, bytes_done: Math.min((index + 1) * CHUNK_BYTES, bundle.size), progress: Math.floor((index + 1) / total * 90) });
|
|
223
295
|
}
|
|
224
296
|
const { outputs, said } = await finish({ vendor, name, origin, uploadId, bundle, fresh });
|
|
@@ -23,9 +23,9 @@ No export runs merely by importing these modules. No account export was run
|
|
|
23
23
|
during implementation.
|
|
24
24
|
|
|
25
25
|
`VENDORS` is the closed vendor enum (`sentry`, `uptimerobot`, `pagerduty`,
|
|
26
|
-
`calendly`, `sorry`). `VENDOR_CAPABILITIES` describes the adapter,
|
|
26
|
+
`calendly`, `sorry`, `zendesk`, `happytohelp-legacy`). `VENDOR_CAPABILITIES` describes the adapter,
|
|
27
27
|
required vendor-key capability, native importer format, layout and read scopes.
|
|
28
|
-
|
|
28
|
+
Except for Zendesk's checked OAuth scopes, adapters cannot verify that a supplied key itself has read-only scope;
|
|
29
29
|
the owner must create the documented read-only key where the vendor offers one.
|
|
30
30
|
|
|
31
31
|
## Transport and resume
|
|
@@ -174,3 +174,42 @@ Seed documentation and official APIs were inspected on 16 October 2026.
|
|
|
174
174
|
These sources document shapes; no real-account parity or migration success is
|
|
175
175
|
claimed. Parent integration must provide seed assembly and transport under the
|
|
176
176
|
actual import grant protocol, plus network-edge acceptance coverage.
|
|
177
|
+
|
|
178
|
+
## Zendesk's retained snapshot
|
|
179
|
+
|
|
180
|
+
`zendesk` is a keyed vendor with `layout: 'snapshot'`. Its credential is the
|
|
181
|
+
hidden-input JSON client credential documented in [the CLI guide](../../README.md#zendesk-offline-snapshots),
|
|
182
|
+
rather than a bare API token. `snapshot({key,inputs,directory,runId,signal})` in
|
|
183
|
+
`zendesk.mjs` drives phases A–F into the durable store. `exportVendor` accepts
|
|
184
|
+
`directory` and `runId` for this vendor; the Business door provides a directory
|
|
185
|
+
keyed by Business and account. It does not apply the older vendors' 2,000-item
|
|
186
|
+
or one-MiB item limits. The complete deliverable is checked against HH v1 limits
|
|
187
|
+
before any upload. Package-required results use the shared package sender with path bindings and
|
|
188
|
+
sidecars. It probes the manifest before sending segments, resumes the transfer
|
|
189
|
+
journal and plans one complete snapshot; no per-part plan or fallback is used.
|
|
190
|
+
|
|
191
|
+
Responsibilities are separated into token/scope checking, transport, pages,
|
|
192
|
+
tickets/comments, people, attachments, Guide, store and HH encoding. Requests
|
|
193
|
+
stop at 250,000, responses at 32 MiB, records/blobs at 64 MiB, snapshots at
|
|
194
|
+
32 GiB. The opt-in `pacedClient` in `common.mjs` supports bounded 429 retries
|
|
195
|
+
(1–60 seconds, at most three per request and ten minutes per run), rate headers,
|
|
196
|
+
and manual attachment redirects. Without rate headers it waits at least six
|
|
197
|
+
seconds between incremental requests and 300 ms otherwise. Other vendors keep
|
|
198
|
+
`client()` and its previous no-retry behavior.
|
|
199
|
+
|
|
200
|
+
Only the account origin receives Authorization. The first attachment URL must
|
|
201
|
+
be on that origin's attachment path. One redirect may go to a public HTTPS DNS
|
|
202
|
+
host, with no userinfo, IP, explicit port or local/private address, and receives
|
|
203
|
+
no credential even when it returns to the account origin. Further redirects
|
|
204
|
+
are errors. Pagination is independently checked for account origin, exact
|
|
205
|
+
collection and allowed query parameters.
|
|
206
|
+
|
|
207
|
+
Checkpoints point at immutable SHA-256 journal nodes and raw objects. Each page
|
|
208
|
+
is fsynced before its journal and atomic checkpoint; a kill before acknowledgement
|
|
209
|
+
leaves only unreachable objects. Resume validates the reachable graph and
|
|
210
|
+
rebuilds last-version records and census from it. Repeated attachment bytes
|
|
211
|
+
have one blob. The census declares tombstones, inaccessible/deleted/malware/large
|
|
212
|
+
attachments and omissions explicitly. Guide 404 records unavailable coverage.
|
|
213
|
+
The JSON census and Guide sidecars stay local and in the package; Business
|
|
214
|
+
receives only the door's ids and SHA for a successful bundle. See the CLI guide
|
|
215
|
+
for the separate Business census/retire output dependencies.
|
|
@@ -197,3 +197,84 @@ export function client({ origin, headers, route, signal, limits, counters, key }
|
|
|
197
197
|
return { body, text, headers: response.headers };
|
|
198
198
|
};
|
|
199
199
|
}
|
|
200
|
+
|
|
201
|
+
/** Opt-in transport for long-lived snapshots. Existing adapters retain client()'s
|
|
202
|
+
* no-retry contract. The caller validates every target before credentials or a
|
|
203
|
+
* request are constructed, including manually followed attachment redirects. */
|
|
204
|
+
export function pacedClient({ origin, signal, validate, secrets = [], limits = {}, beforeRequest }) {
|
|
205
|
+
const maximum = { requests: 250_000, response: 32 * 1024 * 1024, bytes: 32 * 1024 ** 3, ...limits };
|
|
206
|
+
const counters = { requests: 0, bytes: 0, retryMs: 0 };
|
|
207
|
+
let token, last = 0, next = 0, advertisedBefore = false;
|
|
208
|
+
const wait = async milliseconds => { if (milliseconds > 0) await delay(milliseconds, undefined, { signal }); };
|
|
209
|
+
const request = async (path, options = {}) => {
|
|
210
|
+
let url;
|
|
211
|
+
try { url = new URL(path, origin); } catch { fail('INVALID_REFERENCE'); }
|
|
212
|
+
await validate(url, options);
|
|
213
|
+
const method = options.method ?? 'GET';
|
|
214
|
+
if (method !== 'GET' && !(method === 'POST' && url.origin === origin && url.pathname === '/oauth/tokens')) fail('INVALID_REFERENCE');
|
|
215
|
+
for (let retries = 0; ; retries++) {
|
|
216
|
+
signal?.throwIfAborted();
|
|
217
|
+
await wait(Math.max(next, last + (advertisedBefore ? 0 : url.pathname.includes('/incremental/') ? 6000 : 300)) - Date.now());
|
|
218
|
+
if (++counters.requests > maximum.requests) fail('REQUEST_LIMIT');
|
|
219
|
+
await beforeRequest?.();
|
|
220
|
+
last = Date.now();
|
|
221
|
+
const headers = { accept: 'application/json', ...(options.json ? { 'content-type': 'application/json' } : {}),
|
|
222
|
+
...(token && url.origin === origin && options.authenticated !== false ? { authorization: `Bearer ${token}` } : {}) };
|
|
223
|
+
let response;
|
|
224
|
+
try { response = await fetch(url, { method, headers, redirect: options.redirect ?? 'error',
|
|
225
|
+
...(options.json ? { body: JSON.stringify(options.json) } : {}), signal: signal ? AbortSignal.any([signal, AbortSignal.timeout(30_000)]) : AbortSignal.timeout(30_000) }); }
|
|
226
|
+
catch { fail(signal?.aborted ? 'CANCELLED' : 'NETWORK_FAILED'); }
|
|
227
|
+
const waits = [];
|
|
228
|
+
const remaining = response.headers.get('ratelimit-remaining') ?? response.headers.get('x-rate-limit-remaining');
|
|
229
|
+
const reset = response.headers.get('ratelimit-reset') ?? '60';
|
|
230
|
+
const capacity = (remaining, seconds) => {
|
|
231
|
+
if (!/^\d+$/u.test(remaining ?? '') || !/^\d+$/u.test(seconds ?? '') || Number(seconds) > 3600) return;
|
|
232
|
+
waits.push(Math.max(300, Math.ceil(Number(seconds) * 1000 / Math.max(1, Number(remaining)))));
|
|
233
|
+
};
|
|
234
|
+
capacity(remaining, reset);
|
|
235
|
+
for (const [name,value] of response.headers) if (name.toLowerCase().startsWith('zendesk-ratelimit-')) capacity(/remaining=(\d+)/u.exec(value)?.[1], /resets=(\d+)/u.exec(value)?.[1]);
|
|
236
|
+
const interval = response.headers.get('ratelimit-limit') ?? response.headers.get('x-rate-limit');
|
|
237
|
+
if (!waits.length && /^\d+$/u.test(interval ?? '')) waits.push(Math.max(300, Math.ceil(60_000 / Math.max(1, Number(interval)))));
|
|
238
|
+
// Obey the tightest account/endpoint budget. Without valid advertised
|
|
239
|
+
// capacity the next request retains the conservative fallback spacing.
|
|
240
|
+
advertisedBefore = waits.length > 0;
|
|
241
|
+
next = Date.now() + Math.max(0, ...waits);
|
|
242
|
+
if (response.status === 429) {
|
|
243
|
+
const after = response.headers.get('retry-after');
|
|
244
|
+
await response.body?.cancel().catch(() => {});
|
|
245
|
+
const seconds = /^\d+$/u.test(after ?? '') ? Number(after) : 0;
|
|
246
|
+
if (retries >= 3 || seconds < 1 || seconds > 60 || counters.retryMs + seconds * 1000 > 600_000) throw new VendorExportError('RATE_LIMITED', 429);
|
|
247
|
+
counters.retryMs += seconds * 1000;
|
|
248
|
+
await wait(seconds * 1000);
|
|
249
|
+
continue;
|
|
250
|
+
}
|
|
251
|
+
if (options.redirect === 'manual' && [301, 302, 303, 307, 308].includes(response.status)) {
|
|
252
|
+
const location = response.headers.get('location');
|
|
253
|
+
await response.body?.cancel().catch(() => {});
|
|
254
|
+
return { status: response.status, location, headers: response.headers };
|
|
255
|
+
}
|
|
256
|
+
if (!response.ok) {
|
|
257
|
+
await response.body?.cancel().catch(() => {});
|
|
258
|
+
if (options.allow?.includes(response.status)) return { status: response.status };
|
|
259
|
+
throw new VendorExportError('HTTP_REFUSED', response.status);
|
|
260
|
+
}
|
|
261
|
+
const chunks = []; let size = 0;
|
|
262
|
+
try {
|
|
263
|
+
for await (const chunk of response.body ?? []) {
|
|
264
|
+
size += chunk.byteLength; counters.bytes += chunk.byteLength;
|
|
265
|
+
if (size > (options.maxBytes ?? maximum.response) || counters.bytes > maximum.bytes) fail('BYTE_LIMIT');
|
|
266
|
+
chunks.push(chunk);
|
|
267
|
+
}
|
|
268
|
+
} catch (error) { if (error instanceof VendorExportError) throw error; fail(signal?.aborted ? 'CANCELLED' : 'NETWORK_FAILED'); }
|
|
269
|
+
const bytes = Buffer.concat(chunks);
|
|
270
|
+
if (!options.tokenResponse && [...secrets, token].some(secret => secret && bytes.includes(Buffer.from(secret)))) fail('CREDENTIAL_IN_RESPONSE');
|
|
271
|
+
if (options.binary) return { bytes, headers: response.headers, status: response.status };
|
|
272
|
+
let body;
|
|
273
|
+
try { body = JSON.parse(new TextDecoder('utf-8', { fatal: true }).decode(bytes)); } catch { fail('INVALID_JSON'); }
|
|
274
|
+
return { body, bytes, headers: response.headers, status: response.status };
|
|
275
|
+
}
|
|
276
|
+
};
|
|
277
|
+
request.authorize = value => { token = value; };
|
|
278
|
+
request.counters = counters;
|
|
279
|
+
return request;
|
|
280
|
+
}
|
|
@@ -4,11 +4,14 @@ import * as uptimerobot from './uptimerobot.mjs';
|
|
|
4
4
|
import * as pagerduty from './pagerduty.mjs';
|
|
5
5
|
import * as calendly from './calendly.mjs';
|
|
6
6
|
import * as sorry from './sorry.mjs';
|
|
7
|
+
import * as zendesk from './zendesk.mjs';
|
|
7
8
|
|
|
8
9
|
export { VendorExportError } from './common.mjs';
|
|
9
|
-
export const VENDORS = Object.freeze(['sentry','uptimerobot','pagerduty','calendly','sorry']);
|
|
10
|
+
export const VENDORS = Object.freeze(['sentry','uptimerobot','pagerduty','calendly','sorry','zendesk','happytohelp-legacy']);
|
|
10
11
|
const adapters = { sentry, uptimerobot, pagerduty, calendly, sorry };
|
|
11
12
|
export const VENDOR_CAPABILITIES = Object.freeze({
|
|
13
|
+
'happytohelp-legacy': Object.freeze({ vendor: 'happytohelp-legacy', adapter: 'rer:happytohelp-legacy', capability: null, format: 'happytohelp-legacy-chunks-v1', layout: 'package', output: 'rer-package.json', readOnly: true, resume: true, scopes: [], origins: [] }),
|
|
14
|
+
zendesk: Object.freeze({ vendor: 'zendesk', adapter: 'rer:zendesk', capability: 'vendor_key:zendesk', format: 'happytohelp-zendesk-offline-v1', layout: 'snapshot', output: 'zendesk.ndjson', readOnly: true, resume: true, scopes: ['tickets:read','users:read','organizations:read','hc:read','ticket_attachments:read'], origins: ['https://<subdomain>.zendesk.com'] }),
|
|
12
15
|
sentry: Object.freeze({ vendor: 'sentry', adapter: 'rer:sentry', capability: 'vendor_key:sentry', format: 'spool-sentry-org-export-v1', layout: 'concatenate', output: 'sentry-export.json', readOnly: true, resume: true, scopes: ['org:read','project:read','event:read'], origins: Object.values(sentry.origins) }),
|
|
13
16
|
uptimerobot: Object.freeze({ vendor: 'uptimerobot', adapter: 'rer:uptimerobot', capability: 'vendor_key:uptimerobot', format: 'uptimerobot-v3-monitors', layout: 'concatenate', output: 'uptimerobot-monitors.json', readOnly: true, resume: true, scopes: ['read-only API key'], origins: [uptimerobot.origin] }),
|
|
14
17
|
pagerduty: Object.freeze({ vendor: 'pagerduty', adapter: 'rer:pagerduty', capability: 'vendor_key:pagerduty', format: 'pagerduty-saved-responses', layout: 'concatenate', output: 'pagerduty-export.json', readOnly: true, resume: true, scopes: ['read-only REST API key'], origins: [pagerduty.origin] }),
|
|
@@ -33,9 +36,16 @@ const LIMITS = Object.freeze({
|
|
|
33
36
|
/** Persist each checkpoint only after the corresponding bytes have been acknowledged.
|
|
34
37
|
* Bytes are private seed-bound artifacts; checkpoints are metadata, never API bodies.
|
|
35
38
|
*/
|
|
36
|
-
export async function* exportVendor(vendor, { key, inputs = {}, checkpoint, signal } = {}) {
|
|
39
|
+
export async function* exportVendor(vendor, { key, inputs = {}, checkpoint, signal, directory, runId } = {}) {
|
|
37
40
|
try {
|
|
38
41
|
if (!VENDORS.includes(vendor)) fail('UNSUPPORTED_VENDOR');
|
|
42
|
+
if (vendor === 'happytohelp-legacy') fail('LOCAL_PACKAGE_REQUIRED');
|
|
43
|
+
if (vendor === 'zendesk') {
|
|
44
|
+
const result = await zendesk.snapshot({ key, inputs, directory, runId, signal });
|
|
45
|
+
if (result.kind !== 'bundle') fail(result.kind === 'package' ? 'SEED_PACKAGE_UNSUPPORTED' : 'INVALID_INPUT');
|
|
46
|
+
yield { name: 'zendesk.ndjson', bytes: result.bytes, checkpoint: { version: 1, vendor, head: result.store.head } };
|
|
47
|
+
return;
|
|
48
|
+
}
|
|
39
49
|
const keyed = KEYED_VENDORS.includes(vendor);
|
|
40
50
|
if (keyed ? typeof key !== 'string' || !key || key.length > 8192 || /[\u0000-\u0020\u007f]/u.test(key) : key != null) fail('INVALID_KEY');
|
|
41
51
|
object(inputs);
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
import { VendorExportError, fail } from './common.mjs';
|
|
2
|
+
import { materialize, numericId, ordered } from './zendesk-store.mjs';
|
|
3
|
+
import { publicLocation, attachmentPath } from './zendesk-transport.mjs';
|
|
4
|
+
|
|
5
|
+
export function attachmentReferences(data) {
|
|
6
|
+
const refs = [];
|
|
7
|
+
for (const [ticket, comments] of data.comments) {
|
|
8
|
+
if (data.tickets.get(ticket)?.row.status === 'deleted' || data.tickets.get(ticket)?.row.deleted) continue;
|
|
9
|
+
for (const [comment, { row }] of comments) for (const attachment of row.attachments ?? []) refs.push({ key: `ticket:${ticket}:${comment}:${numericId(attachment.id)}`, ticket, comment, attachment });
|
|
10
|
+
}
|
|
11
|
+
for (const [id, { row }] of data.article_attachments) refs.push({ key: `article:${id}`, article: id.split(':')[1], attachment: row });
|
|
12
|
+
return refs.sort((a,b) => a.key.localeCompare(b.key));
|
|
13
|
+
}
|
|
14
|
+
export async function download(request, origin, attachment) {
|
|
15
|
+
if (attachment.deleted) return { reason: 'deleted' };
|
|
16
|
+
if (attachment.malware_scan_result === 'malware_found' || attachment.malware_found) return { reason: 'malware_found' };
|
|
17
|
+
const size = attachment.size;
|
|
18
|
+
if (!Number.isSafeInteger(size) || size < 0) fail('INVALID_RESPONSE');
|
|
19
|
+
if (size > 64 * 1024 * 1024) return { reason: 'too_large' };
|
|
20
|
+
let url;
|
|
21
|
+
try { url = new URL(attachment.content_url); } catch { return { reason: 'refused_location' }; }
|
|
22
|
+
if (url.origin !== origin || !attachmentPath(url.pathname) || !await publicLocation(attachment.content_url, origin)) return { reason: 'refused_location' };
|
|
23
|
+
let response;
|
|
24
|
+
try {
|
|
25
|
+
response = await request(url.href, { attachment: true, binary: true, maxBytes: 64 * 1024 * 1024, redirect: 'manual', allow: [403,404,410] });
|
|
26
|
+
if (response.location !== undefined) {
|
|
27
|
+
const next = await publicLocation(response.location, origin);
|
|
28
|
+
if (!next) return { reason: 'refused_location' };
|
|
29
|
+
response = await request(next.href, { attachment: true, binary: true, authenticated: false, maxBytes: 64 * 1024 * 1024, redirect: 'error', allow: [403,404,410] });
|
|
30
|
+
}
|
|
31
|
+
} catch (error) {
|
|
32
|
+
if (error instanceof VendorExportError && error.code === 'REFUSED_LOCATION') return { reason: 'refused_location' };
|
|
33
|
+
throw error;
|
|
34
|
+
}
|
|
35
|
+
if ([403,404,410].includes(response.status)) return { reason: `http_${response.status}` };
|
|
36
|
+
if (!response.bytes || response.bytes.length !== size) fail('ATTACHMENT_SIZE_MISMATCH');
|
|
37
|
+
return { bytes: response.bytes };
|
|
38
|
+
}
|
|
39
|
+
export async function attachments(ctx) {
|
|
40
|
+
const { store, request, origin, inputs } = ctx, state = store.state, data = await materialize(store);
|
|
41
|
+
const ids = ordered([...data.tickets].filter(([, { row }]) => row.status !== 'deleted' && !row.deleted && (inputs.mode !== 'sweep' || row.generated_timestamp > inputs.highWater)).map(([id]) => id));
|
|
42
|
+
const ticket = ids[state.index];
|
|
43
|
+
const refs = attachmentReferences(data).filter(ref => state.phase === 'guide_attachments' ? ref.article : ref.ticket === ticket);
|
|
44
|
+
const ref = refs[state.attachmentIndex];
|
|
45
|
+
if (!ref) {
|
|
46
|
+
await store.commit(state.phase === 'guide_attachments' ? { ...state, phase: 'done', attachmentIndex: 0 } : { ...state, phase: 'B', path: null, index: state.index + 1, attachmentIndex: 0 });
|
|
47
|
+
return;
|
|
48
|
+
}
|
|
49
|
+
const previous = data.attachments.get(ref.key);
|
|
50
|
+
if (previous && previous.sourceHash === JSON.stringify(ref.attachment)) {
|
|
51
|
+
await store.commit({ ...state, attachmentIndex: state.attachmentIndex + 1 }); return;
|
|
52
|
+
}
|
|
53
|
+
const result = await download(request, origin, ref.attachment);
|
|
54
|
+
const blob = result.bytes ? await store.put(result.bytes) : null;
|
|
55
|
+
await store.commit({ ...state, attachmentIndex: state.attachmentIndex + 1, bytes: state.bytes + (blob?.bytes ?? 0) },
|
|
56
|
+
{ kind: 'attachment', key: ref.key, sourceHash: JSON.stringify(ref.attachment), ...(blob ? { blob } : { reason: result.reason }) }, blob ? [blob] : []);
|
|
57
|
+
}
|
|
@@ -0,0 +1,133 @@
|
|
|
1
|
+
import { credentialShaped } from '../detector.mjs';
|
|
2
|
+
import { assertChunk } from '../outbound.mjs';
|
|
3
|
+
import { fail, hash } from './common.mjs';
|
|
4
|
+
import { materialize, encode, ordered } from './zendesk-store.mjs';
|
|
5
|
+
import { attachmentReferences } from './zendesk-attachments.mjs';
|
|
6
|
+
import { attachmentPath } from './zendesk-transport.mjs';
|
|
7
|
+
|
|
8
|
+
export const FORMAT = 'happytohelp-zendesk-offline-v1';
|
|
9
|
+
const omissions = ['audits','metrics','satisfaction','side conversations','groups','brands','forms','macros','triggers','views','automations','photos','thumbnails','archived articles','community'];
|
|
10
|
+
const escapePointer = value => value.replace(/~/gu, '~0').replace(/\//gu, '~1');
|
|
11
|
+
/** Redact only the delivered copy. Raw responses remain private evidence in
|
|
12
|
+
* the store. Expand prefix detector hits to their whole token and PEM hits to
|
|
13
|
+
* their whole block; assertChunk remains the final authority on every line. */
|
|
14
|
+
function cleaned(value, kind, id, report, pointer = '') {
|
|
15
|
+
const removed = () => report.removals.push({ kind, id, pointer: pointer || '/' });
|
|
16
|
+
if (Array.isArray(value)) return value.map((child,index) => cleaned(child, kind, id, report, `${pointer}/${index}`));
|
|
17
|
+
if (value && typeof value === 'object') {
|
|
18
|
+
const result = Object.create(null);
|
|
19
|
+
for (const [key,child] of Object.entries(value)) {
|
|
20
|
+
if (['content_url','download_url','original_url'].includes(key)) { report.redactedUrls++; continue; }
|
|
21
|
+
if (credentialShaped(key)) fail('CREDENTIAL_SHAPE_UNREMOVABLE');
|
|
22
|
+
result[key] = cleaned(child, kind, id, report, `${pointer}/${escapePointer(key)}`);
|
|
23
|
+
}
|
|
24
|
+
return result;
|
|
25
|
+
}
|
|
26
|
+
if (typeof value !== 'string') return value;
|
|
27
|
+
let text = value.replace(/https?:\/\/[^\s<>"']+/giu, match => {
|
|
28
|
+
let url; try { url = new URL(match); } catch { return match; }
|
|
29
|
+
if (attachmentPath(url.pathname)) { report.redactedUrls++; return '[removed]'; }
|
|
30
|
+
if (!url.username && !url.password && !url.search && !url.hash) return match;
|
|
31
|
+
url.username = ''; url.password = ''; url.search = ''; url.hash = ''; report.redactedUrls++; return url.href;
|
|
32
|
+
});
|
|
33
|
+
text = text.replace(/(?:\/\/[^\s<>"'/]+)?\/(?:api\/v2\/)?attachments\/[^\s<>"']*/giu, () => { report.redactedUrls++; return '[removed]'; });
|
|
34
|
+
text = text.replace(/-----\s*BEGIN\s+([A-Z0-9 ]+)-----[\s\S]*?-----\s*END\s+\1-----/giu, () => { removed(); return '[removed]'; });
|
|
35
|
+
// SSH material includes a scheme and key separated by whitespace.
|
|
36
|
+
text = text.replace(/\b(?:ssh-(?:rsa|ed25519)|ecdsa-sha2-[\w-]+)\s+[A-Za-z0-9+/=]+/gu, () => { removed(); return '[removed]'; });
|
|
37
|
+
text = text.replace(/[^\s<>"'`,;(){}\[\]]+/gu, token => {
|
|
38
|
+
if (!credentialShaped(token)) return token;
|
|
39
|
+
removed(); return '[removed]';
|
|
40
|
+
});
|
|
41
|
+
text = text.replace(/[\u0000-\u0008\u000b\u000c\u000e-\u001f\u007f]/gu, () => { removed(); return '[removed]'; });
|
|
42
|
+
text = text.replace(/https?:\/\/[^\s<>"']+/giu, match => {
|
|
43
|
+
let url;
|
|
44
|
+
try { url = new URL(match); } catch { return match; }
|
|
45
|
+
if (!url.username && !url.password && !url.search && !url.hash) return match;
|
|
46
|
+
url.username = ''; url.password = ''; url.search = ''; url.hash = ''; report.redactedUrls++; return url.href;
|
|
47
|
+
});
|
|
48
|
+
if (credentialShaped(text.replace(/[\r\n\t]/gu, ' '))) fail('CREDENTIAL_SHAPE_UNREMOVABLE');
|
|
49
|
+
return text;
|
|
50
|
+
}
|
|
51
|
+
const deleted = row => row.status === 'deleted' || row.deleted === true;
|
|
52
|
+
const sortedRows = map => ordered(map.keys()).map(id => [id, map.get(id)]);
|
|
53
|
+
function attachmentCensus(references, data, { article = false } = {}) {
|
|
54
|
+
const blobs = new Map(), undelivered = {};
|
|
55
|
+
let fetched = 0, bytes = 0;
|
|
56
|
+
for (const ref of references) {
|
|
57
|
+
const item = data.attachments.get(ref.key);
|
|
58
|
+
if (item?.blob) { fetched++; bytes += item.blob.bytes; blobs.set(item.blob.sha256, item.blob.bytes); }
|
|
59
|
+
const reason = item?.reason ?? (!item ? 'not_fetched' : article ? 'pending_guide' : null);
|
|
60
|
+
if (reason) undelivered[reason] = (undelivered[reason] ?? 0) + 1;
|
|
61
|
+
}
|
|
62
|
+
return { referenced: references.length, fetched, bytes, uniqueBlobs: blobs.size, undelivered };
|
|
63
|
+
}
|
|
64
|
+
export async function encodeSnapshot(store, { origin, packageBindings = false } = {}) {
|
|
65
|
+
const data = await materialize(store), report = { removals: [], redactedUrls: 0 }, lines = [], byKind = {}, blobs = new Map();
|
|
66
|
+
const add = (kind, id, value) => {
|
|
67
|
+
// Bindings are integrity-checked original bytes, never redaction input.
|
|
68
|
+
const clean = kind === 'ticket' ? { ...cleaned({ ...value, _offline: undefined }, kind, id, report), _offline: value._offline } : cleaned(value, kind, id, report);
|
|
69
|
+
const line = encode({ [kind]: clean });
|
|
70
|
+
if (line.length > 64 * 1024 * 1024) fail('RECORD_LIMIT');
|
|
71
|
+
try { assertChunk(line, []); } catch { fail('CREDENTIAL_SHAPE_UNREMOVABLE'); }
|
|
72
|
+
lines.push(line); byKind[kind] = (byKind[kind] ?? 0) + 1;
|
|
73
|
+
};
|
|
74
|
+
let needsPackage = packageBindings, commentTotal = 0, publicComments = 0;
|
|
75
|
+
const tombstones = [], byStatus = {}, refs = attachmentReferences(data);
|
|
76
|
+
for (const [id,{ row, provenance }] of sortedRows(data.tickets)) {
|
|
77
|
+
byStatus[row.status] = (byStatus[row.status] ?? 0) + 1;
|
|
78
|
+
if (deleted(row)) { tombstones.push({ id: row.id, generated_timestamp: row.generated_timestamp }); continue; }
|
|
79
|
+
const comments = sortedRows(data.comments.get(id) ?? new Map()).map(([,item]) => item.row), bindings = [];
|
|
80
|
+
for (const comment of comments) {
|
|
81
|
+
commentTotal++; if (comment.public === true) publicComments++;
|
|
82
|
+
for (const attachment of comment.attachments ?? []) {
|
|
83
|
+
const stored = data.attachments.get(`ticket:${id}:${comment.id}:${attachment.id}`);
|
|
84
|
+
if (!stored?.blob) continue;
|
|
85
|
+
const blob = stored.blob;
|
|
86
|
+
if (blob.bytes > 48 * 1024) needsPackage = true;
|
|
87
|
+
blobs.set(`blobs/${blob.sha256}`, blob);
|
|
88
|
+
bindings.push({ attachment_id: attachment.id, comment_id: comment.id, path: `blobs/${blob.sha256}`, sha256: blob.sha256,
|
|
89
|
+
...(!packageBindings ? { base64: (await store.get(blob)).toString('base64') } : {}) });
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
if (bindings.length > 20) needsPackage = true;
|
|
93
|
+
const value = { ...row, comments, _offline: { format: FORMAT, comments_complete: true, attachments: bindings,
|
|
94
|
+
history: { metrics: 'missing', audits: 'missing' }, metric_events: [], audits: [], provenance: [provenance, ...[...(data.comments.get(id)?.values() ?? [])].map(item => item.provenance)] } };
|
|
95
|
+
if (encode(value).length > 128 * 1024) needsPackage = true;
|
|
96
|
+
add('ticket', id, value);
|
|
97
|
+
}
|
|
98
|
+
for (const [kind, map] of [['user', data.users], ['organization', data.organizations], ['ticket_field', data.ticket_fields]]) {
|
|
99
|
+
for (const [id,{ row, provenance }] of sortedRows(map)) add(kind, id, { ...row, _offline_provenance: [provenance] });
|
|
100
|
+
}
|
|
101
|
+
let restricted = 0, draft = 0;
|
|
102
|
+
for (const [id,{ row, provenance }] of sortedRows(data.articles)) {
|
|
103
|
+
if (row.user_segment_id != null || row.user_segment_ids?.length) restricted++;
|
|
104
|
+
if (row.draft) draft++;
|
|
105
|
+
add('article', id, { ...row, draft: row.draft === true, user_segment_id: row.user_segment_id ?? null, user_segment_ids: row.user_segment_ids ?? [], _offline_provenance: [provenance] });
|
|
106
|
+
}
|
|
107
|
+
const records = Buffer.concat(lines);
|
|
108
|
+
if (records.length > 2 * 1024 * 1024 || lines.length > 5000) needsPackage = true;
|
|
109
|
+
const census = { format: 'rer-zendesk-census/1', tickets: { unique: data.tickets.size, deleted: tombstones.length,
|
|
110
|
+
deletedIdsSha256: hash(tombstones.map(row => `${row.id}\n`).join('')), byStatus, maxId: ordered(data.tickets.keys()).at(-1) ?? null },
|
|
111
|
+
comments: { total: commentTotal, public: publicComments, private: commentTotal - publicComments },
|
|
112
|
+
attachments: attachmentCensus(refs.filter(ref => ref.ticket), data), articleAttachments: attachmentCensus(refs.filter(ref => ref.article), data, { article: true }),
|
|
113
|
+
users: { unique: data.users.size, inactive: [...data.users.values()].filter(item => item.row.active === false).length, withoutEmail: [...data.users.values()].filter(item => !item.row.email).length },
|
|
114
|
+
organizations: data.organizations.size, fields: data.ticket_fields.size,
|
|
115
|
+
guide: { available: data.available, articles: data.articles.size, restricted, draft, translations: data.translations.size, sections: data.sections.size, categories: data.categories.size, segments: data.user_segments.size },
|
|
116
|
+
records: byKind, undelivered: { tombstones: tombstones.length, translations: data.translations.size, sections: data.sections.size, categories: data.categories.size, segments: data.user_segments.size, articleAttachments: data.article_attachments.size }, credentialShapesRemoved: report.removals.length, credentialRemovals: report.removals, redactedUrls: report.redactedUrls, highWater: data.highWater, omissions };
|
|
117
|
+
const files = new Map([['records-000001.ndjson', records], ['tombstones.ndjson', Buffer.concat(tombstones.map(row => encode(row)))]]);
|
|
118
|
+
for (const kind of ['translations','sections','categories','user_segments','article_attachments']) {
|
|
119
|
+
const bytes = Buffer.concat([...data[kind]].sort(([a],[b]) => a.localeCompare(b)).map(([id,{ row }]) => encode(cleaned(row, kind, id, report))));
|
|
120
|
+
try { assertChunk(bytes, []); } catch { fail('CREDENTIAL_SHAPE_UNREMOVABLE'); }
|
|
121
|
+
files.set(`guide/${kind}.ndjson`, bytes);
|
|
122
|
+
}
|
|
123
|
+
// Article media remains in the package/store even before HH can import it.
|
|
124
|
+
for (const ref of refs.filter(ref => ref.article)) {
|
|
125
|
+
const blob = data.attachments.get(ref.key)?.blob;
|
|
126
|
+
if (blob) blobs.set(`blobs/${blob.sha256}`, blob);
|
|
127
|
+
}
|
|
128
|
+
census.credentialShapesRemoved = report.removals.length; census.redactedUrls = report.redactedUrls;
|
|
129
|
+
files.set('census.json', encode(census));
|
|
130
|
+
if (needsPackage && !packageBindings) return encodeSnapshot(store, { origin, packageBindings: true });
|
|
131
|
+
return { kind: needsPackage ? 'package' : 'bundle', bytes: records, files, blobs, census,
|
|
132
|
+
source: { format: FORMAT, version: 1, origin, exportedAt: store.state.exportedAt, synthetic: false, adapter: 'rer:zendesk' } };
|
|
133
|
+
}
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
import { fail, object } from './common.mjs';
|
|
2
|
+
import { materialize, ordered } from './zendesk-store.mjs';
|
|
3
|
+
import { fetchPage } from './zendesk-pages.mjs';
|
|
4
|
+
|
|
5
|
+
const stages = [
|
|
6
|
+
['user_segments'], ['categories'], ['sections'], ['articles'],
|
|
7
|
+
['source_articles'], ['translations', 'categories'], ['translations', 'sections'], ['translations', 'articles'], ['article_attachments', 'articles'],
|
|
8
|
+
];
|
|
9
|
+
export async function guide(ctx) {
|
|
10
|
+
const state = ctx.store.state, stage = stages[state.stage];
|
|
11
|
+
if (!stage) { await ctx.store.commit({ ...state, phase: 'guide_attachments', attachmentIndex: 0 }); return; }
|
|
12
|
+
const [kind, resource] = stage;
|
|
13
|
+
let article, path;
|
|
14
|
+
if (state.stage < 4) path = `/api/v2/help_center/${kind}?per_page=100`;
|
|
15
|
+
else {
|
|
16
|
+
const data = await materialize(ctx.store), records = data[resource ?? 'articles'];
|
|
17
|
+
article = ordered(records.keys())[state.index];
|
|
18
|
+
if (article === undefined) { await ctx.store.commit({ ...state, stage: state.stage + 1, index: 0, path: null }); return; }
|
|
19
|
+
if (kind === 'source_articles') {
|
|
20
|
+
const row = records.get(article).row;
|
|
21
|
+
if (typeof row.source_locale !== 'string' || !/^[a-z]{2,3}(?:-[a-z0-9]{2,8})?$/u.test(row.source_locale)) fail('INVALID_RESPONSE');
|
|
22
|
+
if (row.locale !== row.source_locale) {
|
|
23
|
+
const response = await ctx.request(`/api/v2/help_center/${row.source_locale}/articles/${article}`);
|
|
24
|
+
const source = object(response.body.article);
|
|
25
|
+
if (String(source.id) !== article || source.locale !== row.source_locale) fail('INVALID_RESPONSE');
|
|
26
|
+
await ctx.store.page('articles', response, { ...state, index: state.index + 1 }, { single: 'article' });
|
|
27
|
+
} else await ctx.store.commit({ ...state, index: state.index + 1 });
|
|
28
|
+
return;
|
|
29
|
+
}
|
|
30
|
+
path = `/api/v2/help_center/${resource}/${article}/${kind === 'article_attachments' ? 'attachments' : kind}?per_page=100`;
|
|
31
|
+
}
|
|
32
|
+
await fetchPage({ ...ctx, state, kind, path: state.path ?? path, allow404: true, extra: article ? { article, resource } : {},
|
|
33
|
+
onDone: state => ({ ...state, stage: state.stage + (article ? 0 : 1), index: article ? state.index + 1 : 0 }) });
|
|
34
|
+
}
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
import { join } from 'node:path';
|
|
2
|
+
import { buildPackage } from '../package-transfer.mjs';
|
|
3
|
+
|
|
4
|
+
/** Materialize checked HH lines and sidecars as immutable files, and reference
|
|
5
|
+
* the already verified original blobs. The shared sender owns segmentation,
|
|
6
|
+
* manifest probing and the resumable upload journal. */
|
|
7
|
+
export async function prepareZendeskPackage({ result, quota, directory, signal, mode = 'snapshot' }) {
|
|
8
|
+
const files = [], store = result.store;
|
|
9
|
+
const add = async (path, bytes) => {
|
|
10
|
+
const ref = await store.put(bytes);
|
|
11
|
+
files.push({ path, file: join(store.path, ref.file), bytes: ref.bytes, sha256: ref.sha256 });
|
|
12
|
+
};
|
|
13
|
+
let start = 0, position = 0, ordinal = 1;
|
|
14
|
+
while (position < result.bytes.length) {
|
|
15
|
+
signal?.throwIfAborted();
|
|
16
|
+
const newline = result.bytes.indexOf(10, position);
|
|
17
|
+
const end = newline < 0 ? result.bytes.length : newline + 1;
|
|
18
|
+
if (position > start && end - start > 8 * 1024 * 1024) {
|
|
19
|
+
await add(`records-${String(ordinal++).padStart(6,'0')}.ndjson`, result.bytes.subarray(start,position)); start = position;
|
|
20
|
+
}
|
|
21
|
+
position = end;
|
|
22
|
+
}
|
|
23
|
+
if (position > start) await add(`records-${String(ordinal).padStart(6,'0')}.ndjson`, result.bytes.subarray(start,position));
|
|
24
|
+
for (const [path, bytes] of result.files) if (!path.startsWith('records-')) await add(path,bytes);
|
|
25
|
+
for (const [path,ref] of result.blobs) { await store.get(ref); files.push({ path, file: join(store.path,ref.file), bytes: ref.bytes, sha256: ref.sha256 }); }
|
|
26
|
+
return buildPackage({ files, quota, directory, signal, mode, source: result.source,
|
|
27
|
+
records: { total: Object.values(result.census.records).reduce((a,b) => a+b,0), byKind: result.census.records }, totals: result.census });
|
|
28
|
+
}
|