@runeditrun/rer 0.2.99 → 0.2.100

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (72) hide show
  1. package/cli/README.md +88 -2
  2. package/cli/business/bounded-waves.d.mts +23 -0
  3. package/cli/business/bounded-waves.mjs +57 -0
  4. package/cli/business/catchup-task.mjs +9 -2
  5. package/cli/business/command.mjs +8 -3
  6. package/cli/business/export-inputs.mjs +12 -2
  7. package/cli/business/export-task.mjs +9 -2
  8. package/cli/business/legacy-package.mjs +117 -0
  9. package/cli/business/local-commands.mjs +10 -1
  10. package/cli/business/package-transfer.mjs +127 -0
  11. package/cli/business/state.mjs +3 -3
  12. package/cli/business/vendor-door.mjs +83 -11
  13. package/cli/business/vendors/README.md +41 -2
  14. package/cli/business/vendors/common.mjs +81 -0
  15. package/cli/business/vendors/index.mjs +12 -2
  16. package/cli/business/vendors/zendesk-attachments.mjs +57 -0
  17. package/cli/business/vendors/zendesk-encode.mjs +133 -0
  18. package/cli/business/vendors/zendesk-guide.mjs +34 -0
  19. package/cli/business/vendors/zendesk-package.mjs +28 -0
  20. package/cli/business/vendors/zendesk-pages.mjs +61 -0
  21. package/cli/business/vendors/zendesk-people.mjs +7 -0
  22. package/cli/business/vendors/zendesk-store.mjs +154 -0
  23. package/cli/business/vendors/zendesk-tickets.mjs +21 -0
  24. package/cli/business/vendors/zendesk-token.mjs +31 -0
  25. package/cli/business/vendors/zendesk-transport.mjs +48 -0
  26. package/cli/business/vendors/zendesk.mjs +61 -0
  27. package/cli/deploy-command.mjs +5 -4
  28. package/cli/deploy-inputs.mjs +1 -1
  29. package/cli/deploy.mjs +6 -4
  30. package/cli/production-auth.mjs +16 -3
  31. package/cli/production-mail.mjs +18 -1
  32. package/cli/site.mjs +1 -1
  33. package/global/CHANGELOG.md +21 -0
  34. package/modules/deploy-kit/README.md +10 -0
  35. package/modules/deploy-kit/provider.mjs +18 -2
  36. package/modules/import-kit/src/d1/jobs.ts +3 -1
  37. package/modules/import-kit/src/d1/retirement-scope.ts +2 -2
  38. package/modules/import-kit/src/d1/retirement-tally.ts +1 -0
  39. package/modules/import-kit/src/d1/retirement.ts +15 -5
  40. package/modules/import-kit/src/d1/types.ts +3 -0
  41. package/modules/import-kit/src/d1/wave-boundary.ts +33 -0
  42. package/modules/import-kit/src/d1/wave-retirement.ts +47 -0
  43. package/modules/import-kit/src/door/ADOPTING.md +37 -0
  44. package/modules/import-kit/src/door/README.md +66 -0
  45. package/modules/import-kit/src/door/admission.ts +13 -3
  46. package/modules/import-kit/src/door/authority.ts +1 -0
  47. package/modules/import-kit/src/door/failures.ts +1 -1
  48. package/modules/import-kit/src/door/grants.ts +19 -5
  49. package/modules/import-kit/src/door/handler.ts +12 -1
  50. package/modules/import-kit/src/door/index.ts +6 -0
  51. package/modules/import-kit/src/door/package.ts +121 -0
  52. package/modules/import-kit/src/door/planning.ts +44 -19
  53. package/modules/import-kit/src/door/protocol.ts +4 -2
  54. package/modules/import-kit/src/door/schema.ts +2 -0
  55. package/modules/import-kit/src/door/store.ts +2 -2
  56. package/modules/import-kit/src/door/types.ts +24 -1
  57. package/modules/import-kit/src/door/waves/README.md +158 -0
  58. package/modules/import-kit/src/door/waves/digests.ts +38 -0
  59. package/modules/import-kit/src/door/waves/inventory.ts +84 -0
  60. package/modules/import-kit/src/door/waves/journal.ts +19 -0
  61. package/modules/import-kit/src/door/waves/lifecycle.ts +242 -0
  62. package/modules/import-kit/src/door/waves/preparation.ts +190 -0
  63. package/modules/import-kit/src/door/waves/schema.ts +73 -0
  64. package/modules/import-kit/src/door/waves/store.ts +112 -0
  65. package/modules/import-kit/src/door/waves/types.ts +76 -0
  66. package/onboarding/adapter.mjs +7 -1
  67. package/onboarding/operator-installer.mjs +3 -1
  68. package/package.json +1 -1
  69. package/scripts/publish-check/README.md +20 -0
  70. package/scripts/publish-check/diagnostics.mjs +73 -0
  71. package/scripts/publish-check/slots.py +37 -7
  72. package/scripts/publish-check.mjs +64 -37
@@ -1,7 +1,12 @@
1
+ import { buildPackage, transferPackage } from './package-transfer.mjs';
2
+ import { registeredLegacyPackage } from './legacy-package.mjs';
3
+ import { prepareZendeskPackage } from './vendors/zendesk-package.mjs';
1
4
  import { createHash } from 'node:crypto';
2
5
  import { constants } from 'node:fs';
3
6
  import { open, unlink } from 'node:fs/promises';
4
7
  import { join } from 'node:path';
8
+ import { loadState, saveState } from './state.mjs';
9
+ import { snapshot as zendeskSnapshot, storePath as zendeskStorePath, removeZendeskStores } from './vendors/zendesk.mjs';
5
10
  import { taskPath } from './api.mjs';
6
11
  import { assertChunk } from './outbound.mjs';
7
12
  import { createCredentialStore } from './keychain.mjs';
@@ -18,7 +23,7 @@ import { businessCommand } from './invocation.mjs';
18
23
  const CHUNK_BYTES = 65_536;
19
24
  const DOOR_RESPONSE_BYTES = 4 * 1024 * 1024;
20
25
  // The catalogue names the platform builds vendor task titles from.
21
- export const vendorNames = Object.freeze({ sentry: 'Sentry', uptimerobot: 'UptimeRobot', pagerduty: 'PagerDuty', calendly: 'Calendly', sorry: 'Sorry' });
26
+ export const vendorNames = Object.freeze({ sentry: 'Sentry', uptimerobot: 'UptimeRobot', pagerduty: 'PagerDuty', calendly: 'Calendly', sorry: 'Sorry', zendesk: 'Zendesk', 'happytohelp-legacy': 'HappyToHelp legacy' });
22
27
  /** Whether a vendor's export needs the owner's read-only key (Sorry's public page does not). */
23
28
  export const keyed = vendor => KEYED_VENDORS.includes(vendor);
24
29
  export const vendorHandle = vendor => `vendor:${vendor}:read`;
@@ -97,12 +102,12 @@ function adapterInputs(vendor, task) {
97
102
  /** One RER-signed import grant for one door request. `review` asks for a
98
103
  * catch-up plan grant without the owner's standing approval; a plan grant
99
104
  * Business minted under that approval says so with `catchup: 'bounded'`. */
100
- async function doorGrant(lease, action, origin, vendor, { review = false } = {}) {
101
- const grant = await lease.importGrant(action, review ? { review: true } : {});
105
+ async function doorGrant(lease, action, origin, vendor, { review = false, bounded } = {}) {
106
+ const grant = await lease.importGrant(action, { ...(review ? { review: true } : {}), ...(bounded ? { bounded } : {}) });
102
107
  if (!grant || grant.action !== action || grant.destination !== new URL(`/api/rer/import/${action}`, origin).href
103
108
  || typeof grant.authorization !== 'string' || grant.authorization.length > 8192 || !/^Bearer [A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+$/.test(grant.authorization)
104
109
  || !VENDORS.includes(grant.namespace) || vendor && grant.namespace !== vendor || !(Date.parse(grant.expires_at) > Date.now())
105
- || Object.hasOwn(grant, 'catchup') && (grant.catchup !== 'bounded' || action !== 'plan' || review)) throw Error('Business returned an invalid import grant.');
110
+ || Object.hasOwn(grant, 'catchup') && (grant.catchup !== 'bounded' || !['plan','prepare'].includes(action) || review)) throw Error('Business returned an invalid import grant.');
106
111
  return grant;
107
112
  }
108
113
 
@@ -130,7 +135,7 @@ export async function door(grant, body, signal) {
130
135
  if (!response.ok) {
131
136
  // A door code is snake_case, sometimes with one named reason (`catchup_requires_proposal:source_records_gone`).
132
137
  const doorCode = typeof result?.error === 'string' && /^[a-z][a-z0-9_]{0,79}(?::[a-z][a-z0-9_]{0,79})?$/.test(result.error) ? result.error : null;
133
- throw Object.assign(Error(`The app's import door refused the ${action} request (HTTP ${response.status}${doorCode ? `: ${doorCode}` : ''}).`), { doorStatus: response.status, doorCode });
138
+ throw Object.assign(Error(`The app's import door refused the ${action} request (HTTP ${response.status}${doorCode ? `: ${doorCode}` : ''}).`), { doorStatus: response.status, doorCode, ...(doorCode === 'package_segment_missing' ? { doorDetails: result.details } : {}) });
134
139
  }
135
140
  if (!result || typeof result !== 'object' || Array.isArray(result)) throw Error(`The app's import door returned an invalid ${action} answer.`);
136
141
  return result;
@@ -138,12 +143,21 @@ export async function door(grant, body, signal) {
138
143
 
139
144
  /** Run the shipped adapter into a private 0600 bundle file, checking every
140
145
  * item with the outbound detector before it is written. */
141
- async function exportBundle({ vendor, key = null, inputs, path, quota, signal }) {
146
+ async function exportBundle({ vendor, key = null, inputs, path, quota, signal, directory, runId, business }) {
142
147
  await unlink(path).catch(error => { if (error.code !== 'ENOENT') throw error; });
143
148
  const file = await open(path, constants.O_WRONLY | constants.O_CREAT | constants.O_EXCL | constants.O_NOFOLLOW, 0o600);
144
- const digest = createHash('sha256'); let size = 0;
149
+ const digest = createHash('sha256'); let size = 0, zendeskStore;
145
150
  try {
146
- for await (const item of exportVendor(vendor, { key, inputs, signal })) {
151
+ let items;
152
+ if (vendor === 'zendesk') {
153
+ const root = zendeskStorePath(directory, business, `https://${inputs.account.subdomain}.zendesk.com`);
154
+ const result = await zendeskSnapshot({ key, inputs, directory: inputs.mode === 'census' ? join(root, 'probe') : root, runId, signal });
155
+ if (result.kind === 'census') throw new ExportWaiting(`Zendesk census: ${result.census.tickets.unique} tickets; ${result.census.unswept} unswept. CENSUS_OUTPUTS_UNSUPPORTED: waiting for Business census task outputs.`, 'connected_runner_required');
156
+ if (result.kind === 'package') return { package: result, store: result.store, mode: inputs.mode };
157
+ zendeskStore = result.store;
158
+ items = [{ bytes: result.bytes }];
159
+ } else items = exportVendor(vendor, { key, inputs, signal });
160
+ for await (const item of items) {
147
161
  assertChunk(item.bytes, key ? [key] : []);
148
162
  size += item.bytes.length;
149
163
  if (size > quota) throw Error(`Business vendor export is larger than the app's import door accepts (${quota} bytes). Nothing was uploaded.`);
@@ -159,7 +173,7 @@ async function exportBundle({ vendor, key = null, inputs, path, quota, signal })
159
173
  throw Error(`${vendorNames[vendor]} export stopped (${error.code}${error.status ? `, HTTP ${error.status}` : ''}). Nothing was uploaded.`);
160
174
  } finally { await file.close(); }
161
175
  if (!size) throw Error(`${vendorNames[vendor]} export produced nothing to upload.`);
162
- return { size, sha256: digest.digest('hex') };
176
+ return { size, sha256: digest.digest('hex'), ...(zendeskStore ? { store: zendeskStore, mode: inputs.mode } : {}) };
163
177
  }
164
178
 
165
179
  async function readChunk(path, index, size) {
@@ -199,19 +213,72 @@ export async function withUploadedBundle({ lease, task, grant: claimedGrant, dir
199
213
  const fresh = async (action, options) => {
200
214
  if (action === 'uploads' && spare && Date.parse(spare.expires_at) - Date.now() > 60_000) { const grant = spare; spare = null; return grant; }
201
215
  spare = null;
202
- return doorGrant(lease, action, origin, vendor, options);
216
+ return doorGrant(lease, action, origin, vendor, options ?? (task.bounded_waves && action==='prepare' ? {bounded:{protocol:task.bounded_waves.protocol,migration:task.bounded_waves.migration,revision:task.bounded_waves.revision,policy:task.bounded_waves.policy}} : undefined));
203
217
  };
204
218
  const inputs = adapterInputs(vendor, task);
219
+ const state = vendor === 'zendesk' ? await loadState() : null;
220
+ if (vendor === 'zendesk' && inputs.mode === 'retire') {
221
+ // The retire signal is local revocation, never an OAuth/provider write.
222
+ await createCredentialStore().delete(vendorHandle(vendor));
223
+ await removeZendeskStores(directory);
224
+ await saveState({ ...state, vendorHandles: (state.vendorHandles ?? []).filter(handle => handle !== vendorHandle(vendor)), vendorRevocations: { ...state.vendorRevocations, zendesk: true } });
225
+ throw new ExportWaiting('Zendesk retired locally (VENDOR_REVOKED). RETIRE_OUTPUTS_UNSUPPORTED: waiting for Business retire task acknowledgement.', 'connected_runner_required');
226
+ }
227
+ if (vendor === 'zendesk' && state?.vendorRevocations?.zendesk) throw new VendorExportError('VENDOR_REVOKED');
205
228
  let key = keyed(vendor) ? await createCredentialStore().get(vendorHandle(vendor)) : null;
206
229
  if (keyed(vendor) && !key) throw new ExportWaiting(`Business vendor export needs the owner's read-only ${name} key on this computer: ${keyStep(vendor)}\nThen run ${businessCommand('run')} again.`, 'owner_required', { keyMissing: vendor });
230
+ if (vendor === 'happytohelp-legacy') {
231
+ if (!['export.vendor', 'catchup.vendor'].includes(task.kind)) throw new ExportWaiting('Legacy package tasks require a package-aware export or catch-up reader.', 'configuration_required');
232
+ let legacy;
233
+ try { legacy = await registeredLegacyPackage(); }
234
+ catch (error) { throw new ExportWaiting(error.message, 'configuration_required'); }
235
+ say(doing(name, origin, false));
236
+ const packageDirectory = join(directory, `package-${hash(task.id).slice(0, 32)}`);
237
+ let prepared;
238
+ try {
239
+ prepared = await buildPackage({ files: legacy.files, quota, directory: packageDirectory, signal,
240
+ source: { format: legacy.manifest.format, version: 1, origin: 'legacy-database', exportedAt: legacy.manifest.exportedAt ?? legacy.manifest.source.exportedAt ?? null,
241
+ synthetic: legacy.manifest.source.synthetic === true, adapter: 'rer:happytohelp-legacy' },
242
+ records: { total: null, byKind: {} }, totals: { rowCounts: legacy.rowCounts, objects: legacy.objects } });
243
+ } catch { throw new ExportWaiting(`Legacy package changed or could not be prepared: ${legacy.path}. Recheck and register that package again.`, 'configuration_required'); }
244
+ let uploaded;
245
+ try { uploaded = await transferPackage({ prepared, fresh, request: door, signal, checkpoint: cp => lease.checkpointSaved(cp), catchup: task.kind === 'catchup.vendor', ...(task.bounded_waves ? { planAction:'prepare', planInput:{ordinal:0} } : {}) }); }
246
+ catch (error) { if (!error.message.startsWith('SEED_PACKAGE_UNSUPPORTED')) throw error; throw new ExportWaiting(error.message, 'configuration_required'); }
247
+ const { outputs, said } = await finish({ vendor, name, origin, ...uploaded, fresh });
248
+ await lease.complete(outputs);
249
+ return said;
250
+ }
207
251
  const path = join(directory, `export-${hash(task.id).slice(0, 32)}.bundle`);
208
252
  try {
209
253
  say(doing(name, origin, keyed(vendor)));
210
254
  let bundle;
211
- try { bundle = await exportBundle({ vendor, key, inputs, path, quota, signal }); } finally { key = undefined; }
255
+ try { bundle = await exportBundle({ vendor, key, inputs, path, quota, signal, directory, runId: task.id, business: state?.businessId }); } finally { key = undefined; }
256
+ if (bundle.package) {
257
+ const prepared = await prepareZendeskPackage({ result: bundle.package, quota, directory: join(bundle.store.path, `package-${hash(task.id).slice(0,32)}`), signal, mode: inputs.mode });
258
+ const uploaded = await transferPackage({ prepared, fresh, request: door, signal, checkpoint: cp => lease.checkpointSaved(cp), catchup: task.kind === 'catchup.vendor' || inputs.mode === 'sweep' });
259
+ const finished = await finish({ vendor, name, origin, ...uploaded, fresh });
260
+ await lease.complete(finished.outputs);
261
+ return finished.said;
262
+ }
212
263
  const total = Math.ceil(bundle.size / CHUNK_BYTES);
213
264
  let uploadId;
265
+ const received = new Set();
266
+ const journal = bundle.store?.events.findLast(event => event.kind === 'transfer');
267
+ if (journal && journal.origin === origin && journal.bundleSha256 === bundle.sha256 && journal.totalBytes === bundle.size) {
268
+ try {
269
+ const inventory = await door(await fresh('uploads'), { uploadId: journal.uploadId }, signal);
270
+ if (inventory.uploadId !== journal.uploadId || inventory.bundleSha256 !== bundle.sha256 || !Array.isArray(inventory.received)) throw new VendorExportError('INVALID_CHECKPOINT');
271
+ for (const item of inventory.received) {
272
+ if (!Number.isSafeInteger(item.ordinal) || item.ordinal < 0 || item.ordinal >= total || received.has(item.ordinal)) throw new VendorExportError('INVALID_CHECKPOINT');
273
+ const bytes = await readChunk(path, item.ordinal, bundle.size);
274
+ if (item.sha256 !== hash(bytes) || item.size !== bytes.length) throw new VendorExportError('INVALID_CHECKPOINT');
275
+ received.add(item.ordinal);
276
+ }
277
+ uploadId = journal.uploadId;
278
+ } catch (error) { if (error.doorStatus !== 404) throw error; }
279
+ }
214
280
  for (let index = 0; index < total; index++) {
281
+ if (received.has(index)) continue;
215
282
  signal.throwIfAborted();
216
283
  const bytes = await readChunk(path, index, bundle.size);
217
284
  const result = await door(await fresh('uploads'), { ...(uploadId ? { uploadId } : {}), name: capability.output, bundleSha256: bundle.sha256,
@@ -219,6 +286,11 @@ export async function withUploadedBundle({ lease, task, grant: claimedGrant, dir
219
286
  if (typeof result.uploadId !== 'string' || !DOOR_ID.test(result.uploadId) || uploadId && result.uploadId !== uploadId
220
287
  || result.bundleSha256 !== bundle.sha256 || index === total - 1 && result.complete !== true) throw Error("The app's import door returned an invalid upload answer.");
221
288
  uploadId = result.uploadId;
289
+ if (bundle.store) {
290
+ const transfer = { kind: 'transfer', origin, bundleSha256: bundle.sha256, totalBytes: bundle.size, uploadId };
291
+ await bundle.store.commit(bundle.store.state, transfer);
292
+ await bundle.store.write('transfer.json', transfer);
293
+ }
222
294
  await lease.checkpointSaved({ page: index + 1, total, bytes_done: Math.min((index + 1) * CHUNK_BYTES, bundle.size), progress: Math.floor((index + 1) / total * 90) });
223
295
  }
224
296
  const { outputs, said } = await finish({ vendor, name, origin, uploadId, bundle, fresh });
@@ -23,9 +23,9 @@ No export runs merely by importing these modules. No account export was run
23
23
  during implementation.
24
24
 
25
25
  `VENDORS` is the closed vendor enum (`sentry`, `uptimerobot`, `pagerduty`,
26
- `calendly`, `sorry`). `VENDOR_CAPABILITIES` describes the adapter,
26
+ `calendly`, `sorry`, `zendesk`, `happytohelp-legacy`). `VENDOR_CAPABILITIES` describes the adapter,
27
27
  required vendor-key capability, native importer format, layout and read scopes.
28
- These adapters cannot verify that a supplied key itself has read-only scope;
28
+ Except for Zendesk's checked OAuth scopes, adapters cannot verify that a supplied key itself has read-only scope;
29
29
  the owner must create the documented read-only key where the vendor offers one.
30
30
 
31
31
  ## Transport and resume
@@ -174,3 +174,42 @@ Seed documentation and official APIs were inspected on 16 October 2026.
174
174
  These sources document shapes; no real-account parity or migration success is
175
175
  claimed. Parent integration must provide seed assembly and transport under the
176
176
  actual import grant protocol, plus network-edge acceptance coverage.
177
+
178
+ ## Zendesk's retained snapshot
179
+
180
+ `zendesk` is a keyed vendor with `layout: 'snapshot'`. Its credential is the
181
+ hidden-input JSON client credential documented in [the CLI guide](../../README.md#zendesk-offline-snapshots),
182
+ rather than a bare API token. `snapshot({key,inputs,directory,runId,signal})` in
183
+ `zendesk.mjs` drives phases A–F into the durable store. `exportVendor` accepts
184
+ `directory` and `runId` for this vendor; the Business door provides a directory
185
+ keyed by Business and account. It does not apply the older vendors' 2,000-item
186
+ or one-MiB item limits. The complete deliverable is checked against HH v1 limits
187
+ before any upload. Package-required results use the shared package sender with path bindings and
188
+ sidecars. It probes the manifest before sending segments, resumes the transfer
189
+ journal and plans one complete snapshot; no per-part plan or fallback is used.
190
+
191
+ Responsibilities are separated into token/scope checking, transport, pages,
192
+ tickets/comments, people, attachments, Guide, store and HH encoding. Requests
193
+ stop at 250,000, responses at 32 MiB, records/blobs at 64 MiB, snapshots at
194
+ 32 GiB. The opt-in `pacedClient` in `common.mjs` supports bounded 429 retries
195
+ (1–60 seconds, at most three per request and ten minutes per run), rate headers,
196
+ and manual attachment redirects. Without rate headers it waits at least six
197
+ seconds between incremental requests and 300 ms otherwise. Other vendors keep
198
+ `client()` and its previous no-retry behavior.
199
+
200
+ Only the account origin receives Authorization. The first attachment URL must
201
+ be on that origin's attachment path. One redirect may go to a public HTTPS DNS
202
+ host, with no userinfo, IP, explicit port or local/private address, and receives
203
+ no credential even when it returns to the account origin. Further redirects
204
+ are errors. Pagination is independently checked for account origin, exact
205
+ collection and allowed query parameters.
206
+
207
+ Checkpoints point at immutable SHA-256 journal nodes and raw objects. Each page
208
+ is fsynced before its journal and atomic checkpoint; a kill before acknowledgement
209
+ leaves only unreachable objects. Resume validates the reachable graph and
210
+ rebuilds last-version records and census from it. Repeated attachment bytes
211
+ have one blob. The census declares tombstones, inaccessible/deleted/malware/large
212
+ attachments and omissions explicitly. Guide 404 records unavailable coverage.
213
+ The JSON census and Guide sidecars stay local and in the package; Business
214
+ receives only the door's ids and SHA for a successful bundle. See the CLI guide
215
+ for the separate Business census/retire output dependencies.
@@ -197,3 +197,84 @@ export function client({ origin, headers, route, signal, limits, counters, key }
197
197
  return { body, text, headers: response.headers };
198
198
  };
199
199
  }
200
+
201
+ /** Opt-in transport for long-lived snapshots. Existing adapters retain client()'s
202
+ * no-retry contract. The caller validates every target before credentials or a
203
+ * request are constructed, including manually followed attachment redirects. */
204
+ export function pacedClient({ origin, signal, validate, secrets = [], limits = {}, beforeRequest }) {
205
+ const maximum = { requests: 250_000, response: 32 * 1024 * 1024, bytes: 32 * 1024 ** 3, ...limits };
206
+ const counters = { requests: 0, bytes: 0, retryMs: 0 };
207
+ let token, last = 0, next = 0, advertisedBefore = false;
208
+ const wait = async milliseconds => { if (milliseconds > 0) await delay(milliseconds, undefined, { signal }); };
209
+ const request = async (path, options = {}) => {
210
+ let url;
211
+ try { url = new URL(path, origin); } catch { fail('INVALID_REFERENCE'); }
212
+ await validate(url, options);
213
+ const method = options.method ?? 'GET';
214
+ if (method !== 'GET' && !(method === 'POST' && url.origin === origin && url.pathname === '/oauth/tokens')) fail('INVALID_REFERENCE');
215
+ for (let retries = 0; ; retries++) {
216
+ signal?.throwIfAborted();
217
+ await wait(Math.max(next, last + (advertisedBefore ? 0 : url.pathname.includes('/incremental/') ? 6000 : 300)) - Date.now());
218
+ if (++counters.requests > maximum.requests) fail('REQUEST_LIMIT');
219
+ await beforeRequest?.();
220
+ last = Date.now();
221
+ const headers = { accept: 'application/json', ...(options.json ? { 'content-type': 'application/json' } : {}),
222
+ ...(token && url.origin === origin && options.authenticated !== false ? { authorization: `Bearer ${token}` } : {}) };
223
+ let response;
224
+ try { response = await fetch(url, { method, headers, redirect: options.redirect ?? 'error',
225
+ ...(options.json ? { body: JSON.stringify(options.json) } : {}), signal: signal ? AbortSignal.any([signal, AbortSignal.timeout(30_000)]) : AbortSignal.timeout(30_000) }); }
226
+ catch { fail(signal?.aborted ? 'CANCELLED' : 'NETWORK_FAILED'); }
227
+ const waits = [];
228
+ const remaining = response.headers.get('ratelimit-remaining') ?? response.headers.get('x-rate-limit-remaining');
229
+ const reset = response.headers.get('ratelimit-reset') ?? '60';
230
+ const capacity = (remaining, seconds) => {
231
+ if (!/^\d+$/u.test(remaining ?? '') || !/^\d+$/u.test(seconds ?? '') || Number(seconds) > 3600) return;
232
+ waits.push(Math.max(300, Math.ceil(Number(seconds) * 1000 / Math.max(1, Number(remaining)))));
233
+ };
234
+ capacity(remaining, reset);
235
+ for (const [name,value] of response.headers) if (name.toLowerCase().startsWith('zendesk-ratelimit-')) capacity(/remaining=(\d+)/u.exec(value)?.[1], /resets=(\d+)/u.exec(value)?.[1]);
236
+ const interval = response.headers.get('ratelimit-limit') ?? response.headers.get('x-rate-limit');
237
+ if (!waits.length && /^\d+$/u.test(interval ?? '')) waits.push(Math.max(300, Math.ceil(60_000 / Math.max(1, Number(interval)))));
238
+ // Obey the tightest account/endpoint budget. Without valid advertised
239
+ // capacity the next request retains the conservative fallback spacing.
240
+ advertisedBefore = waits.length > 0;
241
+ next = Date.now() + Math.max(0, ...waits);
242
+ if (response.status === 429) {
243
+ const after = response.headers.get('retry-after');
244
+ await response.body?.cancel().catch(() => {});
245
+ const seconds = /^\d+$/u.test(after ?? '') ? Number(after) : 0;
246
+ if (retries >= 3 || seconds < 1 || seconds > 60 || counters.retryMs + seconds * 1000 > 600_000) throw new VendorExportError('RATE_LIMITED', 429);
247
+ counters.retryMs += seconds * 1000;
248
+ await wait(seconds * 1000);
249
+ continue;
250
+ }
251
+ if (options.redirect === 'manual' && [301, 302, 303, 307, 308].includes(response.status)) {
252
+ const location = response.headers.get('location');
253
+ await response.body?.cancel().catch(() => {});
254
+ return { status: response.status, location, headers: response.headers };
255
+ }
256
+ if (!response.ok) {
257
+ await response.body?.cancel().catch(() => {});
258
+ if (options.allow?.includes(response.status)) return { status: response.status };
259
+ throw new VendorExportError('HTTP_REFUSED', response.status);
260
+ }
261
+ const chunks = []; let size = 0;
262
+ try {
263
+ for await (const chunk of response.body ?? []) {
264
+ size += chunk.byteLength; counters.bytes += chunk.byteLength;
265
+ if (size > (options.maxBytes ?? maximum.response) || counters.bytes > maximum.bytes) fail('BYTE_LIMIT');
266
+ chunks.push(chunk);
267
+ }
268
+ } catch (error) { if (error instanceof VendorExportError) throw error; fail(signal?.aborted ? 'CANCELLED' : 'NETWORK_FAILED'); }
269
+ const bytes = Buffer.concat(chunks);
270
+ if (!options.tokenResponse && [...secrets, token].some(secret => secret && bytes.includes(Buffer.from(secret)))) fail('CREDENTIAL_IN_RESPONSE');
271
+ if (options.binary) return { bytes, headers: response.headers, status: response.status };
272
+ let body;
273
+ try { body = JSON.parse(new TextDecoder('utf-8', { fatal: true }).decode(bytes)); } catch { fail('INVALID_JSON'); }
274
+ return { body, bytes, headers: response.headers, status: response.status };
275
+ }
276
+ };
277
+ request.authorize = value => { token = value; };
278
+ request.counters = counters;
279
+ return request;
280
+ }
@@ -4,11 +4,14 @@ import * as uptimerobot from './uptimerobot.mjs';
4
4
  import * as pagerduty from './pagerduty.mjs';
5
5
  import * as calendly from './calendly.mjs';
6
6
  import * as sorry from './sorry.mjs';
7
+ import * as zendesk from './zendesk.mjs';
7
8
 
8
9
  export { VendorExportError } from './common.mjs';
9
- export const VENDORS = Object.freeze(['sentry','uptimerobot','pagerduty','calendly','sorry']);
10
+ export const VENDORS = Object.freeze(['sentry','uptimerobot','pagerduty','calendly','sorry','zendesk','happytohelp-legacy']);
10
11
  const adapters = { sentry, uptimerobot, pagerduty, calendly, sorry };
11
12
  export const VENDOR_CAPABILITIES = Object.freeze({
13
+ 'happytohelp-legacy': Object.freeze({ vendor: 'happytohelp-legacy', adapter: 'rer:happytohelp-legacy', capability: null, format: 'happytohelp-legacy-chunks-v1', layout: 'package', output: 'rer-package.json', readOnly: true, resume: true, scopes: [], origins: [] }),
14
+ zendesk: Object.freeze({ vendor: 'zendesk', adapter: 'rer:zendesk', capability: 'vendor_key:zendesk', format: 'happytohelp-zendesk-offline-v1', layout: 'snapshot', output: 'zendesk.ndjson', readOnly: true, resume: true, scopes: ['tickets:read','users:read','organizations:read','hc:read','ticket_attachments:read'], origins: ['https://<subdomain>.zendesk.com'] }),
12
15
  sentry: Object.freeze({ vendor: 'sentry', adapter: 'rer:sentry', capability: 'vendor_key:sentry', format: 'spool-sentry-org-export-v1', layout: 'concatenate', output: 'sentry-export.json', readOnly: true, resume: true, scopes: ['org:read','project:read','event:read'], origins: Object.values(sentry.origins) }),
13
16
  uptimerobot: Object.freeze({ vendor: 'uptimerobot', adapter: 'rer:uptimerobot', capability: 'vendor_key:uptimerobot', format: 'uptimerobot-v3-monitors', layout: 'concatenate', output: 'uptimerobot-monitors.json', readOnly: true, resume: true, scopes: ['read-only API key'], origins: [uptimerobot.origin] }),
14
17
  pagerduty: Object.freeze({ vendor: 'pagerduty', adapter: 'rer:pagerduty', capability: 'vendor_key:pagerduty', format: 'pagerduty-saved-responses', layout: 'concatenate', output: 'pagerduty-export.json', readOnly: true, resume: true, scopes: ['read-only REST API key'], origins: [pagerduty.origin] }),
@@ -33,9 +36,16 @@ const LIMITS = Object.freeze({
33
36
  /** Persist each checkpoint only after the corresponding bytes have been acknowledged.
34
37
  * Bytes are private seed-bound artifacts; checkpoints are metadata, never API bodies.
35
38
  */
36
- export async function* exportVendor(vendor, { key, inputs = {}, checkpoint, signal } = {}) {
39
+ export async function* exportVendor(vendor, { key, inputs = {}, checkpoint, signal, directory, runId } = {}) {
37
40
  try {
38
41
  if (!VENDORS.includes(vendor)) fail('UNSUPPORTED_VENDOR');
42
+ if (vendor === 'happytohelp-legacy') fail('LOCAL_PACKAGE_REQUIRED');
43
+ if (vendor === 'zendesk') {
44
+ const result = await zendesk.snapshot({ key, inputs, directory, runId, signal });
45
+ if (result.kind !== 'bundle') fail(result.kind === 'package' ? 'SEED_PACKAGE_UNSUPPORTED' : 'INVALID_INPUT');
46
+ yield { name: 'zendesk.ndjson', bytes: result.bytes, checkpoint: { version: 1, vendor, head: result.store.head } };
47
+ return;
48
+ }
39
49
  const keyed = KEYED_VENDORS.includes(vendor);
40
50
  if (keyed ? typeof key !== 'string' || !key || key.length > 8192 || /[\u0000-\u0020\u007f]/u.test(key) : key != null) fail('INVALID_KEY');
41
51
  object(inputs);
@@ -0,0 +1,57 @@
1
+ import { VendorExportError, fail } from './common.mjs';
2
+ import { materialize, numericId, ordered } from './zendesk-store.mjs';
3
+ import { publicLocation, attachmentPath } from './zendesk-transport.mjs';
4
+
5
+ export function attachmentReferences(data) {
6
+ const refs = [];
7
+ for (const [ticket, comments] of data.comments) {
8
+ if (data.tickets.get(ticket)?.row.status === 'deleted' || data.tickets.get(ticket)?.row.deleted) continue;
9
+ for (const [comment, { row }] of comments) for (const attachment of row.attachments ?? []) refs.push({ key: `ticket:${ticket}:${comment}:${numericId(attachment.id)}`, ticket, comment, attachment });
10
+ }
11
+ for (const [id, { row }] of data.article_attachments) refs.push({ key: `article:${id}`, article: id.split(':')[1], attachment: row });
12
+ return refs.sort((a,b) => a.key.localeCompare(b.key));
13
+ }
14
+ export async function download(request, origin, attachment) {
15
+ if (attachment.deleted) return { reason: 'deleted' };
16
+ if (attachment.malware_scan_result === 'malware_found' || attachment.malware_found) return { reason: 'malware_found' };
17
+ const size = attachment.size;
18
+ if (!Number.isSafeInteger(size) || size < 0) fail('INVALID_RESPONSE');
19
+ if (size > 64 * 1024 * 1024) return { reason: 'too_large' };
20
+ let url;
21
+ try { url = new URL(attachment.content_url); } catch { return { reason: 'refused_location' }; }
22
+ if (url.origin !== origin || !attachmentPath(url.pathname) || !await publicLocation(attachment.content_url, origin)) return { reason: 'refused_location' };
23
+ let response;
24
+ try {
25
+ response = await request(url.href, { attachment: true, binary: true, maxBytes: 64 * 1024 * 1024, redirect: 'manual', allow: [403,404,410] });
26
+ if (response.location !== undefined) {
27
+ const next = await publicLocation(response.location, origin);
28
+ if (!next) return { reason: 'refused_location' };
29
+ response = await request(next.href, { attachment: true, binary: true, authenticated: false, maxBytes: 64 * 1024 * 1024, redirect: 'error', allow: [403,404,410] });
30
+ }
31
+ } catch (error) {
32
+ if (error instanceof VendorExportError && error.code === 'REFUSED_LOCATION') return { reason: 'refused_location' };
33
+ throw error;
34
+ }
35
+ if ([403,404,410].includes(response.status)) return { reason: `http_${response.status}` };
36
+ if (!response.bytes || response.bytes.length !== size) fail('ATTACHMENT_SIZE_MISMATCH');
37
+ return { bytes: response.bytes };
38
+ }
39
+ export async function attachments(ctx) {
40
+ const { store, request, origin, inputs } = ctx, state = store.state, data = await materialize(store);
41
+ const ids = ordered([...data.tickets].filter(([, { row }]) => row.status !== 'deleted' && !row.deleted && (inputs.mode !== 'sweep' || row.generated_timestamp > inputs.highWater)).map(([id]) => id));
42
+ const ticket = ids[state.index];
43
+ const refs = attachmentReferences(data).filter(ref => state.phase === 'guide_attachments' ? ref.article : ref.ticket === ticket);
44
+ const ref = refs[state.attachmentIndex];
45
+ if (!ref) {
46
+ await store.commit(state.phase === 'guide_attachments' ? { ...state, phase: 'done', attachmentIndex: 0 } : { ...state, phase: 'B', path: null, index: state.index + 1, attachmentIndex: 0 });
47
+ return;
48
+ }
49
+ const previous = data.attachments.get(ref.key);
50
+ if (previous && previous.sourceHash === JSON.stringify(ref.attachment)) {
51
+ await store.commit({ ...state, attachmentIndex: state.attachmentIndex + 1 }); return;
52
+ }
53
+ const result = await download(request, origin, ref.attachment);
54
+ const blob = result.bytes ? await store.put(result.bytes) : null;
55
+ await store.commit({ ...state, attachmentIndex: state.attachmentIndex + 1, bytes: state.bytes + (blob?.bytes ?? 0) },
56
+ { kind: 'attachment', key: ref.key, sourceHash: JSON.stringify(ref.attachment), ...(blob ? { blob } : { reason: result.reason }) }, blob ? [blob] : []);
57
+ }
@@ -0,0 +1,133 @@
1
+ import { credentialShaped } from '../detector.mjs';
2
+ import { assertChunk } from '../outbound.mjs';
3
+ import { fail, hash } from './common.mjs';
4
+ import { materialize, encode, ordered } from './zendesk-store.mjs';
5
+ import { attachmentReferences } from './zendesk-attachments.mjs';
6
+ import { attachmentPath } from './zendesk-transport.mjs';
7
+
8
+ export const FORMAT = 'happytohelp-zendesk-offline-v1';
9
+ const omissions = ['audits','metrics','satisfaction','side conversations','groups','brands','forms','macros','triggers','views','automations','photos','thumbnails','archived articles','community'];
10
+ const escapePointer = value => value.replace(/~/gu, '~0').replace(/\//gu, '~1');
11
+ /** Redact only the delivered copy. Raw responses remain private evidence in
12
+ * the store. Expand prefix detector hits to their whole token and PEM hits to
13
+ * their whole block; assertChunk remains the final authority on every line. */
14
+ function cleaned(value, kind, id, report, pointer = '') {
15
+ const removed = () => report.removals.push({ kind, id, pointer: pointer || '/' });
16
+ if (Array.isArray(value)) return value.map((child,index) => cleaned(child, kind, id, report, `${pointer}/${index}`));
17
+ if (value && typeof value === 'object') {
18
+ const result = Object.create(null);
19
+ for (const [key,child] of Object.entries(value)) {
20
+ if (['content_url','download_url','original_url'].includes(key)) { report.redactedUrls++; continue; }
21
+ if (credentialShaped(key)) fail('CREDENTIAL_SHAPE_UNREMOVABLE');
22
+ result[key] = cleaned(child, kind, id, report, `${pointer}/${escapePointer(key)}`);
23
+ }
24
+ return result;
25
+ }
26
+ if (typeof value !== 'string') return value;
27
+ let text = value.replace(/https?:\/\/[^\s<>"']+/giu, match => {
28
+ let url; try { url = new URL(match); } catch { return match; }
29
+ if (attachmentPath(url.pathname)) { report.redactedUrls++; return '[removed]'; }
30
+ if (!url.username && !url.password && !url.search && !url.hash) return match;
31
+ url.username = ''; url.password = ''; url.search = ''; url.hash = ''; report.redactedUrls++; return url.href;
32
+ });
33
+ text = text.replace(/(?:\/\/[^\s<>"'/]+)?\/(?:api\/v2\/)?attachments\/[^\s<>"']*/giu, () => { report.redactedUrls++; return '[removed]'; });
34
+ text = text.replace(/-----\s*BEGIN\s+([A-Z0-9 ]+)-----[\s\S]*?-----\s*END\s+\1-----/giu, () => { removed(); return '[removed]'; });
35
+ // SSH material includes a scheme and key separated by whitespace.
36
+ text = text.replace(/\b(?:ssh-(?:rsa|ed25519)|ecdsa-sha2-[\w-]+)\s+[A-Za-z0-9+/=]+/gu, () => { removed(); return '[removed]'; });
37
+ text = text.replace(/[^\s<>"'`,;(){}\[\]]+/gu, token => {
38
+ if (!credentialShaped(token)) return token;
39
+ removed(); return '[removed]';
40
+ });
41
+ text = text.replace(/[\u0000-\u0008\u000b\u000c\u000e-\u001f\u007f]/gu, () => { removed(); return '[removed]'; });
42
+ text = text.replace(/https?:\/\/[^\s<>"']+/giu, match => {
43
+ let url;
44
+ try { url = new URL(match); } catch { return match; }
45
+ if (!url.username && !url.password && !url.search && !url.hash) return match;
46
+ url.username = ''; url.password = ''; url.search = ''; url.hash = ''; report.redactedUrls++; return url.href;
47
+ });
48
+ if (credentialShaped(text.replace(/[\r\n\t]/gu, ' '))) fail('CREDENTIAL_SHAPE_UNREMOVABLE');
49
+ return text;
50
+ }
51
+ const deleted = row => row.status === 'deleted' || row.deleted === true;
52
+ const sortedRows = map => ordered(map.keys()).map(id => [id, map.get(id)]);
53
+ function attachmentCensus(references, data, { article = false } = {}) {
54
+ const blobs = new Map(), undelivered = {};
55
+ let fetched = 0, bytes = 0;
56
+ for (const ref of references) {
57
+ const item = data.attachments.get(ref.key);
58
+ if (item?.blob) { fetched++; bytes += item.blob.bytes; blobs.set(item.blob.sha256, item.blob.bytes); }
59
+ const reason = item?.reason ?? (!item ? 'not_fetched' : article ? 'pending_guide' : null);
60
+ if (reason) undelivered[reason] = (undelivered[reason] ?? 0) + 1;
61
+ }
62
+ return { referenced: references.length, fetched, bytes, uniqueBlobs: blobs.size, undelivered };
63
+ }
64
+ export async function encodeSnapshot(store, { origin, packageBindings = false } = {}) {
65
+ const data = await materialize(store), report = { removals: [], redactedUrls: 0 }, lines = [], byKind = {}, blobs = new Map();
66
+ const add = (kind, id, value) => {
67
+ // Bindings are integrity-checked original bytes, never redaction input.
68
+ const clean = kind === 'ticket' ? { ...cleaned({ ...value, _offline: undefined }, kind, id, report), _offline: value._offline } : cleaned(value, kind, id, report);
69
+ const line = encode({ [kind]: clean });
70
+ if (line.length > 64 * 1024 * 1024) fail('RECORD_LIMIT');
71
+ try { assertChunk(line, []); } catch { fail('CREDENTIAL_SHAPE_UNREMOVABLE'); }
72
+ lines.push(line); byKind[kind] = (byKind[kind] ?? 0) + 1;
73
+ };
74
+ let needsPackage = packageBindings, commentTotal = 0, publicComments = 0;
75
+ const tombstones = [], byStatus = {}, refs = attachmentReferences(data);
76
+ for (const [id,{ row, provenance }] of sortedRows(data.tickets)) {
77
+ byStatus[row.status] = (byStatus[row.status] ?? 0) + 1;
78
+ if (deleted(row)) { tombstones.push({ id: row.id, generated_timestamp: row.generated_timestamp }); continue; }
79
+ const comments = sortedRows(data.comments.get(id) ?? new Map()).map(([,item]) => item.row), bindings = [];
80
+ for (const comment of comments) {
81
+ commentTotal++; if (comment.public === true) publicComments++;
82
+ for (const attachment of comment.attachments ?? []) {
83
+ const stored = data.attachments.get(`ticket:${id}:${comment.id}:${attachment.id}`);
84
+ if (!stored?.blob) continue;
85
+ const blob = stored.blob;
86
+ if (blob.bytes > 48 * 1024) needsPackage = true;
87
+ blobs.set(`blobs/${blob.sha256}`, blob);
88
+ bindings.push({ attachment_id: attachment.id, comment_id: comment.id, path: `blobs/${blob.sha256}`, sha256: blob.sha256,
89
+ ...(!packageBindings ? { base64: (await store.get(blob)).toString('base64') } : {}) });
90
+ }
91
+ }
92
+ if (bindings.length > 20) needsPackage = true;
93
+ const value = { ...row, comments, _offline: { format: FORMAT, comments_complete: true, attachments: bindings,
94
+ history: { metrics: 'missing', audits: 'missing' }, metric_events: [], audits: [], provenance: [provenance, ...[...(data.comments.get(id)?.values() ?? [])].map(item => item.provenance)] } };
95
+ if (encode(value).length > 128 * 1024) needsPackage = true;
96
+ add('ticket', id, value);
97
+ }
98
+ for (const [kind, map] of [['user', data.users], ['organization', data.organizations], ['ticket_field', data.ticket_fields]]) {
99
+ for (const [id,{ row, provenance }] of sortedRows(map)) add(kind, id, { ...row, _offline_provenance: [provenance] });
100
+ }
101
+ let restricted = 0, draft = 0;
102
+ for (const [id,{ row, provenance }] of sortedRows(data.articles)) {
103
+ if (row.user_segment_id != null || row.user_segment_ids?.length) restricted++;
104
+ if (row.draft) draft++;
105
+ add('article', id, { ...row, draft: row.draft === true, user_segment_id: row.user_segment_id ?? null, user_segment_ids: row.user_segment_ids ?? [], _offline_provenance: [provenance] });
106
+ }
107
+ const records = Buffer.concat(lines);
108
+ if (records.length > 2 * 1024 * 1024 || lines.length > 5000) needsPackage = true;
109
+ const census = { format: 'rer-zendesk-census/1', tickets: { unique: data.tickets.size, deleted: tombstones.length,
110
+ deletedIdsSha256: hash(tombstones.map(row => `${row.id}\n`).join('')), byStatus, maxId: ordered(data.tickets.keys()).at(-1) ?? null },
111
+ comments: { total: commentTotal, public: publicComments, private: commentTotal - publicComments },
112
+ attachments: attachmentCensus(refs.filter(ref => ref.ticket), data), articleAttachments: attachmentCensus(refs.filter(ref => ref.article), data, { article: true }),
113
+ users: { unique: data.users.size, inactive: [...data.users.values()].filter(item => item.row.active === false).length, withoutEmail: [...data.users.values()].filter(item => !item.row.email).length },
114
+ organizations: data.organizations.size, fields: data.ticket_fields.size,
115
+ guide: { available: data.available, articles: data.articles.size, restricted, draft, translations: data.translations.size, sections: data.sections.size, categories: data.categories.size, segments: data.user_segments.size },
116
+ records: byKind, undelivered: { tombstones: tombstones.length, translations: data.translations.size, sections: data.sections.size, categories: data.categories.size, segments: data.user_segments.size, articleAttachments: data.article_attachments.size }, credentialShapesRemoved: report.removals.length, credentialRemovals: report.removals, redactedUrls: report.redactedUrls, highWater: data.highWater, omissions };
117
+ const files = new Map([['records-000001.ndjson', records], ['tombstones.ndjson', Buffer.concat(tombstones.map(row => encode(row)))]]);
118
+ for (const kind of ['translations','sections','categories','user_segments','article_attachments']) {
119
+ const bytes = Buffer.concat([...data[kind]].sort(([a],[b]) => a.localeCompare(b)).map(([id,{ row }]) => encode(cleaned(row, kind, id, report))));
120
+ try { assertChunk(bytes, []); } catch { fail('CREDENTIAL_SHAPE_UNREMOVABLE'); }
121
+ files.set(`guide/${kind}.ndjson`, bytes);
122
+ }
123
+ // Article media remains in the package/store even before HH can import it.
124
+ for (const ref of refs.filter(ref => ref.article)) {
125
+ const blob = data.attachments.get(ref.key)?.blob;
126
+ if (blob) blobs.set(`blobs/${blob.sha256}`, blob);
127
+ }
128
+ census.credentialShapesRemoved = report.removals.length; census.redactedUrls = report.redactedUrls;
129
+ files.set('census.json', encode(census));
130
+ if (needsPackage && !packageBindings) return encodeSnapshot(store, { origin, packageBindings: true });
131
+ return { kind: needsPackage ? 'package' : 'bundle', bytes: records, files, blobs, census,
132
+ source: { format: FORMAT, version: 1, origin, exportedAt: store.state.exportedAt, synthetic: false, adapter: 'rer:zendesk' } };
133
+ }
@@ -0,0 +1,34 @@
1
+ import { fail, object } from './common.mjs';
2
+ import { materialize, ordered } from './zendesk-store.mjs';
3
+ import { fetchPage } from './zendesk-pages.mjs';
4
+
5
+ const stages = [
6
+ ['user_segments'], ['categories'], ['sections'], ['articles'],
7
+ ['source_articles'], ['translations', 'categories'], ['translations', 'sections'], ['translations', 'articles'], ['article_attachments', 'articles'],
8
+ ];
9
+ export async function guide(ctx) {
10
+ const state = ctx.store.state, stage = stages[state.stage];
11
+ if (!stage) { await ctx.store.commit({ ...state, phase: 'guide_attachments', attachmentIndex: 0 }); return; }
12
+ const [kind, resource] = stage;
13
+ let article, path;
14
+ if (state.stage < 4) path = `/api/v2/help_center/${kind}?per_page=100`;
15
+ else {
16
+ const data = await materialize(ctx.store), records = data[resource ?? 'articles'];
17
+ article = ordered(records.keys())[state.index];
18
+ if (article === undefined) { await ctx.store.commit({ ...state, stage: state.stage + 1, index: 0, path: null }); return; }
19
+ if (kind === 'source_articles') {
20
+ const row = records.get(article).row;
21
+ if (typeof row.source_locale !== 'string' || !/^[a-z]{2,3}(?:-[a-z0-9]{2,8})?$/u.test(row.source_locale)) fail('INVALID_RESPONSE');
22
+ if (row.locale !== row.source_locale) {
23
+ const response = await ctx.request(`/api/v2/help_center/${row.source_locale}/articles/${article}`);
24
+ const source = object(response.body.article);
25
+ if (String(source.id) !== article || source.locale !== row.source_locale) fail('INVALID_RESPONSE');
26
+ await ctx.store.page('articles', response, { ...state, index: state.index + 1 }, { single: 'article' });
27
+ } else await ctx.store.commit({ ...state, index: state.index + 1 });
28
+ return;
29
+ }
30
+ path = `/api/v2/help_center/${resource}/${article}/${kind === 'article_attachments' ? 'attachments' : kind}?per_page=100`;
31
+ }
32
+ await fetchPage({ ...ctx, state, kind, path: state.path ?? path, allow404: true, extra: article ? { article, resource } : {},
33
+ onDone: state => ({ ...state, stage: state.stage + (article ? 0 : 1), index: article ? state.index + 1 : 0 }) });
34
+ }
@@ -0,0 +1,28 @@
1
+ import { join } from 'node:path';
2
+ import { buildPackage } from '../package-transfer.mjs';
3
+
4
+ /** Materialize checked HH lines and sidecars as immutable files, and reference
5
+ * the already verified original blobs. The shared sender owns segmentation,
6
+ * manifest probing and the resumable upload journal. */
7
+ export async function prepareZendeskPackage({ result, quota, directory, signal, mode = 'snapshot' }) {
8
+ const files = [], store = result.store;
9
+ const add = async (path, bytes) => {
10
+ const ref = await store.put(bytes);
11
+ files.push({ path, file: join(store.path, ref.file), bytes: ref.bytes, sha256: ref.sha256 });
12
+ };
13
+ let start = 0, position = 0, ordinal = 1;
14
+ while (position < result.bytes.length) {
15
+ signal?.throwIfAborted();
16
+ const newline = result.bytes.indexOf(10, position);
17
+ const end = newline < 0 ? result.bytes.length : newline + 1;
18
+ if (position > start && end - start > 8 * 1024 * 1024) {
19
+ await add(`records-${String(ordinal++).padStart(6,'0')}.ndjson`, result.bytes.subarray(start,position)); start = position;
20
+ }
21
+ position = end;
22
+ }
23
+ if (position > start) await add(`records-${String(ordinal).padStart(6,'0')}.ndjson`, result.bytes.subarray(start,position));
24
+ for (const [path, bytes] of result.files) if (!path.startsWith('records-')) await add(path,bytes);
25
+ for (const [path,ref] of result.blobs) { await store.get(ref); files.push({ path, file: join(store.path,ref.file), bytes: ref.bytes, sha256: ref.sha256 }); }
26
+ return buildPackage({ files, quota, directory, signal, mode, source: result.source,
27
+ records: { total: Object.values(result.census.records).reduce((a,b) => a+b,0), byKind: result.census.records }, totals: result.census });
28
+ }