@runeditrun/rer 0.2.98 → 0.2.100
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/cli/README.md +88 -2
- package/cli/business/api.mjs +4 -2
- package/cli/business/bounded-waves.d.mts +23 -0
- package/cli/business/bounded-waves.mjs +57 -0
- package/cli/business/catchup-task.mjs +9 -2
- package/cli/business/command.mjs +16 -4
- package/cli/business/connect.mjs +39 -33
- package/cli/business/export-inputs.mjs +12 -2
- package/cli/business/export-task.mjs +9 -2
- package/cli/business/legacy-package.mjs +117 -0
- package/cli/business/local-commands.mjs +10 -1
- package/cli/business/package-transfer.mjs +127 -0
- package/cli/business/questions.mjs +2 -2
- package/cli/business/state.mjs +47 -5
- package/cli/business/supervisor.mjs +46 -7
- package/cli/business/vendor-door.mjs +86 -13
- package/cli/business/vendors/README.md +41 -2
- package/cli/business/vendors/common.mjs +81 -0
- package/cli/business/vendors/index.mjs +12 -2
- package/cli/business/vendors/zendesk-attachments.mjs +57 -0
- package/cli/business/vendors/zendesk-encode.mjs +133 -0
- package/cli/business/vendors/zendesk-guide.mjs +34 -0
- package/cli/business/vendors/zendesk-package.mjs +28 -0
- package/cli/business/vendors/zendesk-pages.mjs +61 -0
- package/cli/business/vendors/zendesk-people.mjs +7 -0
- package/cli/business/vendors/zendesk-store.mjs +154 -0
- package/cli/business/vendors/zendesk-tickets.mjs +21 -0
- package/cli/business/vendors/zendesk-token.mjs +31 -0
- package/cli/business/vendors/zendesk-transport.mjs +48 -0
- package/cli/business/vendors/zendesk.mjs +61 -0
- package/cli/deploy-command.mjs +5 -4
- package/cli/deploy-inputs.mjs +1 -1
- package/cli/deploy.mjs +6 -4
- package/cli/production-auth.mjs +16 -3
- package/cli/production-mail.mjs +18 -1
- package/cli/site.mjs +1 -1
- package/global/CHANGELOG.md +42 -0
- package/global/README.md +1 -1
- package/modules/deploy-kit/README.md +10 -0
- package/modules/deploy-kit/provider.mjs +18 -2
- package/modules/import-kit/src/d1/jobs.ts +3 -1
- package/modules/import-kit/src/d1/retirement-scope.ts +2 -2
- package/modules/import-kit/src/d1/retirement-tally.ts +1 -0
- package/modules/import-kit/src/d1/retirement.ts +15 -5
- package/modules/import-kit/src/d1/types.ts +3 -0
- package/modules/import-kit/src/d1/wave-boundary.ts +33 -0
- package/modules/import-kit/src/d1/wave-retirement.ts +47 -0
- package/modules/import-kit/src/door/ADOPTING.md +37 -0
- package/modules/import-kit/src/door/README.md +66 -0
- package/modules/import-kit/src/door/admission.ts +13 -3
- package/modules/import-kit/src/door/authority.ts +1 -0
- package/modules/import-kit/src/door/failures.ts +1 -1
- package/modules/import-kit/src/door/grants.ts +19 -5
- package/modules/import-kit/src/door/handler.ts +12 -1
- package/modules/import-kit/src/door/index.ts +6 -0
- package/modules/import-kit/src/door/package.ts +121 -0
- package/modules/import-kit/src/door/planning.ts +44 -19
- package/modules/import-kit/src/door/protocol.ts +4 -2
- package/modules/import-kit/src/door/schema.ts +2 -0
- package/modules/import-kit/src/door/store.ts +2 -2
- package/modules/import-kit/src/door/types.ts +24 -1
- package/modules/import-kit/src/door/waves/README.md +158 -0
- package/modules/import-kit/src/door/waves/digests.ts +38 -0
- package/modules/import-kit/src/door/waves/inventory.ts +84 -0
- package/modules/import-kit/src/door/waves/journal.ts +19 -0
- package/modules/import-kit/src/door/waves/lifecycle.ts +242 -0
- package/modules/import-kit/src/door/waves/preparation.ts +190 -0
- package/modules/import-kit/src/door/waves/schema.ts +73 -0
- package/modules/import-kit/src/door/waves/store.ts +112 -0
- package/modules/import-kit/src/door/waves/types.ts +76 -0
- package/onboarding/adapter.mjs +7 -1
- package/onboarding/operator-installer.mjs +3 -1
- package/package.json +1 -1
- package/scripts/publish-check/README.md +20 -0
- package/scripts/publish-check/diagnostics.mjs +73 -0
- package/scripts/publish-check/slots.py +37 -7
- package/scripts/publish-check.mjs +64 -37
|
@@ -197,3 +197,84 @@ export function client({ origin, headers, route, signal, limits, counters, key }
|
|
|
197
197
|
return { body, text, headers: response.headers };
|
|
198
198
|
};
|
|
199
199
|
}
|
|
200
|
+
|
|
201
|
+
/** Opt-in transport for long-lived snapshots. Existing adapters retain client()'s
|
|
202
|
+
* no-retry contract. The caller validates every target before credentials or a
|
|
203
|
+
* request are constructed, including manually followed attachment redirects. */
|
|
204
|
+
export function pacedClient({ origin, signal, validate, secrets = [], limits = {}, beforeRequest }) {
|
|
205
|
+
const maximum = { requests: 250_000, response: 32 * 1024 * 1024, bytes: 32 * 1024 ** 3, ...limits };
|
|
206
|
+
const counters = { requests: 0, bytes: 0, retryMs: 0 };
|
|
207
|
+
let token, last = 0, next = 0, advertisedBefore = false;
|
|
208
|
+
const wait = async milliseconds => { if (milliseconds > 0) await delay(milliseconds, undefined, { signal }); };
|
|
209
|
+
const request = async (path, options = {}) => {
|
|
210
|
+
let url;
|
|
211
|
+
try { url = new URL(path, origin); } catch { fail('INVALID_REFERENCE'); }
|
|
212
|
+
await validate(url, options);
|
|
213
|
+
const method = options.method ?? 'GET';
|
|
214
|
+
if (method !== 'GET' && !(method === 'POST' && url.origin === origin && url.pathname === '/oauth/tokens')) fail('INVALID_REFERENCE');
|
|
215
|
+
for (let retries = 0; ; retries++) {
|
|
216
|
+
signal?.throwIfAborted();
|
|
217
|
+
await wait(Math.max(next, last + (advertisedBefore ? 0 : url.pathname.includes('/incremental/') ? 6000 : 300)) - Date.now());
|
|
218
|
+
if (++counters.requests > maximum.requests) fail('REQUEST_LIMIT');
|
|
219
|
+
await beforeRequest?.();
|
|
220
|
+
last = Date.now();
|
|
221
|
+
const headers = { accept: 'application/json', ...(options.json ? { 'content-type': 'application/json' } : {}),
|
|
222
|
+
...(token && url.origin === origin && options.authenticated !== false ? { authorization: `Bearer ${token}` } : {}) };
|
|
223
|
+
let response;
|
|
224
|
+
try { response = await fetch(url, { method, headers, redirect: options.redirect ?? 'error',
|
|
225
|
+
...(options.json ? { body: JSON.stringify(options.json) } : {}), signal: signal ? AbortSignal.any([signal, AbortSignal.timeout(30_000)]) : AbortSignal.timeout(30_000) }); }
|
|
226
|
+
catch { fail(signal?.aborted ? 'CANCELLED' : 'NETWORK_FAILED'); }
|
|
227
|
+
const waits = [];
|
|
228
|
+
const remaining = response.headers.get('ratelimit-remaining') ?? response.headers.get('x-rate-limit-remaining');
|
|
229
|
+
const reset = response.headers.get('ratelimit-reset') ?? '60';
|
|
230
|
+
const capacity = (remaining, seconds) => {
|
|
231
|
+
if (!/^\d+$/u.test(remaining ?? '') || !/^\d+$/u.test(seconds ?? '') || Number(seconds) > 3600) return;
|
|
232
|
+
waits.push(Math.max(300, Math.ceil(Number(seconds) * 1000 / Math.max(1, Number(remaining)))));
|
|
233
|
+
};
|
|
234
|
+
capacity(remaining, reset);
|
|
235
|
+
for (const [name,value] of response.headers) if (name.toLowerCase().startsWith('zendesk-ratelimit-')) capacity(/remaining=(\d+)/u.exec(value)?.[1], /resets=(\d+)/u.exec(value)?.[1]);
|
|
236
|
+
const interval = response.headers.get('ratelimit-limit') ?? response.headers.get('x-rate-limit');
|
|
237
|
+
if (!waits.length && /^\d+$/u.test(interval ?? '')) waits.push(Math.max(300, Math.ceil(60_000 / Math.max(1, Number(interval)))));
|
|
238
|
+
// Obey the tightest account/endpoint budget. Without valid advertised
|
|
239
|
+
// capacity the next request retains the conservative fallback spacing.
|
|
240
|
+
advertisedBefore = waits.length > 0;
|
|
241
|
+
next = Date.now() + Math.max(0, ...waits);
|
|
242
|
+
if (response.status === 429) {
|
|
243
|
+
const after = response.headers.get('retry-after');
|
|
244
|
+
await response.body?.cancel().catch(() => {});
|
|
245
|
+
const seconds = /^\d+$/u.test(after ?? '') ? Number(after) : 0;
|
|
246
|
+
if (retries >= 3 || seconds < 1 || seconds > 60 || counters.retryMs + seconds * 1000 > 600_000) throw new VendorExportError('RATE_LIMITED', 429);
|
|
247
|
+
counters.retryMs += seconds * 1000;
|
|
248
|
+
await wait(seconds * 1000);
|
|
249
|
+
continue;
|
|
250
|
+
}
|
|
251
|
+
if (options.redirect === 'manual' && [301, 302, 303, 307, 308].includes(response.status)) {
|
|
252
|
+
const location = response.headers.get('location');
|
|
253
|
+
await response.body?.cancel().catch(() => {});
|
|
254
|
+
return { status: response.status, location, headers: response.headers };
|
|
255
|
+
}
|
|
256
|
+
if (!response.ok) {
|
|
257
|
+
await response.body?.cancel().catch(() => {});
|
|
258
|
+
if (options.allow?.includes(response.status)) return { status: response.status };
|
|
259
|
+
throw new VendorExportError('HTTP_REFUSED', response.status);
|
|
260
|
+
}
|
|
261
|
+
const chunks = []; let size = 0;
|
|
262
|
+
try {
|
|
263
|
+
for await (const chunk of response.body ?? []) {
|
|
264
|
+
size += chunk.byteLength; counters.bytes += chunk.byteLength;
|
|
265
|
+
if (size > (options.maxBytes ?? maximum.response) || counters.bytes > maximum.bytes) fail('BYTE_LIMIT');
|
|
266
|
+
chunks.push(chunk);
|
|
267
|
+
}
|
|
268
|
+
} catch (error) { if (error instanceof VendorExportError) throw error; fail(signal?.aborted ? 'CANCELLED' : 'NETWORK_FAILED'); }
|
|
269
|
+
const bytes = Buffer.concat(chunks);
|
|
270
|
+
if (!options.tokenResponse && [...secrets, token].some(secret => secret && bytes.includes(Buffer.from(secret)))) fail('CREDENTIAL_IN_RESPONSE');
|
|
271
|
+
if (options.binary) return { bytes, headers: response.headers, status: response.status };
|
|
272
|
+
let body;
|
|
273
|
+
try { body = JSON.parse(new TextDecoder('utf-8', { fatal: true }).decode(bytes)); } catch { fail('INVALID_JSON'); }
|
|
274
|
+
return { body, bytes, headers: response.headers, status: response.status };
|
|
275
|
+
}
|
|
276
|
+
};
|
|
277
|
+
request.authorize = value => { token = value; };
|
|
278
|
+
request.counters = counters;
|
|
279
|
+
return request;
|
|
280
|
+
}
|
|
@@ -4,11 +4,14 @@ import * as uptimerobot from './uptimerobot.mjs';
|
|
|
4
4
|
import * as pagerduty from './pagerduty.mjs';
|
|
5
5
|
import * as calendly from './calendly.mjs';
|
|
6
6
|
import * as sorry from './sorry.mjs';
|
|
7
|
+
import * as zendesk from './zendesk.mjs';
|
|
7
8
|
|
|
8
9
|
export { VendorExportError } from './common.mjs';
|
|
9
|
-
export const VENDORS = Object.freeze(['sentry','uptimerobot','pagerduty','calendly','sorry']);
|
|
10
|
+
export const VENDORS = Object.freeze(['sentry','uptimerobot','pagerduty','calendly','sorry','zendesk','happytohelp-legacy']);
|
|
10
11
|
const adapters = { sentry, uptimerobot, pagerduty, calendly, sorry };
|
|
11
12
|
export const VENDOR_CAPABILITIES = Object.freeze({
|
|
13
|
+
'happytohelp-legacy': Object.freeze({ vendor: 'happytohelp-legacy', adapter: 'rer:happytohelp-legacy', capability: null, format: 'happytohelp-legacy-chunks-v1', layout: 'package', output: 'rer-package.json', readOnly: true, resume: true, scopes: [], origins: [] }),
|
|
14
|
+
zendesk: Object.freeze({ vendor: 'zendesk', adapter: 'rer:zendesk', capability: 'vendor_key:zendesk', format: 'happytohelp-zendesk-offline-v1', layout: 'snapshot', output: 'zendesk.ndjson', readOnly: true, resume: true, scopes: ['tickets:read','users:read','organizations:read','hc:read','ticket_attachments:read'], origins: ['https://<subdomain>.zendesk.com'] }),
|
|
12
15
|
sentry: Object.freeze({ vendor: 'sentry', adapter: 'rer:sentry', capability: 'vendor_key:sentry', format: 'spool-sentry-org-export-v1', layout: 'concatenate', output: 'sentry-export.json', readOnly: true, resume: true, scopes: ['org:read','project:read','event:read'], origins: Object.values(sentry.origins) }),
|
|
13
16
|
uptimerobot: Object.freeze({ vendor: 'uptimerobot', adapter: 'rer:uptimerobot', capability: 'vendor_key:uptimerobot', format: 'uptimerobot-v3-monitors', layout: 'concatenate', output: 'uptimerobot-monitors.json', readOnly: true, resume: true, scopes: ['read-only API key'], origins: [uptimerobot.origin] }),
|
|
14
17
|
pagerduty: Object.freeze({ vendor: 'pagerduty', adapter: 'rer:pagerduty', capability: 'vendor_key:pagerduty', format: 'pagerduty-saved-responses', layout: 'concatenate', output: 'pagerduty-export.json', readOnly: true, resume: true, scopes: ['read-only REST API key'], origins: [pagerduty.origin] }),
|
|
@@ -33,9 +36,16 @@ const LIMITS = Object.freeze({
|
|
|
33
36
|
/** Persist each checkpoint only after the corresponding bytes have been acknowledged.
|
|
34
37
|
* Bytes are private seed-bound artifacts; checkpoints are metadata, never API bodies.
|
|
35
38
|
*/
|
|
36
|
-
export async function* exportVendor(vendor, { key, inputs = {}, checkpoint, signal } = {}) {
|
|
39
|
+
export async function* exportVendor(vendor, { key, inputs = {}, checkpoint, signal, directory, runId } = {}) {
|
|
37
40
|
try {
|
|
38
41
|
if (!VENDORS.includes(vendor)) fail('UNSUPPORTED_VENDOR');
|
|
42
|
+
if (vendor === 'happytohelp-legacy') fail('LOCAL_PACKAGE_REQUIRED');
|
|
43
|
+
if (vendor === 'zendesk') {
|
|
44
|
+
const result = await zendesk.snapshot({ key, inputs, directory, runId, signal });
|
|
45
|
+
if (result.kind !== 'bundle') fail(result.kind === 'package' ? 'SEED_PACKAGE_UNSUPPORTED' : 'INVALID_INPUT');
|
|
46
|
+
yield { name: 'zendesk.ndjson', bytes: result.bytes, checkpoint: { version: 1, vendor, head: result.store.head } };
|
|
47
|
+
return;
|
|
48
|
+
}
|
|
39
49
|
const keyed = KEYED_VENDORS.includes(vendor);
|
|
40
50
|
if (keyed ? typeof key !== 'string' || !key || key.length > 8192 || /[\u0000-\u0020\u007f]/u.test(key) : key != null) fail('INVALID_KEY');
|
|
41
51
|
object(inputs);
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
import { VendorExportError, fail } from './common.mjs';
|
|
2
|
+
import { materialize, numericId, ordered } from './zendesk-store.mjs';
|
|
3
|
+
import { publicLocation, attachmentPath } from './zendesk-transport.mjs';
|
|
4
|
+
|
|
5
|
+
export function attachmentReferences(data) {
|
|
6
|
+
const refs = [];
|
|
7
|
+
for (const [ticket, comments] of data.comments) {
|
|
8
|
+
if (data.tickets.get(ticket)?.row.status === 'deleted' || data.tickets.get(ticket)?.row.deleted) continue;
|
|
9
|
+
for (const [comment, { row }] of comments) for (const attachment of row.attachments ?? []) refs.push({ key: `ticket:${ticket}:${comment}:${numericId(attachment.id)}`, ticket, comment, attachment });
|
|
10
|
+
}
|
|
11
|
+
for (const [id, { row }] of data.article_attachments) refs.push({ key: `article:${id}`, article: id.split(':')[1], attachment: row });
|
|
12
|
+
return refs.sort((a,b) => a.key.localeCompare(b.key));
|
|
13
|
+
}
|
|
14
|
+
export async function download(request, origin, attachment) {
|
|
15
|
+
if (attachment.deleted) return { reason: 'deleted' };
|
|
16
|
+
if (attachment.malware_scan_result === 'malware_found' || attachment.malware_found) return { reason: 'malware_found' };
|
|
17
|
+
const size = attachment.size;
|
|
18
|
+
if (!Number.isSafeInteger(size) || size < 0) fail('INVALID_RESPONSE');
|
|
19
|
+
if (size > 64 * 1024 * 1024) return { reason: 'too_large' };
|
|
20
|
+
let url;
|
|
21
|
+
try { url = new URL(attachment.content_url); } catch { return { reason: 'refused_location' }; }
|
|
22
|
+
if (url.origin !== origin || !attachmentPath(url.pathname) || !await publicLocation(attachment.content_url, origin)) return { reason: 'refused_location' };
|
|
23
|
+
let response;
|
|
24
|
+
try {
|
|
25
|
+
response = await request(url.href, { attachment: true, binary: true, maxBytes: 64 * 1024 * 1024, redirect: 'manual', allow: [403,404,410] });
|
|
26
|
+
if (response.location !== undefined) {
|
|
27
|
+
const next = await publicLocation(response.location, origin);
|
|
28
|
+
if (!next) return { reason: 'refused_location' };
|
|
29
|
+
response = await request(next.href, { attachment: true, binary: true, authenticated: false, maxBytes: 64 * 1024 * 1024, redirect: 'error', allow: [403,404,410] });
|
|
30
|
+
}
|
|
31
|
+
} catch (error) {
|
|
32
|
+
if (error instanceof VendorExportError && error.code === 'REFUSED_LOCATION') return { reason: 'refused_location' };
|
|
33
|
+
throw error;
|
|
34
|
+
}
|
|
35
|
+
if ([403,404,410].includes(response.status)) return { reason: `http_${response.status}` };
|
|
36
|
+
if (!response.bytes || response.bytes.length !== size) fail('ATTACHMENT_SIZE_MISMATCH');
|
|
37
|
+
return { bytes: response.bytes };
|
|
38
|
+
}
|
|
39
|
+
export async function attachments(ctx) {
|
|
40
|
+
const { store, request, origin, inputs } = ctx, state = store.state, data = await materialize(store);
|
|
41
|
+
const ids = ordered([...data.tickets].filter(([, { row }]) => row.status !== 'deleted' && !row.deleted && (inputs.mode !== 'sweep' || row.generated_timestamp > inputs.highWater)).map(([id]) => id));
|
|
42
|
+
const ticket = ids[state.index];
|
|
43
|
+
const refs = attachmentReferences(data).filter(ref => state.phase === 'guide_attachments' ? ref.article : ref.ticket === ticket);
|
|
44
|
+
const ref = refs[state.attachmentIndex];
|
|
45
|
+
if (!ref) {
|
|
46
|
+
await store.commit(state.phase === 'guide_attachments' ? { ...state, phase: 'done', attachmentIndex: 0 } : { ...state, phase: 'B', path: null, index: state.index + 1, attachmentIndex: 0 });
|
|
47
|
+
return;
|
|
48
|
+
}
|
|
49
|
+
const previous = data.attachments.get(ref.key);
|
|
50
|
+
if (previous && previous.sourceHash === JSON.stringify(ref.attachment)) {
|
|
51
|
+
await store.commit({ ...state, attachmentIndex: state.attachmentIndex + 1 }); return;
|
|
52
|
+
}
|
|
53
|
+
const result = await download(request, origin, ref.attachment);
|
|
54
|
+
const blob = result.bytes ? await store.put(result.bytes) : null;
|
|
55
|
+
await store.commit({ ...state, attachmentIndex: state.attachmentIndex + 1, bytes: state.bytes + (blob?.bytes ?? 0) },
|
|
56
|
+
{ kind: 'attachment', key: ref.key, sourceHash: JSON.stringify(ref.attachment), ...(blob ? { blob } : { reason: result.reason }) }, blob ? [blob] : []);
|
|
57
|
+
}
|
|
@@ -0,0 +1,133 @@
|
|
|
1
|
+
import { credentialShaped } from '../detector.mjs';
|
|
2
|
+
import { assertChunk } from '../outbound.mjs';
|
|
3
|
+
import { fail, hash } from './common.mjs';
|
|
4
|
+
import { materialize, encode, ordered } from './zendesk-store.mjs';
|
|
5
|
+
import { attachmentReferences } from './zendesk-attachments.mjs';
|
|
6
|
+
import { attachmentPath } from './zendesk-transport.mjs';
|
|
7
|
+
|
|
8
|
+
export const FORMAT = 'happytohelp-zendesk-offline-v1';
|
|
9
|
+
const omissions = ['audits','metrics','satisfaction','side conversations','groups','brands','forms','macros','triggers','views','automations','photos','thumbnails','archived articles','community'];
|
|
10
|
+
const escapePointer = value => value.replace(/~/gu, '~0').replace(/\//gu, '~1');
|
|
11
|
+
/** Redact only the delivered copy. Raw responses remain private evidence in
|
|
12
|
+
* the store. Expand prefix detector hits to their whole token and PEM hits to
|
|
13
|
+
* their whole block; assertChunk remains the final authority on every line. */
|
|
14
|
+
function cleaned(value, kind, id, report, pointer = '') {
|
|
15
|
+
const removed = () => report.removals.push({ kind, id, pointer: pointer || '/' });
|
|
16
|
+
if (Array.isArray(value)) return value.map((child,index) => cleaned(child, kind, id, report, `${pointer}/${index}`));
|
|
17
|
+
if (value && typeof value === 'object') {
|
|
18
|
+
const result = Object.create(null);
|
|
19
|
+
for (const [key,child] of Object.entries(value)) {
|
|
20
|
+
if (['content_url','download_url','original_url'].includes(key)) { report.redactedUrls++; continue; }
|
|
21
|
+
if (credentialShaped(key)) fail('CREDENTIAL_SHAPE_UNREMOVABLE');
|
|
22
|
+
result[key] = cleaned(child, kind, id, report, `${pointer}/${escapePointer(key)}`);
|
|
23
|
+
}
|
|
24
|
+
return result;
|
|
25
|
+
}
|
|
26
|
+
if (typeof value !== 'string') return value;
|
|
27
|
+
let text = value.replace(/https?:\/\/[^\s<>"']+/giu, match => {
|
|
28
|
+
let url; try { url = new URL(match); } catch { return match; }
|
|
29
|
+
if (attachmentPath(url.pathname)) { report.redactedUrls++; return '[removed]'; }
|
|
30
|
+
if (!url.username && !url.password && !url.search && !url.hash) return match;
|
|
31
|
+
url.username = ''; url.password = ''; url.search = ''; url.hash = ''; report.redactedUrls++; return url.href;
|
|
32
|
+
});
|
|
33
|
+
text = text.replace(/(?:\/\/[^\s<>"'/]+)?\/(?:api\/v2\/)?attachments\/[^\s<>"']*/giu, () => { report.redactedUrls++; return '[removed]'; });
|
|
34
|
+
text = text.replace(/-----\s*BEGIN\s+([A-Z0-9 ]+)-----[\s\S]*?-----\s*END\s+\1-----/giu, () => { removed(); return '[removed]'; });
|
|
35
|
+
// SSH material includes a scheme and key separated by whitespace.
|
|
36
|
+
text = text.replace(/\b(?:ssh-(?:rsa|ed25519)|ecdsa-sha2-[\w-]+)\s+[A-Za-z0-9+/=]+/gu, () => { removed(); return '[removed]'; });
|
|
37
|
+
text = text.replace(/[^\s<>"'`,;(){}\[\]]+/gu, token => {
|
|
38
|
+
if (!credentialShaped(token)) return token;
|
|
39
|
+
removed(); return '[removed]';
|
|
40
|
+
});
|
|
41
|
+
text = text.replace(/[\u0000-\u0008\u000b\u000c\u000e-\u001f\u007f]/gu, () => { removed(); return '[removed]'; });
|
|
42
|
+
text = text.replace(/https?:\/\/[^\s<>"']+/giu, match => {
|
|
43
|
+
let url;
|
|
44
|
+
try { url = new URL(match); } catch { return match; }
|
|
45
|
+
if (!url.username && !url.password && !url.search && !url.hash) return match;
|
|
46
|
+
url.username = ''; url.password = ''; url.search = ''; url.hash = ''; report.redactedUrls++; return url.href;
|
|
47
|
+
});
|
|
48
|
+
if (credentialShaped(text.replace(/[\r\n\t]/gu, ' '))) fail('CREDENTIAL_SHAPE_UNREMOVABLE');
|
|
49
|
+
return text;
|
|
50
|
+
}
|
|
51
|
+
const deleted = row => row.status === 'deleted' || row.deleted === true;
|
|
52
|
+
const sortedRows = map => ordered(map.keys()).map(id => [id, map.get(id)]);
|
|
53
|
+
function attachmentCensus(references, data, { article = false } = {}) {
|
|
54
|
+
const blobs = new Map(), undelivered = {};
|
|
55
|
+
let fetched = 0, bytes = 0;
|
|
56
|
+
for (const ref of references) {
|
|
57
|
+
const item = data.attachments.get(ref.key);
|
|
58
|
+
if (item?.blob) { fetched++; bytes += item.blob.bytes; blobs.set(item.blob.sha256, item.blob.bytes); }
|
|
59
|
+
const reason = item?.reason ?? (!item ? 'not_fetched' : article ? 'pending_guide' : null);
|
|
60
|
+
if (reason) undelivered[reason] = (undelivered[reason] ?? 0) + 1;
|
|
61
|
+
}
|
|
62
|
+
return { referenced: references.length, fetched, bytes, uniqueBlobs: blobs.size, undelivered };
|
|
63
|
+
}
|
|
64
|
+
export async function encodeSnapshot(store, { origin, packageBindings = false } = {}) {
|
|
65
|
+
const data = await materialize(store), report = { removals: [], redactedUrls: 0 }, lines = [], byKind = {}, blobs = new Map();
|
|
66
|
+
const add = (kind, id, value) => {
|
|
67
|
+
// Bindings are integrity-checked original bytes, never redaction input.
|
|
68
|
+
const clean = kind === 'ticket' ? { ...cleaned({ ...value, _offline: undefined }, kind, id, report), _offline: value._offline } : cleaned(value, kind, id, report);
|
|
69
|
+
const line = encode({ [kind]: clean });
|
|
70
|
+
if (line.length > 64 * 1024 * 1024) fail('RECORD_LIMIT');
|
|
71
|
+
try { assertChunk(line, []); } catch { fail('CREDENTIAL_SHAPE_UNREMOVABLE'); }
|
|
72
|
+
lines.push(line); byKind[kind] = (byKind[kind] ?? 0) + 1;
|
|
73
|
+
};
|
|
74
|
+
let needsPackage = packageBindings, commentTotal = 0, publicComments = 0;
|
|
75
|
+
const tombstones = [], byStatus = {}, refs = attachmentReferences(data);
|
|
76
|
+
for (const [id,{ row, provenance }] of sortedRows(data.tickets)) {
|
|
77
|
+
byStatus[row.status] = (byStatus[row.status] ?? 0) + 1;
|
|
78
|
+
if (deleted(row)) { tombstones.push({ id: row.id, generated_timestamp: row.generated_timestamp }); continue; }
|
|
79
|
+
const comments = sortedRows(data.comments.get(id) ?? new Map()).map(([,item]) => item.row), bindings = [];
|
|
80
|
+
for (const comment of comments) {
|
|
81
|
+
commentTotal++; if (comment.public === true) publicComments++;
|
|
82
|
+
for (const attachment of comment.attachments ?? []) {
|
|
83
|
+
const stored = data.attachments.get(`ticket:${id}:${comment.id}:${attachment.id}`);
|
|
84
|
+
if (!stored?.blob) continue;
|
|
85
|
+
const blob = stored.blob;
|
|
86
|
+
if (blob.bytes > 48 * 1024) needsPackage = true;
|
|
87
|
+
blobs.set(`blobs/${blob.sha256}`, blob);
|
|
88
|
+
bindings.push({ attachment_id: attachment.id, comment_id: comment.id, path: `blobs/${blob.sha256}`, sha256: blob.sha256,
|
|
89
|
+
...(!packageBindings ? { base64: (await store.get(blob)).toString('base64') } : {}) });
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
if (bindings.length > 20) needsPackage = true;
|
|
93
|
+
const value = { ...row, comments, _offline: { format: FORMAT, comments_complete: true, attachments: bindings,
|
|
94
|
+
history: { metrics: 'missing', audits: 'missing' }, metric_events: [], audits: [], provenance: [provenance, ...[...(data.comments.get(id)?.values() ?? [])].map(item => item.provenance)] } };
|
|
95
|
+
if (encode(value).length > 128 * 1024) needsPackage = true;
|
|
96
|
+
add('ticket', id, value);
|
|
97
|
+
}
|
|
98
|
+
for (const [kind, map] of [['user', data.users], ['organization', data.organizations], ['ticket_field', data.ticket_fields]]) {
|
|
99
|
+
for (const [id,{ row, provenance }] of sortedRows(map)) add(kind, id, { ...row, _offline_provenance: [provenance] });
|
|
100
|
+
}
|
|
101
|
+
let restricted = 0, draft = 0;
|
|
102
|
+
for (const [id,{ row, provenance }] of sortedRows(data.articles)) {
|
|
103
|
+
if (row.user_segment_id != null || row.user_segment_ids?.length) restricted++;
|
|
104
|
+
if (row.draft) draft++;
|
|
105
|
+
add('article', id, { ...row, draft: row.draft === true, user_segment_id: row.user_segment_id ?? null, user_segment_ids: row.user_segment_ids ?? [], _offline_provenance: [provenance] });
|
|
106
|
+
}
|
|
107
|
+
const records = Buffer.concat(lines);
|
|
108
|
+
if (records.length > 2 * 1024 * 1024 || lines.length > 5000) needsPackage = true;
|
|
109
|
+
const census = { format: 'rer-zendesk-census/1', tickets: { unique: data.tickets.size, deleted: tombstones.length,
|
|
110
|
+
deletedIdsSha256: hash(tombstones.map(row => `${row.id}\n`).join('')), byStatus, maxId: ordered(data.tickets.keys()).at(-1) ?? null },
|
|
111
|
+
comments: { total: commentTotal, public: publicComments, private: commentTotal - publicComments },
|
|
112
|
+
attachments: attachmentCensus(refs.filter(ref => ref.ticket), data), articleAttachments: attachmentCensus(refs.filter(ref => ref.article), data, { article: true }),
|
|
113
|
+
users: { unique: data.users.size, inactive: [...data.users.values()].filter(item => item.row.active === false).length, withoutEmail: [...data.users.values()].filter(item => !item.row.email).length },
|
|
114
|
+
organizations: data.organizations.size, fields: data.ticket_fields.size,
|
|
115
|
+
guide: { available: data.available, articles: data.articles.size, restricted, draft, translations: data.translations.size, sections: data.sections.size, categories: data.categories.size, segments: data.user_segments.size },
|
|
116
|
+
records: byKind, undelivered: { tombstones: tombstones.length, translations: data.translations.size, sections: data.sections.size, categories: data.categories.size, segments: data.user_segments.size, articleAttachments: data.article_attachments.size }, credentialShapesRemoved: report.removals.length, credentialRemovals: report.removals, redactedUrls: report.redactedUrls, highWater: data.highWater, omissions };
|
|
117
|
+
const files = new Map([['records-000001.ndjson', records], ['tombstones.ndjson', Buffer.concat(tombstones.map(row => encode(row)))]]);
|
|
118
|
+
for (const kind of ['translations','sections','categories','user_segments','article_attachments']) {
|
|
119
|
+
const bytes = Buffer.concat([...data[kind]].sort(([a],[b]) => a.localeCompare(b)).map(([id,{ row }]) => encode(cleaned(row, kind, id, report))));
|
|
120
|
+
try { assertChunk(bytes, []); } catch { fail('CREDENTIAL_SHAPE_UNREMOVABLE'); }
|
|
121
|
+
files.set(`guide/${kind}.ndjson`, bytes);
|
|
122
|
+
}
|
|
123
|
+
// Article media remains in the package/store even before HH can import it.
|
|
124
|
+
for (const ref of refs.filter(ref => ref.article)) {
|
|
125
|
+
const blob = data.attachments.get(ref.key)?.blob;
|
|
126
|
+
if (blob) blobs.set(`blobs/${blob.sha256}`, blob);
|
|
127
|
+
}
|
|
128
|
+
census.credentialShapesRemoved = report.removals.length; census.redactedUrls = report.redactedUrls;
|
|
129
|
+
files.set('census.json', encode(census));
|
|
130
|
+
if (needsPackage && !packageBindings) return encodeSnapshot(store, { origin, packageBindings: true });
|
|
131
|
+
return { kind: needsPackage ? 'package' : 'bundle', bytes: records, files, blobs, census,
|
|
132
|
+
source: { format: FORMAT, version: 1, origin, exportedAt: store.state.exportedAt, synthetic: false, adapter: 'rer:zendesk' } };
|
|
133
|
+
}
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
import { fail, object } from './common.mjs';
|
|
2
|
+
import { materialize, ordered } from './zendesk-store.mjs';
|
|
3
|
+
import { fetchPage } from './zendesk-pages.mjs';
|
|
4
|
+
|
|
5
|
+
const stages = [
|
|
6
|
+
['user_segments'], ['categories'], ['sections'], ['articles'],
|
|
7
|
+
['source_articles'], ['translations', 'categories'], ['translations', 'sections'], ['translations', 'articles'], ['article_attachments', 'articles'],
|
|
8
|
+
];
|
|
9
|
+
export async function guide(ctx) {
|
|
10
|
+
const state = ctx.store.state, stage = stages[state.stage];
|
|
11
|
+
if (!stage) { await ctx.store.commit({ ...state, phase: 'guide_attachments', attachmentIndex: 0 }); return; }
|
|
12
|
+
const [kind, resource] = stage;
|
|
13
|
+
let article, path;
|
|
14
|
+
if (state.stage < 4) path = `/api/v2/help_center/${kind}?per_page=100`;
|
|
15
|
+
else {
|
|
16
|
+
const data = await materialize(ctx.store), records = data[resource ?? 'articles'];
|
|
17
|
+
article = ordered(records.keys())[state.index];
|
|
18
|
+
if (article === undefined) { await ctx.store.commit({ ...state, stage: state.stage + 1, index: 0, path: null }); return; }
|
|
19
|
+
if (kind === 'source_articles') {
|
|
20
|
+
const row = records.get(article).row;
|
|
21
|
+
if (typeof row.source_locale !== 'string' || !/^[a-z]{2,3}(?:-[a-z0-9]{2,8})?$/u.test(row.source_locale)) fail('INVALID_RESPONSE');
|
|
22
|
+
if (row.locale !== row.source_locale) {
|
|
23
|
+
const response = await ctx.request(`/api/v2/help_center/${row.source_locale}/articles/${article}`);
|
|
24
|
+
const source = object(response.body.article);
|
|
25
|
+
if (String(source.id) !== article || source.locale !== row.source_locale) fail('INVALID_RESPONSE');
|
|
26
|
+
await ctx.store.page('articles', response, { ...state, index: state.index + 1 }, { single: 'article' });
|
|
27
|
+
} else await ctx.store.commit({ ...state, index: state.index + 1 });
|
|
28
|
+
return;
|
|
29
|
+
}
|
|
30
|
+
path = `/api/v2/help_center/${resource}/${article}/${kind === 'article_attachments' ? 'attachments' : kind}?per_page=100`;
|
|
31
|
+
}
|
|
32
|
+
await fetchPage({ ...ctx, state, kind, path: state.path ?? path, allow404: true, extra: article ? { article, resource } : {},
|
|
33
|
+
onDone: state => ({ ...state, stage: state.stage + (article ? 0 : 1), index: article ? state.index + 1 : 0 }) });
|
|
34
|
+
}
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
import { join } from 'node:path';
|
|
2
|
+
import { buildPackage } from '../package-transfer.mjs';
|
|
3
|
+
|
|
4
|
+
/** Materialize checked HH lines and sidecars as immutable files, and reference
|
|
5
|
+
* the already verified original blobs. The shared sender owns segmentation,
|
|
6
|
+
* manifest probing and the resumable upload journal. */
|
|
7
|
+
export async function prepareZendeskPackage({ result, quota, directory, signal, mode = 'snapshot' }) {
|
|
8
|
+
const files = [], store = result.store;
|
|
9
|
+
const add = async (path, bytes) => {
|
|
10
|
+
const ref = await store.put(bytes);
|
|
11
|
+
files.push({ path, file: join(store.path, ref.file), bytes: ref.bytes, sha256: ref.sha256 });
|
|
12
|
+
};
|
|
13
|
+
let start = 0, position = 0, ordinal = 1;
|
|
14
|
+
while (position < result.bytes.length) {
|
|
15
|
+
signal?.throwIfAborted();
|
|
16
|
+
const newline = result.bytes.indexOf(10, position);
|
|
17
|
+
const end = newline < 0 ? result.bytes.length : newline + 1;
|
|
18
|
+
if (position > start && end - start > 8 * 1024 * 1024) {
|
|
19
|
+
await add(`records-${String(ordinal++).padStart(6,'0')}.ndjson`, result.bytes.subarray(start,position)); start = position;
|
|
20
|
+
}
|
|
21
|
+
position = end;
|
|
22
|
+
}
|
|
23
|
+
if (position > start) await add(`records-${String(ordinal).padStart(6,'0')}.ndjson`, result.bytes.subarray(start,position));
|
|
24
|
+
for (const [path, bytes] of result.files) if (!path.startsWith('records-')) await add(path,bytes);
|
|
25
|
+
for (const [path,ref] of result.blobs) { await store.get(ref); files.push({ path, file: join(store.path,ref.file), bytes: ref.bytes, sha256: ref.sha256 }); }
|
|
26
|
+
return buildPackage({ files, quota, directory, signal, mode, source: result.source,
|
|
27
|
+
records: { total: Object.values(result.census.records).reduce((a,b) => a+b,0), byKind: result.census.records }, totals: result.census });
|
|
28
|
+
}
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
import { fail, object, list } from './common.mjs';
|
|
2
|
+
import { pagination } from './zendesk-transport.mjs';
|
|
3
|
+
|
|
4
|
+
/** Both documented comments envelopes, and Guide's offset envelopes, are
|
|
5
|
+
* explicit: a missing end marker is an error rather than a short snapshot. */
|
|
6
|
+
export function nextPage(body, path, origin, { incremental = false, time = false } = {}) {
|
|
7
|
+
object(body);
|
|
8
|
+
const pathname = new URL(path, origin).pathname;
|
|
9
|
+
if (incremental) {
|
|
10
|
+
if (typeof body.end_of_stream !== 'boolean') fail('INVALID_PAGINATION');
|
|
11
|
+
if (body.end_of_stream) return null;
|
|
12
|
+
if (time) {
|
|
13
|
+
if (!Number.isSafeInteger(body.end_time) || body.end_time < 1 || body.end_time <= Number(new URL(path, origin).searchParams.get('start_time') ?? 0)) fail('INVALID_PAGINATION');
|
|
14
|
+
return `${pathname}?start_time=${body.end_time}&per_page=1000`;
|
|
15
|
+
}
|
|
16
|
+
if (typeof body.after_cursor !== 'string' || !body.after_cursor || body.after_cursor.length > 2048 || /[\u0000-\u0020\u007f]/u.test(body.after_cursor)) fail('INVALID_PAGINATION');
|
|
17
|
+
const next = `${pathname}?cursor=${encodeURIComponent(body.after_cursor)}&per_page=1000`;
|
|
18
|
+
if (new URL(path, origin).searchParams.get('cursor') === body.after_cursor) fail('INVALID_PAGINATION');
|
|
19
|
+
if (body.after_url != null && pagination(body.after_url, origin, pathname, path) !== next) fail('INVALID_PAGINATION');
|
|
20
|
+
return next;
|
|
21
|
+
}
|
|
22
|
+
if (body.meta && Object.hasOwn(body.meta, 'has_more')) {
|
|
23
|
+
if (typeof body.meta.has_more !== 'boolean' || !body.links || !Object.hasOwn(body.links, 'next')) fail('INVALID_PAGINATION');
|
|
24
|
+
if (!body.meta.has_more) { if (body.links.next !== null) fail('INVALID_PAGINATION'); return null; }
|
|
25
|
+
return pagination(body.links.next, origin, pathname, path);
|
|
26
|
+
}
|
|
27
|
+
if (Object.hasOwn(body, 'next_page')) {
|
|
28
|
+
const next = pagination(body.next_page, origin, pathname, path);
|
|
29
|
+
if (next) {
|
|
30
|
+
const current = new URL(path, origin), url = new URL(next, origin);
|
|
31
|
+
if (url.searchParams.get('page') !== String(Number(current.searchParams.get('page') ?? 1) + 1)) fail('INVALID_PAGINATION');
|
|
32
|
+
}
|
|
33
|
+
return next;
|
|
34
|
+
}
|
|
35
|
+
fail('INVALID_PAGINATION');
|
|
36
|
+
}
|
|
37
|
+
export async function fetchPage({ store, request, origin, state, kind, path, extra = {}, incremental, time, allow404 = false, onDone }) {
|
|
38
|
+
const response = await request(path, { allow: allow404 ? [404] : [] });
|
|
39
|
+
if (response.status === 404) {
|
|
40
|
+
const next = onDone({ ...state, path: null });
|
|
41
|
+
await store.commit(next, { kind: 'guide_unavailable' });
|
|
42
|
+
return next;
|
|
43
|
+
}
|
|
44
|
+
list(object(response.body)[kind], 1000);
|
|
45
|
+
const seenKey = JSON.stringify([state.runId, kind, extra.ticket ?? extra.article ?? '', extra.resource ?? '']);
|
|
46
|
+
if (!store.pagePaths) {
|
|
47
|
+
store.pagePaths = new Map();
|
|
48
|
+
for (const event of store.events) if (event.requestPath) {
|
|
49
|
+
const paths = store.pagePaths.get(event.seenKey) ?? new Set(); paths.add(event.requestPath); store.pagePaths.set(event.seenKey, paths);
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
const seen = store.pagePaths.get(seenKey) ?? new Set();
|
|
53
|
+
if (seen.has(path)) fail('INVALID_PAGINATION');
|
|
54
|
+
const next = nextPage(response.body, path, origin, { incremental, time });
|
|
55
|
+
if (next && seen.has(next)) fail('INVALID_PAGINATION');
|
|
56
|
+
const nextState = next ? { ...state, path: next } : onDone({ ...state, path: null });
|
|
57
|
+
nextState.requests = state.requests + 1;
|
|
58
|
+
await store.page(kind, response, nextState, { ...extra, requestPath: path, seenKey });
|
|
59
|
+
seen.add(path); store.pagePaths.set(seenKey, seen);
|
|
60
|
+
return store.state;
|
|
61
|
+
}
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
import { fetchPage } from './zendesk-pages.mjs';
|
|
2
|
+
export async function people(ctx) {
|
|
3
|
+
const state = ctx.store.state;
|
|
4
|
+
const resources = { C: ['users', '/api/v2/incremental/users/cursor?start_time=0&per_page=1000', 'D'], D: ['organizations', '/api/v2/incremental/organizations?start_time=0&per_page=1000', 'E'], E: ['ticket_fields', '/api/v2/ticket_fields?page[size]=100', 'F'] };
|
|
5
|
+
const [kind, path, next] = resources[state.phase];
|
|
6
|
+
await fetchPage({ ...ctx, state, kind, path: state.path ?? path, incremental: state.phase !== 'E', time: state.phase === 'D', onDone: state => ({ ...state, phase: next, index: 0, stage: 0 }) });
|
|
7
|
+
}
|
|
@@ -0,0 +1,154 @@
|
|
|
1
|
+
import { constants } from 'node:fs';
|
|
2
|
+
import { mkdir, lstat, open, rename, rm } from 'node:fs/promises';
|
|
3
|
+
import { join } from 'node:path';
|
|
4
|
+
import { randomUUID } from 'node:crypto';
|
|
5
|
+
import { hash, fail, stable } from './common.mjs';
|
|
6
|
+
|
|
7
|
+
export const encode = value => Buffer.from(JSON.stringify(stable(value)) + '\n');
|
|
8
|
+
export const numericId = value => {
|
|
9
|
+
if (typeof value === 'number' && (!Number.isSafeInteger(value) || value < 0)) fail('INVALID_RESPONSE');
|
|
10
|
+
const id = String(value);
|
|
11
|
+
if (!/^(?:0|[1-9]\d{0,19})$/u.test(id)) fail('INVALID_RESPONSE');
|
|
12
|
+
return id;
|
|
13
|
+
};
|
|
14
|
+
export const ordered = values => [...values].sort((a,b) => BigInt(a) < BigInt(b) ? -1 : BigInt(a) > BigInt(b) ? 1 : 0);
|
|
15
|
+
export const storePath = (directory, business, origin) => join(directory, `zendesk-${hash(JSON.stringify([business, origin])).slice(0,32)}`);
|
|
16
|
+
async function privateDirectory(path) {
|
|
17
|
+
await mkdir(path, { mode: 0o700, recursive: true });
|
|
18
|
+
const info = await lstat(path);
|
|
19
|
+
if (!info.isDirectory() || (info.mode & 0o777) !== 0o700) fail('INVALID_CHECKPOINT');
|
|
20
|
+
}
|
|
21
|
+
async function read(path, maximum = 64 * 1024 * 1024) {
|
|
22
|
+
const file = await open(path, constants.O_RDONLY | constants.O_NOFOLLOW);
|
|
23
|
+
try {
|
|
24
|
+
const info = await file.stat();
|
|
25
|
+
if (!info.isFile() || info.size > maximum || (info.mode & 0o777) !== 0o600) fail('INVALID_CHECKPOINT');
|
|
26
|
+
return await file.readFile();
|
|
27
|
+
} finally { await file.close(); }
|
|
28
|
+
}
|
|
29
|
+
async function atomic(path, bytes) {
|
|
30
|
+
const temporary = `${path}.${randomUUID()}.tmp`;
|
|
31
|
+
const file = await open(temporary, constants.O_WRONLY | constants.O_CREAT | constants.O_EXCL | constants.O_NOFOLLOW, 0o600);
|
|
32
|
+
try { await file.writeFile(bytes); await file.sync(); } finally { await file.close(); }
|
|
33
|
+
await rename(temporary, path);
|
|
34
|
+
const parent = await open(join(path, '..'), constants.O_RDONLY);
|
|
35
|
+
try { await parent.sync(); } finally { await parent.close(); }
|
|
36
|
+
}
|
|
37
|
+
/** Immutable, content-addressed objects and journal nodes precede one atomic
|
|
38
|
+
* checkpoint pointer. A kill can leave unreferenced objects, never a half-
|
|
39
|
+
* acknowledged page. Reopen verifies the complete reachable graph, including
|
|
40
|
+
* blobs, before making a network request. Checkpoints stay constant size. */
|
|
41
|
+
export class ZendeskStore {
|
|
42
|
+
constructor(path) { this.path = path; this.events = []; this.head = null; this.state = null; }
|
|
43
|
+
async start() {
|
|
44
|
+
await privateDirectory(this.path); await privateDirectory(join(this.path, 'objects'));
|
|
45
|
+
try {
|
|
46
|
+
const checkpoint = JSON.parse((await read(join(this.path, 'checkpoint.json'), 128 * 1024 - 1)).toString());
|
|
47
|
+
if (checkpoint.format !== 'rer-zendesk-checkpoint/1' || Object.keys(checkpoint).sort().join() !== 'format,head') fail('INVALID_CHECKPOINT');
|
|
48
|
+
this.head = checkpoint.head;
|
|
49
|
+
let reference = this.head;
|
|
50
|
+
const seen = new Set(), backwards = [];
|
|
51
|
+
while (reference) {
|
|
52
|
+
if (seen.has(reference.sha256)) fail('INVALID_CHECKPOINT');
|
|
53
|
+
seen.add(reference.sha256);
|
|
54
|
+
const node = JSON.parse((await this.get(reference)).toString());
|
|
55
|
+
if (node.format !== 'rer-zendesk-journal/1' || !node.state || !Array.isArray(node.refs)) fail('INVALID_CHECKPOINT');
|
|
56
|
+
for (const ref of node.refs) await this.get(ref);
|
|
57
|
+
backwards.push(node); reference = node.previous;
|
|
58
|
+
}
|
|
59
|
+
backwards.reverse();
|
|
60
|
+
this.events = backwards.flatMap(node => node.event ? [node.event] : []);
|
|
61
|
+
this.state = backwards.at(-1)?.state;
|
|
62
|
+
if (!this.state) fail('INVALID_CHECKPOINT');
|
|
63
|
+
} catch (error) {
|
|
64
|
+
if (error.code !== 'ENOENT' || this.head) fail('INVALID_CHECKPOINT');
|
|
65
|
+
}
|
|
66
|
+
return this;
|
|
67
|
+
}
|
|
68
|
+
async get(ref) {
|
|
69
|
+
if (!ref || !/^[a-f0-9]{64}$/u.test(ref.sha256) || !Number.isSafeInteger(ref.bytes) || ref.bytes < 0 || ref.bytes > 64 * 1024 * 1024 || ref.file !== `objects/${ref.sha256}`) fail('INVALID_CHECKPOINT');
|
|
70
|
+
const bytes = await read(join(this.path, ref.file));
|
|
71
|
+
if (bytes.length !== ref.bytes || hash(bytes) !== ref.sha256) fail('INVALID_CHECKPOINT');
|
|
72
|
+
return bytes;
|
|
73
|
+
}
|
|
74
|
+
async put(bytes) {
|
|
75
|
+
const ref = { file: `objects/${hash(bytes)}`, sha256: hash(bytes), bytes: bytes.length };
|
|
76
|
+
if (bytes.length > 64 * 1024 * 1024) fail('RECORD_LIMIT');
|
|
77
|
+
const path = join(this.path, ref.file);
|
|
78
|
+
let file;
|
|
79
|
+
try { file = await open(path, constants.O_WRONLY | constants.O_CREAT | constants.O_EXCL | constants.O_NOFOLLOW, 0o600); }
|
|
80
|
+
catch (error) { if (error.code !== 'EEXIST') throw error; await this.get(ref); return ref; }
|
|
81
|
+
try { await file.writeFile(bytes); await file.sync(); } finally { await file.close(); }
|
|
82
|
+
const parent = await open(join(this.path, 'objects'), constants.O_RDONLY);
|
|
83
|
+
try { await parent.sync(); } finally { await parent.close(); }
|
|
84
|
+
return ref;
|
|
85
|
+
}
|
|
86
|
+
async commit(state, event = null, refs = []) {
|
|
87
|
+
state = { ...state, requests: this.requestTotal ?? state.requests };
|
|
88
|
+
if (!Number.isSafeInteger(state.bytes) || state.bytes > 32 * 1024 ** 3 || state.requests > 250_000) fail('BYTE_LIMIT');
|
|
89
|
+
const node = { format: 'rer-zendesk-journal/1', previous: this.head, state, event, refs };
|
|
90
|
+
const head = await this.put(encode(node));
|
|
91
|
+
const bytes = encode({ format: 'rer-zendesk-checkpoint/1', head });
|
|
92
|
+
if (bytes.length >= 128 * 1024) fail('CHECKPOINT_LIMIT');
|
|
93
|
+
await atomic(join(this.path, 'checkpoint.json'), bytes);
|
|
94
|
+
this.head = head; this.state = structuredClone(state);
|
|
95
|
+
if (event) this.events.push(event);
|
|
96
|
+
}
|
|
97
|
+
async page(kind, response, state, extra = {}) {
|
|
98
|
+
const raw = await this.put(response.bytes);
|
|
99
|
+
await this.commit({ ...state, bytes: state.bytes + raw.bytes }, { kind, raw, ...extra }, [raw]);
|
|
100
|
+
}
|
|
101
|
+
async reset(state, { retain = false } = {}) {
|
|
102
|
+
if (!retain) { this.events = []; this.head = null; this.cache = null; this.pagePaths = null; }
|
|
103
|
+
await this.commit(state);
|
|
104
|
+
}
|
|
105
|
+
async write(name, value) {
|
|
106
|
+
if (!['census.json', 'transfer.json', 'baseline.json'].includes(name)) fail('INVALID_CHECKPOINT');
|
|
107
|
+
await atomic(join(this.path, name), encode(value));
|
|
108
|
+
}
|
|
109
|
+
async baseline() {
|
|
110
|
+
try {
|
|
111
|
+
const baseline = JSON.parse((await read(join(this.path, 'baseline.json'), 128 * 1024)).toString());
|
|
112
|
+
const node = JSON.parse((await this.get(baseline.head)).toString());
|
|
113
|
+
if (node.state?.phase !== 'done' || node.state.highWater !== baseline.highWater) fail('INVALID_CHECKPOINT');
|
|
114
|
+
return baseline;
|
|
115
|
+
}
|
|
116
|
+
catch (error) { if (error.code !== 'ENOENT') fail('INVALID_CHECKPOINT'); return null; }
|
|
117
|
+
}
|
|
118
|
+
async discard() { await rm(this.path, { recursive: true, force: true }); }
|
|
119
|
+
}
|
|
120
|
+
export async function removeZendeskStores(directory) {
|
|
121
|
+
const { readdir } = await import('node:fs/promises');
|
|
122
|
+
for (const name of await readdir(directory).catch(error => { if (error.code === 'ENOENT') return []; throw error; })) {
|
|
123
|
+
if (/^zendesk-[a-f0-9]{32}$/u.test(name)) await rm(join(directory, name), { recursive: true, force: true });
|
|
124
|
+
}
|
|
125
|
+
}
|
|
126
|
+
/** Collapse replayed ids and retain their last source row and provenance. */
|
|
127
|
+
export async function materialize(store) {
|
|
128
|
+
const data = store.cache?.data ?? Object.fromEntries(['tickets','users','organizations','ticket_fields','user_segments','categories','sections','articles','translations','article_attachments'].map(kind => [kind, new Map()]));
|
|
129
|
+
if (!store.cache) { data.comments = new Map(); data.attachments = new Map(); data.available = true; data.highWater = 0; }
|
|
130
|
+
for (const event of store.events.slice(store.cache?.length ?? 0)) {
|
|
131
|
+
if (event.kind === 'guide_unavailable') { data.available = false; continue; }
|
|
132
|
+
if (event.kind === 'attachment') { data.attachments.set(event.key, event); continue; }
|
|
133
|
+
if (!event.raw) continue;
|
|
134
|
+
const body = JSON.parse((await store.get(event.raw)).toString());
|
|
135
|
+
const rows = event.single ? [body[event.single]] : body[event.kind];
|
|
136
|
+
if (!Array.isArray(rows)) fail('INVALID_CHECKPOINT');
|
|
137
|
+
const destination = event.kind === 'comments' ? (event.reset ? new Map() : data.comments.get(event.ticket) ?? new Map()) : data[event.kind];
|
|
138
|
+
if (!destination) fail('INVALID_CHECKPOINT');
|
|
139
|
+
rows.forEach((row,index) => {
|
|
140
|
+
const id = numericId(row.id);
|
|
141
|
+
if (Buffer.byteLength(JSON.stringify(row)) > 64 * 1024 * 1024) fail('RECORD_LIMIT');
|
|
142
|
+
const key = ['translations', 'article_attachments'].includes(event.kind) ? `${event.resource ?? "articles"}:${event.article}:${id}` : id;
|
|
143
|
+
destination.set(key, { row, provenance: { file: event.raw.file, sha256: event.raw.sha256, row: index + 1, pointer: event.single ? `/${event.single}` : `/${event.kind}/${index}` } });
|
|
144
|
+
if (event.kind === 'tickets') {
|
|
145
|
+
const stamp = row.generated_timestamp;
|
|
146
|
+
if (!Number.isSafeInteger(stamp) || stamp < 0) fail('INVALID_RESPONSE');
|
|
147
|
+
data.highWater = Math.max(data.highWater, stamp);
|
|
148
|
+
}
|
|
149
|
+
});
|
|
150
|
+
if (event.kind === 'comments') data.comments.set(event.ticket, destination);
|
|
151
|
+
}
|
|
152
|
+
store.cache = { data, length: store.events.length };
|
|
153
|
+
return data;
|
|
154
|
+
}
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
import { materialize, ordered } from './zendesk-store.mjs';
|
|
2
|
+
import { fetchPage } from './zendesk-pages.mjs';
|
|
3
|
+
|
|
4
|
+
export async function tickets(ctx) {
|
|
5
|
+
const { store, inputs } = ctx;
|
|
6
|
+
let state = store.state;
|
|
7
|
+
const path = state.path ?? `/api/v2/incremental/tickets/cursor?start_time=${inputs.mode === 'sweep' ? inputs.highWater : 0}&per_page=1000`;
|
|
8
|
+
state = await fetchPage({ ...ctx, state, kind: 'tickets', path, incremental: true,
|
|
9
|
+
onDone: state => ({ ...state, phase: inputs.mode === 'census' ? 'done' : 'B', index: 0 }) });
|
|
10
|
+
return state;
|
|
11
|
+
}
|
|
12
|
+
export async function comments(ctx) {
|
|
13
|
+
const { store, inputs } = ctx, state = store.state, data = await materialize(store);
|
|
14
|
+
// A sweep refetches complete comments only for changed tickets; the retained
|
|
15
|
+
// store still supplies every older ticket to the eventual complete snapshot.
|
|
16
|
+
const ids = ordered([...data.tickets].filter(([, { row }]) => row.status !== 'deleted' && !row.deleted && (inputs.mode !== 'sweep' || row.generated_timestamp > inputs.highWater)).map(([id]) => id));
|
|
17
|
+
const ticket = ids[state.index];
|
|
18
|
+
if (ticket === undefined) { await store.commit({ ...state, phase: inputs.mode === 'sweep' ? 'done' : 'C', path: null, index: 0 }); return; }
|
|
19
|
+
await fetchPage({ ...ctx, state, kind: 'comments', path: state.path ?? `/api/v2/tickets/${ticket}/comments?page[size]=100&include_inline_images=true`, extra: { ticket, reset: state.path === null },
|
|
20
|
+
onDone: state => ({ ...state, phase: 'attachments', index: state.index, attachmentIndex: 0 }) });
|
|
21
|
+
}
|