@runeditrun/rer 0.2.100 → 0.2.101
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +30 -102
- package/cli/README.md +16 -89
- package/cli/app-address.mjs +2 -0
- package/cli/business/api.mjs +2 -1
- package/cli/business/choices.mjs +1 -1
- package/cli/business/command.mjs +36 -30
- package/cli/business/connect.mjs +1 -1
- package/cli/business/lease.mjs +0 -9
- package/cli/business/local-commands.mjs +1 -27
- package/cli/business/outbound.mjs +2 -14
- package/cli/business/questions.mjs +85 -48
- package/cli/business/service.mjs +1 -24
- package/cli/business/setup-command.mjs +33 -26
- package/cli/business/state.mjs +1 -1
- package/cli/business/supervisor.mjs +26 -91
- package/cli/companion/command.mjs +2 -0
- package/cli/demo-native.mjs +1 -1
- package/cli/parse.mjs +2 -1
- package/global/CHANGELOG.md +24 -0
- package/global/README.md +30 -102
- package/global/main.mjs +2 -0
- package/modules/container/adapters/d1-limits.mjs +2 -10
- package/modules/d1-transport/limits.d.mts +10 -0
- package/modules/d1-transport/limits.mjs +10 -0
- package/modules/demo-host/native/prepare.mjs +2 -1
- package/modules/import-kit/schema/importers.schema.json +104 -2
- package/modules/import-kit/src/d1/README.md +13 -0
- package/modules/import-kit/src/d1/apply-reads.ts +48 -0
- package/modules/import-kit/src/d1/apply.ts +21 -8
- package/modules/import-kit/src/d1/history.ts +2 -2
- package/modules/import-kit/src/d1/types.ts +11 -0
- package/modules/import-kit/src/door/ADOPTING.md +0 -37
- package/modules/import-kit/src/door/README.md +0 -66
- package/modules/import-kit/src/door/admission.ts +3 -13
- package/modules/import-kit/src/door/authority.ts +0 -1
- package/modules/import-kit/src/door/failures.ts +1 -1
- package/modules/import-kit/src/door/grants.ts +5 -19
- package/modules/import-kit/src/door/handler.ts +1 -12
- package/modules/import-kit/src/door/index.ts +0 -6
- package/modules/import-kit/src/door/planning.ts +19 -44
- package/modules/import-kit/src/door/protocol.ts +2 -4
- package/modules/import-kit/src/door/schema.ts +0 -2
- package/modules/import-kit/src/door/store.ts +2 -2
- package/modules/import-kit/src/door/types.ts +1 -24
- package/modules/import-kit/src/provenance/credentials.ts +14 -5
- package/modules/import-kit/src/status/evidence.ts +3 -1
- package/modules/import-kit/src/status/index.ts +17 -2
- package/modules/import-kit/src/status/schema.ts +8 -0
- package/modules/import-kit/src/status/types.ts +25 -4
- package/package.json +1 -1
- package/cli/business/archive-task.mjs +0 -32
- package/cli/business/bounded-waves.d.mts +0 -23
- package/cli/business/bounded-waves.mjs +0 -57
- package/cli/business/catchup-task.mjs +0 -49
- package/cli/business/export-inputs.mjs +0 -79
- package/cli/business/export-task.mjs +0 -24
- package/cli/business/legacy-package.mjs +0 -117
- package/cli/business/package-transfer.mjs +0 -127
- package/cli/business/setup-demo.mjs +0 -95
- package/cli/business/vendor-door.mjs +0 -303
- package/cli/business/vendors/README.md +0 -215
- package/cli/business/vendors/calendly.mjs +0 -80
- package/cli/business/vendors/common.mjs +0 -280
- package/cli/business/vendors/credential-names.mjs +0 -124
- package/cli/business/vendors/index.mjs +0 -96
- package/cli/business/vendors/pagerduty.mjs +0 -61
- package/cli/business/vendors/sentry.mjs +0 -85
- package/cli/business/vendors/sorry.mjs +0 -106
- package/cli/business/vendors/uptimerobot.mjs +0 -21
- package/cli/business/vendors/zendesk-attachments.mjs +0 -57
- package/cli/business/vendors/zendesk-encode.mjs +0 -133
- package/cli/business/vendors/zendesk-guide.mjs +0 -34
- package/cli/business/vendors/zendesk-package.mjs +0 -28
- package/cli/business/vendors/zendesk-pages.mjs +0 -61
- package/cli/business/vendors/zendesk-people.mjs +0 -7
- package/cli/business/vendors/zendesk-store.mjs +0 -154
- package/cli/business/vendors/zendesk-tickets.mjs +0 -21
- package/cli/business/vendors/zendesk-token.mjs +0 -31
- package/cli/business/vendors/zendesk-transport.mjs +0 -48
- package/cli/business/vendors/zendesk.mjs +0 -61
- package/modules/import-kit/src/door/package.ts +0 -121
- package/modules/import-kit/src/door/waves/README.md +0 -158
- package/modules/import-kit/src/door/waves/digests.ts +0 -38
- package/modules/import-kit/src/door/waves/inventory.ts +0 -84
- package/modules/import-kit/src/door/waves/journal.ts +0 -19
- package/modules/import-kit/src/door/waves/lifecycle.ts +0 -242
- package/modules/import-kit/src/door/waves/preparation.ts +0 -190
- package/modules/import-kit/src/door/waves/schema.ts +0 -73
- package/modules/import-kit/src/door/waves/store.ts +0 -112
- package/modules/import-kit/src/door/waves/types.ts +0 -76
|
@@ -1,61 +0,0 @@
|
|
|
1
|
-
import { object, list, id, windowInputs, assertCount, shape, fail, withoutCredentials } from './common.mjs';
|
|
2
|
-
|
|
3
|
-
export const origin = 'https://api.pagerduty.com';
|
|
4
|
-
export const route = url => /^\/(?:users|escalation_policies|schedules(?:\/[a-zA-Z0-9_-]+(?:\/overrides)?)?)$/u.test(url.pathname) && [...url.searchParams.keys()].every(x => ['limit','offset','since','until','overflow'].includes(x));
|
|
5
|
-
function advance(body, state) {
|
|
6
|
-
if (typeof body.more !== 'boolean') fail();
|
|
7
|
-
if (body.more) {
|
|
8
|
-
const limit = assertCount(body.limit, 100);
|
|
9
|
-
if (!limit || body.offset !== state.offset) fail('INVALID_PAGINATION');
|
|
10
|
-
state.offset += limit;
|
|
11
|
-
assertCount(state.offset, 100000);
|
|
12
|
-
} else state.offset = 0;
|
|
13
|
-
return body.more;
|
|
14
|
-
}
|
|
15
|
-
export async function* run(ctx) {
|
|
16
|
-
const { start, end } = windowInputs(ctx.inputs);
|
|
17
|
-
const window = new URLSearchParams({ since: start, until: end });
|
|
18
|
-
const s = ctx.state ?? { phase: 0, offset: 0, schedules: [], index: 0, first: true };
|
|
19
|
-
shape(s, ['phase','offset','schedules','index','first']);
|
|
20
|
-
if (typeof s.first !== 'boolean') fail('INVALID_CHECKPOINT');
|
|
21
|
-
assertCount(s.phase, 5); assertCount(s.offset, 100000); assertCount(s.index, 500);
|
|
22
|
-
if (!Array.isArray(s.schedules) || s.schedules.length > 500) fail('INVALID_CHECKPOINT');
|
|
23
|
-
s.schedules.forEach(id);
|
|
24
|
-
while (s.phase < 5) {
|
|
25
|
-
let bytes;
|
|
26
|
-
if (s.phase === 0 || s.phase === 1 || s.phase === 4) {
|
|
27
|
-
const kind = ['users','schedules',null,null,'escalation_policies'][s.phase];
|
|
28
|
-
const { body } = await ctx.get(`/${kind}?limit=100&offset=${s.offset}`);
|
|
29
|
-
object(body); const rows = list(body[kind], 100); ctx.records(s.phase === 1 ? 0 : rows.length);
|
|
30
|
-
if (s.phase === 1) {
|
|
31
|
-
for (const row of rows) { const value = id(row.id); if (s.schedules.includes(value)) fail('DUPLICATE_ID'); s.schedules.push(value); }
|
|
32
|
-
if (s.schedules.length > 500) fail('RECORD_LIMIT');
|
|
33
|
-
bytes = '';
|
|
34
|
-
} else {
|
|
35
|
-
bytes = (s.first ? (s.phase === 0 ? '{"users":[' : '],"escalation_policies":[') : ',') + JSON.stringify(withoutCredentials(body));
|
|
36
|
-
s.first = false;
|
|
37
|
-
}
|
|
38
|
-
if (!advance(body, s)) { s.phase++; s.first = true; }
|
|
39
|
-
} else if (s.phase === 2) {
|
|
40
|
-
if (s.index === s.schedules.length) { s.phase = 3; s.index = 0; s.first = true; bytes = s.schedules.length ? '' : '],"schedules":['; }
|
|
41
|
-
else {
|
|
42
|
-
const { body } = await ctx.get(`/schedules/${id(s.schedules[s.index])}?${window}`);
|
|
43
|
-
object(body); object(body.schedule); ctx.records(1);
|
|
44
|
-
bytes = (s.first ? '],"schedules":[' : ',') + JSON.stringify(withoutCredentials(body));
|
|
45
|
-
s.first = false; s.index++;
|
|
46
|
-
}
|
|
47
|
-
} else {
|
|
48
|
-
if (s.index === s.schedules.length) { s.phase = 4; s.first = true; bytes = s.schedules.length ? '' : '],"overrides":['; }
|
|
49
|
-
else {
|
|
50
|
-
const schedule = id(s.schedules[s.index]);
|
|
51
|
-
const { body } = await ctx.get(`/schedules/${schedule}/overrides?${window}&overflow=true`);
|
|
52
|
-
object(body); ctx.records(list(body.overrides).length);
|
|
53
|
-
bytes = (s.first ? '],"overrides":[' : ',') + JSON.stringify(withoutCredentials({ ...body, schedule_id: schedule }));
|
|
54
|
-
s.first = false;
|
|
55
|
-
s.index++;
|
|
56
|
-
}
|
|
57
|
-
}
|
|
58
|
-
if (s.phase === 5) bytes += ']}\n';
|
|
59
|
-
yield ctx.item(bytes, s);
|
|
60
|
-
}
|
|
61
|
-
}
|
|
@@ -1,85 +0,0 @@
|
|
|
1
|
-
import { object, list, id, windowInputs, cursor, assertCount, shape, fail, withoutCredentials } from './common.mjs';
|
|
2
|
-
|
|
3
|
-
export const origins = Object.freeze({ us: 'https://us.sentry.io', de: 'https://de.sentry.io' });
|
|
4
|
-
const queries = ['per_page','limit','cursor','query','project','start','end','sort'];
|
|
5
|
-
export const route = url => /^\/api\/0\/(?:organizations\/[a-zA-Z0-9_-]+\/(?:projects|releases|issues)|projects\/[a-zA-Z0-9_-]+\/[a-zA-Z0-9_-]+\/events\/[a-fA-F0-9]{32})\/$/u.test(url.pathname) && [...url.searchParams.keys()].every(x => queries.includes(x));
|
|
6
|
-
function nextCursor(headers) {
|
|
7
|
-
const link = headers.get('link');
|
|
8
|
-
if (!link) return null;
|
|
9
|
-
const next = link.split(',').find(part => /\brel="next"/u.test(part));
|
|
10
|
-
if (!next || /\bresults="false"/u.test(next)) return null;
|
|
11
|
-
if (!/\bresults="true"/u.test(next)) fail('INVALID_PAGINATION');
|
|
12
|
-
const value = next.match(/\bcursor="([^"]+)"/u)?.[1];
|
|
13
|
-
if (!value) fail('INVALID_PAGINATION');
|
|
14
|
-
return cursor(value);
|
|
15
|
-
}
|
|
16
|
-
export async function* run(ctx) {
|
|
17
|
-
const organization = object(ctx.inputs.organization);
|
|
18
|
-
id(organization.id); id(organization.slug);
|
|
19
|
-
if (!Object.hasOwn(origins, organization.region)) fail('INVALID_INPUT');
|
|
20
|
-
const window = windowInputs(ctx.inputs);
|
|
21
|
-
const selected = ctx.inputs.events ?? [];
|
|
22
|
-
list(selected, 100);
|
|
23
|
-
for (const event of selected) {
|
|
24
|
-
id(event.project);
|
|
25
|
-
if (!/^[a-fA-F0-9]{32}$/u.test(event.eventId)) fail('INVALID_INPUT');
|
|
26
|
-
}
|
|
27
|
-
const s = ctx.state ?? { phase: 0, projects: [], index: 0, event: 0, cursor: null, seen: [], collections: 0, page: 0 };
|
|
28
|
-
shape(s, ['phase','projects','index','event','cursor','seen','collections','page']);
|
|
29
|
-
assertCount(s.phase, 5); assertCount(s.index, 100); assertCount(s.event, 100); assertCount(s.collections, 100); assertCount(s.page, 100);
|
|
30
|
-
if (!Array.isArray(s.projects) || s.projects.length > 49 || !Array.isArray(s.seen) || s.seen.length > 100) fail('INVALID_CHECKPOINT');
|
|
31
|
-
s.projects.forEach(id); cursor(s.cursor); s.seen.forEach(cursor);
|
|
32
|
-
if (s.phase === 0) {
|
|
33
|
-
s.phase = 1;
|
|
34
|
-
yield ctx.item(JSON.stringify({ format: 'spool-sentry-org-export', version: 1, organization: { id: organization.id, slug: organization.slug, region: organization.region }, window }).slice(0,-1) + ',"collections":[', s);
|
|
35
|
-
}
|
|
36
|
-
while (s.phase < 5) {
|
|
37
|
-
if (s.phase === 3 && s.index >= s.projects.length) { s.phase = 4; s.index = 0; }
|
|
38
|
-
if (s.phase === 4 && s.index >= s.projects.length) {
|
|
39
|
-
if (selected.some(event => !s.projects.includes(event.project))) fail('EVENT_PROJECT_NOT_EXPORTED');
|
|
40
|
-
s.phase = 5;
|
|
41
|
-
yield ctx.item('],"omissions":' + JSON.stringify([
|
|
42
|
-
'Only explicitly selected event IDs are exported; all other historical events, retention gaps and inaccessible records are outside this bundle.',
|
|
43
|
-
'Issues include all statuses in the requested time window. Projects and releases are not time-filtered.',
|
|
44
|
-
'Attachments, source maps, sessions, tracing, replays, profiles, feedback, check-ins, logs and metrics are not exported.',
|
|
45
|
-
]) + '}\n', s);
|
|
46
|
-
break;
|
|
47
|
-
}
|
|
48
|
-
const kind = [null,'projects','releases','issues','events'][s.phase];
|
|
49
|
-
const project = s.phase >= 3 ? id(s.projects[s.index]) : null;
|
|
50
|
-
const events = selected.filter(event => event.project === project);
|
|
51
|
-
let response, next = null;
|
|
52
|
-
if (s.phase === 4) {
|
|
53
|
-
if (s.event >= events.length && events.length) fail('INVALID_CHECKPOINT');
|
|
54
|
-
if (events.length) {
|
|
55
|
-
const result = await ctx.get(`/api/0/projects/${organization.slug}/${project}/events/${events[s.event].eventId}/`);
|
|
56
|
-
object(result.body);
|
|
57
|
-
response = [result.body]; ctx.records(1);
|
|
58
|
-
const at = Date.parse(result.body.dateCreated ?? result.body.dateReceived);
|
|
59
|
-
if (!Number.isFinite(at) || at < Date.parse(window.start) || at > Date.parse(window.end)) fail('EVENT_OUTSIDE_WINDOW');
|
|
60
|
-
next = s.event + 1 < events.length ? `selected-${s.event + 1}` : null;
|
|
61
|
-
} else response = [];
|
|
62
|
-
} else {
|
|
63
|
-
const query = new URLSearchParams(s.phase === 3 ? { limit: '100', project, query: '', start: window.start, end: window.end, sort: 'date' } : { per_page: '100' });
|
|
64
|
-
if (s.cursor !== null) query.set('cursor', s.cursor);
|
|
65
|
-
const { body, headers } = await ctx.get(`/api/0/organizations/${organization.slug}/${kind}/?${query}`);
|
|
66
|
-
response = list(body, 100); ctx.records(response.length); next = nextCursor(headers);
|
|
67
|
-
if (s.phase === 1) for (const row of response) {
|
|
68
|
-
const slug = id(row.slug);
|
|
69
|
-
if (s.projects.includes(slug)) fail('DUPLICATE_ID');
|
|
70
|
-
s.projects.push(slug);
|
|
71
|
-
if (s.projects.length > 49) fail('COLLECTION_LIMIT');
|
|
72
|
-
}
|
|
73
|
-
}
|
|
74
|
-
if (next !== null && (next === s.cursor || s.seen.includes(next))) fail('PAGINATION_CYCLE');
|
|
75
|
-
if (++s.page > 100) fail('PAGE_LIMIT');
|
|
76
|
-
let bytes = s.cursor === null ? (s.collections ? ',' : '') + JSON.stringify({kind,project}).slice(0,-1) + ',"pages":[' : ',';
|
|
77
|
-
bytes += JSON.stringify({ cursor: s.cursor, nextCursor: next, response: withoutCredentials(response, queries) });
|
|
78
|
-
s.cursor = next;
|
|
79
|
-
if (next === null) {
|
|
80
|
-
bytes += ']}'; s.collections++; s.seen = []; s.page = 0; s.event = 0;
|
|
81
|
-
if (s.phase < 3) s.phase++; else s.index++;
|
|
82
|
-
} else { s.seen.push(next); if (s.phase === 4) s.event++; }
|
|
83
|
-
yield ctx.item(bytes, s);
|
|
84
|
-
}
|
|
85
|
-
}
|
|
@@ -1,106 +0,0 @@
|
|
|
1
|
-
import { object, list, assertCount, shape, fail } from './common.mjs';
|
|
2
|
-
|
|
3
|
-
// Sorry's public status page API, read anonymously from the page's own HTTPS
|
|
4
|
-
// address, as Lamp's Move in reads it (statuspage-import/sorry/fetch.ts,
|
|
5
|
-
// fetchSorryPublicResponses): /api/v1/status, the components and notices lists
|
|
6
|
-
// (25 a page, `meta.next_page`), then every notice's detail with its components
|
|
7
|
-
// and full update history. No key is sent or accepted.
|
|
8
|
-
|
|
9
|
-
const label = /^(?!-)[a-z0-9-]{1,63}(?<!-)$/u;
|
|
10
|
-
/** The page's public HTTPS origin from its host name alone: no scheme, port,
|
|
11
|
-
* path, credentials, IP address or local name can come from the inputs. */
|
|
12
|
-
export function originOf(inputs) {
|
|
13
|
-
const page = object(inputs.page);
|
|
14
|
-
if (Object.keys(page).some(key => key !== 'host')) fail('INVALID_INPUT');
|
|
15
|
-
const host = page.host;
|
|
16
|
-
if (typeof host !== 'string' || host.length > 253) fail('INVALID_INPUT');
|
|
17
|
-
const labels = host.split('.');
|
|
18
|
-
if (labels.length < 2 || !labels.every(part => label.test(part)) || /^[0-9]+$/u.test(labels.at(-1))
|
|
19
|
-
|| ['localhost', 'local', 'internal', 'invalid', 'test'].includes(labels.at(-1))) fail('INVALID_INPUT');
|
|
20
|
-
return `https://${host}`;
|
|
21
|
-
}
|
|
22
|
-
export const route = url => /^\/api\/v1\/(?:status|components|notices(?:\/(?:0|[1-9][0-9]{0,127}))?)$/u.test(url.pathname)
|
|
23
|
-
&& [...url.searchParams.keys()].every(key => key === 'page');
|
|
24
|
-
|
|
25
|
-
const resources = ['components', 'notices'];
|
|
26
|
-
/** The documented next page of one public list: the same collection, page n+1, or null. */
|
|
27
|
-
function nextPage(value, path, current) {
|
|
28
|
-
if (value == null) return null;
|
|
29
|
-
if (typeof value !== 'string' || value.length > 2048) fail('INVALID_PAGINATION');
|
|
30
|
-
let next;
|
|
31
|
-
try { next = new URL(value, 'https://sorry.invalid'); } catch { fail('INVALID_PAGINATION'); }
|
|
32
|
-
const params = [...next.searchParams];
|
|
33
|
-
if (next.username || next.password || next.hash || ![path, path.replace('/api/v1/', '/v1/')].includes(next.pathname)
|
|
34
|
-
|| params.length !== 1 || params[0][0] !== 'page' || params[0][1] !== String(current + 1)) fail('INVALID_PAGINATION');
|
|
35
|
-
return `${path}?page=${current + 1}`;
|
|
36
|
-
}
|
|
37
|
-
/** Notice ids exactly as Sorry wrote them (large integers never pass through Number). */
|
|
38
|
-
function noticeIds(text) {
|
|
39
|
-
const ids = [];
|
|
40
|
-
JSON.parse(text, function (key, value, context) {
|
|
41
|
-
// A notice is the object with a subject; its components and updates have none.
|
|
42
|
-
if (key === 'id' && this && !Array.isArray(this) && Object.hasOwn(this, 'subject')) {
|
|
43
|
-
const lexeme = typeof value === 'string' ? value : context?.source;
|
|
44
|
-
if (typeof lexeme !== 'string' || !/^(?:0|[1-9][0-9]{0,127})$/u.test(lexeme)) fail('INVALID_RESPONSE');
|
|
45
|
-
ids.push(lexeme);
|
|
46
|
-
}
|
|
47
|
-
return value;
|
|
48
|
-
});
|
|
49
|
-
return ids;
|
|
50
|
-
}
|
|
51
|
-
|
|
52
|
-
/** The bundle Lamp's import door reads for its `sorry` source (door-plan.ts
|
|
53
|
-
* `doorExport`): the public page address, the saved responses under the file
|
|
54
|
-
* names Lamp's Sorry reader requires, each with its original text, and the
|
|
55
|
-
* acknowledgement that they are public. Every response here came from the
|
|
56
|
-
* page's own unauthenticated API, so it is public by construction; Lamp still
|
|
57
|
-
* refuses any private, unpublished or audience-scoped record, and the owner
|
|
58
|
-
* reviews the plan before anything is imported. */
|
|
59
|
-
export async function* run(ctx) {
|
|
60
|
-
const origin = originOf(ctx.inputs);
|
|
61
|
-
const s = ctx.state ?? { phase: 0, page: 1, total: null, rows: 0, notices: [], index: 0 };
|
|
62
|
-
shape(s, ['phase', 'page', 'total', 'rows', 'notices', 'index']);
|
|
63
|
-
assertCount(s.phase, 4); assertCount(s.page, 50); assertCount(s.rows, 5000); assertCount(s.index, 5000);
|
|
64
|
-
if (s.total !== null) assertCount(s.total, 5000);
|
|
65
|
-
if (!Array.isArray(s.notices) || s.notices.length > 1250 || s.notices.some(id => typeof id !== 'string' || !/^(?:0|[1-9][0-9]{0,127})$/u.test(id))) fail('INVALID_CHECKPOINT');
|
|
66
|
-
const file = (name, text, first = false) => (first ? '' : ',') + JSON.stringify({ name, text });
|
|
67
|
-
while (s.phase < 4) {
|
|
68
|
-
let bytes;
|
|
69
|
-
if (s.phase === 0) {
|
|
70
|
-
const { body, text } = await ctx.get('/api/v1/status');
|
|
71
|
-
const page = object(object(body).page);
|
|
72
|
-
if (typeof page.name !== 'string' || typeof page.url !== 'string') fail();
|
|
73
|
-
ctx.records(1);
|
|
74
|
-
s.phase = 1;
|
|
75
|
-
bytes = JSON.stringify({ source: 'sorry', publicUrl: origin, publicAccessAcknowledged: true }).slice(0, -1) + ',"files":[' + file('sorry-status.json', text, true);
|
|
76
|
-
} else if (s.phase <= 2) {
|
|
77
|
-
const resource = resources[s.phase - 1], path = `/api/v1/${resource}`;
|
|
78
|
-
const { body, text } = await ctx.get(s.page === 1 ? path : `${path}?page=${s.page}`);
|
|
79
|
-
object(body); const rows = list(body[resource], 25), meta = object(body.meta);
|
|
80
|
-
if (meta.count !== rows.length || !Number.isSafeInteger(meta.total_count) || s.total !== null && meta.total_count !== s.total) fail('INVALID_PAGINATION');
|
|
81
|
-
ctx.records(rows.length);
|
|
82
|
-
if (resource === 'notices') {
|
|
83
|
-
const ids = noticeIds(text);
|
|
84
|
-
if (ids.length !== rows.length || ids.some(id => s.notices.includes(id))) fail('DUPLICATE_ID');
|
|
85
|
-
s.notices.push(...ids);
|
|
86
|
-
}
|
|
87
|
-
s.rows += rows.length; s.total = meta.total_count;
|
|
88
|
-
const next = nextPage(meta.next_page, path, s.page);
|
|
89
|
-
if (s.rows > s.total || Boolean(next) !== s.rows < s.total || next && !rows.length) fail('INVALID_PAGINATION');
|
|
90
|
-
bytes = file(`sorry-${resource}${s.page === 1 ? '' : `-${s.page}`}.json`, text);
|
|
91
|
-
if (next) { if (++s.page > 50) fail('PAGE_LIMIT'); }
|
|
92
|
-
else { s.phase++; s.page = 1; s.total = null; s.rows = 0; }
|
|
93
|
-
} else {
|
|
94
|
-
if (s.index === s.notices.length) { s.phase = 4; yield ctx.item(']}\n', s); break; }
|
|
95
|
-
const id = s.notices[s.index];
|
|
96
|
-
const { body, text } = await ctx.get(`/api/v1/notices/${id}`);
|
|
97
|
-
const notice = object(object(body).notice);
|
|
98
|
-
const ids = noticeIds(text);
|
|
99
|
-
if (ids.length !== 1 || ids[0] !== id || !Array.isArray(notice.components) || !Array.isArray(notice.updates)) fail('INVALID_RESPONSE');
|
|
100
|
-
ctx.records(1 + notice.updates.length);
|
|
101
|
-
bytes = file(`sorry-notice-${id}.json`, text);
|
|
102
|
-
s.index++;
|
|
103
|
-
}
|
|
104
|
-
yield ctx.item(bytes, s);
|
|
105
|
-
}
|
|
106
|
-
}
|
|
@@ -1,21 +0,0 @@
|
|
|
1
|
-
import { object, list, reference, shape, fail, withoutCredentials } from './common.mjs';
|
|
2
|
-
|
|
3
|
-
export const origin = 'https://api.uptimerobot.com';
|
|
4
|
-
const pagination = ['limit','offset','cursor','page','page_token'];
|
|
5
|
-
export const route = url => url.pathname === '/v3/monitors' && [...url.searchParams.keys()].every(x => pagination.includes(x));
|
|
6
|
-
export async function* run(ctx) {
|
|
7
|
-
const state = ctx.state ?? { next: '/v3/monitors?limit=200', seen: [] };
|
|
8
|
-
shape(state, ['next','seen']);
|
|
9
|
-
if (!Array.isArray(state.seen) || state.seen.length > 600) fail('INVALID_CHECKPOINT');
|
|
10
|
-
state.seen.forEach(value => reference(value, origin, /^\/v3\/monitors$/u, pagination));
|
|
11
|
-
while (state.next !== null) {
|
|
12
|
-
const url = reference(state.next, origin, /^\/v3\/monitors$/u, pagination);
|
|
13
|
-
if (state.seen.includes(url.href)) fail('PAGINATION_CYCLE');
|
|
14
|
-
const { body } = await ctx.get(url.href);
|
|
15
|
-
object(body); ctx.records(list(body.data).length);
|
|
16
|
-
if (!Object.hasOwn(body, 'nextLink')) fail();
|
|
17
|
-
state.seen.push(url.href);
|
|
18
|
-
state.next = body.nextLink == null ? null : reference(body.nextLink, origin, /^\/v3\/monitors$/u, pagination).href;
|
|
19
|
-
yield ctx.item(JSON.stringify(withoutCredentials(body, pagination)) + '\n', state);
|
|
20
|
-
}
|
|
21
|
-
}
|
|
@@ -1,57 +0,0 @@
|
|
|
1
|
-
import { VendorExportError, fail } from './common.mjs';
|
|
2
|
-
import { materialize, numericId, ordered } from './zendesk-store.mjs';
|
|
3
|
-
import { publicLocation, attachmentPath } from './zendesk-transport.mjs';
|
|
4
|
-
|
|
5
|
-
export function attachmentReferences(data) {
|
|
6
|
-
const refs = [];
|
|
7
|
-
for (const [ticket, comments] of data.comments) {
|
|
8
|
-
if (data.tickets.get(ticket)?.row.status === 'deleted' || data.tickets.get(ticket)?.row.deleted) continue;
|
|
9
|
-
for (const [comment, { row }] of comments) for (const attachment of row.attachments ?? []) refs.push({ key: `ticket:${ticket}:${comment}:${numericId(attachment.id)}`, ticket, comment, attachment });
|
|
10
|
-
}
|
|
11
|
-
for (const [id, { row }] of data.article_attachments) refs.push({ key: `article:${id}`, article: id.split(':')[1], attachment: row });
|
|
12
|
-
return refs.sort((a,b) => a.key.localeCompare(b.key));
|
|
13
|
-
}
|
|
14
|
-
export async function download(request, origin, attachment) {
|
|
15
|
-
if (attachment.deleted) return { reason: 'deleted' };
|
|
16
|
-
if (attachment.malware_scan_result === 'malware_found' || attachment.malware_found) return { reason: 'malware_found' };
|
|
17
|
-
const size = attachment.size;
|
|
18
|
-
if (!Number.isSafeInteger(size) || size < 0) fail('INVALID_RESPONSE');
|
|
19
|
-
if (size > 64 * 1024 * 1024) return { reason: 'too_large' };
|
|
20
|
-
let url;
|
|
21
|
-
try { url = new URL(attachment.content_url); } catch { return { reason: 'refused_location' }; }
|
|
22
|
-
if (url.origin !== origin || !attachmentPath(url.pathname) || !await publicLocation(attachment.content_url, origin)) return { reason: 'refused_location' };
|
|
23
|
-
let response;
|
|
24
|
-
try {
|
|
25
|
-
response = await request(url.href, { attachment: true, binary: true, maxBytes: 64 * 1024 * 1024, redirect: 'manual', allow: [403,404,410] });
|
|
26
|
-
if (response.location !== undefined) {
|
|
27
|
-
const next = await publicLocation(response.location, origin);
|
|
28
|
-
if (!next) return { reason: 'refused_location' };
|
|
29
|
-
response = await request(next.href, { attachment: true, binary: true, authenticated: false, maxBytes: 64 * 1024 * 1024, redirect: 'error', allow: [403,404,410] });
|
|
30
|
-
}
|
|
31
|
-
} catch (error) {
|
|
32
|
-
if (error instanceof VendorExportError && error.code === 'REFUSED_LOCATION') return { reason: 'refused_location' };
|
|
33
|
-
throw error;
|
|
34
|
-
}
|
|
35
|
-
if ([403,404,410].includes(response.status)) return { reason: `http_${response.status}` };
|
|
36
|
-
if (!response.bytes || response.bytes.length !== size) fail('ATTACHMENT_SIZE_MISMATCH');
|
|
37
|
-
return { bytes: response.bytes };
|
|
38
|
-
}
|
|
39
|
-
export async function attachments(ctx) {
|
|
40
|
-
const { store, request, origin, inputs } = ctx, state = store.state, data = await materialize(store);
|
|
41
|
-
const ids = ordered([...data.tickets].filter(([, { row }]) => row.status !== 'deleted' && !row.deleted && (inputs.mode !== 'sweep' || row.generated_timestamp > inputs.highWater)).map(([id]) => id));
|
|
42
|
-
const ticket = ids[state.index];
|
|
43
|
-
const refs = attachmentReferences(data).filter(ref => state.phase === 'guide_attachments' ? ref.article : ref.ticket === ticket);
|
|
44
|
-
const ref = refs[state.attachmentIndex];
|
|
45
|
-
if (!ref) {
|
|
46
|
-
await store.commit(state.phase === 'guide_attachments' ? { ...state, phase: 'done', attachmentIndex: 0 } : { ...state, phase: 'B', path: null, index: state.index + 1, attachmentIndex: 0 });
|
|
47
|
-
return;
|
|
48
|
-
}
|
|
49
|
-
const previous = data.attachments.get(ref.key);
|
|
50
|
-
if (previous && previous.sourceHash === JSON.stringify(ref.attachment)) {
|
|
51
|
-
await store.commit({ ...state, attachmentIndex: state.attachmentIndex + 1 }); return;
|
|
52
|
-
}
|
|
53
|
-
const result = await download(request, origin, ref.attachment);
|
|
54
|
-
const blob = result.bytes ? await store.put(result.bytes) : null;
|
|
55
|
-
await store.commit({ ...state, attachmentIndex: state.attachmentIndex + 1, bytes: state.bytes + (blob?.bytes ?? 0) },
|
|
56
|
-
{ kind: 'attachment', key: ref.key, sourceHash: JSON.stringify(ref.attachment), ...(blob ? { blob } : { reason: result.reason }) }, blob ? [blob] : []);
|
|
57
|
-
}
|
|
@@ -1,133 +0,0 @@
|
|
|
1
|
-
import { credentialShaped } from '../detector.mjs';
|
|
2
|
-
import { assertChunk } from '../outbound.mjs';
|
|
3
|
-
import { fail, hash } from './common.mjs';
|
|
4
|
-
import { materialize, encode, ordered } from './zendesk-store.mjs';
|
|
5
|
-
import { attachmentReferences } from './zendesk-attachments.mjs';
|
|
6
|
-
import { attachmentPath } from './zendesk-transport.mjs';
|
|
7
|
-
|
|
8
|
-
export const FORMAT = 'happytohelp-zendesk-offline-v1';
|
|
9
|
-
const omissions = ['audits','metrics','satisfaction','side conversations','groups','brands','forms','macros','triggers','views','automations','photos','thumbnails','archived articles','community'];
|
|
10
|
-
const escapePointer = value => value.replace(/~/gu, '~0').replace(/\//gu, '~1');
|
|
11
|
-
/** Redact only the delivered copy. Raw responses remain private evidence in
|
|
12
|
-
* the store. Expand prefix detector hits to their whole token and PEM hits to
|
|
13
|
-
* their whole block; assertChunk remains the final authority on every line. */
|
|
14
|
-
function cleaned(value, kind, id, report, pointer = '') {
|
|
15
|
-
const removed = () => report.removals.push({ kind, id, pointer: pointer || '/' });
|
|
16
|
-
if (Array.isArray(value)) return value.map((child,index) => cleaned(child, kind, id, report, `${pointer}/${index}`));
|
|
17
|
-
if (value && typeof value === 'object') {
|
|
18
|
-
const result = Object.create(null);
|
|
19
|
-
for (const [key,child] of Object.entries(value)) {
|
|
20
|
-
if (['content_url','download_url','original_url'].includes(key)) { report.redactedUrls++; continue; }
|
|
21
|
-
if (credentialShaped(key)) fail('CREDENTIAL_SHAPE_UNREMOVABLE');
|
|
22
|
-
result[key] = cleaned(child, kind, id, report, `${pointer}/${escapePointer(key)}`);
|
|
23
|
-
}
|
|
24
|
-
return result;
|
|
25
|
-
}
|
|
26
|
-
if (typeof value !== 'string') return value;
|
|
27
|
-
let text = value.replace(/https?:\/\/[^\s<>"']+/giu, match => {
|
|
28
|
-
let url; try { url = new URL(match); } catch { return match; }
|
|
29
|
-
if (attachmentPath(url.pathname)) { report.redactedUrls++; return '[removed]'; }
|
|
30
|
-
if (!url.username && !url.password && !url.search && !url.hash) return match;
|
|
31
|
-
url.username = ''; url.password = ''; url.search = ''; url.hash = ''; report.redactedUrls++; return url.href;
|
|
32
|
-
});
|
|
33
|
-
text = text.replace(/(?:\/\/[^\s<>"'/]+)?\/(?:api\/v2\/)?attachments\/[^\s<>"']*/giu, () => { report.redactedUrls++; return '[removed]'; });
|
|
34
|
-
text = text.replace(/-----\s*BEGIN\s+([A-Z0-9 ]+)-----[\s\S]*?-----\s*END\s+\1-----/giu, () => { removed(); return '[removed]'; });
|
|
35
|
-
// SSH material includes a scheme and key separated by whitespace.
|
|
36
|
-
text = text.replace(/\b(?:ssh-(?:rsa|ed25519)|ecdsa-sha2-[\w-]+)\s+[A-Za-z0-9+/=]+/gu, () => { removed(); return '[removed]'; });
|
|
37
|
-
text = text.replace(/[^\s<>"'`,;(){}\[\]]+/gu, token => {
|
|
38
|
-
if (!credentialShaped(token)) return token;
|
|
39
|
-
removed(); return '[removed]';
|
|
40
|
-
});
|
|
41
|
-
text = text.replace(/[\u0000-\u0008\u000b\u000c\u000e-\u001f\u007f]/gu, () => { removed(); return '[removed]'; });
|
|
42
|
-
text = text.replace(/https?:\/\/[^\s<>"']+/giu, match => {
|
|
43
|
-
let url;
|
|
44
|
-
try { url = new URL(match); } catch { return match; }
|
|
45
|
-
if (!url.username && !url.password && !url.search && !url.hash) return match;
|
|
46
|
-
url.username = ''; url.password = ''; url.search = ''; url.hash = ''; report.redactedUrls++; return url.href;
|
|
47
|
-
});
|
|
48
|
-
if (credentialShaped(text.replace(/[\r\n\t]/gu, ' '))) fail('CREDENTIAL_SHAPE_UNREMOVABLE');
|
|
49
|
-
return text;
|
|
50
|
-
}
|
|
51
|
-
const deleted = row => row.status === 'deleted' || row.deleted === true;
|
|
52
|
-
const sortedRows = map => ordered(map.keys()).map(id => [id, map.get(id)]);
|
|
53
|
-
function attachmentCensus(references, data, { article = false } = {}) {
|
|
54
|
-
const blobs = new Map(), undelivered = {};
|
|
55
|
-
let fetched = 0, bytes = 0;
|
|
56
|
-
for (const ref of references) {
|
|
57
|
-
const item = data.attachments.get(ref.key);
|
|
58
|
-
if (item?.blob) { fetched++; bytes += item.blob.bytes; blobs.set(item.blob.sha256, item.blob.bytes); }
|
|
59
|
-
const reason = item?.reason ?? (!item ? 'not_fetched' : article ? 'pending_guide' : null);
|
|
60
|
-
if (reason) undelivered[reason] = (undelivered[reason] ?? 0) + 1;
|
|
61
|
-
}
|
|
62
|
-
return { referenced: references.length, fetched, bytes, uniqueBlobs: blobs.size, undelivered };
|
|
63
|
-
}
|
|
64
|
-
export async function encodeSnapshot(store, { origin, packageBindings = false } = {}) {
|
|
65
|
-
const data = await materialize(store), report = { removals: [], redactedUrls: 0 }, lines = [], byKind = {}, blobs = new Map();
|
|
66
|
-
const add = (kind, id, value) => {
|
|
67
|
-
// Bindings are integrity-checked original bytes, never redaction input.
|
|
68
|
-
const clean = kind === 'ticket' ? { ...cleaned({ ...value, _offline: undefined }, kind, id, report), _offline: value._offline } : cleaned(value, kind, id, report);
|
|
69
|
-
const line = encode({ [kind]: clean });
|
|
70
|
-
if (line.length > 64 * 1024 * 1024) fail('RECORD_LIMIT');
|
|
71
|
-
try { assertChunk(line, []); } catch { fail('CREDENTIAL_SHAPE_UNREMOVABLE'); }
|
|
72
|
-
lines.push(line); byKind[kind] = (byKind[kind] ?? 0) + 1;
|
|
73
|
-
};
|
|
74
|
-
let needsPackage = packageBindings, commentTotal = 0, publicComments = 0;
|
|
75
|
-
const tombstones = [], byStatus = {}, refs = attachmentReferences(data);
|
|
76
|
-
for (const [id,{ row, provenance }] of sortedRows(data.tickets)) {
|
|
77
|
-
byStatus[row.status] = (byStatus[row.status] ?? 0) + 1;
|
|
78
|
-
if (deleted(row)) { tombstones.push({ id: row.id, generated_timestamp: row.generated_timestamp }); continue; }
|
|
79
|
-
const comments = sortedRows(data.comments.get(id) ?? new Map()).map(([,item]) => item.row), bindings = [];
|
|
80
|
-
for (const comment of comments) {
|
|
81
|
-
commentTotal++; if (comment.public === true) publicComments++;
|
|
82
|
-
for (const attachment of comment.attachments ?? []) {
|
|
83
|
-
const stored = data.attachments.get(`ticket:${id}:${comment.id}:${attachment.id}`);
|
|
84
|
-
if (!stored?.blob) continue;
|
|
85
|
-
const blob = stored.blob;
|
|
86
|
-
if (blob.bytes > 48 * 1024) needsPackage = true;
|
|
87
|
-
blobs.set(`blobs/${blob.sha256}`, blob);
|
|
88
|
-
bindings.push({ attachment_id: attachment.id, comment_id: comment.id, path: `blobs/${blob.sha256}`, sha256: blob.sha256,
|
|
89
|
-
...(!packageBindings ? { base64: (await store.get(blob)).toString('base64') } : {}) });
|
|
90
|
-
}
|
|
91
|
-
}
|
|
92
|
-
if (bindings.length > 20) needsPackage = true;
|
|
93
|
-
const value = { ...row, comments, _offline: { format: FORMAT, comments_complete: true, attachments: bindings,
|
|
94
|
-
history: { metrics: 'missing', audits: 'missing' }, metric_events: [], audits: [], provenance: [provenance, ...[...(data.comments.get(id)?.values() ?? [])].map(item => item.provenance)] } };
|
|
95
|
-
if (encode(value).length > 128 * 1024) needsPackage = true;
|
|
96
|
-
add('ticket', id, value);
|
|
97
|
-
}
|
|
98
|
-
for (const [kind, map] of [['user', data.users], ['organization', data.organizations], ['ticket_field', data.ticket_fields]]) {
|
|
99
|
-
for (const [id,{ row, provenance }] of sortedRows(map)) add(kind, id, { ...row, _offline_provenance: [provenance] });
|
|
100
|
-
}
|
|
101
|
-
let restricted = 0, draft = 0;
|
|
102
|
-
for (const [id,{ row, provenance }] of sortedRows(data.articles)) {
|
|
103
|
-
if (row.user_segment_id != null || row.user_segment_ids?.length) restricted++;
|
|
104
|
-
if (row.draft) draft++;
|
|
105
|
-
add('article', id, { ...row, draft: row.draft === true, user_segment_id: row.user_segment_id ?? null, user_segment_ids: row.user_segment_ids ?? [], _offline_provenance: [provenance] });
|
|
106
|
-
}
|
|
107
|
-
const records = Buffer.concat(lines);
|
|
108
|
-
if (records.length > 2 * 1024 * 1024 || lines.length > 5000) needsPackage = true;
|
|
109
|
-
const census = { format: 'rer-zendesk-census/1', tickets: { unique: data.tickets.size, deleted: tombstones.length,
|
|
110
|
-
deletedIdsSha256: hash(tombstones.map(row => `${row.id}\n`).join('')), byStatus, maxId: ordered(data.tickets.keys()).at(-1) ?? null },
|
|
111
|
-
comments: { total: commentTotal, public: publicComments, private: commentTotal - publicComments },
|
|
112
|
-
attachments: attachmentCensus(refs.filter(ref => ref.ticket), data), articleAttachments: attachmentCensus(refs.filter(ref => ref.article), data, { article: true }),
|
|
113
|
-
users: { unique: data.users.size, inactive: [...data.users.values()].filter(item => item.row.active === false).length, withoutEmail: [...data.users.values()].filter(item => !item.row.email).length },
|
|
114
|
-
organizations: data.organizations.size, fields: data.ticket_fields.size,
|
|
115
|
-
guide: { available: data.available, articles: data.articles.size, restricted, draft, translations: data.translations.size, sections: data.sections.size, categories: data.categories.size, segments: data.user_segments.size },
|
|
116
|
-
records: byKind, undelivered: { tombstones: tombstones.length, translations: data.translations.size, sections: data.sections.size, categories: data.categories.size, segments: data.user_segments.size, articleAttachments: data.article_attachments.size }, credentialShapesRemoved: report.removals.length, credentialRemovals: report.removals, redactedUrls: report.redactedUrls, highWater: data.highWater, omissions };
|
|
117
|
-
const files = new Map([['records-000001.ndjson', records], ['tombstones.ndjson', Buffer.concat(tombstones.map(row => encode(row)))]]);
|
|
118
|
-
for (const kind of ['translations','sections','categories','user_segments','article_attachments']) {
|
|
119
|
-
const bytes = Buffer.concat([...data[kind]].sort(([a],[b]) => a.localeCompare(b)).map(([id,{ row }]) => encode(cleaned(row, kind, id, report))));
|
|
120
|
-
try { assertChunk(bytes, []); } catch { fail('CREDENTIAL_SHAPE_UNREMOVABLE'); }
|
|
121
|
-
files.set(`guide/${kind}.ndjson`, bytes);
|
|
122
|
-
}
|
|
123
|
-
// Article media remains in the package/store even before HH can import it.
|
|
124
|
-
for (const ref of refs.filter(ref => ref.article)) {
|
|
125
|
-
const blob = data.attachments.get(ref.key)?.blob;
|
|
126
|
-
if (blob) blobs.set(`blobs/${blob.sha256}`, blob);
|
|
127
|
-
}
|
|
128
|
-
census.credentialShapesRemoved = report.removals.length; census.redactedUrls = report.redactedUrls;
|
|
129
|
-
files.set('census.json', encode(census));
|
|
130
|
-
if (needsPackage && !packageBindings) return encodeSnapshot(store, { origin, packageBindings: true });
|
|
131
|
-
return { kind: needsPackage ? 'package' : 'bundle', bytes: records, files, blobs, census,
|
|
132
|
-
source: { format: FORMAT, version: 1, origin, exportedAt: store.state.exportedAt, synthetic: false, adapter: 'rer:zendesk' } };
|
|
133
|
-
}
|
|
@@ -1,34 +0,0 @@
|
|
|
1
|
-
import { fail, object } from './common.mjs';
|
|
2
|
-
import { materialize, ordered } from './zendesk-store.mjs';
|
|
3
|
-
import { fetchPage } from './zendesk-pages.mjs';
|
|
4
|
-
|
|
5
|
-
const stages = [
|
|
6
|
-
['user_segments'], ['categories'], ['sections'], ['articles'],
|
|
7
|
-
['source_articles'], ['translations', 'categories'], ['translations', 'sections'], ['translations', 'articles'], ['article_attachments', 'articles'],
|
|
8
|
-
];
|
|
9
|
-
export async function guide(ctx) {
|
|
10
|
-
const state = ctx.store.state, stage = stages[state.stage];
|
|
11
|
-
if (!stage) { await ctx.store.commit({ ...state, phase: 'guide_attachments', attachmentIndex: 0 }); return; }
|
|
12
|
-
const [kind, resource] = stage;
|
|
13
|
-
let article, path;
|
|
14
|
-
if (state.stage < 4) path = `/api/v2/help_center/${kind}?per_page=100`;
|
|
15
|
-
else {
|
|
16
|
-
const data = await materialize(ctx.store), records = data[resource ?? 'articles'];
|
|
17
|
-
article = ordered(records.keys())[state.index];
|
|
18
|
-
if (article === undefined) { await ctx.store.commit({ ...state, stage: state.stage + 1, index: 0, path: null }); return; }
|
|
19
|
-
if (kind === 'source_articles') {
|
|
20
|
-
const row = records.get(article).row;
|
|
21
|
-
if (typeof row.source_locale !== 'string' || !/^[a-z]{2,3}(?:-[a-z0-9]{2,8})?$/u.test(row.source_locale)) fail('INVALID_RESPONSE');
|
|
22
|
-
if (row.locale !== row.source_locale) {
|
|
23
|
-
const response = await ctx.request(`/api/v2/help_center/${row.source_locale}/articles/${article}`);
|
|
24
|
-
const source = object(response.body.article);
|
|
25
|
-
if (String(source.id) !== article || source.locale !== row.source_locale) fail('INVALID_RESPONSE');
|
|
26
|
-
await ctx.store.page('articles', response, { ...state, index: state.index + 1 }, { single: 'article' });
|
|
27
|
-
} else await ctx.store.commit({ ...state, index: state.index + 1 });
|
|
28
|
-
return;
|
|
29
|
-
}
|
|
30
|
-
path = `/api/v2/help_center/${resource}/${article}/${kind === 'article_attachments' ? 'attachments' : kind}?per_page=100`;
|
|
31
|
-
}
|
|
32
|
-
await fetchPage({ ...ctx, state, kind, path: state.path ?? path, allow404: true, extra: article ? { article, resource } : {},
|
|
33
|
-
onDone: state => ({ ...state, stage: state.stage + (article ? 0 : 1), index: article ? state.index + 1 : 0 }) });
|
|
34
|
-
}
|
|
@@ -1,28 +0,0 @@
|
|
|
1
|
-
import { join } from 'node:path';
|
|
2
|
-
import { buildPackage } from '../package-transfer.mjs';
|
|
3
|
-
|
|
4
|
-
/** Materialize checked HH lines and sidecars as immutable files, and reference
|
|
5
|
-
* the already verified original blobs. The shared sender owns segmentation,
|
|
6
|
-
* manifest probing and the resumable upload journal. */
|
|
7
|
-
export async function prepareZendeskPackage({ result, quota, directory, signal, mode = 'snapshot' }) {
|
|
8
|
-
const files = [], store = result.store;
|
|
9
|
-
const add = async (path, bytes) => {
|
|
10
|
-
const ref = await store.put(bytes);
|
|
11
|
-
files.push({ path, file: join(store.path, ref.file), bytes: ref.bytes, sha256: ref.sha256 });
|
|
12
|
-
};
|
|
13
|
-
let start = 0, position = 0, ordinal = 1;
|
|
14
|
-
while (position < result.bytes.length) {
|
|
15
|
-
signal?.throwIfAborted();
|
|
16
|
-
const newline = result.bytes.indexOf(10, position);
|
|
17
|
-
const end = newline < 0 ? result.bytes.length : newline + 1;
|
|
18
|
-
if (position > start && end - start > 8 * 1024 * 1024) {
|
|
19
|
-
await add(`records-${String(ordinal++).padStart(6,'0')}.ndjson`, result.bytes.subarray(start,position)); start = position;
|
|
20
|
-
}
|
|
21
|
-
position = end;
|
|
22
|
-
}
|
|
23
|
-
if (position > start) await add(`records-${String(ordinal).padStart(6,'0')}.ndjson`, result.bytes.subarray(start,position));
|
|
24
|
-
for (const [path, bytes] of result.files) if (!path.startsWith('records-')) await add(path,bytes);
|
|
25
|
-
for (const [path,ref] of result.blobs) { await store.get(ref); files.push({ path, file: join(store.path,ref.file), bytes: ref.bytes, sha256: ref.sha256 }); }
|
|
26
|
-
return buildPackage({ files, quota, directory, signal, mode, source: result.source,
|
|
27
|
-
records: { total: Object.values(result.census.records).reduce((a,b) => a+b,0), byKind: result.census.records }, totals: result.census });
|
|
28
|
-
}
|
|
@@ -1,61 +0,0 @@
|
|
|
1
|
-
import { fail, object, list } from './common.mjs';
|
|
2
|
-
import { pagination } from './zendesk-transport.mjs';
|
|
3
|
-
|
|
4
|
-
/** Both documented comments envelopes, and Guide's offset envelopes, are
|
|
5
|
-
* explicit: a missing end marker is an error rather than a short snapshot. */
|
|
6
|
-
export function nextPage(body, path, origin, { incremental = false, time = false } = {}) {
|
|
7
|
-
object(body);
|
|
8
|
-
const pathname = new URL(path, origin).pathname;
|
|
9
|
-
if (incremental) {
|
|
10
|
-
if (typeof body.end_of_stream !== 'boolean') fail('INVALID_PAGINATION');
|
|
11
|
-
if (body.end_of_stream) return null;
|
|
12
|
-
if (time) {
|
|
13
|
-
if (!Number.isSafeInteger(body.end_time) || body.end_time < 1 || body.end_time <= Number(new URL(path, origin).searchParams.get('start_time') ?? 0)) fail('INVALID_PAGINATION');
|
|
14
|
-
return `${pathname}?start_time=${body.end_time}&per_page=1000`;
|
|
15
|
-
}
|
|
16
|
-
if (typeof body.after_cursor !== 'string' || !body.after_cursor || body.after_cursor.length > 2048 || /[\u0000-\u0020\u007f]/u.test(body.after_cursor)) fail('INVALID_PAGINATION');
|
|
17
|
-
const next = `${pathname}?cursor=${encodeURIComponent(body.after_cursor)}&per_page=1000`;
|
|
18
|
-
if (new URL(path, origin).searchParams.get('cursor') === body.after_cursor) fail('INVALID_PAGINATION');
|
|
19
|
-
if (body.after_url != null && pagination(body.after_url, origin, pathname, path) !== next) fail('INVALID_PAGINATION');
|
|
20
|
-
return next;
|
|
21
|
-
}
|
|
22
|
-
if (body.meta && Object.hasOwn(body.meta, 'has_more')) {
|
|
23
|
-
if (typeof body.meta.has_more !== 'boolean' || !body.links || !Object.hasOwn(body.links, 'next')) fail('INVALID_PAGINATION');
|
|
24
|
-
if (!body.meta.has_more) { if (body.links.next !== null) fail('INVALID_PAGINATION'); return null; }
|
|
25
|
-
return pagination(body.links.next, origin, pathname, path);
|
|
26
|
-
}
|
|
27
|
-
if (Object.hasOwn(body, 'next_page')) {
|
|
28
|
-
const next = pagination(body.next_page, origin, pathname, path);
|
|
29
|
-
if (next) {
|
|
30
|
-
const current = new URL(path, origin), url = new URL(next, origin);
|
|
31
|
-
if (url.searchParams.get('page') !== String(Number(current.searchParams.get('page') ?? 1) + 1)) fail('INVALID_PAGINATION');
|
|
32
|
-
}
|
|
33
|
-
return next;
|
|
34
|
-
}
|
|
35
|
-
fail('INVALID_PAGINATION');
|
|
36
|
-
}
|
|
37
|
-
export async function fetchPage({ store, request, origin, state, kind, path, extra = {}, incremental, time, allow404 = false, onDone }) {
|
|
38
|
-
const response = await request(path, { allow: allow404 ? [404] : [] });
|
|
39
|
-
if (response.status === 404) {
|
|
40
|
-
const next = onDone({ ...state, path: null });
|
|
41
|
-
await store.commit(next, { kind: 'guide_unavailable' });
|
|
42
|
-
return next;
|
|
43
|
-
}
|
|
44
|
-
list(object(response.body)[kind], 1000);
|
|
45
|
-
const seenKey = JSON.stringify([state.runId, kind, extra.ticket ?? extra.article ?? '', extra.resource ?? '']);
|
|
46
|
-
if (!store.pagePaths) {
|
|
47
|
-
store.pagePaths = new Map();
|
|
48
|
-
for (const event of store.events) if (event.requestPath) {
|
|
49
|
-
const paths = store.pagePaths.get(event.seenKey) ?? new Set(); paths.add(event.requestPath); store.pagePaths.set(event.seenKey, paths);
|
|
50
|
-
}
|
|
51
|
-
}
|
|
52
|
-
const seen = store.pagePaths.get(seenKey) ?? new Set();
|
|
53
|
-
if (seen.has(path)) fail('INVALID_PAGINATION');
|
|
54
|
-
const next = nextPage(response.body, path, origin, { incremental, time });
|
|
55
|
-
if (next && seen.has(next)) fail('INVALID_PAGINATION');
|
|
56
|
-
const nextState = next ? { ...state, path: next } : onDone({ ...state, path: null });
|
|
57
|
-
nextState.requests = state.requests + 1;
|
|
58
|
-
await store.page(kind, response, nextState, { ...extra, requestPath: path, seenKey });
|
|
59
|
-
seen.add(path); store.pagePaths.set(seenKey, seen);
|
|
60
|
-
return store.state;
|
|
61
|
-
}
|
|
@@ -1,7 +0,0 @@
|
|
|
1
|
-
import { fetchPage } from './zendesk-pages.mjs';
|
|
2
|
-
export async function people(ctx) {
|
|
3
|
-
const state = ctx.store.state;
|
|
4
|
-
const resources = { C: ['users', '/api/v2/incremental/users/cursor?start_time=0&per_page=1000', 'D'], D: ['organizations', '/api/v2/incremental/organizations?start_time=0&per_page=1000', 'E'], E: ['ticket_fields', '/api/v2/ticket_fields?page[size]=100', 'F'] };
|
|
5
|
-
const [kind, path, next] = resources[state.phase];
|
|
6
|
-
await fetchPage({ ...ctx, state, kind, path: state.path ?? path, incremental: state.phase !== 'E', time: state.phase === 'D', onDone: state => ({ ...state, phase: next, index: 0, stage: 0 }) });
|
|
7
|
-
}
|