@runeditrun/rer 0.2.99 → 0.2.100
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/cli/README.md +88 -2
- package/cli/business/bounded-waves.d.mts +23 -0
- package/cli/business/bounded-waves.mjs +57 -0
- package/cli/business/catchup-task.mjs +9 -2
- package/cli/business/command.mjs +8 -3
- package/cli/business/export-inputs.mjs +12 -2
- package/cli/business/export-task.mjs +9 -2
- package/cli/business/legacy-package.mjs +117 -0
- package/cli/business/local-commands.mjs +10 -1
- package/cli/business/package-transfer.mjs +127 -0
- package/cli/business/state.mjs +3 -3
- package/cli/business/vendor-door.mjs +83 -11
- package/cli/business/vendors/README.md +41 -2
- package/cli/business/vendors/common.mjs +81 -0
- package/cli/business/vendors/index.mjs +12 -2
- package/cli/business/vendors/zendesk-attachments.mjs +57 -0
- package/cli/business/vendors/zendesk-encode.mjs +133 -0
- package/cli/business/vendors/zendesk-guide.mjs +34 -0
- package/cli/business/vendors/zendesk-package.mjs +28 -0
- package/cli/business/vendors/zendesk-pages.mjs +61 -0
- package/cli/business/vendors/zendesk-people.mjs +7 -0
- package/cli/business/vendors/zendesk-store.mjs +154 -0
- package/cli/business/vendors/zendesk-tickets.mjs +21 -0
- package/cli/business/vendors/zendesk-token.mjs +31 -0
- package/cli/business/vendors/zendesk-transport.mjs +48 -0
- package/cli/business/vendors/zendesk.mjs +61 -0
- package/cli/deploy-command.mjs +5 -4
- package/cli/deploy-inputs.mjs +1 -1
- package/cli/deploy.mjs +6 -4
- package/cli/production-auth.mjs +16 -3
- package/cli/production-mail.mjs +18 -1
- package/cli/site.mjs +1 -1
- package/global/CHANGELOG.md +21 -0
- package/modules/deploy-kit/README.md +10 -0
- package/modules/deploy-kit/provider.mjs +18 -2
- package/modules/import-kit/src/d1/jobs.ts +3 -1
- package/modules/import-kit/src/d1/retirement-scope.ts +2 -2
- package/modules/import-kit/src/d1/retirement-tally.ts +1 -0
- package/modules/import-kit/src/d1/retirement.ts +15 -5
- package/modules/import-kit/src/d1/types.ts +3 -0
- package/modules/import-kit/src/d1/wave-boundary.ts +33 -0
- package/modules/import-kit/src/d1/wave-retirement.ts +47 -0
- package/modules/import-kit/src/door/ADOPTING.md +37 -0
- package/modules/import-kit/src/door/README.md +66 -0
- package/modules/import-kit/src/door/admission.ts +13 -3
- package/modules/import-kit/src/door/authority.ts +1 -0
- package/modules/import-kit/src/door/failures.ts +1 -1
- package/modules/import-kit/src/door/grants.ts +19 -5
- package/modules/import-kit/src/door/handler.ts +12 -1
- package/modules/import-kit/src/door/index.ts +6 -0
- package/modules/import-kit/src/door/package.ts +121 -0
- package/modules/import-kit/src/door/planning.ts +44 -19
- package/modules/import-kit/src/door/protocol.ts +4 -2
- package/modules/import-kit/src/door/schema.ts +2 -0
- package/modules/import-kit/src/door/store.ts +2 -2
- package/modules/import-kit/src/door/types.ts +24 -1
- package/modules/import-kit/src/door/waves/README.md +158 -0
- package/modules/import-kit/src/door/waves/digests.ts +38 -0
- package/modules/import-kit/src/door/waves/inventory.ts +84 -0
- package/modules/import-kit/src/door/waves/journal.ts +19 -0
- package/modules/import-kit/src/door/waves/lifecycle.ts +242 -0
- package/modules/import-kit/src/door/waves/preparation.ts +190 -0
- package/modules/import-kit/src/door/waves/schema.ts +73 -0
- package/modules/import-kit/src/door/waves/store.ts +112 -0
- package/modules/import-kit/src/door/waves/types.ts +76 -0
- package/onboarding/adapter.mjs +7 -1
- package/onboarding/operator-installer.mjs +3 -1
- package/package.json +1 -1
- package/scripts/publish-check/README.md +20 -0
- package/scripts/publish-check/diagnostics.mjs +73 -0
- package/scripts/publish-check/slots.py +37 -7
- package/scripts/publish-check.mjs +64 -37
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
import { fail, object, list } from './common.mjs';
|
|
2
|
+
import { pagination } from './zendesk-transport.mjs';
|
|
3
|
+
|
|
4
|
+
/** Both documented comments envelopes, and Guide's offset envelopes, are
|
|
5
|
+
* explicit: a missing end marker is an error rather than a short snapshot. */
|
|
6
|
+
export function nextPage(body, path, origin, { incremental = false, time = false } = {}) {
|
|
7
|
+
object(body);
|
|
8
|
+
const pathname = new URL(path, origin).pathname;
|
|
9
|
+
if (incremental) {
|
|
10
|
+
if (typeof body.end_of_stream !== 'boolean') fail('INVALID_PAGINATION');
|
|
11
|
+
if (body.end_of_stream) return null;
|
|
12
|
+
if (time) {
|
|
13
|
+
if (!Number.isSafeInteger(body.end_time) || body.end_time < 1 || body.end_time <= Number(new URL(path, origin).searchParams.get('start_time') ?? 0)) fail('INVALID_PAGINATION');
|
|
14
|
+
return `${pathname}?start_time=${body.end_time}&per_page=1000`;
|
|
15
|
+
}
|
|
16
|
+
if (typeof body.after_cursor !== 'string' || !body.after_cursor || body.after_cursor.length > 2048 || /[\u0000-\u0020\u007f]/u.test(body.after_cursor)) fail('INVALID_PAGINATION');
|
|
17
|
+
const next = `${pathname}?cursor=${encodeURIComponent(body.after_cursor)}&per_page=1000`;
|
|
18
|
+
if (new URL(path, origin).searchParams.get('cursor') === body.after_cursor) fail('INVALID_PAGINATION');
|
|
19
|
+
if (body.after_url != null && pagination(body.after_url, origin, pathname, path) !== next) fail('INVALID_PAGINATION');
|
|
20
|
+
return next;
|
|
21
|
+
}
|
|
22
|
+
if (body.meta && Object.hasOwn(body.meta, 'has_more')) {
|
|
23
|
+
if (typeof body.meta.has_more !== 'boolean' || !body.links || !Object.hasOwn(body.links, 'next')) fail('INVALID_PAGINATION');
|
|
24
|
+
if (!body.meta.has_more) { if (body.links.next !== null) fail('INVALID_PAGINATION'); return null; }
|
|
25
|
+
return pagination(body.links.next, origin, pathname, path);
|
|
26
|
+
}
|
|
27
|
+
if (Object.hasOwn(body, 'next_page')) {
|
|
28
|
+
const next = pagination(body.next_page, origin, pathname, path);
|
|
29
|
+
if (next) {
|
|
30
|
+
const current = new URL(path, origin), url = new URL(next, origin);
|
|
31
|
+
if (url.searchParams.get('page') !== String(Number(current.searchParams.get('page') ?? 1) + 1)) fail('INVALID_PAGINATION');
|
|
32
|
+
}
|
|
33
|
+
return next;
|
|
34
|
+
}
|
|
35
|
+
fail('INVALID_PAGINATION');
|
|
36
|
+
}
|
|
37
|
+
export async function fetchPage({ store, request, origin, state, kind, path, extra = {}, incremental, time, allow404 = false, onDone }) {
|
|
38
|
+
const response = await request(path, { allow: allow404 ? [404] : [] });
|
|
39
|
+
if (response.status === 404) {
|
|
40
|
+
const next = onDone({ ...state, path: null });
|
|
41
|
+
await store.commit(next, { kind: 'guide_unavailable' });
|
|
42
|
+
return next;
|
|
43
|
+
}
|
|
44
|
+
list(object(response.body)[kind], 1000);
|
|
45
|
+
const seenKey = JSON.stringify([state.runId, kind, extra.ticket ?? extra.article ?? '', extra.resource ?? '']);
|
|
46
|
+
if (!store.pagePaths) {
|
|
47
|
+
store.pagePaths = new Map();
|
|
48
|
+
for (const event of store.events) if (event.requestPath) {
|
|
49
|
+
const paths = store.pagePaths.get(event.seenKey) ?? new Set(); paths.add(event.requestPath); store.pagePaths.set(event.seenKey, paths);
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
const seen = store.pagePaths.get(seenKey) ?? new Set();
|
|
53
|
+
if (seen.has(path)) fail('INVALID_PAGINATION');
|
|
54
|
+
const next = nextPage(response.body, path, origin, { incremental, time });
|
|
55
|
+
if (next && seen.has(next)) fail('INVALID_PAGINATION');
|
|
56
|
+
const nextState = next ? { ...state, path: next } : onDone({ ...state, path: null });
|
|
57
|
+
nextState.requests = state.requests + 1;
|
|
58
|
+
await store.page(kind, response, nextState, { ...extra, requestPath: path, seenKey });
|
|
59
|
+
seen.add(path); store.pagePaths.set(seenKey, seen);
|
|
60
|
+
return store.state;
|
|
61
|
+
}
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
import { fetchPage } from './zendesk-pages.mjs';
|
|
2
|
+
export async function people(ctx) {
|
|
3
|
+
const state = ctx.store.state;
|
|
4
|
+
const resources = { C: ['users', '/api/v2/incremental/users/cursor?start_time=0&per_page=1000', 'D'], D: ['organizations', '/api/v2/incremental/organizations?start_time=0&per_page=1000', 'E'], E: ['ticket_fields', '/api/v2/ticket_fields?page[size]=100', 'F'] };
|
|
5
|
+
const [kind, path, next] = resources[state.phase];
|
|
6
|
+
await fetchPage({ ...ctx, state, kind, path: state.path ?? path, incremental: state.phase !== 'E', time: state.phase === 'D', onDone: state => ({ ...state, phase: next, index: 0, stage: 0 }) });
|
|
7
|
+
}
|
|
@@ -0,0 +1,154 @@
|
|
|
1
|
+
import { constants } from 'node:fs';
|
|
2
|
+
import { mkdir, lstat, open, rename, rm } from 'node:fs/promises';
|
|
3
|
+
import { join } from 'node:path';
|
|
4
|
+
import { randomUUID } from 'node:crypto';
|
|
5
|
+
import { hash, fail, stable } from './common.mjs';
|
|
6
|
+
|
|
7
|
+
export const encode = value => Buffer.from(JSON.stringify(stable(value)) + '\n');
|
|
8
|
+
export const numericId = value => {
|
|
9
|
+
if (typeof value === 'number' && (!Number.isSafeInteger(value) || value < 0)) fail('INVALID_RESPONSE');
|
|
10
|
+
const id = String(value);
|
|
11
|
+
if (!/^(?:0|[1-9]\d{0,19})$/u.test(id)) fail('INVALID_RESPONSE');
|
|
12
|
+
return id;
|
|
13
|
+
};
|
|
14
|
+
export const ordered = values => [...values].sort((a,b) => BigInt(a) < BigInt(b) ? -1 : BigInt(a) > BigInt(b) ? 1 : 0);
|
|
15
|
+
export const storePath = (directory, business, origin) => join(directory, `zendesk-${hash(JSON.stringify([business, origin])).slice(0,32)}`);
|
|
16
|
+
async function privateDirectory(path) {
|
|
17
|
+
await mkdir(path, { mode: 0o700, recursive: true });
|
|
18
|
+
const info = await lstat(path);
|
|
19
|
+
if (!info.isDirectory() || (info.mode & 0o777) !== 0o700) fail('INVALID_CHECKPOINT');
|
|
20
|
+
}
|
|
21
|
+
async function read(path, maximum = 64 * 1024 * 1024) {
|
|
22
|
+
const file = await open(path, constants.O_RDONLY | constants.O_NOFOLLOW);
|
|
23
|
+
try {
|
|
24
|
+
const info = await file.stat();
|
|
25
|
+
if (!info.isFile() || info.size > maximum || (info.mode & 0o777) !== 0o600) fail('INVALID_CHECKPOINT');
|
|
26
|
+
return await file.readFile();
|
|
27
|
+
} finally { await file.close(); }
|
|
28
|
+
}
|
|
29
|
+
async function atomic(path, bytes) {
|
|
30
|
+
const temporary = `${path}.${randomUUID()}.tmp`;
|
|
31
|
+
const file = await open(temporary, constants.O_WRONLY | constants.O_CREAT | constants.O_EXCL | constants.O_NOFOLLOW, 0o600);
|
|
32
|
+
try { await file.writeFile(bytes); await file.sync(); } finally { await file.close(); }
|
|
33
|
+
await rename(temporary, path);
|
|
34
|
+
const parent = await open(join(path, '..'), constants.O_RDONLY);
|
|
35
|
+
try { await parent.sync(); } finally { await parent.close(); }
|
|
36
|
+
}
|
|
37
|
+
/** Immutable, content-addressed objects and journal nodes precede one atomic
|
|
38
|
+
* checkpoint pointer. A kill can leave unreferenced objects, never a half-
|
|
39
|
+
* acknowledged page. Reopen verifies the complete reachable graph, including
|
|
40
|
+
* blobs, before making a network request. Checkpoints stay constant size. */
|
|
41
|
+
export class ZendeskStore {
|
|
42
|
+
constructor(path) { this.path = path; this.events = []; this.head = null; this.state = null; }
|
|
43
|
+
async start() {
|
|
44
|
+
await privateDirectory(this.path); await privateDirectory(join(this.path, 'objects'));
|
|
45
|
+
try {
|
|
46
|
+
const checkpoint = JSON.parse((await read(join(this.path, 'checkpoint.json'), 128 * 1024 - 1)).toString());
|
|
47
|
+
if (checkpoint.format !== 'rer-zendesk-checkpoint/1' || Object.keys(checkpoint).sort().join() !== 'format,head') fail('INVALID_CHECKPOINT');
|
|
48
|
+
this.head = checkpoint.head;
|
|
49
|
+
let reference = this.head;
|
|
50
|
+
const seen = new Set(), backwards = [];
|
|
51
|
+
while (reference) {
|
|
52
|
+
if (seen.has(reference.sha256)) fail('INVALID_CHECKPOINT');
|
|
53
|
+
seen.add(reference.sha256);
|
|
54
|
+
const node = JSON.parse((await this.get(reference)).toString());
|
|
55
|
+
if (node.format !== 'rer-zendesk-journal/1' || !node.state || !Array.isArray(node.refs)) fail('INVALID_CHECKPOINT');
|
|
56
|
+
for (const ref of node.refs) await this.get(ref);
|
|
57
|
+
backwards.push(node); reference = node.previous;
|
|
58
|
+
}
|
|
59
|
+
backwards.reverse();
|
|
60
|
+
this.events = backwards.flatMap(node => node.event ? [node.event] : []);
|
|
61
|
+
this.state = backwards.at(-1)?.state;
|
|
62
|
+
if (!this.state) fail('INVALID_CHECKPOINT');
|
|
63
|
+
} catch (error) {
|
|
64
|
+
if (error.code !== 'ENOENT' || this.head) fail('INVALID_CHECKPOINT');
|
|
65
|
+
}
|
|
66
|
+
return this;
|
|
67
|
+
}
|
|
68
|
+
async get(ref) {
|
|
69
|
+
if (!ref || !/^[a-f0-9]{64}$/u.test(ref.sha256) || !Number.isSafeInteger(ref.bytes) || ref.bytes < 0 || ref.bytes > 64 * 1024 * 1024 || ref.file !== `objects/${ref.sha256}`) fail('INVALID_CHECKPOINT');
|
|
70
|
+
const bytes = await read(join(this.path, ref.file));
|
|
71
|
+
if (bytes.length !== ref.bytes || hash(bytes) !== ref.sha256) fail('INVALID_CHECKPOINT');
|
|
72
|
+
return bytes;
|
|
73
|
+
}
|
|
74
|
+
async put(bytes) {
|
|
75
|
+
const ref = { file: `objects/${hash(bytes)}`, sha256: hash(bytes), bytes: bytes.length };
|
|
76
|
+
if (bytes.length > 64 * 1024 * 1024) fail('RECORD_LIMIT');
|
|
77
|
+
const path = join(this.path, ref.file);
|
|
78
|
+
let file;
|
|
79
|
+
try { file = await open(path, constants.O_WRONLY | constants.O_CREAT | constants.O_EXCL | constants.O_NOFOLLOW, 0o600); }
|
|
80
|
+
catch (error) { if (error.code !== 'EEXIST') throw error; await this.get(ref); return ref; }
|
|
81
|
+
try { await file.writeFile(bytes); await file.sync(); } finally { await file.close(); }
|
|
82
|
+
const parent = await open(join(this.path, 'objects'), constants.O_RDONLY);
|
|
83
|
+
try { await parent.sync(); } finally { await parent.close(); }
|
|
84
|
+
return ref;
|
|
85
|
+
}
|
|
86
|
+
async commit(state, event = null, refs = []) {
|
|
87
|
+
state = { ...state, requests: this.requestTotal ?? state.requests };
|
|
88
|
+
if (!Number.isSafeInteger(state.bytes) || state.bytes > 32 * 1024 ** 3 || state.requests > 250_000) fail('BYTE_LIMIT');
|
|
89
|
+
const node = { format: 'rer-zendesk-journal/1', previous: this.head, state, event, refs };
|
|
90
|
+
const head = await this.put(encode(node));
|
|
91
|
+
const bytes = encode({ format: 'rer-zendesk-checkpoint/1', head });
|
|
92
|
+
if (bytes.length >= 128 * 1024) fail('CHECKPOINT_LIMIT');
|
|
93
|
+
await atomic(join(this.path, 'checkpoint.json'), bytes);
|
|
94
|
+
this.head = head; this.state = structuredClone(state);
|
|
95
|
+
if (event) this.events.push(event);
|
|
96
|
+
}
|
|
97
|
+
async page(kind, response, state, extra = {}) {
|
|
98
|
+
const raw = await this.put(response.bytes);
|
|
99
|
+
await this.commit({ ...state, bytes: state.bytes + raw.bytes }, { kind, raw, ...extra }, [raw]);
|
|
100
|
+
}
|
|
101
|
+
async reset(state, { retain = false } = {}) {
|
|
102
|
+
if (!retain) { this.events = []; this.head = null; this.cache = null; this.pagePaths = null; }
|
|
103
|
+
await this.commit(state);
|
|
104
|
+
}
|
|
105
|
+
async write(name, value) {
|
|
106
|
+
if (!['census.json', 'transfer.json', 'baseline.json'].includes(name)) fail('INVALID_CHECKPOINT');
|
|
107
|
+
await atomic(join(this.path, name), encode(value));
|
|
108
|
+
}
|
|
109
|
+
async baseline() {
|
|
110
|
+
try {
|
|
111
|
+
const baseline = JSON.parse((await read(join(this.path, 'baseline.json'), 128 * 1024)).toString());
|
|
112
|
+
const node = JSON.parse((await this.get(baseline.head)).toString());
|
|
113
|
+
if (node.state?.phase !== 'done' || node.state.highWater !== baseline.highWater) fail('INVALID_CHECKPOINT');
|
|
114
|
+
return baseline;
|
|
115
|
+
}
|
|
116
|
+
catch (error) { if (error.code !== 'ENOENT') fail('INVALID_CHECKPOINT'); return null; }
|
|
117
|
+
}
|
|
118
|
+
async discard() { await rm(this.path, { recursive: true, force: true }); }
|
|
119
|
+
}
|
|
120
|
+
export async function removeZendeskStores(directory) {
|
|
121
|
+
const { readdir } = await import('node:fs/promises');
|
|
122
|
+
for (const name of await readdir(directory).catch(error => { if (error.code === 'ENOENT') return []; throw error; })) {
|
|
123
|
+
if (/^zendesk-[a-f0-9]{32}$/u.test(name)) await rm(join(directory, name), { recursive: true, force: true });
|
|
124
|
+
}
|
|
125
|
+
}
|
|
126
|
+
/** Collapse replayed ids and retain their last source row and provenance. */
|
|
127
|
+
export async function materialize(store) {
|
|
128
|
+
const data = store.cache?.data ?? Object.fromEntries(['tickets','users','organizations','ticket_fields','user_segments','categories','sections','articles','translations','article_attachments'].map(kind => [kind, new Map()]));
|
|
129
|
+
if (!store.cache) { data.comments = new Map(); data.attachments = new Map(); data.available = true; data.highWater = 0; }
|
|
130
|
+
for (const event of store.events.slice(store.cache?.length ?? 0)) {
|
|
131
|
+
if (event.kind === 'guide_unavailable') { data.available = false; continue; }
|
|
132
|
+
if (event.kind === 'attachment') { data.attachments.set(event.key, event); continue; }
|
|
133
|
+
if (!event.raw) continue;
|
|
134
|
+
const body = JSON.parse((await store.get(event.raw)).toString());
|
|
135
|
+
const rows = event.single ? [body[event.single]] : body[event.kind];
|
|
136
|
+
if (!Array.isArray(rows)) fail('INVALID_CHECKPOINT');
|
|
137
|
+
const destination = event.kind === 'comments' ? (event.reset ? new Map() : data.comments.get(event.ticket) ?? new Map()) : data[event.kind];
|
|
138
|
+
if (!destination) fail('INVALID_CHECKPOINT');
|
|
139
|
+
rows.forEach((row,index) => {
|
|
140
|
+
const id = numericId(row.id);
|
|
141
|
+
if (Buffer.byteLength(JSON.stringify(row)) > 64 * 1024 * 1024) fail('RECORD_LIMIT');
|
|
142
|
+
const key = ['translations', 'article_attachments'].includes(event.kind) ? `${event.resource ?? "articles"}:${event.article}:${id}` : id;
|
|
143
|
+
destination.set(key, { row, provenance: { file: event.raw.file, sha256: event.raw.sha256, row: index + 1, pointer: event.single ? `/${event.single}` : `/${event.kind}/${index}` } });
|
|
144
|
+
if (event.kind === 'tickets') {
|
|
145
|
+
const stamp = row.generated_timestamp;
|
|
146
|
+
if (!Number.isSafeInteger(stamp) || stamp < 0) fail('INVALID_RESPONSE');
|
|
147
|
+
data.highWater = Math.max(data.highWater, stamp);
|
|
148
|
+
}
|
|
149
|
+
});
|
|
150
|
+
if (event.kind === 'comments') data.comments.set(event.ticket, destination);
|
|
151
|
+
}
|
|
152
|
+
store.cache = { data, length: store.events.length };
|
|
153
|
+
return data;
|
|
154
|
+
}
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
import { materialize, ordered } from './zendesk-store.mjs';
|
|
2
|
+
import { fetchPage } from './zendesk-pages.mjs';
|
|
3
|
+
|
|
4
|
+
export async function tickets(ctx) {
|
|
5
|
+
const { store, inputs } = ctx;
|
|
6
|
+
let state = store.state;
|
|
7
|
+
const path = state.path ?? `/api/v2/incremental/tickets/cursor?start_time=${inputs.mode === 'sweep' ? inputs.highWater : 0}&per_page=1000`;
|
|
8
|
+
state = await fetchPage({ ...ctx, state, kind: 'tickets', path, incremental: true,
|
|
9
|
+
onDone: state => ({ ...state, phase: inputs.mode === 'census' ? 'done' : 'B', index: 0 }) });
|
|
10
|
+
return state;
|
|
11
|
+
}
|
|
12
|
+
export async function comments(ctx) {
|
|
13
|
+
const { store, inputs } = ctx, state = store.state, data = await materialize(store);
|
|
14
|
+
// A sweep refetches complete comments only for changed tickets; the retained
|
|
15
|
+
// store still supplies every older ticket to the eventual complete snapshot.
|
|
16
|
+
const ids = ordered([...data.tickets].filter(([, { row }]) => row.status !== 'deleted' && !row.deleted && (inputs.mode !== 'sweep' || row.generated_timestamp > inputs.highWater)).map(([id]) => id));
|
|
17
|
+
const ticket = ids[state.index];
|
|
18
|
+
if (ticket === undefined) { await store.commit({ ...state, phase: inputs.mode === 'sweep' ? 'done' : 'C', path: null, index: 0 }); return; }
|
|
19
|
+
await fetchPage({ ...ctx, state, kind: 'comments', path: state.path ?? `/api/v2/tickets/${ticket}/comments?page[size]=100&include_inline_images=true`, extra: { ticket, reset: state.path === null },
|
|
20
|
+
onDone: state => ({ ...state, phase: 'attachments', index: state.index, attachmentIndex: 0 }) });
|
|
21
|
+
}
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
import { fail, object } from './common.mjs';
|
|
2
|
+
|
|
3
|
+
export const SCOPES = Object.freeze(['tickets:read', 'users:read', 'organizations:read', 'hc:read', 'ticket_attachments:read']);
|
|
4
|
+
export function clientCredential(key) {
|
|
5
|
+
let value;
|
|
6
|
+
try { value = JSON.parse(key); } catch { fail('INVALID_KEY'); }
|
|
7
|
+
if (!value || Array.isArray(value) || typeof value !== 'object' || Object.keys(value).sort().join() !== 'client_id,client_secret'
|
|
8
|
+
|| Object.values(value).some(part => typeof part !== 'string' || !part || part.length > 4096 || /[\u0000-\u0020\u007f]/u.test(part))) fail('INVALID_KEY');
|
|
9
|
+
return value;
|
|
10
|
+
}
|
|
11
|
+
export function originOf(inputs) {
|
|
12
|
+
const account = inputs?.account;
|
|
13
|
+
if (!account || Object.keys(account).join() !== 'subdomain' || typeof account.subdomain !== 'string'
|
|
14
|
+
|| !/^(?!-)[a-z0-9-]{1,63}(?<!-)$/u.test(account.subdomain)) fail('INVALID_INPUT');
|
|
15
|
+
if (!['snapshot', 'sweep', 'census', 'retire'].includes(inputs.mode ?? 'snapshot')
|
|
16
|
+
|| inputs.highWater !== undefined && (!Number.isSafeInteger(inputs.highWater) || inputs.highWater < 0)) fail('INVALID_INPUT');
|
|
17
|
+
return `https://${account.subdomain}.zendesk.com`;
|
|
18
|
+
}
|
|
19
|
+
export async function mintToken(request, key) {
|
|
20
|
+
const credential = clientCredential(key);
|
|
21
|
+
const { body } = await request('/oauth/tokens', { method: 'POST', json: { ...credential, grant_type: 'client_credentials', scope: SCOPES.join(' '), expires_in: 1800 }, authenticated: false, tokenResponse: true });
|
|
22
|
+
object(body);
|
|
23
|
+
if (typeof body.access_token !== 'string' || !body.access_token || body.access_token.length > 8192 || /[\u0000-\u0020\u007f]/u.test(body.access_token)
|
|
24
|
+
|| body.token_type?.toLowerCase() !== 'bearer') fail('INVALID_TOKEN');
|
|
25
|
+
request.authorize(body.access_token);
|
|
26
|
+
const current = await request('/api/v2/oauth/tokens/current', { tokenResponse: true });
|
|
27
|
+
const scopes = current.body?.token?.scopes ?? current.body?.scopes;
|
|
28
|
+
const values = typeof scopes === 'string' ? scopes.split(/\s+/u) : scopes;
|
|
29
|
+
if (!Array.isArray(values) || !values.length || values.some(scope => !['read', ...SCOPES].includes(scope))) fail('KEY_NOT_READ_ONLY');
|
|
30
|
+
return body.access_token;
|
|
31
|
+
}
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
import { isIP } from 'node:net';
|
|
2
|
+
import { lookup } from 'node:dns/promises';
|
|
3
|
+
import { pacedClient, fail } from './common.mjs';
|
|
4
|
+
import { clientCredential, originOf, mintToken } from './zendesk-token.mjs';
|
|
5
|
+
|
|
6
|
+
const routes = /^\/(?:oauth\/tokens|api\/v2\/(?:oauth\/tokens\/current|incremental\/(?:tickets\/cursor|users\/cursor|organizations)(?:\.json)?|tickets\/\d+\/comments(?:\.json)?|ticket_fields(?:\.json)?|help_center\/(?:[a-z]{2,3}(?:-[a-z0-9]{2,8})?\/articles\/\d+|(?:user_segments|categories|sections|articles)(?:\/\d+\/(?:translations|attachments))?(?:\.json)?)))$/u;
|
|
7
|
+
const queries = new Set(['start_time', 'per_page', 'cursor', 'page', 'page[size]', 'page[after]', 'include_inline_images']);
|
|
8
|
+
export const attachmentPath = path => /^\/(?:attachments\/|api\/v2\/attachments\/)/u.test(path);
|
|
9
|
+
const privateAddress = address => {
|
|
10
|
+
if (isIP(address) === 4) {
|
|
11
|
+
const [a,b] = address.split('.').map(Number);
|
|
12
|
+
return a === 0 || a === 10 || a === 127 || a === 169 && b === 254 || a === 172 && b >= 16 && b <= 31 || a === 192 && (b === 168 || b === 0) || a === 100 && b >= 64 && b <= 127 || a >= 224 || a === 198 && (b === 18 || b === 19 || b === 51) || a === 203 && b === 0 || a === 192 && (b === 0 || b === 88);
|
|
13
|
+
}
|
|
14
|
+
// Only global unicast IPv6; mapped IPv4, loopback, multicast and ULA fail.
|
|
15
|
+
return !/^2[0-9a-f]{3}:/iu.test(address) || /^2001:(?:db8|0):/iu.test(address);
|
|
16
|
+
};
|
|
17
|
+
export async function publicLocation(value, origin) {
|
|
18
|
+
let url;
|
|
19
|
+
try { url = new URL(value, origin); } catch { return null; }
|
|
20
|
+
if (/^https:\/\/[^/?#]*:\d+(?:[/?#]|$)/iu.test(String(value)) || url.protocol !== 'https:' || url.port || url.username || url.password || url.hash || isIP(url.hostname) || url.hostname.startsWith('[')
|
|
21
|
+
|| url.hostname.length > 253 || !url.hostname.split('.').every(part => /^(?!-)[a-z0-9-]{1,63}(?<!-)$/u.test(part)) || !url.hostname.includes('.')
|
|
22
|
+
|| /\.(?:localhost|local|internal|invalid|test|home|lan)$/u.test(url.hostname) || /^\d+$/u.test(url.hostname.split('.').at(-1))) return null;
|
|
23
|
+
if (url.origin !== origin) {
|
|
24
|
+
let addresses;
|
|
25
|
+
try { addresses = await lookup(url.hostname, { all: true }); } catch { return null; }
|
|
26
|
+
if (!addresses.length || addresses.some(item => privateAddress(item.address))) return null;
|
|
27
|
+
}
|
|
28
|
+
return url;
|
|
29
|
+
}
|
|
30
|
+
export function pagination(value, origin, pathname, current) {
|
|
31
|
+
if (value === null) return null;
|
|
32
|
+
if (typeof value !== 'string' || !value || value.length > 4096) fail('INVALID_PAGINATION');
|
|
33
|
+
let next;
|
|
34
|
+
try { next = new URL(value, origin); } catch { fail('INVALID_PAGINATION'); }
|
|
35
|
+
if (next.origin !== origin || next.username || next.password || next.hash || next.pathname.replace(/\.json$/u, '') !== pathname.replace(/\.json$/u, '')
|
|
36
|
+
|| [...next.searchParams.keys()].some(key => !queries.has(key) || next.searchParams.getAll(key).length !== 1) || next.href === new URL(current, origin).href) fail('INVALID_PAGINATION');
|
|
37
|
+
return pathname + next.search;
|
|
38
|
+
}
|
|
39
|
+
export async function transport({ inputs, key, signal, beforeRequest }) {
|
|
40
|
+
const origin = originOf(inputs), credentials = clientCredential(key);
|
|
41
|
+
const request = pacedClient({ origin, signal, beforeRequest, secrets: [key, credentials.client_id, credentials.client_secret], validate: async (url, options) => {
|
|
42
|
+
if (options.attachment) {
|
|
43
|
+
if (!await publicLocation(url.href, origin) || url.origin === origin && !attachmentPath(url.pathname)) fail('REFUSED_LOCATION');
|
|
44
|
+
} else if (url.origin !== origin || url.username || url.password || url.hash || !routes.test(url.pathname) || [...url.searchParams.keys()].some(key => !queries.has(key))) fail('INVALID_REFERENCE');
|
|
45
|
+
} });
|
|
46
|
+
await mintToken(request, key);
|
|
47
|
+
return { request, origin };
|
|
48
|
+
}
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
import { VendorExportError, fail } from './common.mjs';
|
|
2
|
+
import { originOf } from './zendesk-token.mjs';
|
|
3
|
+
import { transport } from './zendesk-transport.mjs';
|
|
4
|
+
import { ZendeskStore, materialize, ordered } from './zendesk-store.mjs';
|
|
5
|
+
import { tickets, comments } from './zendesk-tickets.mjs';
|
|
6
|
+
import { people } from './zendesk-people.mjs';
|
|
7
|
+
import { guide } from './zendesk-guide.mjs';
|
|
8
|
+
import { attachments } from './zendesk-attachments.mjs';
|
|
9
|
+
import { encodeSnapshot } from './zendesk-encode.mjs';
|
|
10
|
+
|
|
11
|
+
export { originOf } from './zendesk-token.mjs';
|
|
12
|
+
export { clientCredential } from './zendesk-token.mjs';
|
|
13
|
+
export { storePath, removeZendeskStores } from './zendesk-store.mjs';
|
|
14
|
+
/** The durable local snapshot is independent of door transfer. A new task
|
|
15
|
+
* starts a fresh snapshot or extends the complete retained store for a sweep;
|
|
16
|
+
* retries of the same task resume its last acknowledged page. */
|
|
17
|
+
export async function snapshot({ key, inputs, directory, runId, signal }) {
|
|
18
|
+
try {
|
|
19
|
+
const origin = originOf(inputs);
|
|
20
|
+
if (!directory || typeof runId !== 'string' || !runId || inputs.mode === 'retire') fail('INVALID_INPUT');
|
|
21
|
+
const store = await new ZendeskStore(directory).start();
|
|
22
|
+
const mode = inputs.mode ?? 'snapshot', baseline = await store.baseline();
|
|
23
|
+
if (mode === 'sweep' && store.state?.runId !== runId && (!baseline || inputs.highWater !== baseline.highWater)) fail('INVALID_HIGH_WATER');
|
|
24
|
+
if (store.state?.runId !== runId) {
|
|
25
|
+
if (mode === 'sweep' && store.state?.phase !== 'done') fail('INCOMPLETE_SNAPSHOT');
|
|
26
|
+
await store.reset({ phase: 'A', path: null, index: 0, attachmentIndex: 0, stage: 0, runId, origin, mode, inputHighWater: inputs.highWater ?? null,
|
|
27
|
+
exportedAt: new Date().toISOString(), bytes: mode === 'sweep' ? store.state.bytes : 0, requests: 0 }, { retain: mode === 'sweep' });
|
|
28
|
+
} else if (store.state.origin !== origin || store.state.mode !== mode || store.state.inputHighWater !== (inputs.highWater ?? null)) fail('INVALID_CHECKPOINT');
|
|
29
|
+
store.requestTotal = store.state.requests;
|
|
30
|
+
const ctx = { store, inputs, ...await transport({ inputs, key, signal, beforeRequest: async () => {
|
|
31
|
+
if (store.requestTotal >= 250_000) fail('REQUEST_LIMIT');
|
|
32
|
+
store.requestTotal++;
|
|
33
|
+
await store.commit(store.state);
|
|
34
|
+
} }) };
|
|
35
|
+
while (store.state.phase !== 'done') {
|
|
36
|
+
signal?.throwIfAborted();
|
|
37
|
+
const phase = store.state.phase;
|
|
38
|
+
if (phase === 'A') await tickets(ctx);
|
|
39
|
+
else if (phase === 'B') await comments(ctx);
|
|
40
|
+
else if (['C','D','E'].includes(phase)) await people(ctx);
|
|
41
|
+
else if (phase === 'F') await guide(ctx);
|
|
42
|
+
else if (['attachments', 'guide_attachments'].includes(phase)) await attachments(ctx);
|
|
43
|
+
else fail('INVALID_CHECKPOINT');
|
|
44
|
+
}
|
|
45
|
+
if (mode === 'census') {
|
|
46
|
+
const data = await materialize(store), unswept = [...data.tickets.values()].filter(({ row }) => row.generated_timestamp > (inputs.highWater ?? 0)).length;
|
|
47
|
+
const census = { format: 'rer-zendesk-census/1', tickets: { unique: data.tickets.size, deleted: [...data.tickets.values()].filter(({row}) => row.status === 'deleted' || row.deleted).length,
|
|
48
|
+
maxId: ordered(data.tickets.keys()).at(-1) ?? null }, unswept, highWater: data.highWater };
|
|
49
|
+
await store.write('census.json', census);
|
|
50
|
+
return { kind: 'census', census, store };
|
|
51
|
+
}
|
|
52
|
+
const result = await encodeSnapshot(store, { origin });
|
|
53
|
+
if (store.state.highWater !== result.census.highWater) await store.commit({ ...store.state, highWater: result.census.highWater });
|
|
54
|
+
await store.write('census.json', result.census);
|
|
55
|
+
await store.write('baseline.json', { highWater: result.census.highWater, head: store.head });
|
|
56
|
+
return { ...result, store };
|
|
57
|
+
} catch (error) {
|
|
58
|
+
if (error instanceof VendorExportError) throw error;
|
|
59
|
+
throw new VendorExportError(signal?.aborted ? 'CANCELLED' : 'INVALID_EXPORT');
|
|
60
|
+
}
|
|
61
|
+
}
|
package/cli/deploy-command.mjs
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { requireProductionAuth } from './production-auth.mjs';
|
|
1
|
+
import { requireProductionAuth, mailOffNotice } from './production-auth.mjs';
|
|
2
2
|
import { observedReadiness } from '../modules/deploy-kit/readiness-window.mjs';
|
|
3
3
|
import { readFile, readdir } from 'node:fs/promises';
|
|
4
4
|
import { contained } from './project.mjs';
|
|
@@ -85,7 +85,8 @@ export async function deployCommand(context) {
|
|
|
85
85
|
const production = await deployEnvironment({ root, options, declaration, setupManifest, settings, localOnlyKeys });
|
|
86
86
|
const plan = result?.plan ?? (await owner(['local-plan', '--declaration', path, '--environment', options.environment ?? 'production', ...await deployProductArguments(root, deployProductDirectory(declaration))], production.env, { cwd: root, localOnlyKeys })).plan;
|
|
87
87
|
result = { ...result, dryRun: true, deployed: false, providerChecked: false, plan };
|
|
88
|
-
const { signIn } = await requireProductionAuth({ root, declaration, env: production.env, setupManifest, setupSeed: context.setupSeed, adapter, declarationPath: path, options }) ?? {};
|
|
88
|
+
const { signIn, mailOff } = await requireProductionAuth({ root, declaration, env: production.env, setupManifest, setupSeed: context.setupSeed, adapter, declarationPath: path, options }) ?? {};
|
|
89
|
+
if (mailOff) store.report(mailOffNotice(mailOff, options.environment ?? 'production'));
|
|
89
90
|
const needed = [...new Set([...neededForRealDeploy({ declaration, env: production.env, signIn, secretsChecked: false }), ...(result.neededForRealDeploy ?? [])])];
|
|
90
91
|
reportNeeded(store, needed);
|
|
91
92
|
if (declaration.secrets?.some(item => item.binding === 'INSTALLATION_KEY')) store.report('deploy: INSTALLATION_KEY is a private generated binding; INSTALLATION_KEY_PREVIOUS is optional and managed only during rotation or an explicit retired-key reopen. Values are withheld.');
|
|
@@ -110,7 +111,7 @@ export async function deployCommand(context) {
|
|
|
110
111
|
if (adapter.deploy.managesHostedOwner) { const result = await runHook(context); return options.dryRun ? dryRun(result) : result; }
|
|
111
112
|
const productDirectory = deployProductDirectory(declaration);
|
|
112
113
|
const production = await deployEnvironment({ root, options, declaration, setupManifest, settings, localOnlyKeys });
|
|
113
|
-
await requireProductionAuth({ root, declaration, env: production.env, setupManifest, setupSeed: context.setupSeed, adapter, declarationPath: path, options });
|
|
114
|
+
const { mailOff } = await requireProductionAuth({ root, declaration, env: production.env, setupManifest, setupSeed: context.setupSeed, adapter, declarationPath: path, options }) ?? {};
|
|
114
115
|
const sourceSha = options.dryRun ? undefined : committedSource(root, options);
|
|
115
116
|
const ownerInput = await deployOwnerInput(context);
|
|
116
117
|
const { env, supplied, generated, reused, rotated } = await deployEnvironment({ root, options, declaration, setupManifest, settings, localOnlyKeys });
|
|
@@ -180,7 +181,7 @@ export async function deployCommand(context) {
|
|
|
180
181
|
const { setupToken: _, ...result } = deployed;
|
|
181
182
|
return { ...result, ...(claimed ? { owner: claimed } : {}), dependenciesLinked: true };
|
|
182
183
|
});
|
|
183
|
-
if (claimed) store.report(`${claimed === 'claimed' ? 'Claimed the hosted owner' : 'The hosted owner was already claimed and was kept'}${recorded ? '; the one-time setup token was removed' : ''}. Sign in by email at ${result.origin}.`);
|
|
184
|
+
if (claimed) store.report(`${claimed === 'claimed' ? 'Claimed the hosted owner' : 'The hosted owner was already claimed and was kept'}${recorded ? '; the one-time setup token was removed' : ''}. ${mailOff ? mailOffNotice(mailOff, environment) : `Sign in by email at ${result.origin}.`}`);
|
|
184
185
|
return result;
|
|
185
186
|
} catch (error) {
|
|
186
187
|
if (error instanceof DeployProductError) throw error;
|
package/cli/deploy-inputs.mjs
CHANGED
|
@@ -79,7 +79,7 @@ export async function withDeployEnvironment(project, options, out, run) {
|
|
|
79
79
|
const redaction = createRedaction();
|
|
80
80
|
// Provider names are public enum values. Masking "resend" would also hide
|
|
81
81
|
// RESEND_API_KEY in the preflight's instruction for the missing credential.
|
|
82
|
-
const publicProvider = (key, value) => key === 'MAIL_PROVIDER' && /^(resend|sendgrid|cloudflare|rer|later)$/.test(value);
|
|
82
|
+
const publicProvider = (key, value) => key === 'MAIL_PROVIDER' && /^(resend|sendgrid|cloudflare|rer|later|none)$/.test(value);
|
|
83
83
|
const supplied = Object.entries(env).filter(([key, value]) => saved[key] !== value);
|
|
84
84
|
redaction.addSecrets([...kinds.privateValues(env), ...supplied.filter(([key, value]) => !kinds.shown(key, value) && !publicProvider(key, value)).map(([, value]) => value)]);
|
|
85
85
|
const undeclared = declared.declaration ? kinds.undeclared(supplied.map(([key]) => key)) : [];
|
package/cli/deploy.mjs
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
|
-
import { requireProductionAuth } from './production-auth.mjs';
|
|
1
|
+
import { requireProductionAuth, mailOffNotice } from './production-auth.mjs';
|
|
2
|
+
import { mailOffSignInCommand } from './production-mail.mjs';
|
|
2
3
|
import { readFile } from 'node:fs/promises';
|
|
3
4
|
import { deployDiagnostic, migrationFailure, providerFailure, assetFailure } from '../modules/deploy-kit/diagnostics.mjs';
|
|
4
5
|
import { shared } from './shared.mjs';
|
|
@@ -41,10 +42,10 @@ export function ownerDeploy(declarationPath, lifecycle) {
|
|
|
41
42
|
const recordedState = change.length ? await deploymentRecord(root, options) : undefined;
|
|
42
43
|
// A dry run reads the settings file too, so a refused key fails here and not only on the real deploy.
|
|
43
44
|
if (options.dryRun && options.envFile) await deployEnvironment({ root, options, declaration, localOnlyKeys });
|
|
44
|
-
let production, ownerInput, signIn;
|
|
45
|
+
let production, ownerInput, signIn, mailOff;
|
|
45
46
|
if (!change.length) {
|
|
46
47
|
production = await deployEnvironment({ root, options, declaration, setupManifest, settings: lifecycle.settings, localOnlyKeys });
|
|
47
|
-
({signIn} = await requireProductionAuth({ root, declaration, env: production.env, setupManifest, setupSeed, adapter: lifecycle, declarationPath, options }) ?? {});
|
|
48
|
+
({signIn, mailOff} = await requireProductionAuth({ root, declaration, env: production.env, setupManifest, setupSeed, adapter: lifecycle, declarationPath, options, mailOffSignIn: mailOffSignInCommand(adapter) ?? mailOffSignInCommand(lifecycle) }) ?? {});
|
|
48
49
|
if (!options.dryRun) committedSource(root, options);
|
|
49
50
|
ownerInput = await deployOwnerInput({ root, adapter, setupSeed, options, signal });
|
|
50
51
|
await withoutLocalSettings(root, { settings: lifecycle.settings, operationToken }, () => lifecycle.run(root, {}, store, () => {}, signal, { verb: 'build', operationToken }));
|
|
@@ -132,7 +133,8 @@ export function ownerDeploy(declarationPath, lifecycle) {
|
|
|
132
133
|
// The app claims the OWNER_EMAIL the Worker holds: the local one only when this run uploaded it.
|
|
133
134
|
const email = declared(claim?.emailField);
|
|
134
135
|
const named = !email ? '' : deployed.sentSecrets.includes(email.binding) && env[email.environment] ? ` (${env[email.environment]})` : ` (the ${email.binding} already set on the Worker)`;
|
|
135
|
-
if (claimed) store.report(`${claimed === 'claimed' ? 'Claimed the hosted owner' : 'The hosted owner was already claimed and was kept'}${named}; the one-time setup token was removed. Sign in by email at ${deployed.origin}.`);
|
|
136
|
+
if (claimed) store.report(`${claimed === 'claimed' ? 'Claimed the hosted owner' : 'The hosted owner was already claimed and was kept'}${named}; the one-time setup token was removed. ${mailOff ? mailOffNotice(mailOff, environment) : `Sign in by email at ${deployed.origin}.`}`);
|
|
137
|
+
else if (mailOff) store.report(mailOffNotice(mailOff, environment));
|
|
136
138
|
return { deployed: true, origin: deployed.origin, ...(claimed ? { owner: claimed } : {}), dependenciesLinked: true };
|
|
137
139
|
} catch (error) {
|
|
138
140
|
if (error instanceof RerError) throw error;
|
package/cli/production-auth.mjs
CHANGED
|
@@ -2,16 +2,29 @@ import { refuse } from './errors.mjs';
|
|
|
2
2
|
import { main as owner } from '../modules/deploy-kit/owner-cli.mjs';
|
|
3
3
|
import { hookState } from './deploy-hook-record.mjs';
|
|
4
4
|
import { savedSetupAnswers } from './setup-answers.mjs';
|
|
5
|
-
import { productionMail } from './production-mail.mjs';
|
|
5
|
+
import { productionMail, mailOffSignInCommand } from './production-mail.mjs';
|
|
6
|
+
|
|
7
|
+
// What a deploy says when outbound email is deliberately off: the state and the way in.
|
|
8
|
+
export function mailOffNotice({ signIn }, environment = 'production') {
|
|
9
|
+
return `Outbound email is off (MAIL_PROVIDER=none): no mail credential is bound and this installation sends nothing. Sign in without mail with \`${signIn}\` for the ${environment} environment, from this checkout with the Cloudflare credentials that deployed it. Turn mail on later with rer deploy --environment ${environment} --set-secret and the provider inputs.`;
|
|
10
|
+
}
|
|
6
11
|
|
|
7
12
|
// Validate the shared email/identity contract using the bindings that will reach
|
|
8
13
|
// the runtime. Independently authored authentication remains the adapter's job.
|
|
9
|
-
|
|
14
|
+
// `mailOffSignIn` names the seed's mail-less sign-in command when the caller holds it apart
|
|
15
|
+
// from `adapter` (the owner deploy runs on the lifecycle, the seed declares on its command adapter).
|
|
16
|
+
export async function requireProductionAuth({ root, declaration, env, setupManifest, setupSeed, adapter, declarationPath, options = {}, mailOffSignIn: declaredSignIn = mailOffSignInCommand(adapter) }) {
|
|
10
17
|
const dependencies = setupManifest?.dependencies?.filter(item => item.interface === 'oidc-provider@1') ?? [];
|
|
11
18
|
const bindings = { ...declaration.vars };
|
|
12
19
|
for (const secret of declaration.secrets ?? []) if (env[secret.environment]?.trim()) bindings[secret.binding] = env[secret.environment];
|
|
13
|
-
const mail = await productionMail(root, declaration, env, declarationPath, adapter);
|
|
20
|
+
const mail = await productionMail(root, declaration, env, declarationPath, adapter, declaredSignIn);
|
|
14
21
|
if (!setupManifest?.steps?.some(step => step.type === 'owner-claim') && !dependencies.length && !mail.declared) return;
|
|
22
|
+
// MAIL_PROVIDER=none is accepted only for a seed that declares how its owner signs in without mail.
|
|
23
|
+
const mailOffSignIn = mail.off ? declaredSignIn : undefined;
|
|
24
|
+
if (mailOffSignIn) {
|
|
25
|
+
if (mail.offConflicts.length) throw refuse(`MAIL_PROVIDER=none turns outbound email off, and ${mail.offConflicts.join(', ')} would bind a way to send. Remove ${mail.offConflicts.length > 1 ? 'them' : 'it'} or choose a mail provider. Nothing was built or uploaded.`);
|
|
26
|
+
return { mailOff: { signIn: mailOffSignIn } };
|
|
27
|
+
}
|
|
15
28
|
if (mail.senderIssue && options.verb === 'deploy' && !(options.dryRun && mail.fixtureSender)) throw refuse(`${mail.senderIssue} Nothing was built or uploaded.`);
|
|
16
29
|
if (mail.configured()) return;
|
|
17
30
|
if (mail.canInspect && !options.dryRun && declarationPath) {
|
package/cli/production-mail.mjs
CHANGED
|
@@ -8,6 +8,8 @@ import { validateMailSenders } from '../modules/delivery-producer/mail.mjs';
|
|
|
8
8
|
import { sourceMailSenders } from './deploy-variables.mjs';
|
|
9
9
|
|
|
10
10
|
const providers = ['resend', 'sendgrid', 'cloudflare', 'rer'];
|
|
11
|
+
// Bindings that can send mail. MAIL_PROVIDER=none contradicts every one of them.
|
|
12
|
+
const sendingBindings = ['MAIL_API_KEY', 'SENDGRID_API_KEY', 'RESEND_API_KEY', 'MAIL_PROVIDER_API_KEY', 'MAIL_ACCOUNT_ID', 'MAIL_RELAY_URL'];
|
|
11
13
|
const nativeCredential = { sendgrid: 'SENDGRID_API_KEY', resend: 'RESEND_API_KEY' };
|
|
12
14
|
const present = value => typeof value === 'string' && Boolean(value.trim());
|
|
13
15
|
// The sender forms the mail runtime sends from: a bare address or "Display Name <address>".
|
|
@@ -27,10 +29,17 @@ const placeholderSender = value => {
|
|
|
27
29
|
|| /^(?:fake|placeholder|change-?me|replace-?me|your-?email|your-?name|you)(?:[+._-]|$)/.test(local);
|
|
28
30
|
};
|
|
29
31
|
|
|
32
|
+
// A seed that signs its owner in without mail declares the command on its adapter
|
|
33
|
+
// (`mailOffSignIn`: one line naming a command the person runs from the checkout).
|
|
34
|
+
export function mailOffSignInCommand(adapter) {
|
|
35
|
+
const command = adapter?.mailOffSignIn;
|
|
36
|
+
return typeof command === 'string' && command.trim() && command.length <= 300 && !/[\u0000-\u001f\u007f]/.test(command) ? command.trim() : undefined;
|
|
37
|
+
}
|
|
38
|
+
|
|
30
39
|
// Packaging can declare inputs that a product's deploy adapter adds to the
|
|
31
40
|
// root declaration later. Read that source contract, never arbitrary ambient
|
|
32
41
|
// keys or the adapter's executable source, before any build or provider write.
|
|
33
|
-
export async function productionMail(root, declaration, env, declarationPath = 'deploy.json', adapter) {
|
|
42
|
+
export async function productionMail(root, declaration, env, declarationPath = 'deploy.json', adapter, mailOffSignIn = mailOffSignInCommand(adapter)) {
|
|
34
43
|
const bindings = { ...declaration.vars }, secrets = new Set(), publicInputs = new Set(), declaredProviders = new Set();
|
|
35
44
|
const senders = [{ source: `${declarationPath} vars.MAIL_FROM`, value: declaration.vars?.MAIL_FROM }];
|
|
36
45
|
senders.push(...await sourceMailSenders(root, adapter));
|
|
@@ -76,6 +85,12 @@ export async function productionMail(root, declaration, env, declarationPath = '
|
|
|
76
85
|
bindings[name] = env[name];
|
|
77
86
|
if (name === 'MAIL_FROM') senderFix = environmentFix(name);
|
|
78
87
|
}
|
|
88
|
+
// Outbound email is off only when the deployment says so in words: a seed that
|
|
89
|
+
// declares a mail-less sign-in command and a MAIL_PROVIDER setting, with an
|
|
90
|
+
// explicit `none` for it. A missing or deferred value ('later', 'direct') never
|
|
91
|
+
// turns mail off.
|
|
92
|
+
if (mailOffSignIn && present(env.MAIL_PROVIDER) && env.MAIL_PROVIDER.trim() === 'none' && 'MAIL_PROVIDER' in bindings) bindings.MAIL_PROVIDER = 'none';
|
|
93
|
+
const off = bindings.MAIL_PROVIDER === 'none';
|
|
79
94
|
try { validateMailSenders(senders); }
|
|
80
95
|
catch (error) { throw refuse(`${error.message} Nothing was built or uploaded.`); }
|
|
81
96
|
let senderIssue;
|
|
@@ -112,6 +127,8 @@ export async function productionMail(root, declaration, env, declarationPath = '
|
|
|
112
127
|
};
|
|
113
128
|
});
|
|
114
129
|
return {
|
|
130
|
+
off,
|
|
131
|
+
offConflicts: off ? sendingBindings.filter(key => present(bindings[key])) : [],
|
|
115
132
|
declared: 'MAIL_PROVIDER' in bindings || publicInputs.has('MAIL_PROVIDER') || secrets.has('MAIL_API_KEY') || selected.length > 0,
|
|
116
133
|
senderIssue,
|
|
117
134
|
fixtureSender: fixtureSender(bindings.MAIL_FROM),
|
package/cli/site.mjs
CHANGED
|
@@ -149,7 +149,7 @@ export async function siteCommand(project, options, store, signal, out) {
|
|
|
149
149
|
const { siteDeploy, siteReadbackSummary } = await import('../modules/deploy-kit/site.mjs');
|
|
150
150
|
const environment = options.environment ?? 'production';
|
|
151
151
|
try {
|
|
152
|
-
const api = cloudflare({ accountId: process.env.CLOUDFLARE_ACCOUNT_ID, token: process.env.CLOUDFLARE_API_TOKEN });
|
|
152
|
+
const api = cloudflare({ accountId: process.env.CLOUDFLARE_ACCOUNT_ID, token: process.env.CLOUDFLARE_API_TOKEN, settleLists: true, requestSignal: () => signal });
|
|
153
153
|
// The receipt is for the person; ownership is read from the provider, so a fresh checkout deploys too.
|
|
154
154
|
const result = await siteDeploy({ api, plan, files, receiptPath: join(project.root, '.rer', 'deploy', `site-${environment}.json`) });
|
|
155
155
|
return { dryRun: false, deployed: true, origin: result.origin, site: summary };
|
package/global/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,26 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## Unreleased
|
|
4
|
+
|
|
5
|
+
## 0.2.100
|
|
6
|
+
|
|
7
|
+
### Deploy with outbound email explicitly off
|
|
8
|
+
|
|
9
|
+
- `rer deploy` and `rer deploy --dry-run` accept explicit `MAIL_PROVIDER=none` only when the seed's adapter declares `mailOffSignIn`, naming its sign-in command without mail. They bind no mail credential and name the hosted sign-in command and environment. Missing mail configuration is still refused; `none` with a sending credential, account or relay binding is refused before build or upload. The seed owns its behavior with mail off; turning mail on later uses the documented provider inputs and `rer deploy --set-secret`.
|
|
10
|
+
|
|
11
|
+
### Import transport and opt-in bounded waves
|
|
12
|
+
|
|
13
|
+
- The import door supports a logical package whose files travel in bounded, verified segments. Completed-file hashes, declared aggregate counts and existing source/native quotas are checked before planning; refused packages withdraw their segment custody. Segment transport does not approve an import or raise a quota.
|
|
14
|
+
- Opt-in complete-parent imports can prepare, seal and approve ordered bounded waves, preserving parent lineage, complete membership and native before-images. Unchanged catch-up dispositions and completed reversal retries remain stable; concurrent parent preparation fences reversal. Adopters supply their source/native integrations and a new numbered migration for the additive bounded-wave schema. The reference includes the schema; no edition adoption, live migration, import or cutover is claimed.
|
|
15
|
+
- Business can register a verified local legacy package and transfer its exact files to the granted import door. The accepted Zendesk exporter captures paginated people, tickets, comments, attachments and Guide content into the documented package, with bounded read-only transport, credential removal and resumable checkpoints. These are the inherited accepted package, bounded-wave and Zendesk changes; this release adds no further Business command changes.
|
|
16
|
+
|
|
17
|
+
### Startup, approval replay and deployment recovery
|
|
18
|
+
|
|
19
|
+
- Setup and dev let the managed Vite listener use its finite startup budget instead of stopping at 60 seconds. Custom start targets retain their declared deadlines; a missing listener announcement reports elapsed and allowed time with startup-log guidance.
|
|
20
|
+
- Legacy MCP tool calls held for approval return their generated `requestState` alongside `approval_required`, so retrying an approved read resumes the original operation without asking for a second approval.
|
|
21
|
+
- `rer deploy --site` retries a changing or partial Cloudflare list from page one up to three times, waiting 20 seconds between failed reads. It discards incomplete snapshots, never replays deploy writes, preserves other refusals and stops waiting on cancellation.
|
|
22
|
+
- Publication scans retain value-free worker, report, version-stamping and cleanup failure diagnostics. Cleanup and version-stamping failures still refuse publication; diagnostic handling does not turn a failed scan into a pass.
|
|
23
|
+
|
|
3
24
|
## 0.2.99
|
|
4
25
|
|
|
5
26
|
### Agents wait for the owner instead of failing
|
|
@@ -207,6 +207,16 @@ No zone ruleset is read or written. Every request now invokes the Worker: one pa
|
|
|
207
207
|
|
|
208
208
|
A read-only preflight reads the zone (it must be in this account, else `site-zone-missing`), the Worker and the zone's Custom Domains, and refuses before any write: `resource-name-collision` for a Worker of the name whose live version the kit did not upload, or for a hostname attached to another service (or to a Worker that is not the kit's); `owned-resource-replaced` for a kit Worker whose latest deployment is not the kit's upload (a dashboard rollback or a split deployment). It then uploads the assets and the site Worker, checks that the new version is live, turns workers.dev off, attaches each missing hostname, and reads the Worker and both domains back. Both post-upload Worker checks share a 90-second upload window: missing visibility or the exact preflight version waits with backoff until the upload response's version serves alone as newest with its expected fingerprint. A foreign version, changed fingerprint or split deployment refuses immediately. A timeout names the last serving/newest versions and fingerprint. A repeat attaches nothing new. `.rer/deploy/site-<environment>.json` receives a receipt for the person; it is never read. Provider propagation cases are tested against the fake Cloudflare API in `tests/site-provider.mjs`, where every zone ruleset path answers 403.
|
|
209
209
|
|
|
210
|
+
Site deploy retries only `provider-partial-list` and `provider-pagination-changed`
|
|
211
|
+
from read-only list collection: three total attempts, with 20 seconds between
|
|
212
|
+
failed reads. Each attempt discards the incomplete snapshot and restarts at page
|
|
213
|
+
one, retaining the original filters; pages from different attempts are never
|
|
214
|
+
combined. This covers zone, Custom Domain and Worker version/deployment lists,
|
|
215
|
+
including post-upload readback. It never retries the whole deploy or replays a
|
|
216
|
+
write. Other refusals stop immediately; persistent inconsistency keeps the final
|
|
217
|
+
original refusal and its GET diagnostic. Cancellation stops the wait and prevents
|
|
218
|
+
the next read. Owner-mode and teardown settling defaults are unchanged.
|
|
219
|
+
|
|
210
220
|
A seed can opt into docs search through its command adapter's `site.runtime`:
|
|
211
221
|
|
|
212
222
|
```ts
|