@runeditrun/rer 0.2.100 → 0.2.102
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +30 -102
- package/cli/README.md +16 -89
- package/cli/app-address.mjs +2 -0
- package/cli/business/api.mjs +2 -1
- package/cli/business/choices.mjs +1 -1
- package/cli/business/command.mjs +36 -30
- package/cli/business/connect.mjs +1 -1
- package/cli/business/lease.mjs +0 -9
- package/cli/business/local-commands.mjs +1 -27
- package/cli/business/outbound.mjs +2 -14
- package/cli/business/questions.mjs +85 -48
- package/cli/business/service.mjs +1 -24
- package/cli/business/setup-command.mjs +33 -26
- package/cli/business/state.mjs +1 -1
- package/cli/business/supervisor.mjs +26 -91
- package/cli/companion/command.mjs +2 -0
- package/cli/demo-native.mjs +1 -1
- package/cli/parse.mjs +2 -1
- package/global/CHANGELOG.md +32 -0
- package/global/README.md +30 -102
- package/global/main.mjs +2 -0
- package/modules/container/adapters/d1-limits.mjs +2 -10
- package/modules/d1-transport/limits.d.mts +10 -0
- package/modules/d1-transport/limits.mjs +10 -0
- package/modules/demo-host/native/prepare.mjs +2 -1
- package/modules/deploy-kit/d1-import.mjs +11 -1
- package/modules/deploy-kit/migrations.mjs +14 -1
- package/modules/import-kit/schema/importers.schema.json +104 -2
- package/modules/import-kit/src/d1/README.md +14 -0
- package/modules/import-kit/src/d1/apply-reads.ts +74 -0
- package/modules/import-kit/src/d1/apply.ts +32 -9
- package/modules/import-kit/src/d1/history.ts +2 -2
- package/modules/import-kit/src/d1/redaction-step.ts +17 -7
- package/modules/import-kit/src/d1/types.ts +11 -0
- package/modules/import-kit/src/door/ADOPTING.md +0 -37
- package/modules/import-kit/src/door/README.md +0 -66
- package/modules/import-kit/src/door/admission.ts +3 -13
- package/modules/import-kit/src/door/authority.ts +0 -1
- package/modules/import-kit/src/door/failures.ts +1 -1
- package/modules/import-kit/src/door/grants.ts +5 -19
- package/modules/import-kit/src/door/handler.ts +1 -12
- package/modules/import-kit/src/door/index.ts +0 -6
- package/modules/import-kit/src/door/planning.ts +19 -44
- package/modules/import-kit/src/door/protocol.ts +2 -4
- package/modules/import-kit/src/door/schema.ts +0 -2
- package/modules/import-kit/src/door/store.ts +2 -2
- package/modules/import-kit/src/door/types.ts +1 -24
- package/modules/import-kit/src/provenance/credentials.ts +14 -5
- package/modules/import-kit/src/status/evidence.ts +3 -1
- package/modules/import-kit/src/status/index.ts +17 -2
- package/modules/import-kit/src/status/schema.ts +8 -0
- package/modules/import-kit/src/status/types.ts +25 -4
- package/package.json +1 -1
- package/cli/business/archive-task.mjs +0 -32
- package/cli/business/bounded-waves.d.mts +0 -23
- package/cli/business/bounded-waves.mjs +0 -57
- package/cli/business/catchup-task.mjs +0 -49
- package/cli/business/export-inputs.mjs +0 -79
- package/cli/business/export-task.mjs +0 -24
- package/cli/business/legacy-package.mjs +0 -117
- package/cli/business/package-transfer.mjs +0 -127
- package/cli/business/setup-demo.mjs +0 -95
- package/cli/business/vendor-door.mjs +0 -303
- package/cli/business/vendors/README.md +0 -215
- package/cli/business/vendors/calendly.mjs +0 -80
- package/cli/business/vendors/common.mjs +0 -280
- package/cli/business/vendors/credential-names.mjs +0 -124
- package/cli/business/vendors/index.mjs +0 -96
- package/cli/business/vendors/pagerduty.mjs +0 -61
- package/cli/business/vendors/sentry.mjs +0 -85
- package/cli/business/vendors/sorry.mjs +0 -106
- package/cli/business/vendors/uptimerobot.mjs +0 -21
- package/cli/business/vendors/zendesk-attachments.mjs +0 -57
- package/cli/business/vendors/zendesk-encode.mjs +0 -133
- package/cli/business/vendors/zendesk-guide.mjs +0 -34
- package/cli/business/vendors/zendesk-package.mjs +0 -28
- package/cli/business/vendors/zendesk-pages.mjs +0 -61
- package/cli/business/vendors/zendesk-people.mjs +0 -7
- package/cli/business/vendors/zendesk-store.mjs +0 -154
- package/cli/business/vendors/zendesk-tickets.mjs +0 -21
- package/cli/business/vendors/zendesk-token.mjs +0 -31
- package/cli/business/vendors/zendesk-transport.mjs +0 -48
- package/cli/business/vendors/zendesk.mjs +0 -61
- package/modules/import-kit/src/door/package.ts +0 -121
- package/modules/import-kit/src/door/waves/README.md +0 -158
- package/modules/import-kit/src/door/waves/digests.ts +0 -38
- package/modules/import-kit/src/door/waves/inventory.ts +0 -84
- package/modules/import-kit/src/door/waves/journal.ts +0 -19
- package/modules/import-kit/src/door/waves/lifecycle.ts +0 -242
- package/modules/import-kit/src/door/waves/preparation.ts +0 -190
- package/modules/import-kit/src/door/waves/schema.ts +0 -73
- package/modules/import-kit/src/door/waves/store.ts +0 -112
- package/modules/import-kit/src/door/waves/types.ts +0 -76
|
@@ -1,154 +0,0 @@
|
|
|
1
|
-
import { constants } from 'node:fs';
|
|
2
|
-
import { mkdir, lstat, open, rename, rm } from 'node:fs/promises';
|
|
3
|
-
import { join } from 'node:path';
|
|
4
|
-
import { randomUUID } from 'node:crypto';
|
|
5
|
-
import { hash, fail, stable } from './common.mjs';
|
|
6
|
-
|
|
7
|
-
export const encode = value => Buffer.from(JSON.stringify(stable(value)) + '\n');
|
|
8
|
-
export const numericId = value => {
|
|
9
|
-
if (typeof value === 'number' && (!Number.isSafeInteger(value) || value < 0)) fail('INVALID_RESPONSE');
|
|
10
|
-
const id = String(value);
|
|
11
|
-
if (!/^(?:0|[1-9]\d{0,19})$/u.test(id)) fail('INVALID_RESPONSE');
|
|
12
|
-
return id;
|
|
13
|
-
};
|
|
14
|
-
export const ordered = values => [...values].sort((a,b) => BigInt(a) < BigInt(b) ? -1 : BigInt(a) > BigInt(b) ? 1 : 0);
|
|
15
|
-
export const storePath = (directory, business, origin) => join(directory, `zendesk-${hash(JSON.stringify([business, origin])).slice(0,32)}`);
|
|
16
|
-
async function privateDirectory(path) {
|
|
17
|
-
await mkdir(path, { mode: 0o700, recursive: true });
|
|
18
|
-
const info = await lstat(path);
|
|
19
|
-
if (!info.isDirectory() || (info.mode & 0o777) !== 0o700) fail('INVALID_CHECKPOINT');
|
|
20
|
-
}
|
|
21
|
-
async function read(path, maximum = 64 * 1024 * 1024) {
|
|
22
|
-
const file = await open(path, constants.O_RDONLY | constants.O_NOFOLLOW);
|
|
23
|
-
try {
|
|
24
|
-
const info = await file.stat();
|
|
25
|
-
if (!info.isFile() || info.size > maximum || (info.mode & 0o777) !== 0o600) fail('INVALID_CHECKPOINT');
|
|
26
|
-
return await file.readFile();
|
|
27
|
-
} finally { await file.close(); }
|
|
28
|
-
}
|
|
29
|
-
async function atomic(path, bytes) {
|
|
30
|
-
const temporary = `${path}.${randomUUID()}.tmp`;
|
|
31
|
-
const file = await open(temporary, constants.O_WRONLY | constants.O_CREAT | constants.O_EXCL | constants.O_NOFOLLOW, 0o600);
|
|
32
|
-
try { await file.writeFile(bytes); await file.sync(); } finally { await file.close(); }
|
|
33
|
-
await rename(temporary, path);
|
|
34
|
-
const parent = await open(join(path, '..'), constants.O_RDONLY);
|
|
35
|
-
try { await parent.sync(); } finally { await parent.close(); }
|
|
36
|
-
}
|
|
37
|
-
/** Immutable, content-addressed objects and journal nodes precede one atomic
|
|
38
|
-
* checkpoint pointer. A kill can leave unreferenced objects, never a half-
|
|
39
|
-
* acknowledged page. Reopen verifies the complete reachable graph, including
|
|
40
|
-
* blobs, before making a network request. Checkpoints stay constant size. */
|
|
41
|
-
export class ZendeskStore {
|
|
42
|
-
constructor(path) { this.path = path; this.events = []; this.head = null; this.state = null; }
|
|
43
|
-
async start() {
|
|
44
|
-
await privateDirectory(this.path); await privateDirectory(join(this.path, 'objects'));
|
|
45
|
-
try {
|
|
46
|
-
const checkpoint = JSON.parse((await read(join(this.path, 'checkpoint.json'), 128 * 1024 - 1)).toString());
|
|
47
|
-
if (checkpoint.format !== 'rer-zendesk-checkpoint/1' || Object.keys(checkpoint).sort().join() !== 'format,head') fail('INVALID_CHECKPOINT');
|
|
48
|
-
this.head = checkpoint.head;
|
|
49
|
-
let reference = this.head;
|
|
50
|
-
const seen = new Set(), backwards = [];
|
|
51
|
-
while (reference) {
|
|
52
|
-
if (seen.has(reference.sha256)) fail('INVALID_CHECKPOINT');
|
|
53
|
-
seen.add(reference.sha256);
|
|
54
|
-
const node = JSON.parse((await this.get(reference)).toString());
|
|
55
|
-
if (node.format !== 'rer-zendesk-journal/1' || !node.state || !Array.isArray(node.refs)) fail('INVALID_CHECKPOINT');
|
|
56
|
-
for (const ref of node.refs) await this.get(ref);
|
|
57
|
-
backwards.push(node); reference = node.previous;
|
|
58
|
-
}
|
|
59
|
-
backwards.reverse();
|
|
60
|
-
this.events = backwards.flatMap(node => node.event ? [node.event] : []);
|
|
61
|
-
this.state = backwards.at(-1)?.state;
|
|
62
|
-
if (!this.state) fail('INVALID_CHECKPOINT');
|
|
63
|
-
} catch (error) {
|
|
64
|
-
if (error.code !== 'ENOENT' || this.head) fail('INVALID_CHECKPOINT');
|
|
65
|
-
}
|
|
66
|
-
return this;
|
|
67
|
-
}
|
|
68
|
-
async get(ref) {
|
|
69
|
-
if (!ref || !/^[a-f0-9]{64}$/u.test(ref.sha256) || !Number.isSafeInteger(ref.bytes) || ref.bytes < 0 || ref.bytes > 64 * 1024 * 1024 || ref.file !== `objects/${ref.sha256}`) fail('INVALID_CHECKPOINT');
|
|
70
|
-
const bytes = await read(join(this.path, ref.file));
|
|
71
|
-
if (bytes.length !== ref.bytes || hash(bytes) !== ref.sha256) fail('INVALID_CHECKPOINT');
|
|
72
|
-
return bytes;
|
|
73
|
-
}
|
|
74
|
-
async put(bytes) {
|
|
75
|
-
const ref = { file: `objects/${hash(bytes)}`, sha256: hash(bytes), bytes: bytes.length };
|
|
76
|
-
if (bytes.length > 64 * 1024 * 1024) fail('RECORD_LIMIT');
|
|
77
|
-
const path = join(this.path, ref.file);
|
|
78
|
-
let file;
|
|
79
|
-
try { file = await open(path, constants.O_WRONLY | constants.O_CREAT | constants.O_EXCL | constants.O_NOFOLLOW, 0o600); }
|
|
80
|
-
catch (error) { if (error.code !== 'EEXIST') throw error; await this.get(ref); return ref; }
|
|
81
|
-
try { await file.writeFile(bytes); await file.sync(); } finally { await file.close(); }
|
|
82
|
-
const parent = await open(join(this.path, 'objects'), constants.O_RDONLY);
|
|
83
|
-
try { await parent.sync(); } finally { await parent.close(); }
|
|
84
|
-
return ref;
|
|
85
|
-
}
|
|
86
|
-
async commit(state, event = null, refs = []) {
|
|
87
|
-
state = { ...state, requests: this.requestTotal ?? state.requests };
|
|
88
|
-
if (!Number.isSafeInteger(state.bytes) || state.bytes > 32 * 1024 ** 3 || state.requests > 250_000) fail('BYTE_LIMIT');
|
|
89
|
-
const node = { format: 'rer-zendesk-journal/1', previous: this.head, state, event, refs };
|
|
90
|
-
const head = await this.put(encode(node));
|
|
91
|
-
const bytes = encode({ format: 'rer-zendesk-checkpoint/1', head });
|
|
92
|
-
if (bytes.length >= 128 * 1024) fail('CHECKPOINT_LIMIT');
|
|
93
|
-
await atomic(join(this.path, 'checkpoint.json'), bytes);
|
|
94
|
-
this.head = head; this.state = structuredClone(state);
|
|
95
|
-
if (event) this.events.push(event);
|
|
96
|
-
}
|
|
97
|
-
async page(kind, response, state, extra = {}) {
|
|
98
|
-
const raw = await this.put(response.bytes);
|
|
99
|
-
await this.commit({ ...state, bytes: state.bytes + raw.bytes }, { kind, raw, ...extra }, [raw]);
|
|
100
|
-
}
|
|
101
|
-
async reset(state, { retain = false } = {}) {
|
|
102
|
-
if (!retain) { this.events = []; this.head = null; this.cache = null; this.pagePaths = null; }
|
|
103
|
-
await this.commit(state);
|
|
104
|
-
}
|
|
105
|
-
async write(name, value) {
|
|
106
|
-
if (!['census.json', 'transfer.json', 'baseline.json'].includes(name)) fail('INVALID_CHECKPOINT');
|
|
107
|
-
await atomic(join(this.path, name), encode(value));
|
|
108
|
-
}
|
|
109
|
-
async baseline() {
|
|
110
|
-
try {
|
|
111
|
-
const baseline = JSON.parse((await read(join(this.path, 'baseline.json'), 128 * 1024)).toString());
|
|
112
|
-
const node = JSON.parse((await this.get(baseline.head)).toString());
|
|
113
|
-
if (node.state?.phase !== 'done' || node.state.highWater !== baseline.highWater) fail('INVALID_CHECKPOINT');
|
|
114
|
-
return baseline;
|
|
115
|
-
}
|
|
116
|
-
catch (error) { if (error.code !== 'ENOENT') fail('INVALID_CHECKPOINT'); return null; }
|
|
117
|
-
}
|
|
118
|
-
async discard() { await rm(this.path, { recursive: true, force: true }); }
|
|
119
|
-
}
|
|
120
|
-
export async function removeZendeskStores(directory) {
|
|
121
|
-
const { readdir } = await import('node:fs/promises');
|
|
122
|
-
for (const name of await readdir(directory).catch(error => { if (error.code === 'ENOENT') return []; throw error; })) {
|
|
123
|
-
if (/^zendesk-[a-f0-9]{32}$/u.test(name)) await rm(join(directory, name), { recursive: true, force: true });
|
|
124
|
-
}
|
|
125
|
-
}
|
|
126
|
-
/** Collapse replayed ids and retain their last source row and provenance. */
|
|
127
|
-
export async function materialize(store) {
|
|
128
|
-
const data = store.cache?.data ?? Object.fromEntries(['tickets','users','organizations','ticket_fields','user_segments','categories','sections','articles','translations','article_attachments'].map(kind => [kind, new Map()]));
|
|
129
|
-
if (!store.cache) { data.comments = new Map(); data.attachments = new Map(); data.available = true; data.highWater = 0; }
|
|
130
|
-
for (const event of store.events.slice(store.cache?.length ?? 0)) {
|
|
131
|
-
if (event.kind === 'guide_unavailable') { data.available = false; continue; }
|
|
132
|
-
if (event.kind === 'attachment') { data.attachments.set(event.key, event); continue; }
|
|
133
|
-
if (!event.raw) continue;
|
|
134
|
-
const body = JSON.parse((await store.get(event.raw)).toString());
|
|
135
|
-
const rows = event.single ? [body[event.single]] : body[event.kind];
|
|
136
|
-
if (!Array.isArray(rows)) fail('INVALID_CHECKPOINT');
|
|
137
|
-
const destination = event.kind === 'comments' ? (event.reset ? new Map() : data.comments.get(event.ticket) ?? new Map()) : data[event.kind];
|
|
138
|
-
if (!destination) fail('INVALID_CHECKPOINT');
|
|
139
|
-
rows.forEach((row,index) => {
|
|
140
|
-
const id = numericId(row.id);
|
|
141
|
-
if (Buffer.byteLength(JSON.stringify(row)) > 64 * 1024 * 1024) fail('RECORD_LIMIT');
|
|
142
|
-
const key = ['translations', 'article_attachments'].includes(event.kind) ? `${event.resource ?? "articles"}:${event.article}:${id}` : id;
|
|
143
|
-
destination.set(key, { row, provenance: { file: event.raw.file, sha256: event.raw.sha256, row: index + 1, pointer: event.single ? `/${event.single}` : `/${event.kind}/${index}` } });
|
|
144
|
-
if (event.kind === 'tickets') {
|
|
145
|
-
const stamp = row.generated_timestamp;
|
|
146
|
-
if (!Number.isSafeInteger(stamp) || stamp < 0) fail('INVALID_RESPONSE');
|
|
147
|
-
data.highWater = Math.max(data.highWater, stamp);
|
|
148
|
-
}
|
|
149
|
-
});
|
|
150
|
-
if (event.kind === 'comments') data.comments.set(event.ticket, destination);
|
|
151
|
-
}
|
|
152
|
-
store.cache = { data, length: store.events.length };
|
|
153
|
-
return data;
|
|
154
|
-
}
|
|
@@ -1,21 +0,0 @@
|
|
|
1
|
-
import { materialize, ordered } from './zendesk-store.mjs';
|
|
2
|
-
import { fetchPage } from './zendesk-pages.mjs';
|
|
3
|
-
|
|
4
|
-
export async function tickets(ctx) {
|
|
5
|
-
const { store, inputs } = ctx;
|
|
6
|
-
let state = store.state;
|
|
7
|
-
const path = state.path ?? `/api/v2/incremental/tickets/cursor?start_time=${inputs.mode === 'sweep' ? inputs.highWater : 0}&per_page=1000`;
|
|
8
|
-
state = await fetchPage({ ...ctx, state, kind: 'tickets', path, incremental: true,
|
|
9
|
-
onDone: state => ({ ...state, phase: inputs.mode === 'census' ? 'done' : 'B', index: 0 }) });
|
|
10
|
-
return state;
|
|
11
|
-
}
|
|
12
|
-
export async function comments(ctx) {
|
|
13
|
-
const { store, inputs } = ctx, state = store.state, data = await materialize(store);
|
|
14
|
-
// A sweep refetches complete comments only for changed tickets; the retained
|
|
15
|
-
// store still supplies every older ticket to the eventual complete snapshot.
|
|
16
|
-
const ids = ordered([...data.tickets].filter(([, { row }]) => row.status !== 'deleted' && !row.deleted && (inputs.mode !== 'sweep' || row.generated_timestamp > inputs.highWater)).map(([id]) => id));
|
|
17
|
-
const ticket = ids[state.index];
|
|
18
|
-
if (ticket === undefined) { await store.commit({ ...state, phase: inputs.mode === 'sweep' ? 'done' : 'C', path: null, index: 0 }); return; }
|
|
19
|
-
await fetchPage({ ...ctx, state, kind: 'comments', path: state.path ?? `/api/v2/tickets/${ticket}/comments?page[size]=100&include_inline_images=true`, extra: { ticket, reset: state.path === null },
|
|
20
|
-
onDone: state => ({ ...state, phase: 'attachments', index: state.index, attachmentIndex: 0 }) });
|
|
21
|
-
}
|
|
@@ -1,31 +0,0 @@
|
|
|
1
|
-
import { fail, object } from './common.mjs';
|
|
2
|
-
|
|
3
|
-
export const SCOPES = Object.freeze(['tickets:read', 'users:read', 'organizations:read', 'hc:read', 'ticket_attachments:read']);
|
|
4
|
-
export function clientCredential(key) {
|
|
5
|
-
let value;
|
|
6
|
-
try { value = JSON.parse(key); } catch { fail('INVALID_KEY'); }
|
|
7
|
-
if (!value || Array.isArray(value) || typeof value !== 'object' || Object.keys(value).sort().join() !== 'client_id,client_secret'
|
|
8
|
-
|| Object.values(value).some(part => typeof part !== 'string' || !part || part.length > 4096 || /[\u0000-\u0020\u007f]/u.test(part))) fail('INVALID_KEY');
|
|
9
|
-
return value;
|
|
10
|
-
}
|
|
11
|
-
export function originOf(inputs) {
|
|
12
|
-
const account = inputs?.account;
|
|
13
|
-
if (!account || Object.keys(account).join() !== 'subdomain' || typeof account.subdomain !== 'string'
|
|
14
|
-
|| !/^(?!-)[a-z0-9-]{1,63}(?<!-)$/u.test(account.subdomain)) fail('INVALID_INPUT');
|
|
15
|
-
if (!['snapshot', 'sweep', 'census', 'retire'].includes(inputs.mode ?? 'snapshot')
|
|
16
|
-
|| inputs.highWater !== undefined && (!Number.isSafeInteger(inputs.highWater) || inputs.highWater < 0)) fail('INVALID_INPUT');
|
|
17
|
-
return `https://${account.subdomain}.zendesk.com`;
|
|
18
|
-
}
|
|
19
|
-
export async function mintToken(request, key) {
|
|
20
|
-
const credential = clientCredential(key);
|
|
21
|
-
const { body } = await request('/oauth/tokens', { method: 'POST', json: { ...credential, grant_type: 'client_credentials', scope: SCOPES.join(' '), expires_in: 1800 }, authenticated: false, tokenResponse: true });
|
|
22
|
-
object(body);
|
|
23
|
-
if (typeof body.access_token !== 'string' || !body.access_token || body.access_token.length > 8192 || /[\u0000-\u0020\u007f]/u.test(body.access_token)
|
|
24
|
-
|| body.token_type?.toLowerCase() !== 'bearer') fail('INVALID_TOKEN');
|
|
25
|
-
request.authorize(body.access_token);
|
|
26
|
-
const current = await request('/api/v2/oauth/tokens/current', { tokenResponse: true });
|
|
27
|
-
const scopes = current.body?.token?.scopes ?? current.body?.scopes;
|
|
28
|
-
const values = typeof scopes === 'string' ? scopes.split(/\s+/u) : scopes;
|
|
29
|
-
if (!Array.isArray(values) || !values.length || values.some(scope => !['read', ...SCOPES].includes(scope))) fail('KEY_NOT_READ_ONLY');
|
|
30
|
-
return body.access_token;
|
|
31
|
-
}
|
|
@@ -1,48 +0,0 @@
|
|
|
1
|
-
import { isIP } from 'node:net';
|
|
2
|
-
import { lookup } from 'node:dns/promises';
|
|
3
|
-
import { pacedClient, fail } from './common.mjs';
|
|
4
|
-
import { clientCredential, originOf, mintToken } from './zendesk-token.mjs';
|
|
5
|
-
|
|
6
|
-
const routes = /^\/(?:oauth\/tokens|api\/v2\/(?:oauth\/tokens\/current|incremental\/(?:tickets\/cursor|users\/cursor|organizations)(?:\.json)?|tickets\/\d+\/comments(?:\.json)?|ticket_fields(?:\.json)?|help_center\/(?:[a-z]{2,3}(?:-[a-z0-9]{2,8})?\/articles\/\d+|(?:user_segments|categories|sections|articles)(?:\/\d+\/(?:translations|attachments))?(?:\.json)?)))$/u;
|
|
7
|
-
const queries = new Set(['start_time', 'per_page', 'cursor', 'page', 'page[size]', 'page[after]', 'include_inline_images']);
|
|
8
|
-
export const attachmentPath = path => /^\/(?:attachments\/|api\/v2\/attachments\/)/u.test(path);
|
|
9
|
-
const privateAddress = address => {
|
|
10
|
-
if (isIP(address) === 4) {
|
|
11
|
-
const [a,b] = address.split('.').map(Number);
|
|
12
|
-
return a === 0 || a === 10 || a === 127 || a === 169 && b === 254 || a === 172 && b >= 16 && b <= 31 || a === 192 && (b === 168 || b === 0) || a === 100 && b >= 64 && b <= 127 || a >= 224 || a === 198 && (b === 18 || b === 19 || b === 51) || a === 203 && b === 0 || a === 192 && (b === 0 || b === 88);
|
|
13
|
-
}
|
|
14
|
-
// Only global unicast IPv6; mapped IPv4, loopback, multicast and ULA fail.
|
|
15
|
-
return !/^2[0-9a-f]{3}:/iu.test(address) || /^2001:(?:db8|0):/iu.test(address);
|
|
16
|
-
};
|
|
17
|
-
export async function publicLocation(value, origin) {
|
|
18
|
-
let url;
|
|
19
|
-
try { url = new URL(value, origin); } catch { return null; }
|
|
20
|
-
if (/^https:\/\/[^/?#]*:\d+(?:[/?#]|$)/iu.test(String(value)) || url.protocol !== 'https:' || url.port || url.username || url.password || url.hash || isIP(url.hostname) || url.hostname.startsWith('[')
|
|
21
|
-
|| url.hostname.length > 253 || !url.hostname.split('.').every(part => /^(?!-)[a-z0-9-]{1,63}(?<!-)$/u.test(part)) || !url.hostname.includes('.')
|
|
22
|
-
|| /\.(?:localhost|local|internal|invalid|test|home|lan)$/u.test(url.hostname) || /^\d+$/u.test(url.hostname.split('.').at(-1))) return null;
|
|
23
|
-
if (url.origin !== origin) {
|
|
24
|
-
let addresses;
|
|
25
|
-
try { addresses = await lookup(url.hostname, { all: true }); } catch { return null; }
|
|
26
|
-
if (!addresses.length || addresses.some(item => privateAddress(item.address))) return null;
|
|
27
|
-
}
|
|
28
|
-
return url;
|
|
29
|
-
}
|
|
30
|
-
export function pagination(value, origin, pathname, current) {
|
|
31
|
-
if (value === null) return null;
|
|
32
|
-
if (typeof value !== 'string' || !value || value.length > 4096) fail('INVALID_PAGINATION');
|
|
33
|
-
let next;
|
|
34
|
-
try { next = new URL(value, origin); } catch { fail('INVALID_PAGINATION'); }
|
|
35
|
-
if (next.origin !== origin || next.username || next.password || next.hash || next.pathname.replace(/\.json$/u, '') !== pathname.replace(/\.json$/u, '')
|
|
36
|
-
|| [...next.searchParams.keys()].some(key => !queries.has(key) || next.searchParams.getAll(key).length !== 1) || next.href === new URL(current, origin).href) fail('INVALID_PAGINATION');
|
|
37
|
-
return pathname + next.search;
|
|
38
|
-
}
|
|
39
|
-
export async function transport({ inputs, key, signal, beforeRequest }) {
|
|
40
|
-
const origin = originOf(inputs), credentials = clientCredential(key);
|
|
41
|
-
const request = pacedClient({ origin, signal, beforeRequest, secrets: [key, credentials.client_id, credentials.client_secret], validate: async (url, options) => {
|
|
42
|
-
if (options.attachment) {
|
|
43
|
-
if (!await publicLocation(url.href, origin) || url.origin === origin && !attachmentPath(url.pathname)) fail('REFUSED_LOCATION');
|
|
44
|
-
} else if (url.origin !== origin || url.username || url.password || url.hash || !routes.test(url.pathname) || [...url.searchParams.keys()].some(key => !queries.has(key))) fail('INVALID_REFERENCE');
|
|
45
|
-
} });
|
|
46
|
-
await mintToken(request, key);
|
|
47
|
-
return { request, origin };
|
|
48
|
-
}
|
|
@@ -1,61 +0,0 @@
|
|
|
1
|
-
import { VendorExportError, fail } from './common.mjs';
|
|
2
|
-
import { originOf } from './zendesk-token.mjs';
|
|
3
|
-
import { transport } from './zendesk-transport.mjs';
|
|
4
|
-
import { ZendeskStore, materialize, ordered } from './zendesk-store.mjs';
|
|
5
|
-
import { tickets, comments } from './zendesk-tickets.mjs';
|
|
6
|
-
import { people } from './zendesk-people.mjs';
|
|
7
|
-
import { guide } from './zendesk-guide.mjs';
|
|
8
|
-
import { attachments } from './zendesk-attachments.mjs';
|
|
9
|
-
import { encodeSnapshot } from './zendesk-encode.mjs';
|
|
10
|
-
|
|
11
|
-
export { originOf } from './zendesk-token.mjs';
|
|
12
|
-
export { clientCredential } from './zendesk-token.mjs';
|
|
13
|
-
export { storePath, removeZendeskStores } from './zendesk-store.mjs';
|
|
14
|
-
/** The durable local snapshot is independent of door transfer. A new task
|
|
15
|
-
* starts a fresh snapshot or extends the complete retained store for a sweep;
|
|
16
|
-
* retries of the same task resume its last acknowledged page. */
|
|
17
|
-
export async function snapshot({ key, inputs, directory, runId, signal }) {
|
|
18
|
-
try {
|
|
19
|
-
const origin = originOf(inputs);
|
|
20
|
-
if (!directory || typeof runId !== 'string' || !runId || inputs.mode === 'retire') fail('INVALID_INPUT');
|
|
21
|
-
const store = await new ZendeskStore(directory).start();
|
|
22
|
-
const mode = inputs.mode ?? 'snapshot', baseline = await store.baseline();
|
|
23
|
-
if (mode === 'sweep' && store.state?.runId !== runId && (!baseline || inputs.highWater !== baseline.highWater)) fail('INVALID_HIGH_WATER');
|
|
24
|
-
if (store.state?.runId !== runId) {
|
|
25
|
-
if (mode === 'sweep' && store.state?.phase !== 'done') fail('INCOMPLETE_SNAPSHOT');
|
|
26
|
-
await store.reset({ phase: 'A', path: null, index: 0, attachmentIndex: 0, stage: 0, runId, origin, mode, inputHighWater: inputs.highWater ?? null,
|
|
27
|
-
exportedAt: new Date().toISOString(), bytes: mode === 'sweep' ? store.state.bytes : 0, requests: 0 }, { retain: mode === 'sweep' });
|
|
28
|
-
} else if (store.state.origin !== origin || store.state.mode !== mode || store.state.inputHighWater !== (inputs.highWater ?? null)) fail('INVALID_CHECKPOINT');
|
|
29
|
-
store.requestTotal = store.state.requests;
|
|
30
|
-
const ctx = { store, inputs, ...await transport({ inputs, key, signal, beforeRequest: async () => {
|
|
31
|
-
if (store.requestTotal >= 250_000) fail('REQUEST_LIMIT');
|
|
32
|
-
store.requestTotal++;
|
|
33
|
-
await store.commit(store.state);
|
|
34
|
-
} }) };
|
|
35
|
-
while (store.state.phase !== 'done') {
|
|
36
|
-
signal?.throwIfAborted();
|
|
37
|
-
const phase = store.state.phase;
|
|
38
|
-
if (phase === 'A') await tickets(ctx);
|
|
39
|
-
else if (phase === 'B') await comments(ctx);
|
|
40
|
-
else if (['C','D','E'].includes(phase)) await people(ctx);
|
|
41
|
-
else if (phase === 'F') await guide(ctx);
|
|
42
|
-
else if (['attachments', 'guide_attachments'].includes(phase)) await attachments(ctx);
|
|
43
|
-
else fail('INVALID_CHECKPOINT');
|
|
44
|
-
}
|
|
45
|
-
if (mode === 'census') {
|
|
46
|
-
const data = await materialize(store), unswept = [...data.tickets.values()].filter(({ row }) => row.generated_timestamp > (inputs.highWater ?? 0)).length;
|
|
47
|
-
const census = { format: 'rer-zendesk-census/1', tickets: { unique: data.tickets.size, deleted: [...data.tickets.values()].filter(({row}) => row.status === 'deleted' || row.deleted).length,
|
|
48
|
-
maxId: ordered(data.tickets.keys()).at(-1) ?? null }, unswept, highWater: data.highWater };
|
|
49
|
-
await store.write('census.json', census);
|
|
50
|
-
return { kind: 'census', census, store };
|
|
51
|
-
}
|
|
52
|
-
const result = await encodeSnapshot(store, { origin });
|
|
53
|
-
if (store.state.highWater !== result.census.highWater) await store.commit({ ...store.state, highWater: result.census.highWater });
|
|
54
|
-
await store.write('census.json', result.census);
|
|
55
|
-
await store.write('baseline.json', { highWater: result.census.highWater, head: store.head });
|
|
56
|
-
return { ...result, store };
|
|
57
|
-
} catch (error) {
|
|
58
|
-
if (error instanceof VendorExportError) throw error;
|
|
59
|
-
throw new VendorExportError(signal?.aborted ? 'CANCELLED' : 'INVALID_EXPORT');
|
|
60
|
-
}
|
|
61
|
-
}
|
|
@@ -1,121 +0,0 @@
|
|
|
1
|
-
import { Buffer } from 'node:buffer';
|
|
2
|
-
import { createHash } from 'node:crypto';
|
|
3
|
-
import { sha256 } from '../provenance/identity.ts';
|
|
4
|
-
import { DEFAULT_LIMITS, limitsFor, RecordCollector, safeValue } from '../readers/records.ts';
|
|
5
|
-
import type { ParsedRecord, SourceBundle } from '../types.ts';
|
|
6
|
-
import type { DoorPackageFile, DoorPackageManifest, DoorRequest, ImportDoorPackageUpload } from './types.ts';
|
|
7
|
-
import { DoorError } from './protocol.ts';
|
|
8
|
-
import { bundleBytes, CHUNK_BYTES, type UploadRow } from './uploads.ts';
|
|
9
|
-
|
|
10
|
-
const positive = (n: unknown): n is number => Number.isSafeInteger(n) && (n as number) > 0;
|
|
11
|
-
const count = (n: unknown): n is number => Number.isSafeInteger(n) && (n as number) >= 0;
|
|
12
|
-
const digest = (s: unknown): s is string => typeof s === 'string' && /^[a-f0-9]{64}$/.test(s);
|
|
13
|
-
const filename = (s: unknown): s is string => typeof s === 'string' && /^[A-Za-z0-9_][A-Za-z0-9_./-]*$/.test(s) && !s.split('/').some(p=>!p || p==='.' || p==='..');
|
|
14
|
-
const invalid = (): never => { throw new DoorError('invalid_package',422); };
|
|
15
|
-
const missing = (segments: {bytes: number; sha256: string}[]): never => { throw new DoorError('package_segment_missing',409,{missing:segments}); };
|
|
16
|
-
const json = (bytes: Uint8Array): any => { try { return JSON.parse(new TextDecoder('utf-8',{fatal:true}).decode(bytes)); } catch { return invalid(); } };
|
|
17
|
-
|
|
18
|
-
/** Resolve only verified complete segments in this installation AND namespace. No schema or grant changes. */
|
|
19
|
-
export async function packageUpload(request: DoorRequest, upload: UploadRow, resolved?: (rows: readonly UploadRow[]) => void): Promise<ImportDoorPackageUpload> {
|
|
20
|
-
const declaration = request.options.package;
|
|
21
|
-
if (!declaration) throw new DoorError('seed_package_unsupported',415);
|
|
22
|
-
const raw = await bundleBytes(request,upload), value = json(raw);
|
|
23
|
-
safeValue(value);
|
|
24
|
-
if (value?.format!=='rer-door-package/1' || !declaration.formats.includes(value.source?.format) || typeof value.mode!=='string'
|
|
25
|
-
|| !positive(value.stream?.bytes) || value.stream.bytes > declaration.maxBytes || !digest(value.stream.sha256)
|
|
26
|
-
|| !Array.isArray(value.segments) || !value.segments.length || value.segments.length > declaration.maxSegments
|
|
27
|
-
|| value.index?.path!=='index.ndjson' || !positive(value.index.bytes) || value.index.bytes > request.access.maxBytes || !digest(value.index.sha256)
|
|
28
|
-
|| !value.records || !(value.records.total===null || count(value.records.total)) || !value.records.byKind || typeof value.records.byKind!=='object' || Array.isArray(value.records.byKind)
|
|
29
|
-
|| Object.values(value.records.byKind).some(n=>!count(n))) invalid();
|
|
30
|
-
const manifest = value as DoorPackageManifest;
|
|
31
|
-
let total = 0;
|
|
32
|
-
for (const [i,segment] of manifest.segments.entries()) {
|
|
33
|
-
if (!positive(segment?.bytes) || segment.bytes > request.access.maxBytes || Math.ceil(segment.bytes / CHUNK_BYTES) > request.access.maxChunks
|
|
34
|
-
|| i < manifest.segments.length - 1 && segment.bytes !== request.access.maxBytes || !digest(segment.sha256)) invalid();
|
|
35
|
-
total += segment.bytes;
|
|
36
|
-
}
|
|
37
|
-
if (!Number.isSafeInteger(total) || total!==manifest.stream.bytes || manifest.index.bytes > total) invalid();
|
|
38
|
-
const rows: UploadRow[] = [], absent: {bytes:number;sha256:string}[] = [];
|
|
39
|
-
const streamHash = createHash('sha256');
|
|
40
|
-
for (const segment of manifest.segments) {
|
|
41
|
-
const candidates = await request.options.database.prepare(`SELECT * FROM ik_door_uploads WHERE installation_id=? AND namespace=? AND name='rer-segment' AND bundle_sha256=? AND total_bytes=? ORDER BY id`)
|
|
42
|
-
.bind(request.access.installationId,request.access.namespace,segment.sha256,segment.bytes).all<UploadRow>();
|
|
43
|
-
let complete: UploadRow | undefined, bytes: Uint8Array | undefined;
|
|
44
|
-
for (const row of candidates.results) {
|
|
45
|
-
if (row.total_chunks!==Math.ceil(segment.bytes / CHUNK_BYTES) || row.intake_json) continue;
|
|
46
|
-
try { bytes = await bundleBytes(request,row); complete = row; break; }
|
|
47
|
-
catch (error) { if (!(error instanceof DoorError) || !['upload_incomplete','upload_custody_mismatch','bundle_hash_mismatch'].includes(error.code)) throw error; }
|
|
48
|
-
}
|
|
49
|
-
if (!complete || !bytes) absent.push(segment);
|
|
50
|
-
else { rows.push(complete); streamHash.update(bytes); }
|
|
51
|
-
}
|
|
52
|
-
if (absent.length) missing(absent);
|
|
53
|
-
if (streamHash.digest('hex')!==manifest.stream.sha256) invalid();
|
|
54
|
-
// Planning owns these verified custody identities even if the index or reader refuses.
|
|
55
|
-
resolved?.(rows);
|
|
56
|
-
// Range reads materialize at most one segment plus the bounded requested file.
|
|
57
|
-
async function range(start: number, size: number): Promise<Uint8Array> {
|
|
58
|
-
const result = new Uint8Array(size); let offset = 0, written = 0;
|
|
59
|
-
for (let i = 0; i < rows.length && written < size; i++) {
|
|
60
|
-
const end = offset + manifest.segments[i].bytes;
|
|
61
|
-
if (end > start && offset < start + size) {
|
|
62
|
-
let bytes: Uint8Array;
|
|
63
|
-
try { bytes = await bundleBytes(request,rows[i]); } catch (error) { if (error instanceof DoorError && ['upload_incomplete','upload_custody_mismatch','bundle_hash_mismatch'].includes(error.code)) missing([manifest.segments[i]]); throw error; }
|
|
64
|
-
const from = Math.max(start,offset), to = Math.min(start+size,end);
|
|
65
|
-
result.set(bytes.subarray(from-offset,to-offset),from-start); written += to-from;
|
|
66
|
-
}
|
|
67
|
-
offset = end;
|
|
68
|
-
}
|
|
69
|
-
if (written!==size) invalid();
|
|
70
|
-
return result;
|
|
71
|
-
}
|
|
72
|
-
const indexBytes = await range(0,manifest.index.bytes);
|
|
73
|
-
if (sha256(indexBytes)!==manifest.index.sha256) invalid();
|
|
74
|
-
let indexText: string;
|
|
75
|
-
try { indexText = new TextDecoder('utf-8',{fatal:true}).decode(indexBytes); } catch { return invalid(); }
|
|
76
|
-
if (!indexText.endsWith('\n')) invalid();
|
|
77
|
-
const files: DoorPackageFile[] = [], names = new Set<string>(); let offset = 0;
|
|
78
|
-
for (const line of indexText.slice(0,-1).split('\n')) {
|
|
79
|
-
const row = json(Buffer.from(line));
|
|
80
|
-
if (!filename(row?.path) || row.path==='index.ndjson' || names.has(row.path) || !count(row.bytes) || !digest(row.sha256) || row.offset!==offset) invalid();
|
|
81
|
-
offset += row.bytes;
|
|
82
|
-
if (!Number.isSafeInteger(offset) || offset > total - manifest.index.bytes) invalid();
|
|
83
|
-
names.add(row.path); files.push(Object.freeze({path:row.path,bytes:row.bytes,sha256:row.sha256,offset:row.offset}));
|
|
84
|
-
}
|
|
85
|
-
if (!files.length || offset + manifest.index.bytes!==total) invalid();
|
|
86
|
-
// Detach the accessor's custody data from callback-owned metadata.
|
|
87
|
-
const exposed = JSON.parse(JSON.stringify(manifest)) as DoorPackageManifest;
|
|
88
|
-
const fileLimit = Math.min(declaration.maxBytes,request.options.limits?.maxBytes ?? DEFAULT_LIMITS.maxBytes);
|
|
89
|
-
return {manifest:exposed,files:Object.freeze(files),async read(path, maxBytes = fileLimit) {
|
|
90
|
-
const file = files.find(f=>f.path===path);
|
|
91
|
-
if (!file) throw new DoorError('package_file_missing',422);
|
|
92
|
-
if (!count(maxBytes) || file.bytes > Math.min(maxBytes,fileLimit)) throw new DoorError('package_file_exceeds_limit',413);
|
|
93
|
-
const bytes = await range(manifest.index.bytes+file.offset,file.bytes);
|
|
94
|
-
if (sha256(bytes)!==file.sha256) throw new DoorError('package_file_hash_mismatch',409);
|
|
95
|
-
return bytes;
|
|
96
|
-
}};
|
|
97
|
-
}
|
|
98
|
-
|
|
99
|
-
export async function readPackageSource(request: DoorRequest, upload: UploadRow, namespace: string): Promise<SourceBundle> {
|
|
100
|
-
const pkg = await packageUpload(request,upload), declaration = request.options.package!;
|
|
101
|
-
const limits = limitsFor({...request.options.limits,maxBytes:Math.min(declaration.maxBytes,request.options.limits?.maxBytes ?? DEFAULT_LIMITS.maxBytes),
|
|
102
|
-
maxRecords:Math.min(request.options.limits?.maxRecords ?? request.access.maxRecords,request.access.maxRecords)});
|
|
103
|
-
const expectedTotal = pkg.manifest.records.total, expectedKinds = {...pkg.manifest.records.byKind};
|
|
104
|
-
let records: ParsedRecord[];
|
|
105
|
-
try { records = await declaration.read(pkg,Object.freeze({...limits})); }
|
|
106
|
-
catch (error) { if (error instanceof DoorError) throw error; throw new DoorError('package_reader_failed',422); }
|
|
107
|
-
if (!Array.isArray(records)) invalid();
|
|
108
|
-
// Count the reader's aggregate before mapping or saving ANY plan, including when total is null (legacy).
|
|
109
|
-
if (records.length > request.access.maxRecords) throw new DoorError('plan_exceeds_grant_quota');
|
|
110
|
-
if (expectedTotal!==null && records.length!==expectedTotal) throw new DoorError('package_record_count_mismatch',422);
|
|
111
|
-
if (expectedTotal!==null) {
|
|
112
|
-
const actual: Record<string,number> = Object.create(null);
|
|
113
|
-
for (const record of records) actual[record.kind] = (actual[record.kind] ?? 0) + 1;
|
|
114
|
-
if (Object.keys(actual).length!==Object.keys(expectedKinds).length || Object.entries(actual).some(([kind,n])=>expectedKinds[kind]!==n)) throw new DoorError('package_record_count_mismatch',422);
|
|
115
|
-
}
|
|
116
|
-
const collector = new RecordCollector(namespace,limits);
|
|
117
|
-
collector.add(records,{artifact:upload.name},upload.bundle_sha256);
|
|
118
|
-
collector.bundle.artifacts.push({name:upload.name,sha256:upload.bundle_sha256,size:upload.total_bytes},
|
|
119
|
-
...pkg.files.map(f=>({name:f.path,sha256:f.sha256,size:f.bytes})));
|
|
120
|
-
return collector.bundle;
|
|
121
|
-
}
|
|
@@ -1,158 +0,0 @@
|
|
|
1
|
-
# Opt-in complete-parent imports
|
|
2
|
-
|
|
3
|
-
The bounded-waves protocol shown below is a grant-only source protocol. Ordinary `/plan`, including
|
|
4
|
-
the logical-package path, keeps its existing aggregate refusal and quotas.
|
|
5
|
-
Transport segments never grant execution authority. This source change does not
|
|
6
|
-
adopt the protocol in an edition, raise a grant, approve an aggregate, or authorize
|
|
7
|
-
a real import. An ordinary whole snapshot above its record grant continues to refuse.
|
|
8
|
-
|
|
9
|
-
Apply `importDoorBoundedWavesMigration` in a new numbered edition migration after
|
|
10
|
-
the existing door migrations. The maintained reference includes the additive
|
|
11
|
-
`0042_import_bounded_waves.sql`; it supplies no bounded source or Business tasks.
|
|
12
|
-
Existing jobs, originals, approvals and migration names stay intact. The schema
|
|
13
|
-
adds root, ordered child, hashed membership and approval tables. Source content
|
|
14
|
-
uses the existing door snapshot chunks, and original/attachment bytes use the
|
|
15
|
-
ordinary D1 job custody. Erasure withdraws snapshots and equivalent authorized
|
|
16
|
-
package segments, marks their roots withdrawn, and retires affected root journals
|
|
17
|
-
with the existing stream/reuse/history closure. Peer native rows are retained
|
|
18
|
-
unless the product's erasure statements select them. Withdrawal is not Undo.
|
|
19
|
-
|
|
20
|
-
The edition supplies `ImportDoorBoundedSource` from `types.ts` as `source.bounded`
|
|
21
|
-
alongside its existing `source.package`. This is an actual typed input boundary;
|
|
22
|
-
no HappyToHelp reader, parser, native grouping or Business approval implementation
|
|
23
|
-
is supplied here. Those upstream integrations must exist and be independently
|
|
24
|
-
qualified before adoption. The ordinary whole-array package `read` is not used
|
|
25
|
-
by preparation.
|
|
26
|
-
|
|
27
|
-
`describe(upload)` independently enumerates the complete source and returns one
|
|
28
|
-
finite `WaveCensus`, including source/native counts by kind/table, originals,
|
|
29
|
-
objects, logical/stored/native/blob/context bytes, accounting version, source
|
|
30
|
-
mode and high water, and complete inventory/relationship/baseline digests.
|
|
31
|
-
`page(upload, ordinal, limits)` returns exactly one indivisible `WaveUnit` with
|
|
32
|
-
all its primary identities, file roles and dependencies on earlier units. The
|
|
33
|
-
adapter must include metadata/context/archive/rejected aliases in its reader's
|
|
34
|
-
erasure recognition, and coalesce cycles and publication closures before paging.
|
|
35
|
-
Files, source identities, additional subject aliases and native projection keys each have exactly one owner.
|
|
36
|
-
`aliases` explicitly indexes context, archive, rejected and metadata subjects that
|
|
37
|
-
are not primary source units; their files must be in the verified package. The
|
|
38
|
-
census reports their separate count and its inventory digest includes every
|
|
39
|
-
alias. The server-owned erasure adapter can supply exact
|
|
40
|
-
`D1ImportPerson.aliases: [{installationId, namespace, kind, sourceId}]`, including
|
|
41
|
-
a subject that never became a native target. It persists scoped digest tombstones
|
|
42
|
-
even before preparation. These selectors are never client input. Wave erasure uses the journaled
|
|
43
|
-
four-argument `retireD1ImportCustody` API and the actual restore checkpoint; the
|
|
44
|
-
separate legacy `operationKey` erasure-step interface refuses wave custody with
|
|
45
|
-
`IMPORT_WAVE_JOURNAL_REQUIRED`. The journal API retains its existing 1000-statement
|
|
46
|
-
and 2 MiB RPC bounds.
|
|
47
|
-
An unsupported indivisible unit refuses; no remainder is silently deferred.
|
|
48
|
-
|
|
49
|
-
`native(plan)` enumerates every actual expanded native row, including joins,
|
|
50
|
-
origin/hold rows and native blob chunks, with its stable key, projection digest
|
|
51
|
-
and stored/blob byte sizes. Its totals cannot be smaller than the kit effects.
|
|
52
|
-
It must describe the installed native adapter, including unchanged reused rows;
|
|
53
|
-
an effect/container is not a substitute record unit. The native adapter's
|
|
54
|
-
`read`, exact `matches` and full-set predicates must cover those physical rows,
|
|
55
|
-
references and byte representations. Kit protocol/journal metadata is separate.
|
|
56
|
-
`batchSize` optionally lowers the ordinary engine batch (1..20), without changing
|
|
57
|
-
the independent source and expanded-native execution quotas.
|
|
58
|
-
|
|
59
|
-
`waveInventoryHash({source,native,files})`, exported from the door, defines the
|
|
60
|
-
complete census digest. Source rows supply kind, stable ID and payload hash;
|
|
61
|
-
native rows supply the full `WaveNativeRow`, and `aliases` supplies additional
|
|
62
|
-
`WaveAlias` records; verified files also supply their
|
|
63
|
-
role and logical bytes. `inventoryMember(type, identity, content)` exposes each
|
|
64
|
-
hashed leaf. Sort its hexadecimal digest leaves in ascending ASCII order and
|
|
65
|
-
hash their canonical JSON array. The server computes this from its immutable
|
|
66
|
-
membership index with bounded keyset pages, without building a full inventory
|
|
67
|
-
JSON value. An adapter can do the same in its paged reader or use the convenience
|
|
68
|
-
function for a small complete census.
|
|
69
|
-
|
|
70
|
-
`relationshipsHash` hashes the ordered array of each unit's relationship digest.
|
|
71
|
-
`baselineHash` is the previous complete parent fingerprint, or SHA-256 of
|
|
72
|
-
canonical `[]` for the initial snapshot. Each child separately commits its exact
|
|
73
|
-
native before-images. The sealed parent commits all child fingerprints, census,
|
|
74
|
-
package, destination/claim, migration/revision/policy, mapping and erasure epoch.
|
|
75
|
-
|
|
76
|
-
`authority(binding, action)` MUST return a same-database scalar predicate for the
|
|
77
|
-
CURRENT installation, owner/claim, migration/revision/policy and active exact
|
|
78
|
-
parent/child approvals. It must consult the edition/Business approval rows;
|
|
79
|
-
returning a cached boolean or constant permission defeats the integration.
|
|
80
|
-
The door combines it with current enablement, trust/key revocation, single-use
|
|
81
|
-
nonce, expiry, namespace/destination binding and erasure epoch before and after
|
|
82
|
-
each existing D1 transaction. A root approval has no apply authority.
|
|
83
|
-
|
|
84
|
-
For example, the trusted signer issues fresh grants using these additional
|
|
85
|
-
payload members (the existing issuer/audience/owner/nonce/expiry/quotas remain):
|
|
86
|
-
|
|
87
|
-
```ts
|
|
88
|
-
const bounded = {
|
|
89
|
-
protocol: 'rer-bounded-waves/1', migration: 'migration-17',
|
|
90
|
-
revision: 'revision-2', policy: 'policy-1',
|
|
91
|
-
};
|
|
92
|
-
// prepare / seal-parent / wave-status: { bounded }
|
|
93
|
-
// review-parent:
|
|
94
|
-
const reviewedParent = {
|
|
95
|
-
...bounded, parentFingerprint, parentApprovalRecordId, parentApprovedAt,
|
|
96
|
-
};
|
|
97
|
-
// { bounded: reviewedParent, fingerprint: parentFingerprint,
|
|
98
|
-
// approvalRecordId: parentApprovalRecordId, approvedAt: parentApprovedAt }
|
|
99
|
-
// apply-child:
|
|
100
|
-
// { bounded: {...reviewedParent, childFingerprint}, fingerprint: childFingerprint,
|
|
101
|
-
// approvalRecordId: childApprovalRecordId, approvedAt: childApprovedAt }
|
|
102
|
-
```
|
|
103
|
-
|
|
104
|
-
The concrete POST bodies are:
|
|
105
|
-
|
|
106
|
-
```json
|
|
107
|
-
{"uploadId":"server-upload-id","ordinal":0}
|
|
108
|
-
{"parentId":"server-parent-id","ordinal":1}
|
|
109
|
-
{"parentId":"server-parent-id"}
|
|
110
|
-
{"parentId":"server-parent-id","ordinal":0,"reviewedFingerprint":"<child-sha256>"}
|
|
111
|
-
```
|
|
112
|
-
|
|
113
|
-
The first two are `/prepare`. The third is `/seal-parent`, `/review-parent`,
|
|
114
|
-
`/wave-status`, `/finalize-parent` or `/reverse-parent` under its own action
|
|
115
|
-
grant. The fourth is `/apply-child`. Before approval, `/prepare` with
|
|
116
|
-
`{"parentId":"server-parent-id","cancel":true}` can discard private custody;
|
|
117
|
-
the previous complete head stays authoritative. A cancelled revision is not
|
|
118
|
-
mutated/reused: review a new revision. All routes remain on `/api/rer/import/`.
|
|
119
|
-
|
|
120
|
-
Child responses say `childSuccess` and always `aggregateSuccess:false`. Missing,
|
|
121
|
-
altered, foreign, unapproved, revoked or out-of-order children refuse. Fresh
|
|
122
|
-
grants resolve lost responses through durable inventory/status, operation keys
|
|
123
|
-
and the existing D1 checkpoints. A child uses an ordinary D1 job, preserving
|
|
124
|
-
attachment/original eligibility, strict before-images, apply and rollback. The
|
|
125
|
-
kit's internal source namespace binds the reviewed snapshot revision, not a
|
|
126
|
-
child/upload/attempt; the logical namespace and migration lineage never split.
|
|
127
|
-
|
|
128
|
-
Finalization requires all exact children, current approvals, readable custody,
|
|
129
|
-
complete inventory totals and exact native/provenance reconciliation. Only that
|
|
130
|
-
transaction advances the complete head and releases gone-but-retained reporting.
|
|
131
|
-
Absence never deletes. `wave-status` reports progress/completion history, not a
|
|
132
|
-
fresh aggregate proof; the sender consumes `/finalize-parent` itself.
|
|
133
|
-
Routine grants marked `catchup:'bounded'` require the global strict under-five-
|
|
134
|
-
percent comparison and all existing gone/conflict/rejected/unmapped exclusions.
|
|
135
|
-
A separately reviewed proposal can describe a complete snapshot with absence;
|
|
136
|
-
its previous targets remain attributed and retained. Catch-up Undo preserves
|
|
137
|
-
unchanged original owners. Reverse checks all known conflicts first, then uses
|
|
138
|
-
ordinary strict Undo or partial-job rollback, descending dependants first. It is
|
|
139
|
-
bounded, resumable reversal, not whole-database atomicity. Reversal reserves the
|
|
140
|
-
same pending-parent slot, so concurrent cumulative preparation cannot race its
|
|
141
|
-
preflight or clear another parent during complete-head restoration.
|
|
142
|
-
|
|
143
|
-
Every request retains the existing plan/value/parser/SQL/transport limits. A
|
|
144
|
-
complete reconciliation or baseline read that cannot fit the actual request
|
|
145
|
-
budget refuses before publication; there is no unchecked paged remainder or
|
|
146
|
-
hosted-capacity claim. Independent qualification must establish the capacity of
|
|
147
|
-
the adopting edition and its exact native adapter. The tiny-grant public proof
|
|
148
|
-
does not establish capacity for a production-sized legacy snapshot.
|
|
149
|
-
|
|
150
|
-
The reusable sender is `cli/business/bounded-waves.mjs`, with its actual typed
|
|
151
|
-
`BoundedArrangement` in `bounded-waves.d.mts`. `rer business run` opts in only
|
|
152
|
-
when Business supplies `task.bounded_waves`; ordinary tasks keep their path.
|
|
153
|
-
Business supplies the claimed destination and fresh action grants, then the
|
|
154
|
-
exact server parent fingerprint and ordered immutable child approvals. The
|
|
155
|
-
sender cannot synthesize them or treat task completion as reconciliation.
|
|
156
|
-
An arrangement without approval prepares/seals and pauses truthfully. Approval
|
|
157
|
-
arrangement, grant minting, edition-native grouping, and real adoption remain
|
|
158
|
-
upstream dependencies, not evidence supplied by this Base implementation.
|
|
@@ -1,38 +0,0 @@
|
|
|
1
|
-
import { createHash } from 'node:crypto';
|
|
2
|
-
import type { D1Database } from '../../d1/types.ts';
|
|
3
|
-
import { canonicalHash } from '../../provenance/canonical-stream.ts';
|
|
4
|
-
import type { WaveAlias, WaveNativeRow } from './types.ts';
|
|
5
|
-
|
|
6
|
-
export interface WaveInventoryMember {type: string; identity: string; digest: string}
|
|
7
|
-
export const inventoryMember = (type: string, identity: unknown, content: unknown): WaveInventoryMember => ({
|
|
8
|
-
type,identity:canonicalHash(identity),digest:canonicalHash([type,identity,content]),
|
|
9
|
-
});
|
|
10
|
-
export interface WaveInventoryInput {
|
|
11
|
-
source: readonly {kind:string;sourceId:string;payloadHash:string}[];
|
|
12
|
-
native: readonly WaveNativeRow[];
|
|
13
|
-
aliases?: readonly WaveAlias[];
|
|
14
|
-
files: readonly {path:string;sha256:string;bytes:number;role:'source'|'object'|'context';logicalBytes:number}[];
|
|
15
|
-
}
|
|
16
|
-
/** Adapter convenience for a complete independently enumerated census. An
|
|
17
|
-
* edition can equivalently sort these digest leaves in its paged reader/store. */
|
|
18
|
-
export function waveInventoryHash(input: WaveInventoryInput): string {
|
|
19
|
-
return canonicalHash([
|
|
20
|
-
...input.source.map(row=>inventoryMember('source',[row.kind,row.sourceId],row.payloadHash)),
|
|
21
|
-
...(input.aliases ?? []).map(alias=>inventoryMember('alias',[alias.kind,alias.sourceId],alias)),
|
|
22
|
-
...input.native.map(row=>inventoryMember('native',[row.table,row.key],row)),
|
|
23
|
-
...input.files.map(file=>inventoryMember('file',file.path,[file.sha256,file.bytes,file.role,file.logicalBytes])),
|
|
24
|
-
].map(row=>row.digest).sort());
|
|
25
|
-
}
|
|
26
|
-
/** Hash the complete exact inventory without materializing its JSON or all
|
|
27
|
-
* source/native membership rows. The unique identity index seals coverage;
|
|
28
|
-
* the digest index permits bounded keyset reads. */
|
|
29
|
-
export async function storedInventoryHash(db: D1Database, parentId: string): Promise<string> {
|
|
30
|
-
const digest=createHash('sha256');digest.update('[');
|
|
31
|
-
let after='',first=true;
|
|
32
|
-
for (;;) {
|
|
33
|
-
const {results}=await db.prepare('SELECT digest FROM ik_door_wave_members WHERE parent_id=? AND digest>? ORDER BY digest LIMIT 128').bind(parentId,after).all<{digest:string}>();
|
|
34
|
-
for (const row of results) {digest.update((first?'':',')+JSON.stringify(row.digest));first=false;after=row.digest;}
|
|
35
|
-
if(results.length<128)break;
|
|
36
|
-
}
|
|
37
|
-
digest.update(']');return digest.digest('hex');
|
|
38
|
-
}
|