@runeditrun/rer 0.2.79 → 0.2.80
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +17 -3
- package/cli/README.md +25 -13
- package/cli/authoring.mjs +14 -0
- package/cli/contract.mjs +25 -5
- package/cli/demo-host.mjs +4 -2
- package/cli/dev-identity.mjs +2 -2
- package/cli/export.mjs +1 -1
- package/cli/inspect.mjs +38 -2
- package/cli/main.mjs +8 -2
- package/cli/managed-server.mjs +1 -1
- package/cli/parse.mjs +7 -6
- package/cli/portability-worker.mjs +2 -1
- package/cli/portability.mjs +6 -2
- package/cli/production-auth.mjs +2 -1
- package/cli/production-mail.mjs +36 -9
- package/cli/project.mjs +5 -14
- package/cli/release.mjs +2 -1
- package/cli/site-preview.mjs +4 -1
- package/cli/site.mjs +3 -2
- package/edition-authoring/adopt.mjs +39 -0
- package/edition-authoring/base-acknowledgements.mjs +24 -0
- package/edition-authoring/base-copies.mjs +418 -0
- package/edition-authoring/cli.mjs +85 -0
- package/edition-authoring/copy-checkout.mjs +69 -0
- package/edition-authoring/files.mjs +15 -2
- package/edition-authoring/lock.mjs +48 -0
- package/edition-authoring/offline-git.mjs +28 -0
- package/edition-authoring/parent-comparison.mjs +77 -0
- package/edition-authoring/projection-lock.mjs +38 -0
- package/edition-authoring/release.mjs +48 -0
- package/edition-authoring/sources.mjs +114 -0
- package/edition-authoring/updates.mjs +105 -0
- package/global/CHANGELOG.md +12 -0
- package/global/README.md +17 -3
- package/global/cli/install.mjs +3 -1
- package/global/main.mjs +9 -12
- package/modules/container/build.mjs +1 -1
- package/modules/d1-transport/index.mjs +68 -21
- package/modules/demo/README.md +8 -2
- package/modules/demo/declaration.mjs +21 -0
- package/modules/demo/input-path.mjs +18 -0
- package/modules/demo/label.d.mts +1 -1
- package/modules/demo/label.mjs +10 -5
- package/modules/demo/navigation.mjs +32 -0
- package/modules/demo/run-url.mjs +9 -0
- package/modules/demo/vite.d.mts +7 -2
- package/modules/demo/vite.mjs +17 -2
- package/modules/demo-host/README.md +5 -3
- package/modules/demo-host/page-style.mjs +2 -2
- package/modules/demo-host/pages.mjs +3 -3
- package/modules/demo-host/plan.mjs +6 -4
- package/modules/demo-host/theme.mjs +5 -2
- package/modules/deploy-kit/README.md +19 -15
- package/modules/deploy-kit/owner-cli.mjs +1 -1
- package/modules/deploy-kit/readiness-window.mjs +3 -3
- package/modules/erasure/README.md +10 -5
- package/modules/import-kit/src/d1/README.md +15 -9
- package/modules/import-kit/src/d1/apply.ts +10 -6
- package/modules/import-kit/src/d1/atomic-prefix.ts +64 -0
- package/modules/import-kit/src/d1/budget.ts +48 -26
- package/modules/import-kit/src/d1/compressed-plan.ts +9 -2
- package/modules/import-kit/src/d1/database.ts +2 -1
- package/modules/import-kit/src/d1/jobs.ts +2 -1
- package/modules/import-kit/src/d1/plans.ts +4 -2
- package/modules/import-kit/src/d1/verified-plan.ts +63 -0
- package/modules/integrations/core/README.md +2 -2
- package/modules/mail/README.md +4 -3
- package/modules/mail/pnpm-lock.yaml +16 -0
- package/modules/operator-auth/accounts/native.mjs +3 -1
- package/modules/operator-auth/accounts/provider-control.mjs +7 -4
- package/modules/operator-auth/accounts/provider-setup.mjs +34 -0
- package/modules/operator-auth/accounts/settings-page.mjs +13 -4
- package/modules/operator-auth/conformance/README.md +1 -1
- package/modules/operator-auth/dev/index.mjs +22 -4
- package/modules/operator-auth/dev/refusal.mjs +19 -0
- package/modules/operator-auth/docs/README.md +6 -5
- package/modules/operator-auth/docs/tanstack-start.md +1 -1
- package/modules/operator-auth/docs/vite-spa.md +5 -6
- package/modules/operator-auth/index.d.mts +1 -1
- package/modules/operator-auth/page.mjs +14 -0
- package/modules/operator-auth/representation.mjs +83 -0
- package/modules/operator-auth/routes.mjs +25 -2
- package/modules/portability/README.md +91 -14
- package/modules/portability/bookkeeping.mjs +3 -3
- package/modules/portability/bridge.mjs +5 -4
- package/modules/portability/catalogue.mjs +12 -8
- package/modules/portability/cli.mjs +5 -1
- package/modules/portability/export.mjs +5 -11
- package/modules/portability/format.mjs +7 -1
- package/modules/portability/http-errors.mjs +51 -0
- package/modules/portability/http.mjs +4 -1
- package/modules/portability/import-lease.mjs +13 -2
- package/modules/portability/import.mjs +10 -0
- package/modules/portability/index.d.mts +3 -0
- package/modules/portability/references.mjs +20 -0
- package/modules/portability/schema-identity.mjs +13 -0
- package/modules/portability/schema-legacy.mjs +21 -0
- package/modules/portability/sqlite-tokens.mjs +121 -0
- package/modules/release/README.md +20 -9
- package/modules/release/github-verify.mjs +30 -9
- package/modules/site/docs-search/search.css +1 -1
- package/onboarding/README.md +16 -4
- package/onboarding/adapter.mjs +5 -1
- package/onboarding/schema/SETUP.schema.json +26 -1
- package/onboarding/schema/business-field-inputs.mjs +37 -0
- package/onboarding/schema/json-schema.mjs +6 -2
- package/onboarding/schema/setup-validation.mjs +20 -3
- package/onboarding/server.mjs +1 -1
- package/onboarding/setup-manifest.mjs +2 -0
- package/onboarding/start-origin-diagnostic.mjs +24 -0
- package/onboarding/ui/style.css +9 -9
- package/package.json +1 -1
- package/scripts/publish-check/README.md +1 -1
|
@@ -36,3 +36,17 @@ export async function pageResponse({ branding, title, heading, content, status =
|
|
|
36
36
|
if (referrer) headers['Referrer-Policy'] = referrer;
|
|
37
37
|
return new Response(method === 'HEAD' ? null : html, { status, headers });
|
|
38
38
|
}
|
|
39
|
+
|
|
40
|
+
// Refusals on private document routes use the same frame as sign-in. The API
|
|
41
|
+
// adapter keeps its typed JSON response for callers that did not ask for HTML.
|
|
42
|
+
export function settingsRefusalPage(request, branding, code, status) {
|
|
43
|
+
const ownerOnly = code === 'owner_required';
|
|
44
|
+
const unauthenticated = code === 'unauthenticated';
|
|
45
|
+
const heading = ownerOnly ? 'Only the owner can change sign-in settings' : unauthenticated ? 'Sign in to view these settings' : 'Settings could not be changed';
|
|
46
|
+
const message = ownerOnly ? 'Your account can use the app. Ask the installation owner to manage identity providers.'
|
|
47
|
+
: unauthenticated ? 'Your session has ended or you are signed out. Sign in, then open settings again.'
|
|
48
|
+
: 'The request was refused and no settings were saved. Return to the app, reload settings and try again.';
|
|
49
|
+
const href = unauthenticated ? branding.signInPath : '/app';
|
|
50
|
+
return pageResponse({ branding, heading, status, method: request.method,
|
|
51
|
+
content: `<p>${message}</p><a class="action" href="${escapeHtml(href)}">${unauthenticated ? 'Sign in' : 'Back to app'}</a>` });
|
|
52
|
+
}
|
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
const token = /^[!#$%&'*+\-.^_`|~0-9A-Za-z]+$/;
|
|
2
|
+
const quality = /^(?:0(?:\.[0-9]{0,3})?|1(?:\.0{0,3})?)$/;
|
|
3
|
+
|
|
4
|
+
// Commas and semicolons inside quoted parameter values are not separators.
|
|
5
|
+
function split(value, separator) {
|
|
6
|
+
const parts = [];
|
|
7
|
+
let start = 0, quoted = false, escaped = false;
|
|
8
|
+
for (let i = 0; i < value.length; i++) {
|
|
9
|
+
const char = value[i];
|
|
10
|
+
if (escaped) escaped = false;
|
|
11
|
+
else if (quoted && char === '\\') escaped = true;
|
|
12
|
+
else if (char === '"') quoted = !quoted;
|
|
13
|
+
else if (!quoted && char === separator) { parts.push(value.slice(start, i)); start = i + 1; }
|
|
14
|
+
}
|
|
15
|
+
return quoted || escaped ? null : [...parts, value.slice(start)];
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
function ranges(value) {
|
|
19
|
+
if (/[\u0000-\u0008\u000a-\u001f\u007f]/.test(value)) return null;
|
|
20
|
+
const members = split(value, ',');
|
|
21
|
+
if (!members) return null;
|
|
22
|
+
const result = [];
|
|
23
|
+
for (const member of members) {
|
|
24
|
+
if (!member.trim()) continue; // RFC 9110 list syntax permits empty members.
|
|
25
|
+
const parts = split(member, ';');
|
|
26
|
+
if (!parts) return null;
|
|
27
|
+
const media = parts.shift().trim().toLowerCase().split('/');
|
|
28
|
+
if (media.length !== 2 || !media.every(part => token.test(part)) || (media[0] === '*' && media[1] !== '*')) return null;
|
|
29
|
+
const parameters = new Map();
|
|
30
|
+
let q = 1, weighted = false;
|
|
31
|
+
for (const part of parts) {
|
|
32
|
+
if (!part.trim()) continue;
|
|
33
|
+
const at = part.indexOf('='), name = part.slice(0, at).trim().toLowerCase(), raw = part.slice(at + 1).trim();
|
|
34
|
+
if (at < 0 || !token.test(name)) return null;
|
|
35
|
+
if (name === 'q') {
|
|
36
|
+
if (weighted || !quality.test(raw)) return null;
|
|
37
|
+
q = Number(raw); weighted = true;
|
|
38
|
+
} else {
|
|
39
|
+
if (parameters.has(name)) return null;
|
|
40
|
+
const quoted = /^"(?:[\t !#-\[\]-~\u0080-\u00ff]|\\[\t\x20-\x7e\u0080-\u00ff])*"$/.test(raw);
|
|
41
|
+
if (!token.test(raw) && !quoted) return null;
|
|
42
|
+
parameters.set(name, quoted ? raw.slice(1, -1).replace(/\\(.)/g, '$1') : raw);
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
result.push({ media, parameters, q, order: result.length });
|
|
46
|
+
}
|
|
47
|
+
return result;
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
function preference(accepted, type, subtype) {
|
|
51
|
+
let best = { q: 0, specificity: -1, parameters: -1, order: Infinity };
|
|
52
|
+
for (const range of accepted) {
|
|
53
|
+
if ((range.media[0] !== '*' && range.media[0] !== type) || (range.media[1] !== '*' && range.media[1] !== subtype)) continue;
|
|
54
|
+
// Both responses are UTF-8; other media parameters are not supplied.
|
|
55
|
+
if ([...range.parameters].some(([name, value]) => name !== 'charset' || value.toLowerCase() !== 'utf-8')) continue;
|
|
56
|
+
const specificity = range.media[0] === '*' ? 0 : range.media[1] === '*' ? 1 : 2;
|
|
57
|
+
const parameters = range.parameters.size;
|
|
58
|
+
// Most specific matching range sets quality, including an explicit q=0.
|
|
59
|
+
// Equally specific duplicate ranges use the first occurrence.
|
|
60
|
+
if (specificity > best.specificity || (specificity === best.specificity && parameters > best.parameters)) {
|
|
61
|
+
best = { q: range.q, specificity, parameters, order: range.order };
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
return best;
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
/**
|
|
68
|
+
* RFC 9110 Accept negotiation for the auth HTML/JSON surfaces. Equal quality
|
|
69
|
+
* prefers specificity, then header order. No preference (absent or wildcard),
|
|
70
|
+
* malformed fields, or no acceptable representation retain the non-HTML route
|
|
71
|
+
* default. Request Content-Type never overrides the response preference.
|
|
72
|
+
*/
|
|
73
|
+
export function representation(request, routeDefault = 'json') {
|
|
74
|
+
const accepted = ranges(request.headers.get('accept') ?? '');
|
|
75
|
+
if (!accepted?.length) return routeDefault;
|
|
76
|
+
const html = preference(accepted, 'text', 'html');
|
|
77
|
+
const json = preference(accepted, 'application', 'json');
|
|
78
|
+
if (!html.q) return html.specificity >= 0 || json.q ? 'json' : routeDefault;
|
|
79
|
+
if (html.q !== json.q) return html.q > json.q ? 'html' : 'json';
|
|
80
|
+
if (html.specificity !== json.specificity) return html.specificity > json.specificity ? 'html' : 'json';
|
|
81
|
+
if (html.parameters !== json.parameters) return html.parameters > json.parameters ? 'html' : 'json';
|
|
82
|
+
return html.order === json.order ? routeDefault : html.order < json.order ? 'html' : 'json';
|
|
83
|
+
}
|
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
import { rejectedResponse } from './rejected-response.mjs';
|
|
2
2
|
import { identitySettingsPage } from './accounts/settings-page.mjs';
|
|
3
|
+
import { settingsRefusalPage } from './page.mjs';
|
|
4
|
+
import { representation } from './representation.mjs';
|
|
3
5
|
import { confirmationResponse } from './identity/email-link/confirmation.mjs';
|
|
4
6
|
import { createOidcHandler } from './identity/oidc/index.mjs';
|
|
5
7
|
import { AuthError } from './errors.mjs';
|
|
@@ -64,7 +66,12 @@ export function createAuthHandler(auth) {
|
|
|
64
66
|
if (auth.scheduledPolicies && path === '/auth/policy-runs' && request.method === 'GET') return json(await auth.scheduledPolicies.runs(request));
|
|
65
67
|
if (path === '/auth/sessions' && request.method === 'GET') return json(await auth.listSessions(request, readParameters(url.searchParams).operatorId));
|
|
66
68
|
if (path === '/auth/domain-rules' && request.method === 'GET') return json(await auth.listDomainRules(request));
|
|
67
|
-
if (path === '/auth/settings/identity' &&
|
|
69
|
+
if (path === '/auth/settings/identity' && ['GET', 'HEAD'].includes(request.method)) {
|
|
70
|
+
const value = await auth.identitySettings(request);
|
|
71
|
+
return representation(request) === 'html'
|
|
72
|
+
? await identitySettingsPage(value, auth.presentation, { saved: url.searchParams.get('saved') === '1', method: request.method })
|
|
73
|
+
: request.method === 'HEAD' ? new Response(null, { headers: { ...securityHeaders(), 'content-type': 'application/json' } }) : json(value);
|
|
74
|
+
}
|
|
68
75
|
if (path === '/auth/providers' && request.method === 'GET') return json(await auth.listProviders(request));
|
|
69
76
|
const posts = new Set(['/auth/bootstrap', '/auth/magic-link', '/auth/confirm', '/auth/signout', '/auth/invite', '/auth/invite/cancel', '/auth/review-session',
|
|
70
77
|
'/auth/policy/approve', '/auth/policy/revoke', '/auth/operator/status', '/auth/operator/role', '/auth/session/revoke', '/auth/step-up', '/auth/domain-rules', '/auth/providers/unlink', '/auth/providers/relink', '/auth/settings/identity', '/auth/setup-complete', OWNER_SIGNIN_PATH, '/auth/email', '/auth/display-name']);
|
|
@@ -80,7 +87,10 @@ export function createAuthHandler(auth) {
|
|
|
80
87
|
switch (path) {
|
|
81
88
|
case '/auth/policy/approve': return json(await auth.scheduledPolicies.approve(request, input));
|
|
82
89
|
case '/auth/policy/revoke': await auth.scheduledPolicies.revoke(request, input); return json({ revoked: true });
|
|
83
|
-
case '/auth/settings/identity': {
|
|
90
|
+
case '/auth/settings/identity': {
|
|
91
|
+
const value = await auth.setIdentitySettings(request, formPost(request) ? { idpSignIn: input.idpSignIn === 'true' } : input);
|
|
92
|
+
return formPost(request) ? seeOther('/auth/settings/identity?saved=1') : json(value);
|
|
93
|
+
}
|
|
84
94
|
case '/auth/setup-complete': return json(await auth.completeLocalSetup(request));
|
|
85
95
|
case '/auth/bootstrap': { const value = await auth.bootstrap(request, input); return seeOther(value.returnTo, value.cookie); }
|
|
86
96
|
case '/auth/magic-link': {
|
|
@@ -118,6 +128,15 @@ export function createAuthHandler(auth) {
|
|
|
118
128
|
case '/auth/providers/relink': return json(await auth.relinkProvider(request, input));
|
|
119
129
|
}
|
|
120
130
|
} catch (error) {
|
|
131
|
+
if (path === '/auth/settings/identity' && representation(request) === 'html') {
|
|
132
|
+
const code = error instanceof AuthError ? error.code : 'auth_unavailable';
|
|
133
|
+
const status = error instanceof AuthError ? error.status : 503;
|
|
134
|
+
if (code === 'provider_unlinked') {
|
|
135
|
+
try { return await identitySettingsPage(await auth.identitySettings(request), auth.presentation, { error: code, status, method: request.method }); }
|
|
136
|
+
catch { return settingsRefusalPage(request, auth.presentation, 'auth_unavailable', 503); }
|
|
137
|
+
}
|
|
138
|
+
return settingsRefusalPage(request, auth.presentation, code, status);
|
|
139
|
+
}
|
|
121
140
|
if (form) return seeOther(notice(auth, error instanceof AuthError ? error.code : 'auth_unavailable'));
|
|
122
141
|
if (error instanceof AuthError) return json({ error: error.code }, error.status);
|
|
123
142
|
// Do not put SQL, email addresses, setup tokens or provider exceptions on the wire.
|
|
@@ -126,6 +145,10 @@ export function createAuthHandler(auth) {
|
|
|
126
145
|
};
|
|
127
146
|
return async function(request) {
|
|
128
147
|
const response = await handleAuth(request);
|
|
148
|
+
if (response && request.method === 'HEAD' && new URL(request.url).pathname === '/auth/settings/identity') {
|
|
149
|
+
void response.body?.cancel();
|
|
150
|
+
return rejectedResponse(request, new Response(null, { status: response.status, headers: response.headers }));
|
|
151
|
+
}
|
|
129
152
|
return response ? rejectedResponse(request, response) : response;
|
|
130
153
|
};
|
|
131
154
|
}
|
|
@@ -31,7 +31,8 @@ payload consumers, and prove the integrated round trip.
|
|
|
31
31
|
## Policy
|
|
32
32
|
|
|
33
33
|
`await createPortability({db, buckets, policy})` validates policy and the current
|
|
34
|
-
schema at construction. Apply the bookkeeping migration first.
|
|
34
|
+
schema at construction. Apply the bookkeeping migration first. Export refuses
|
|
35
|
+
with HTTP 409 `import_applying` while an import is applying. Policy fields:
|
|
35
36
|
|
|
36
37
|
| Field | Meaning |
|
|
37
38
|
| --- | --- |
|
|
@@ -47,7 +48,7 @@ schema at construction. Apply the bookkeeping migration first. Policy fields:
|
|
|
47
48
|
| `requiredConfiguration` | Setting names returned after restore for the owner to re-enter. |
|
|
48
49
|
| `validateStaged({db,tables,read,manifest})` | Read-only product validation before any object write; false or a thrown error refuses restore. `tables` names the staging tables for bounded SQL queries; `read` is a convenience for small tables. |
|
|
49
50
|
| `acceptedSchemas` | Map from an earlier full schema hash to `{upgradeRow(table,row)}`. Upgrade functions are pure and synchronous. No older schema is accepted by default. |
|
|
50
|
-
| `buckets` | Binding names mapped to `{prefix?,objectReferences?:{table,keyColumn}}`. Prefixes isolate an object namespace. References require exactly the distinct referenced keys
|
|
51
|
+
| `buckets` | Binding names mapped to `{prefix?,objectReferences?:{table,keyColumn}}`. Prefixes isolate an object namespace. References require exactly the distinct referenced keys: export checks its snapshot, and restore its staged rows (after `importRow`, before any object write), that each key is nonempty, under the prefix, not absolute and without `.` or `..` segments, and that the keys are exactly the bucket's archived objects, none missing and none extra; otherwise `object_references:<binding>`. |
|
|
51
52
|
| `extraPayloads` | Named `{produce,consume}` pairs. Producers return bytes or a byte stream. Consumers receive a verified byte stream and descriptor before D1 commit. |
|
|
52
53
|
| `partBytes` | Maximum part size, default 16 MiB, configurable from 1 KiB to 16 MiB. |
|
|
53
54
|
|
|
@@ -107,7 +108,8 @@ and declaring both `policy` and `policyModule` fail closed. Serializable object
|
|
|
107
108
|
schema-upgrade and payload callbacks) are refused instead of being dropped by
|
|
108
109
|
JSON transport. Container transfer currently refuses `policyModule` explicitly;
|
|
109
110
|
it continues to accept serializable object policies. Malformed primary or later
|
|
110
|
-
journal JSON is reported once as `
|
|
111
|
+
journal JSON is reported once as `invalid_archive` with exit 2 and a re-export
|
|
112
|
+
recovery command, before local data
|
|
111
113
|
migrations, without echoing archive content or parser diagnostics.
|
|
112
114
|
|
|
113
115
|
An installation migrated by Platform has `rer_migrations(name, sha256,
|
|
@@ -207,7 +209,7 @@ RER's native `localPortability` invokes this same entry point in its
|
|
|
207
209
|
history again for transfer.
|
|
208
210
|
|
|
209
211
|
The implementation ships in the RER CLI, not the Worker portability module.
|
|
210
|
-
Package-mode seeds need a
|
|
212
|
+
Package-mode seeds need a pinned `@runeditrun/rer` release containing this
|
|
211
213
|
export and an exact pin/lock update; version 0.2.77 does not expose it.
|
|
212
214
|
Vendored seeds need a reviewed Base CLI-copy refresh with
|
|
213
215
|
`node scripts/export-cli.mjs SEED_ROOT` from the selected Base source and their
|
|
@@ -256,7 +258,7 @@ secret base cells become NULL before generated snapshots are normalized. A
|
|
|
256
258
|
generated column cannot itself be redacted: exclude the table or change the
|
|
257
259
|
product schema/policy if its computed value must be secret. Ordinary-column
|
|
258
260
|
metadata and row encoding are unchanged. Ordinary-table archives using the
|
|
259
|
-
|
|
261
|
+
v2 integrity-checked format continue to import into the same schema; see the version
|
|
260
262
|
1 refusal below.
|
|
261
263
|
|
|
262
264
|
An archived INTEGER-affinity base cell outside JavaScript's safe integer range fails before
|
|
@@ -330,6 +332,59 @@ parts also preserve the original rowid, including gaps, for restoration and
|
|
|
330
332
|
erasure matching. Export time,
|
|
331
333
|
source identity, job ids and boundaries do not affect equivalence.
|
|
332
334
|
|
|
335
|
+
Schema identity covers table definitions (including constraints and generated
|
|
336
|
+
expressions), explicit indexes (including uniqueness and partial predicates),
|
|
337
|
+
views and triggers. Indexes and triggers on excluded tables are excluded with
|
|
338
|
+
their tables, including platform bookkeeping tables. The complete additional
|
|
339
|
+
definitions travel in `schemaObjects`. A single SQLite lexer reads the original
|
|
340
|
+
SQL before removing trivia. Identity records each token's class and canonical
|
|
341
|
+
value, plus a deterministic spelling: a single space separates neighbours
|
|
342
|
+
exactly when concatenating them would change their tokenization. The complete
|
|
343
|
+
joined stream is tokenized again and must reproduce those same tokens.
|
|
344
|
+
Comments and harmless inter-token whitespace therefore compare equally, while
|
|
345
|
+
`x '31'` and `x'31'`, separated minus signs, numbers and combined operators stay
|
|
346
|
+
distinct. No regular expression rewrites SQL.
|
|
347
|
+
|
|
348
|
+
Quoted identifiers (double quotes, backticks, brackets, or a single-quoted string
|
|
349
|
+
used as a name), strings, blobs, numbers, variables and ordinary identifiers keep
|
|
350
|
+
their exact spelling. Only unambiguous reserved keywords fold ASCII case.
|
|
351
|
+
Fallback/JOIN/window keywords retain their spelling: for example, SQLite can
|
|
352
|
+
interpret `DEFAULT ABORT` and `DEFAULT abort` as different string defaults.
|
|
353
|
+
Unicode case and quote styles are never normalized. The lexer follows SQLite's
|
|
354
|
+
[`sqlite3GetToken`](https://github.com/sqlite/sqlite/blob/master/src/tokenize.c)
|
|
355
|
+
and the keyword fallback rules in
|
|
356
|
+
[`parse.y`](https://github.com/sqlite/sqlite/blob/master/src/parse.y).
|
|
357
|
+
The original DDL is authenticated by `archiveHash`.
|
|
358
|
+
|
|
359
|
+
Unsupported or ambiguous SQL refuses with `schema_mismatch:unsupported_sql` and
|
|
360
|
+
the existing original-installation re-export/recovery instructions; it never
|
|
361
|
+
receives a shared or empty identity. This includes NUL and ill-formed Unicode,
|
|
362
|
+
BOM at token start, vertical-tab whitespace, unterminated quotes/brackets/block
|
|
363
|
+
comments (including bare `/*`), malformed/odd-length blobs, invalid numeric
|
|
364
|
+
separators or numeric identifier suffixes, malformed named/Tcl variables, and
|
|
365
|
+
unknown ASCII punctuation. Valid decimal/hex/exponent/underscore numbers,
|
|
366
|
+
SQLite's longest operators, question-mark and named/Tcl variables, LF/EOF line
|
|
367
|
+
comments, and terminated block comments are recognized. SQLite may accept some
|
|
368
|
+
of the refused whitespace or EOF-comment forms; re-export with supported DDL
|
|
369
|
+
rather than silently discarding their bytes. Virtual tables remain unsupported.
|
|
370
|
+
|
|
371
|
+
Older version 2 archives first verify their original raw-DDL, table-token, or
|
|
372
|
+
complete-object-token schema checksum and complete archive proof. A read-only
|
|
373
|
+
historical checksum codec projects the real lexer's tokens into those old wire
|
|
374
|
+
encodings. Its lossy result never decides compatibility: complete definitions
|
|
375
|
+
are always compared through the new typed token identity. Compatibility then uses
|
|
376
|
+
any complete `schemaObjects` inventory the archive carries, without rewriting
|
|
377
|
+
its authenticated manifest or parts. Without that inventory, the original
|
|
378
|
+
presence or absence of indexes, views and triggers cannot be established from
|
|
379
|
+
table SQL or migration names alone, even when the destination has no such
|
|
380
|
+
objects. Normal restore refuses `schema_mismatch` and the CLI gives the existing
|
|
381
|
+
re-export recovery instructions. This also catches removal of an old object's
|
|
382
|
+
definition rather than assuming missing evidence means the objects match.
|
|
383
|
+
Re-export from the original installation to carry the complete definitions.
|
|
384
|
+
A new archive explicitly records even an empty object inventory, so later
|
|
385
|
+
additive migrations can still be placed normally. The separate acknowledged
|
|
386
|
+
v1 recovery path remains an explicitly unverified recovery, as documented below.
|
|
387
|
+
|
|
333
388
|
New exports use archive version 2. The closing `archiveHash` is SHA-256 of
|
|
334
389
|
canonical JSON `{manifest,parts}`, where `manifest` is the complete manifest
|
|
335
390
|
and `parts` is the ordered array of verified part SHA-256 strings. This binds
|
|
@@ -468,7 +523,9 @@ a view resolved from applied migration history), selects `*` or
|
|
|
468
523
|
a secret column, when a trigger on it or a foreign key action from it reaches a
|
|
469
524
|
catalogue table, when any other later statement writes it (directly or through
|
|
470
525
|
a trigger) or changes its columns other than by `ADD COLUMN` (allowed only when
|
|
471
|
-
every replay names its columns),
|
|
526
|
+
every replay names its columns), when the statement cannot be bounded, or when
|
|
527
|
+
it writes a table that a bucket's `objectReferences` names: references are
|
|
528
|
+
checked on the staged rows, before any object write.
|
|
472
529
|
A failed or interrupted apply leaves no replayed row; finalizing again replays
|
|
473
530
|
from the start.
|
|
474
531
|
Read-only trigger bodies, including RAISE-only guards, add no write dependency.
|
|
@@ -612,8 +669,8 @@ part; different content returns 409. Reassembled bytes pass the part checksum
|
|
|
612
669
|
before completion advances. Staged data stays private until finalization.
|
|
613
670
|
Finalize requires the closing `archiveHash` and verifies all
|
|
614
671
|
parts, semantic contents, destination
|
|
615
|
-
emptiness, erasure and product invariants. R2 ownership
|
|
616
|
-
empty-namespace check. Verified object writes precede one atomic D1 batch, with
|
|
672
|
+
emptiness, erasure, object references and product invariants. R2 ownership
|
|
673
|
+
is recorded after the empty-namespace check. Verified object writes precede one atomic D1 batch, with
|
|
617
674
|
deferred FKs, trigger suppression/recreation, replaceable rows, staged inserts,
|
|
618
675
|
journal adoption and completion. FK cycles and self-references restore together.
|
|
619
676
|
An unsuccessful commit removes only owned objects. A lost commit acknowledgement
|
|
@@ -640,7 +697,10 @@ by every upload/finalize request, then every 30 seconds while that request is
|
|
|
640
697
|
working (including verification, R2 transfers and product callbacks). This is a
|
|
641
698
|
recovery window, not a total restore limit: the measured 20,000-row round trip
|
|
642
699
|
below already takes 171 seconds, and callbacks have no wall-time bound. A fresh
|
|
643
|
-
import returns `409 import_running
|
|
700
|
+
import returns `409 import_running`. Only a validated internal lease refusal,
|
|
701
|
+
after authorization and for the actor who owns that import, includes `jobId`
|
|
702
|
+
and ISO `abortableAt`. Authorization failures and other actors receive no job or
|
|
703
|
+
lease metadata, even if an error carries the same code or fields. The owner
|
|
644
704
|
can find its active job with `GET /api/portability/imports` (`{active:status}`
|
|
645
705
|
or `{active:null}`), and inspect `GET .../imports/:id`: `abortableAt` and `stale` describe
|
|
646
706
|
its lease. The existing authenticated `POST .../imports/:id/abort` (or CLI
|
|
@@ -671,6 +731,17 @@ returns an async dispatcher, or null for foreign paths. Default base path is
|
|
|
671
731
|
an actor id or null. Jobs belong to their creating actor. Unsafe methods require
|
|
672
732
|
the exact configured Origin. Responses have `Cache-Control: no-store` and errors
|
|
673
733
|
are `{error:code}`. Request bodies are bounded; `maxBodyBytes` can lower the bound.
|
|
734
|
+
Typed authorization errors retain their HTTP status, directly or through a
|
|
735
|
+
bounded, cycle-safe `cause` chain. Codes come from the existing fixed public
|
|
736
|
+
vocabulary (for example `403 step_up_required`); unknown codes become
|
|
737
|
+
`operation_failed`. Portability diagnostics expose the known code prefix only,
|
|
738
|
+
never caller-supplied suffixes. Aggregate error collections are not authority
|
|
739
|
+
wrappers. Native diagnostics remain private:
|
|
740
|
+
transient D1 availability failures use `503 operation_failed`, transport timeouts
|
|
741
|
+
use `504 operation_failed`, and unexpected faults remain `500 operation_failed`.
|
|
742
|
+
The hosted CLI aborts a completed export after saving and verifying the local
|
|
743
|
+
archive, releasing its snapshots and parts. Interrupted downloads retain their
|
|
744
|
+
job for resume.
|
|
674
745
|
|
|
675
746
|
| Method / relative path | Action |
|
|
676
747
|
| --- | --- |
|
|
@@ -749,13 +820,14 @@ product erasure or remove product objects; the product handles attachments and
|
|
|
749
820
|
any extra state. Do not retain a previously downloaded archive as erasure proof;
|
|
750
821
|
supply the owner's later journal when restoring old offline data.
|
|
751
822
|
|
|
752
|
-
##
|
|
823
|
+
## Historical measured capacity
|
|
753
824
|
|
|
754
|
-
|
|
755
|
-
five tables in 171.169 seconds, with 625 parts at a configured 4 KiB bound and a
|
|
825
|
+
An unverified historical local Miniflare D1/R2 report recorded export, restore
|
|
826
|
+
and export of 20,000 rows across five tables in 171.169 seconds, with 625 parts at a configured 4 KiB bound and a
|
|
756
827
|
1,979,471-byte archive. Restore used one atomic D1 commit. The verification run
|
|
757
|
-
|
|
758
|
-
|
|
828
|
+
can produce fresh measurements; no corresponding historical benchmark receipt
|
|
829
|
+
is retained in this repository. These figures do not establish current production
|
|
830
|
+
capacity. Real D1 CPU, query, statement and storage limits, object mutation,
|
|
759
831
|
destination writers and extra-payload recovery need product-level verification.
|
|
760
832
|
|
|
761
833
|
## Adopting
|
|
@@ -772,7 +844,11 @@ modules/portability/index.d.mts
|
|
|
772
844
|
modules/portability/lines.mjs
|
|
773
845
|
modules/portability/hash.mjs
|
|
774
846
|
modules/portability/format.mjs
|
|
847
|
+
modules/portability/references.mjs
|
|
775
848
|
modules/portability/catalogue.mjs
|
|
849
|
+
modules/portability/schema-identity.mjs
|
|
850
|
+
modules/portability/schema-legacy.mjs
|
|
851
|
+
modules/portability/sqlite-tokens.mjs
|
|
776
852
|
modules/portability/bookkeeping.mjs
|
|
777
853
|
modules/portability/storage.mjs
|
|
778
854
|
modules/portability/copies.mjs
|
|
@@ -798,6 +874,7 @@ modules/portability/import-safety.mjs
|
|
|
798
874
|
modules/portability/payloads.mjs
|
|
799
875
|
modules/portability/journal.mjs
|
|
800
876
|
modules/portability/http.mjs
|
|
877
|
+
modules/portability/http-errors.mjs
|
|
801
878
|
modules/portability/http-client.mjs
|
|
802
879
|
modules/portability/cli.mjs
|
|
803
880
|
modules/portability/cli-options.mjs
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { schemaIdentityOf } from './schema-identity.mjs';
|
|
2
2
|
import { insist } from './storage.mjs';
|
|
3
3
|
|
|
4
4
|
// Installation authority and transient working state, never product data.
|
|
@@ -23,8 +23,8 @@ export async function restorableManifest(manifest) {
|
|
|
23
23
|
for(const table of manifest.tables)if(identifier(table.name)==='ik_d1_assertions')
|
|
24
24
|
insist(table.count===0,'invalid_bookkeeping_rows:'+table.name,409);
|
|
25
25
|
const tables=manifest.tables.filter(table=>!platformTable(table.name));
|
|
26
|
-
|
|
27
|
-
return {...manifest,tables,schema:await
|
|
26
|
+
const schemaObjects=manifest.schemaObjects?.filter(object=>!manifest.tables.some(table=>platformTable(table.name) && table.name===object.table));
|
|
27
|
+
return {...manifest,tables,...(schemaObjects?{schemaObjects}:{}),schema:await schemaIdentityOf(tables,schemaObjects),
|
|
28
28
|
excludedColumns:Object.fromEntries(Object.entries(manifest.excludedColumns).filter(([name])=>!platformTable(name)))};
|
|
29
29
|
}
|
|
30
30
|
export const skippedTablesLine=tables=>'Skipped platform bookkeeping tables: '+tables.join(', ')+'.';
|
|
@@ -4,6 +4,7 @@ import { tokens, identifier, folded } from './generated.mjs';
|
|
|
4
4
|
import { laterData } from './backfills.mjs';
|
|
5
5
|
import { destinationLedger, matchMigrationLedger } from './ledgers.mjs';
|
|
6
6
|
import { checkRebuild } from './rebuilds.mjs';
|
|
7
|
+
import { schemaSQL } from './schema-identity.mjs';
|
|
7
8
|
|
|
8
9
|
// Applied product migrations in the ledger's own order (ledgers.mjs). A
|
|
9
10
|
// state-bearing ledger may hold an unacknowledged migration whose effects are
|
|
@@ -38,7 +39,7 @@ async function placement(db,manifest,cat,replay,ledger,history) {
|
|
|
38
39
|
unapplied=applied.filter(name=>!earlier.includes(name));
|
|
39
40
|
}else{
|
|
40
41
|
// Unrecorded: the latest matching prefix. Migrations that change no archived
|
|
41
|
-
//
|
|
42
|
+
// definition (excluded tables or data-only changes) cannot be told apart
|
|
42
43
|
// and count as later. The source may have stopped at any matching prefix,
|
|
43
44
|
// so every migration after the earliest one may be missing from its rows.
|
|
44
45
|
const matching=[];
|
|
@@ -62,7 +63,7 @@ async function placement(db,manifest,cat,replay,ledger,history) {
|
|
|
62
63
|
return {applied,earlier,replays,rebuilt};
|
|
63
64
|
}
|
|
64
65
|
// An archive with the destination's schema identity can still predate later
|
|
65
|
-
// migrations that changed only data (a purge or a backfill)
|
|
66
|
+
// migrations that changed only data (a purge or a backfill).
|
|
66
67
|
// Those that change restorable data are refused like any other. Without a
|
|
67
68
|
// replayer the statements cannot be read, so an archive whose recorded
|
|
68
69
|
// migrations the destination's ledger has outgrown is refused. An archive that
|
|
@@ -82,7 +83,7 @@ export async function sameSchemaMigrations(db,manifest,cat,migrations,history) {
|
|
|
82
83
|
}
|
|
83
84
|
// An archive made before later migrations restores only when those migrations
|
|
84
85
|
// are additive, or a proven row-preserving CHECK rebuild: new tables, new
|
|
85
|
-
// columns that take a default, or
|
|
86
|
+
// columns that take a default, or later schema objects established by replay.
|
|
86
87
|
// Anything else stays a schema_mismatch refusal.
|
|
87
88
|
export async function additiveBridge(db,manifest,cat,migrations,history) {
|
|
88
89
|
insist(migrations,'schema_mismatch');
|
|
@@ -95,7 +96,7 @@ export async function additiveBridge(db,manifest,cat,migrations,history) {
|
|
|
95
96
|
const kept=target.columns.slice(0,table.columns.length),added=target.columns.slice(table.columns.length);
|
|
96
97
|
insist(canonicalJson(kept)===canonicalJson(table.columns) && table.rowidAlias===target.rowidAlias && table.dependencies.every(d=>target.dependencies.includes(d)),'schema_mismatch:columns'+label);
|
|
97
98
|
const rebuiltChecks=rebuilt.has(folded(table.name)) && checkRebuild(table.sql,target.sql);
|
|
98
|
-
if(!added.length){insist(table.sql===target.sql || rebuiltChecks,'schema_mismatch:definition'+label);continue;}
|
|
99
|
+
if(!added.length){insist(schemaSQL(table.sql)===schemaSQL(target.sql) || rebuiltChecks,'schema_mismatch:definition'+label);continue;}
|
|
99
100
|
const declared=await all(db,`PRAGMA table_xinfo(${quote(table.name)})`);
|
|
100
101
|
insist(added.every(c=>!c.pk && !c.generated && (!c.notnull || declared.find(d=>d.name===c.name)?.dflt_value!=null)),'schema_mismatch:column_default'+label);
|
|
101
102
|
insist(appended(table.sql,target.sql,added.map(c=>c.name)),'schema_mismatch:definition'+label);
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { all, insist, quote, PortabilityError } from './storage.mjs';
|
|
2
|
-
import { canonicalJson,
|
|
2
|
+
import { canonicalJson, base64, unbase64, utf8 } from './hash.mjs';
|
|
3
|
+
import { schemaIdentityOf } from './schema-identity.mjs';
|
|
3
4
|
import { platformTables, platformTable, identifier, matches } from './bookkeeping.mjs';
|
|
4
5
|
/** Platform bookkeeping is mandatory; operator identity remains a seed choice. */
|
|
5
6
|
export const excludedTables=Object.freeze([...platformTables,'oa_%','oc_%']);
|
|
@@ -57,14 +58,17 @@ async function archived(db,config) {
|
|
|
57
58
|
const derived=config.derivedTables;
|
|
58
59
|
const excluded=objects.filter(x=>x.type==='table' && (excludedTable(x.name,config) || (!config.includeTables.includes(x.name) &&
|
|
59
60
|
derived.some(p=>x.name===p || (kinds.find(k=>k.name===x.name)?.type==='shadow' && x.name.startsWith(p+'_'))))));
|
|
60
|
-
objects.sort((a,b)=>a.name<b.name?-1:a.name>b.name?1:0);
|
|
61
|
-
|
|
61
|
+
objects.sort((a,b)=>a.name<b.name?-1:a.name>b.name?1:a.type<b.type?-1:a.type>b.type?1:0);
|
|
62
|
+
const schemaObjects=objects.filter(x=>['index','view','trigger'].includes(x.type)
|
|
63
|
+
&& !excluded.some(t=>t.name===x.tbl_name)).map(({name,type,tbl_name,sql})=>({name,type,table:tbl_name,sql}));
|
|
64
|
+
return {objects,kinds,excluded,schemaObjects,entries:objects.filter(x=>x.type==='table' && !excluded.some(y=>y.name===x.name))};
|
|
62
65
|
}
|
|
63
|
-
const identityOf=tables=>sha256(canonicalJson(tables.map(({name,sql})=>({name,sql}))));
|
|
64
66
|
// The archive schema identity alone, for databases that need not satisfy the policy.
|
|
65
|
-
export async function schemaIdentity(db,config) {
|
|
67
|
+
export async function schemaIdentity(db,config) {
|
|
68
|
+
const {entries,schemaObjects}=await archived(db,config);return schemaIdentityOf(entries,schemaObjects);
|
|
69
|
+
}
|
|
66
70
|
export async function catalogue(db,config) {
|
|
67
|
-
const {objects,kinds,excluded,entries}=await archived(db,config);
|
|
71
|
+
const {objects,kinds,excluded,entries,schemaObjects}=await archived(db,config);
|
|
68
72
|
const tables=[];
|
|
69
73
|
for(const entry of entries) {
|
|
70
74
|
insist(!/^CREATE\s+VIRTUAL\s+TABLE/i.test(entry.sql) && kinds.find(x=>x.name===entry.name)?.type!=='shadow','virtual_table:'+entry.name);
|
|
@@ -99,8 +103,8 @@ export async function catalogue(db,config) {
|
|
|
99
103
|
insist(tables.find(x=>x.name===table)?.columns.some(c=>c.name===keyColumn),'reference_column_missing');
|
|
100
104
|
insist(!config.secretColumns[table]?.includes(keyColumn),'reference_column_excluded');
|
|
101
105
|
}
|
|
102
|
-
const identity=await
|
|
103
|
-
return {identity,tables,excludedTables:excluded.map(x=>x.name).sort(),triggers:objects.filter(x=>x.type==='trigger' && tables.some(t=>t.name===x.tbl_name))};
|
|
106
|
+
const identity=await schemaIdentityOf(tables,schemaObjects);
|
|
107
|
+
return {identity,tables,schemaObjects,excludedTables:excluded.map(x=>x.name).sort(),triggers:objects.filter(x=>x.type==='trigger' && tables.some(t=>t.name===x.tbl_name))};
|
|
104
108
|
}
|
|
105
109
|
export function affinity(type) {
|
|
106
110
|
const text=type.toUpperCase();
|
|
@@ -24,7 +24,11 @@ export async function main(args) {
|
|
|
24
24
|
const records=await client.download(id),file=await privateFile(opts.out+'.partial',Boolean(opts.resume));
|
|
25
25
|
try{for await(const record of records)await file.writeFile(canonicalJson(record)+'\n');await file.sync();}
|
|
26
26
|
finally{await file.close();}
|
|
27
|
-
await promote(opts.out+'.partial',opts.out);
|
|
27
|
+
await promote(opts.out+'.partial',opts.out);
|
|
28
|
+
// The verified local archive is durable: acknowledge it and release the
|
|
29
|
+
// hosted job's snapshots/parts. Failed downloads keep their resumable job.
|
|
30
|
+
await get('exports/'+id+'/abort','POST',{});
|
|
31
|
+
process.stdout.write('Export complete.\n');return;
|
|
28
32
|
}
|
|
29
33
|
if(opts.command==='import') {
|
|
30
34
|
const source=Readable.toWeb(createReadStream(opts.from));let id=opts.resume,closing;
|
|
@@ -1,5 +1,6 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { statement, state, guard, job, save, partStatements, parts, readPart, cleanCopies, quote, stream, insist, run } from './storage.mjs';
|
|
2
2
|
import { catalogue, encodedRow } from './catalogue.mjs';
|
|
3
|
+
import { checkReferences } from './references.mjs';
|
|
3
4
|
import { createCopy,readCopy,normalizeCopy } from './copies.mjs';
|
|
4
5
|
import { canonicalJson, utf8, base64, StreamHash } from './hash.mjs';
|
|
5
6
|
import { FORMAT, makePart, checkPart, ContentVerifier, ArchiveVerifier } from './format.mjs';
|
|
@@ -23,7 +24,7 @@ export async function createExport(context,actor='direct') {
|
|
|
23
24
|
objects.sort((a,b)=>a.bucket<b.bucket?-1:a.bucket>b.bucket?1:a.key<b.key?-1:a.key>b.key?1:0);
|
|
24
25
|
const copies=Object.fromEntries(cat.tables.map((t,i)=>[t.name,'rer_portability_snap_'+id.replaceAll('-','')+'_'+i]));
|
|
25
26
|
const migrations=await appliedMigrations(db);
|
|
26
|
-
const manifest={kind:'manifest',format:FORMAT,version:2,product:config.product,exportedAt:new Date().toISOString(),sourceIdentity:s.source_identity,schema:cat.identity,tables:cat.tables,objects,payloads:[],excludedTables:cat.excludedTables,excludedColumns:config.secretColumns,requiredConfiguration:config.requiredConfiguration,partCount:0,epoch:s.epoch,journal:await journal(db,s.source_identity),...(migrations?{migrations}:{})};
|
|
27
|
+
const manifest={kind:'manifest',format:FORMAT,version:2,product:config.product,exportedAt:new Date().toISOString(),sourceIdentity:s.source_identity,schema:cat.identity,tables:cat.tables,schemaObjects:cat.schemaObjects,objects,payloads:[],excludedTables:cat.excludedTables,excludedColumns:config.secretColumns,requiredConfiguration:config.requiredConfiguration,partCount:0,epoch:s.epoch,journal:await journal(db,s.source_identity),...(migrations?{migrations}:{})};
|
|
27
28
|
const row={id,kind:'export',actor,phase:'building',completed:0,metadata:{manifest,copies,cursor:{table:0,row:0,binary:0,offset:0},partBytes:config.partBytes}};
|
|
28
29
|
await db.batch([guard(db,'EXISTS(SELECT 1 FROM rer_portability_state WHERE id=1 AND applying=0 AND source_identity=? AND epoch=?)',s.source_identity,s.epoch),
|
|
29
30
|
...cat.tables.flatMap(t=>createCopy(db,t,copies[t.name],true)),statement(db,'INSERT INTO rer_portability_jobs(id,kind,actor,phase,metadata) VALUES(?,?,?,?,?)',id,'export',actor,row.phase,canonicalJson(row.metadata)),db.prepare('DELETE FROM rer_portability_guard')]);
|
|
@@ -36,17 +37,10 @@ export async function createExport(context,actor='direct') {
|
|
|
36
37
|
const descriptor=await capturePayload(db,id,manifest.payloads.length,await producer.produce());manifest.payloads.push({name,...descriptor});
|
|
37
38
|
}
|
|
38
39
|
await run(db,'UPDATE rer_portability_jobs SET metadata=? WHERE id=?',canonicalJson(row.metadata),id);
|
|
39
|
-
await checkReferences(context,
|
|
40
|
+
await checkReferences(context,copies,objects);
|
|
40
41
|
return id;
|
|
41
42
|
}catch(e){await abortExport(context,id);throw e;}
|
|
42
43
|
}
|
|
43
|
-
async function checkReferences({db,config},row) {
|
|
44
|
-
for(const [binding,spec] of Object.entries(config.buckets)) if(spec.objectReferences) {
|
|
45
|
-
const {table,keyColumn}=spec.objectReferences,keys=new Set((await all(db,`SELECT DISTINCT ${quote(keyColumn)} AS key FROM ${quote(row.metadata.copies[table])} WHERE ${quote(keyColumn)} IS NOT NULL`)).map(x=>x.key));
|
|
46
|
-
const objects=row.metadata.manifest.objects.filter(x=>x.bucket===binding);
|
|
47
|
-
insist(objects.length===keys.size && objects.every(x=>keys.has(x.key)),'object_references:'+binding);
|
|
48
|
-
}
|
|
49
|
-
}
|
|
50
44
|
async function nextItems(context,row) {
|
|
51
45
|
const {db,config,buckets}=context,{manifest,cursor,copies,partBytes}=row.metadata;
|
|
52
46
|
while(cursor.table<manifest.tables.length && cursor.row===manifest.tables[cursor.table].count){cursor.table++;cursor.row=0;}
|
|
@@ -106,7 +100,7 @@ export async function advanceExport(context,id,{expectedPart,maxParts=1}={}) {
|
|
|
106
100
|
if(expectedPart!==undefined)expectedPart++;
|
|
107
101
|
}else{
|
|
108
102
|
// References are checked again against the archived, normalized snapshot.
|
|
109
|
-
await checkReferences(context,row);
|
|
103
|
+
await checkReferences(context,row.metadata.copies,row.metadata.manifest.objects);
|
|
110
104
|
row.metadata.manifest.partCount=row.completed;
|
|
111
105
|
const verifier=new ContentVerifier(row.metadata.manifest),archive=new ArchiveVerifier(row.metadata.manifest);
|
|
112
106
|
for await(const part of parts(context.db,id,row.completed)){archive.add(part);for(const item of part.items)verifier.add(item);}
|
|
@@ -2,6 +2,8 @@ import { archiveLines } from './lines.mjs';
|
|
|
2
2
|
import { canonicalJson, sha256, StreamHash, unbase64, utf8 } from './hash.mjs';
|
|
3
3
|
import { insist, PortabilityError } from './storage.mjs';
|
|
4
4
|
import { validateRow } from './catalogue.mjs';
|
|
5
|
+
import { schemaIdentityOf, rawSchemaIdentityOf, tableSchemaIdentityOf } from './schema-identity.mjs';
|
|
6
|
+
import { legacySchemaIdentities } from './schema-legacy.mjs';
|
|
5
7
|
export const FORMAT='runeditrun-portability';
|
|
6
8
|
export const MAX_LINE_BYTES=24*1024*1024;
|
|
7
9
|
export const MAX_PART_BYTES=16*1024*1024;
|
|
@@ -36,8 +38,12 @@ async function checkManifestFields(m) {
|
|
|
36
38
|
insist(Array.isArray(m.tables) && Array.isArray(m.objects) && Array.isArray(m.payloads) && Array.isArray(m.journal) && Array.isArray(m.requiredConfiguration),'manifest_inventory');
|
|
37
39
|
// Applied product migrations, recorded by archives since additive restore.
|
|
38
40
|
insist(m.migrations===undefined || (Array.isArray(m.migrations) && m.migrations.every(x=>typeof x==='string')),'manifest_inventory');
|
|
39
|
-
insist(m.schema===await sha256(canonicalJson(m.tables.map(({name,sql})=>({name,sql})))),'schema_checksum');
|
|
40
41
|
insist(m.tables.every((t,i)=>typeof t.name==='string' && typeof t.sql==='string' && Array.isArray(t.columns) && Array.isArray(t.dependencies) && Number.isSafeInteger(t.count) && t.count>=0 && (!i || m.tables[i-1].name<t.name)),'table_inventory');
|
|
42
|
+
insist(m.schemaObjects===undefined || (Array.isArray(m.schemaObjects) && m.schemaObjects.every((o,i)=>
|
|
43
|
+
o && typeof o.name==='string' && ['index','view','trigger'].includes(o.type) && typeof o.table==='string' && typeof o.sql==='string'
|
|
44
|
+
&& (!i || m.schemaObjects[i-1].name<o.name || (m.schemaObjects[i-1].name===o.name && m.schemaObjects[i-1].type<o.type)))),'manifest_inventory');
|
|
45
|
+
insist(m.schema===await schemaIdentityOf(m.tables,m.schemaObjects) || m.schema===await tableSchemaIdentityOf(m.tables)
|
|
46
|
+
|| m.schema===await rawSchemaIdentityOf(m.tables) || (await legacySchemaIdentities(m.tables,m.schemaObjects)).includes(m.schema),'schema_checksum');
|
|
41
47
|
for(const collection of [m.objects,m.payloads]) {
|
|
42
48
|
const keys=new Set();
|
|
43
49
|
for(const entry of collection) {
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
import { importRunningMetadata } from './import-lease.mjs';
|
|
2
|
+
|
|
3
|
+
// A code's spelling is not evidence that it is public. Keep the established
|
|
4
|
+
// vocabulary closed, and discard diagnostic suffixes containing schema/input.
|
|
5
|
+
const publicCodes = new Set(`
|
|
6
|
+
D1_RPC_TRANSPORT_LIMIT advance_limit archive_checksum archive_format archive_incomplete archive_json
|
|
7
|
+
binary_checksum binary_inventory binary_order binary_size body_too_large bucket_inventory bucket_not_empty
|
|
8
|
+
closing_missing column_inventory configuration_inventory confirmation_required content_checksum copy_inventory copy_rowid
|
|
9
|
+
data_after_closing dev_data erased_row erasure_epoch excluded_destination_dependents excluded_foreign_key
|
|
10
|
+
export_incomplete export_phase export_row_must_be_pure export_row_not_idempotent extra_part generated_sql
|
|
11
|
+
import_applying import_complete import_lease_expired import_phase import_row_columns import_row_key import_row_must_be_pure
|
|
12
|
+
import_row_not_deterministic import_row_not_idempotent import_row_undeclared import_row_unsafe_value import_running import_safety_table_missing
|
|
13
|
+
invalid_accepted_schema invalid_bookkeeping_rows invalid_bucket_policy invalid_bucket_prefix invalid_extra_payload
|
|
14
|
+
invalid_handler_configuration invalid_import_safety invalid_import_safety_column invalid_journal invalid_json invalid_object_references
|
|
15
|
+
invalid_part_limit invalid_policy invalid_policy_importSafety invalid_policy_importSafety_pair invalid_secret_column invalid_secret_columns
|
|
16
|
+
invalid_policy_excludeTables invalid_policy_includeTables invalid_policy_derivedTables invalid_policy_replaceableTables
|
|
17
|
+
invalid_policy_requiredConfiguration invalid_policy_exportRow invalid_policy_importRow invalid_policy_validateStaged
|
|
18
|
+
job_not_found later_journal ledger_inventory legacy_acknowledgement_required legacy_archive_unverifiable legacy_bucket_not_empty
|
|
19
|
+
legacy_output_exists legacy_output_required legacy_restored_export_failed legacy_target_not_empty legacy_v1_required live_policy_approval
|
|
20
|
+
manifest_counts manifest_identity manifest_inventory migration_ledger missing_bucket object_changed object_references
|
|
21
|
+
object_write_checksum object_write_failed origin_refused part_byte_limit part_checksum part_conflict part_count part_missing
|
|
22
|
+
part_order part_row_limit parts_missing payload_changed payload_inventory platform_table policy_table_missing
|
|
23
|
+
provider_relink_required provider_session_data provider_sign_in_must_be_off provider_transient_data reference_column_excluded reference_column_missing
|
|
24
|
+
route_not_found row_columns row_inventory row_not_null row_order row_part_too_large row_type rowid_inventory rows_missing
|
|
25
|
+
schema_checksum schema_mismatch secret_inventory secret_table_missing staged_table staged_validation table_inventory target_not_empty
|
|
26
|
+
transfer_sql_limit unauthorized unsafe_integer unsupported_column unsupported_copy_rowid unsupported_restore_width unsupported_rowid
|
|
27
|
+
upgrade_row_must_be_pure virtual_table wrong_product
|
|
28
|
+
unauthenticated forbidden owner_required authority_changed step_up_required rate_limited operation_failed
|
|
29
|
+
`.trim().split(/\s+/));
|
|
30
|
+
const publicCode = code => typeof code === 'string' && publicCodes.has(code.split(':')[0]) ? code.split(':')[0] : 'operation_failed';
|
|
31
|
+
|
|
32
|
+
// Public errors use the existing {error:code} vocabulary. Native messages can
|
|
33
|
+
// contain SQL, row values or credentials; inspect causes only to choose status.
|
|
34
|
+
export function httpFailure(error,{actor=null}={}) {
|
|
35
|
+
const seen = new Set();let typed,nativeStatus;
|
|
36
|
+
for (let cause = error; cause && !seen.has(cause) && seen.size < 5; cause = cause.cause) {
|
|
37
|
+
seen.add(cause);
|
|
38
|
+
if (Number.isInteger(cause.status) && cause.status >= 400 && cause.status <= 599) {
|
|
39
|
+
const code = publicCode(cause.code);
|
|
40
|
+
// A generic server wrapper must not turn a known authority refusal into 500.
|
|
41
|
+
if (!typed || typed.status >= 500 && cause.status < 500) typed = { status: cause.status, body: { error: code,
|
|
42
|
+
...(cause.status === 409 && cause.code === 'import_running' ? importRunningMetadata(cause,actor) : {}) } };
|
|
43
|
+
}
|
|
44
|
+
if (cause.name === 'TimeoutError' || ['ETIMEDOUT', 'UND_ERR_CONNECT_TIMEOUT', 'UND_ERR_HEADERS_TIMEOUT'].includes(cause.code))
|
|
45
|
+
nativeStatus = 504;
|
|
46
|
+
if (['ECONNRESET', 'ECONNREFUSED', 'SQLITE_BUSY', 'SQLITE_LOCKED'].includes(cause.code)
|
|
47
|
+
|| /\bD1_ERROR\b/i.test(cause.message ?? '') && /overloaded|temporarily unavailable|reset|timed? out|timeout|SQLITE_BUSY|SQLITE_LOCKED|database is locked/i.test(cause.message))
|
|
48
|
+
nativeStatus ??= 503;
|
|
49
|
+
}
|
|
50
|
+
return typed ?? { status: nativeStatus ?? 500, body: { error: 'operation_failed' } };
|
|
51
|
+
}
|